buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-arm-bazel-sanitizers-tsan"
>

• 9f5f893e1f0c86a22fda6f7619faa8f486cab1db (2 weeks ago)

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "9f5f893e1f0c86a22fda6f7619faa8f486cab1db"
}

• buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/9f5f893e1f0c86a22fda6f7619faa8f486cab1db
• gitiles_ref:refs/heads/main

[18:43:52.498] New invocation is queued and will start shortly
[18:43:53.572] Starting the invocation (attempt 1)
[18:43:53.606] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/9f5f893e1f0c86a22fda6f7619faa8f486cab1db"
[18:43:53.606] Popped gitiles commit info from properties and tags
[18:43:53.606] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[18:43:53.607] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[18:43:53.608] Buildbucket request:
{
  "requestId": "8866258684181850032",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8866258684181850032",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "9f5f893e1f0c86a22fda6f7619faa8f486cab1db"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@9f5f893e1f0c86a22fda6f7619faa8f486cab1db",
                        "title": "9f5f893e1f0c86a22fda6f7619faa8f486cab1db",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/9f5f893e1f0c86a22fda6f7619faa8f486cab1db"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "9f5f893e1f0c86a22fda6f7619faa8f486cab1db",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8866258684181850032"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[18:43:54.036] Scheduled build:
{
  "id": "8700828622291761089",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-10-16T18:43:53.675989552Z",
  "updateTime": "2025-10-16T18:43:53.675989552Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "9f5f893e1f0c86a22fda6f7619faa8f486cab1db",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[18:43:54.036] Task URL: https://cr-buildbucket.appspot.com/build/8700828622291761089
[18:43:54.036] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:2:0) after 3m33s
[18:44:04.249] Received PubSub notification, asking Buildbucket for the build status
[18:44:04.267] Build status: SCHEDULED
[18:47:27.117] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:2:0)
[18:47:27.132] Build status: SCHEDULED
[18:47:27.132] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:4:0) after 2m39s
[18:50:06.264] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:4:0)
[18:50:06.305] Build status: SCHEDULED
[18:50:06.305] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:5:0) after 6m7s
[18:56:13.421] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:5:0)
[18:56:13.437] Build status: SCHEDULED
[18:56:13.437] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:6:0) after 1m8s
[18:57:21.452] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:6:0)
[18:57:21.468] Build status: SCHEDULED
[18:57:21.468] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:7:0) after 2m16s
[18:59:37.513] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:7:0)
[18:59:37.579] Build status: SCHEDULED
[18:59:37.579] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:8:0) after 3m15s
[19:02:52.564] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:8:0)
[19:02:52.593] Build status: SCHEDULED
[19:02:52.593] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:9:0) after 1m6s
[19:03:58.608] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:9:0)
[19:03:58.630] Build status: SCHEDULED
[19:03:58.630] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:10:0) after 2m19s
[19:06:17.650] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:10:0)
[19:06:17.694] Build status: SCHEDULED
[19:06:17.694] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:11:0) after 9m18s
[19:15:35.708] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:11:0)
[19:15:35.879] Build status: SCHEDULED
[19:15:35.879] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:12:0) after 4m9s
[19:19:44.898] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:12:0)
[19:19:44.917] Build status: SCHEDULED
[19:19:44.917] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:13:0) after 2m31s
[19:22:15.961] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:13:0)
[19:22:15.982] Build status: SCHEDULED
[19:22:15.982] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:14:0) after 7m58s
[19:30:14.004] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:14:0)
[19:30:14.120] Build status: SCHEDULED
[19:30:14.120] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:15:0) after 4m21s
[19:34:35.248] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:15:0)
[19:34:35.283] Build status: SCHEDULED
[19:34:35.283] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:16:0) after 5m17s
[19:39:52.297] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:16:0)
[19:39:52.314] Build status: SCHEDULED
[19:39:52.314] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:17:0) after 5m37s
[19:45:29.330] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:17:0)
[19:45:29.353] Build status: SCHEDULED
[19:45:29.353] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:18:0) after 8m46s
[19:54:15.367] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:18:0)
[19:54:15.398] Build status: SCHEDULED
[19:54:15.398] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:19:0) after 3m12s
[19:57:27.422] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:19:0)
[19:57:27.444] Build status: SCHEDULED
[19:57:27.444] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:20:0) after 4m55s
[20:02:22.458] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:20:0)
[20:02:22.484] Build status: SCHEDULED
[20:02:22.484] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:21:0) after 1m12s
[20:03:34.552] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:21:0)
[20:03:34.570] Build status: SCHEDULED
[20:03:34.570] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:22:0) after 9m21s
[20:12:55.604] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:22:0)
[20:12:55.625] Build status: SCHEDULED
[20:12:55.625] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:23:0) after 4m37s
[20:17:32.658] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:23:0)
[20:17:32.679] Build status: SCHEDULED
[20:17:32.679] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:24:0) after 6m2s
[20:23:34.692] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:24:0)
[20:23:34.707] Build status: SCHEDULED
[20:23:34.707] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:25:0) after 2m14s
[20:25:48.762] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:25:0)
[20:25:48.782] Build status: SCHEDULED
[20:25:48.782] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:26:0) after 2m2s
[20:27:50.946] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:26:0)
[20:27:50.966] Build status: SCHEDULED
[20:27:50.966] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:27:0) after 7m48s
[20:35:38.973] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:27:0)
[20:35:38.991] Build status: SCHEDULED
[20:35:38.991] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:28:0) after 9m29s
[20:45:07.967] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:28:0)
[20:45:07.997] Build status: SCHEDULED
[20:45:07.997] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:29:0) after 1m2s
[20:46:09.985] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:29:0)
[20:46:10.028] Build status: SCHEDULED
[20:46:10.028] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:30:0) after 6m56s
[20:53:06.039] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:30:0)
[20:53:06.074] Build status: SCHEDULED
[20:53:06.074] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:31:0) after 8m42s
[21:01:48.089] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:31:0)
[21:01:48.115] Build status: SCHEDULED
[21:01:48.115] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:32:0) after 5m29s
[21:07:17.132] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:32:0)
[21:07:17.163] Build status: SCHEDULED
[21:07:17.163] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:33:0) after 7m14s
[21:14:31.223] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:33:0)
[21:14:31.245] Build status: SCHEDULED
[21:14:31.245] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:34:0) after 5m21s
[21:19:52.310] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:34:0)
[21:19:52.327] Build status: SCHEDULED
[21:19:52.327] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:35:0) after 1m32s
[21:21:24.349] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:35:0)
[21:21:24.427] Build status: SCHEDULED
[21:21:24.427] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:36:0) after 9m30s
[21:30:54.510] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:36:0)
[21:30:54.680] Build status: SCHEDULED
[21:30:54.680] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:37:0) after 3m38s
[21:34:32.825] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:37:0)
[21:34:32.844] Build status: SCHEDULED
[21:34:32.844] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:38:0) after 1m44s
[21:36:16.881] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:38:0)
[21:36:16.908] Build status: SCHEDULED
[21:36:16.908] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8866258684181850032:39:0) after 6m27s
[21:37:47.823] Received PubSub notification, asking Buildbucket for the build status
[21:37:47.842] Build status: STARTED
[21:42:13.308] Received PubSub notification, asking Buildbucket for the build status
[21:42:13.327] Build:
{
  "id": "8700828622291761089",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-10-16T18:43:53.675989552Z",
  "startTime": "2025-10-16T21:37:34.374365793Z",
  "endTime": "2025-10-16T21:42:12.992060904Z",
  "updateTime": "2025-10-16T21:42:12.992060904Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "9f5f893e1f0c86a22fda6f7619faa8f486cab1db",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[21:42:13.327] Invocation finished in 2h58m20.837807386s with status FAILED