Status: FAILED
Triggered by: -
Duration: 4 hours
Actions:

Definition (rev ed331b29843b850a4bc37ad64e7e3a240af6f486)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-arm-bazel-sanitizers-tsan"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "7d40d0bd85d44f40f6591a288d39cdc667c43c6e"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/7d40d0bd85d44f40f6591a288d39cdc667c43c6e
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[18:32:16.774] New invocation is queued and will start shortly
[18:32:17.990] Starting the invocation (attempt 1)
[18:32:18.015] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/7d40d0bd85d44f40f6591a288d39cdc667c43c6e"
[18:32:18.015] Popped gitiles commit info from properties and tags
[18:32:18.015] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[18:32:18.015] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[18:32:18.015] Buildbucket request:
{
  "requestId": "8860732998756251680",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8860732998756251680",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "7ae69963717795820a26eea99099767c9a2501a2"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@7ae69963717795820a26eea99099767c9a2501a2",
                        "title": "7ae69963717795820a26eea99099767c9a2501a2",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/7ae69963717795820a26eea99099767c9a2501a2"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "7d40d0bd85d44f40f6591a288d39cdc667c43c6e"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@7d40d0bd85d44f40f6591a288d39cdc667c43c6e",
                        "title": "7d40d0bd85d44f40f6591a288d39cdc667c43c6e",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/7d40d0bd85d44f40f6591a288d39cdc667c43c6e"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "7d40d0bd85d44f40f6591a288d39cdc667c43c6e",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8860732998756251680"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[18:32:18.385] Scheduled build:
{
  "id": "8695302936731920193",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-12-16T18:32:18.074610571Z",
  "updateTime": "2025-12-16T18:32:18.074610571Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "7d40d0bd85d44f40f6591a288d39cdc667c43c6e",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[18:32:18.385] Task URL: https://cr-buildbucket.appspot.com/build/8695302936731920193
[18:32:18.385] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:2:0) after 5m23s
[18:32:20.330] Received PubSub notification, asking Buildbucket for the build status
[18:32:20.369] Build status: SCHEDULED
[18:37:41.399] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:2:0)
[18:37:41.415] Build status: SCHEDULED
[18:37:41.415] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:4:0) after 1m32s
[18:39:13.452] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:4:0)
[18:39:13.468] Build status: SCHEDULED
[18:39:13.468] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:5:0) after 4m44s
[18:43:57.531] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:5:0)
[18:43:57.564] Build status: SCHEDULED
[18:43:57.564] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:6:0) after 3m30s
[18:47:27.603] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:6:0)
[18:47:27.627] Build status: SCHEDULED
[18:47:27.627] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:7:0) after 1m10s
[18:48:37.605] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:7:0)
[18:48:37.627] Build status: SCHEDULED
[18:48:37.627] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:8:0) after 2m50s
[18:51:27.648] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:8:0)
[18:51:27.666] Build status: SCHEDULED
[18:51:27.666] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:9:0) after 1m55s
[18:53:22.679] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:9:0)
[18:53:22.693] Build status: SCHEDULED
[18:53:22.693] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:10:0) after 9m4s
[19:02:26.670] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:10:0)
[19:02:26.689] Build status: SCHEDULED
[19:02:26.689] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:11:0) after 9m6s
[19:11:32.703] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:11:0)
[19:11:32.721] Build status: SCHEDULED
[19:11:32.721] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:12:0) after 8m36s
[19:20:08.880] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:12:0)
[19:20:08.961] Build status: SCHEDULED
[19:20:08.961] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:13:0) after 4m37s
[19:24:46.065] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:13:0)
[19:24:46.083] Build status: SCHEDULED
[19:24:46.083] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:14:0) after 8m26s
[19:33:12.143] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:14:0)
[19:33:12.156] Build status: SCHEDULED
[19:33:12.156] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:15:0) after 5m5s
[19:38:17.170] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:15:0)
[19:38:17.184] Build status: SCHEDULED
[19:38:17.184] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:16:0) after 3m12s
[19:41:29.196] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:16:0)
[19:41:29.215] Build status: SCHEDULED
[19:41:29.215] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:17:0) after 8m58s
[19:50:27.294] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:17:0)
[19:50:27.310] Build status: SCHEDULED
[19:50:27.310] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:18:0) after 1m6s
[19:51:33.321] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:18:0)
[19:51:33.351] Build status: SCHEDULED
[19:51:33.351] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:19:0) after 5m56s
[19:57:29.364] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:19:0)
[19:57:29.379] Build status: SCHEDULED
[19:57:29.380] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:20:0) after 7m49s
[20:05:18.394] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:20:0)
[20:05:18.409] Build status: SCHEDULED
[20:05:18.409] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:21:0) after 4m17s
[20:09:35.423] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:21:0)
[20:09:35.467] Build status: SCHEDULED
[20:09:35.467] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:22:0) after 3m56s
[20:13:31.547] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:22:0)
[20:13:31.624] Build status: SCHEDULED
[20:13:31.624] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:23:0) after 6m34s
[20:20:05.686] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:23:0)
[20:20:05.756] Build status: SCHEDULED
[20:20:05.756] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:24:0) after 3m29s
[20:23:34.858] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:24:0)
[20:23:34.879] Build status: SCHEDULED
[20:23:34.879] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:25:0) after 4m6s
[20:27:40.892] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:25:0)
[20:27:40.913] Build status: SCHEDULED
[20:27:40.913] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:26:0) after 4m41s
[20:32:21.967] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:26:0)
[20:32:22.016] Build status: SCHEDULED
[20:32:22.016] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:27:0) after 7m44s
[20:40:06.101] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:27:0)
[20:40:06.118] Build status: SCHEDULED
[20:40:06.118] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:28:0) after 7m23s
[20:47:29.505] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:28:0)
[20:47:29.530] Build status: SCHEDULED
[20:47:29.530] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:29:0) after 5m33s
[20:53:02.528] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:29:0)
[20:53:02.552] Build status: SCHEDULED
[20:53:02.552] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:30:0) after 6m45s
[20:59:47.655] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:30:0)
[20:59:47.678] Build status: SCHEDULED
[20:59:47.678] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:31:0) after 4m58s
[21:04:45.760] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:31:0)
[21:04:45.786] Build status: SCHEDULED
[21:04:45.786] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:32:0) after 7m24s
[21:12:09.910] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:32:0)
[21:12:09.949] Build status: SCHEDULED
[21:12:09.949] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:33:0) after 2m29s
[21:14:38.953] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:33:0)
[21:14:38.974] Build status: SCHEDULED
[21:14:38.974] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:34:0) after 8m32s
[21:23:11.037] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:34:0)
[21:23:11.055] Build status: SCHEDULED
[21:23:11.055] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:35:0) after 8m7s
[21:31:18.141] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:35:0)
[21:31:18.169] Build status: SCHEDULED
[21:31:18.169] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:36:0) after 6m12s
[21:37:30.186] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:36:0)
[21:37:30.204] Build status: SCHEDULED
[21:37:30.204] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:37:0) after 4m53s
[21:42:23.242] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:37:0)
[21:42:23.261] Build status: SCHEDULED
[21:42:23.261] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:38:0) after 8m0s
[21:50:23.411] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:38:0)
[21:50:23.454] Build status: SCHEDULED
[21:50:23.454] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:39:0) after 6m21s
[21:56:44.560] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:39:0)
[21:56:44.669] Build status: SCHEDULED
[21:56:44.669] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:40:0) after 8m28s
[22:05:12.684] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:40:0)
[22:05:12.706] Build status: SCHEDULED
[22:05:12.706] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:41:0) after 7m33s
[22:12:45.826] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:41:0)
[22:12:45.866] Build status: SCHEDULED
[22:12:45.866] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:42:0) after 7m16s
[22:20:02.086] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:42:0)
[22:20:02.107] Build status: SCHEDULED
[22:20:02.107] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:43:0) after 3m47s
[22:20:32.134] Received PubSub notification, asking Buildbucket for the build status
[22:20:32.155] Build status: STARTED
[22:23:49.157] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:43:0)
[22:23:49.167] Build status: STARTED
[22:23:49.167] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:45:0) after 2m17s
[22:26:06.309] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:45:0)
[22:26:06.327] Build status: STARTED
[22:26:06.327] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:46:0) after 3m42s
[22:29:48.458] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:46:0)
[22:29:48.481] Build status: STARTED
[22:29:48.481] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:47:0) after 1m24s
[22:31:12.498] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:47:0)
[22:31:12.551] Build status: STARTED
[22:31:12.551] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:48:0) after 9m59s
[22:41:11.625] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:48:0)
[22:41:11.648] Build status: STARTED
[22:41:11.648] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:49:0) after 8m19s
[22:49:30.710] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:49:0)
[22:49:30.731] Build status: STARTED
[22:49:30.731] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8860732998756251680:50:0) after 4m24s
[22:51:13.544] Received PubSub notification, asking Buildbucket for the build status
[22:51:13.560] Build:
{
  "id": "8695302936731920193",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-12-16T18:32:18.074610571Z",
  "startTime": "2025-12-16T22:20:31.382612124Z",
  "endTime": "2025-12-16T22:51:13.324123053Z",
  "updateTime": "2025-12-16T22:51:13.324123053Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "7d40d0bd85d44f40f6591a288d39cdc667c43c6e",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[22:51:13.560] Invocation finished in 4h18m56.799823957s with status FAILED