Status: FAILED
Triggered by: -
Duration: 8 hours
Actions:

Definition (rev d2b949be221fc8dd3acbfdb77921a85f7b1b8e9d)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-x86-bazel-san-tsan"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "9adb198099f30a18e219fb8b16990c375a4b7c84"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/9adb198099f30a18e219fb8b16990c375a4b7c84
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[00:00:44.654] New invocation is queued and will start shortly
[00:00:45.907] Starting the invocation (attempt 1)
[00:00:45.943] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/9adb198099f30a18e219fb8b16990c375a4b7c84"
[00:00:45.943] Popped gitiles commit info from properties and tags
[00:00:45.943] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[00:00:45.943] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[00:00:45.943] Buildbucket request:
{
  "requestId": "8875117251207465072",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-tsan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8875117251207465072",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "31964ac812fe8b306ca23e4b4db669e9e91b55c4"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@31964ac812fe8b306ca23e4b4db669e9e91b55c4",
                        "title": "31964ac812fe8b306ca23e4b4db669e9e91b55c4",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/31964ac812fe8b306ca23e4b4db669e9e91b55c4"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "60a4cc54044d92647890d5318f5a475d2f26369c"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@60a4cc54044d92647890d5318f5a475d2f26369c",
                        "title": "60a4cc54044d92647890d5318f5a475d2f26369c",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/60a4cc54044d92647890d5318f5a475d2f26369c"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "04aeff8fd3ca62306e3422e43e0bf45a43323d32"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@04aeff8fd3ca62306e3422e43e0bf45a43323d32",
                        "title": "04aeff8fd3ca62306e3422e43e0bf45a43323d32",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/04aeff8fd3ca62306e3422e43e0bf45a43323d32"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "cb5e7a324c13fd7bcc71646ad0c03386b52ea019"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@cb5e7a324c13fd7bcc71646ad0c03386b52ea019",
                        "title": "cb5e7a324c13fd7bcc71646ad0c03386b52ea019",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/cb5e7a324c13fd7bcc71646ad0c03386b52ea019"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "c91fc7c107f021fb967af806fcedc5abc335c2b0"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@c91fc7c107f021fb967af806fcedc5abc335c2b0",
                        "title": "c91fc7c107f021fb967af806fcedc5abc335c2b0",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/c91fc7c107f021fb967af806fcedc5abc335c2b0"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "8f705b4fd4e4712394aeab25dd619e31e7e8e768"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@8f705b4fd4e4712394aeab25dd619e31e7e8e768",
                        "title": "8f705b4fd4e4712394aeab25dd619e31e7e8e768",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/8f705b4fd4e4712394aeab25dd619e31e7e8e768"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "9adb198099f30a18e219fb8b16990c375a4b7c84"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@9adb198099f30a18e219fb8b16990c375a4b7c84",
                        "title": "9adb198099f30a18e219fb8b16990c375a4b7c84",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/9adb198099f30a18e219fb8b16990c375a4b7c84"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "9adb198099f30a18e219fb8b16990c375a4b7c84",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8875117251207465072"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[00:00:46.416] Scheduled build:
{
  "id": "8709687189108268481",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-07-11T00:00:46.022252863Z",
  "updateTime": "2025-07-11T00:00:46.022252863Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "9adb198099f30a18e219fb8b16990c375a4b7c84",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[00:00:46.416] Task URL: https://cr-buildbucket.appspot.com/build/8709687189108268481
[00:00:46.416] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:2:0) after 3m48s
[00:00:49.477] Received PubSub notification, asking Buildbucket for the build status
[00:00:49.509] Build status: SCHEDULED
[00:04:34.551] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:2:0)
[00:04:34.625] Build status: SCHEDULED
[00:04:34.625] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:4:0) after 6m30s
[00:11:04.631] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:4:0)
[00:11:04.656] Build status: SCHEDULED
[00:11:04.656] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:5:0) after 6m50s
[00:17:54.672] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:5:0)
[00:17:54.694] Build status: SCHEDULED
[00:17:54.694] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:6:0) after 7m25s
[00:25:19.743] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:6:0)
[00:25:19.778] Build status: SCHEDULED
[00:25:19.778] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:7:0) after 9m41s
[00:35:00.834] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:7:0)
[00:35:00.850] Build status: SCHEDULED
[00:35:00.850] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:8:0) after 5m20s
[00:40:21.016] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:8:0)
[00:40:21.047] Build status: SCHEDULED
[00:40:21.047] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:9:0) after 3m45s
[00:44:06.045] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:9:0)
[00:44:06.088] Build status: SCHEDULED
[00:44:06.088] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:10:0) after 9m57s
[00:54:03.157] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:10:0)
[00:54:03.175] Build status: SCHEDULED
[00:54:03.176] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:11:0) after 8m0s
[01:02:03.190] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:11:0)
[01:02:03.220] Build status: SCHEDULED
[01:02:03.220] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:12:0) after 2m2s
[01:04:05.295] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:12:0)
[01:04:05.313] Build status: SCHEDULED
[01:04:05.313] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:13:0) after 7m56s
[01:12:01.334] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:13:0)
[01:12:01.352] Build status: SCHEDULED
[01:12:01.352] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:14:0) after 5m45s
[01:17:46.437] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:14:0)
[01:17:46.459] Build status: SCHEDULED
[01:17:46.459] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:15:0) after 6m35s
[01:24:21.459] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:15:0)
[01:24:21.498] Build status: SCHEDULED
[01:24:21.498] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:16:0) after 2m10s
[01:26:31.569] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:16:0)
[01:26:31.770] Build status: SCHEDULED
[01:26:31.770] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:17:0) after 2m36s
[01:29:07.800] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:17:0)
[01:29:07.825] Build status: SCHEDULED
[01:29:07.825] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:18:0) after 6m36s
[01:35:43.943] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:18:0)
[01:35:44.018] Build status: SCHEDULED
[01:35:44.018] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:19:0) after 2m48s
[01:38:32.041] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:19:0)
[01:38:32.059] Build status: SCHEDULED
[01:38:32.060] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:20:0) after 1m18s
[01:39:50.075] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:20:0)
[01:39:50.093] Build status: SCHEDULED
[01:39:50.093] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:21:0) after 7m5s
[01:46:55.155] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:21:0)
[01:46:55.244] Build status: SCHEDULED
[01:46:55.244] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:22:0) after 1m38s
[01:48:33.413] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:22:0)
[01:48:33.499] Build status: SCHEDULED
[01:48:33.499] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:23:0) after 7m0s
[01:55:33.550] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:23:0)
[01:55:33.636] Build status: SCHEDULED
[01:55:33.636] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:24:0) after 1m51s
[01:57:24.655] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:24:0)
[01:57:24.679] Build status: SCHEDULED
[01:57:24.679] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:25:0) after 8m22s
[02:05:46.751] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:25:0)
[02:05:46.772] Build status: SCHEDULED
[02:05:46.772] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:26:0) after 3m43s
[02:09:29.788] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:26:0)
[02:09:29.807] Build status: SCHEDULED
[02:09:29.807] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:27:0) after 9m25s
[02:18:54.887] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:27:0)
[02:18:54.928] Build status: SCHEDULED
[02:18:54.928] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:28:0) after 5m28s
[02:24:22.944] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:28:0)
[02:24:23.010] Build status: SCHEDULED
[02:24:23.010] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:29:0) after 1m56s
[02:26:19.032] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:29:0)
[02:26:19.053] Build status: SCHEDULED
[02:26:19.053] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:30:0) after 5m42s
[02:32:01.088] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:30:0)
[02:32:01.138] Build status: SCHEDULED
[02:32:01.138] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:31:0) after 7m29s
[02:39:30.238] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:31:0)
[02:39:30.279] Build status: SCHEDULED
[02:39:30.279] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:32:0) after 3m38s
[02:43:08.283] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:32:0)
[02:43:08.300] Build status: SCHEDULED
[02:43:08.300] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:33:0) after 4m46s
[02:47:54.411] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:33:0)
[02:47:54.445] Build status: SCHEDULED
[02:47:54.445] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:34:0) after 5m6s
[02:53:00.514] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:34:0)
[02:53:00.535] Build status: SCHEDULED
[02:53:00.535] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:35:0) after 1m0s
[02:54:00.561] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:35:0)
[02:54:00.685] Build status: SCHEDULED
[02:54:00.685] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:36:0) after 6m49s
[03:00:49.711] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:36:0)
[03:00:49.766] Build status: SCHEDULED
[03:00:49.766] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:37:0) after 2m5s
[03:02:54.870] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:37:0)
[03:02:54.888] Build status: SCHEDULED
[03:02:54.888] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:38:0) after 5m52s
[03:08:46.944] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:38:0)
[03:08:46.965] Build status: SCHEDULED
[03:08:46.965] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:39:0) after 4m59s
[03:13:45.981] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:39:0)
[03:13:46.031] Build status: SCHEDULED
[03:13:46.031] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:40:0) after 8m57s
[03:22:43.068] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:40:0)
[03:22:43.091] Build status: SCHEDULED
[03:22:43.091] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:41:0) after 1m15s
[03:23:58.074] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:41:0)
[03:23:58.092] Build status: SCHEDULED
[03:23:58.092] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:42:0) after 2m52s
[03:26:50.117] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:42:0)
[03:26:50.136] Build status: SCHEDULED
[03:26:50.136] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:43:0) after 2m41s
[03:29:31.212] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:43:0)
[03:29:31.320] Build status: SCHEDULED
[03:29:31.320] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:44:0) after 4m1s
[03:33:32.298] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:44:0)
[03:33:32.316] Build status: SCHEDULED
[03:33:32.316] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:45:0) after 2m16s
[03:35:48.376] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:45:0)
[03:35:48.403] Build status: SCHEDULED
[03:35:48.403] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:46:0) after 1m48s
[03:37:36.476] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:46:0)
[03:37:36.498] Build status: SCHEDULED
[03:37:36.498] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:47:0) after 8m5s
[03:45:41.618] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:47:0)
[03:45:41.688] Build status: SCHEDULED
[03:45:41.688] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:48:0) after 9m32s
[03:55:13.761] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:48:0)
[03:55:13.826] Build status: SCHEDULED
[03:55:13.826] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:49:0) after 1m16s
[03:56:29.852] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:49:0)
[03:56:29.913] Build status: SCHEDULED
[03:56:29.913] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:50:0) after 3m9s
[03:59:38.970] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:50:0)
[03:59:38.989] Build status: SCHEDULED
[03:59:38.989] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:51:0) after 7m53s
[04:07:32.159] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:51:0)
[04:07:32.194] Build status: SCHEDULED
[04:07:32.194] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:52:0) after 8m34s
[04:16:06.231] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:52:0)
[04:16:06.248] Build status: SCHEDULED
[04:16:06.248] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:53:0) after 1m24s
[04:17:30.383] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:53:0)
[04:17:30.423] Build status: SCHEDULED
[04:17:30.423] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:54:0) after 2m53s
[04:20:23.490] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:54:0)
[04:20:23.523] Build status: SCHEDULED
[04:20:23.523] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:55:0) after 8m1s
[04:28:24.512] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:55:0)
[04:28:24.543] Build status: SCHEDULED
[04:28:24.543] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:56:0) after 7m40s
[04:36:04.568] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:56:0)
[04:36:04.590] Build status: SCHEDULED
[04:36:04.590] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:57:0) after 9m41s
[04:45:45.670] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:57:0)
[04:45:45.689] Build status: SCHEDULED
[04:45:45.689] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:58:0) after 7m10s
[04:52:55.711] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:58:0)
[04:52:55.762] Build status: SCHEDULED
[04:52:55.762] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:59:0) after 8m44s
[05:01:39.803] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:59:0)
[05:01:39.826] Build status: SCHEDULED
[05:01:39.826] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:60:0) after 7m18s
[05:08:57.901] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:60:0)
[05:08:57.921] Build status: SCHEDULED
[05:08:57.921] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:61:0) after 4m28s
[05:13:25.939] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:61:0)
[05:13:25.977] Build status: SCHEDULED
[05:13:25.978] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:62:0) after 4m9s
[05:17:35.025] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:62:0)
[05:17:35.047] Build status: SCHEDULED
[05:17:35.047] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:63:0) after 2m49s
[05:20:24.071] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:63:0)
[05:20:24.090] Build status: SCHEDULED
[05:20:24.090] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:64:0) after 4m36s
[05:25:00.115] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:64:0)
[05:25:00.158] Build status: SCHEDULED
[05:25:00.158] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:65:0) after 3m23s
[05:28:23.174] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:65:0)
[05:28:23.193] Build status: SCHEDULED
[05:28:23.193] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:66:0) after 7m34s
[05:35:57.264] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:66:0)
[05:35:57.283] Build status: SCHEDULED
[05:35:57.283] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:67:0) after 4m8s
[05:40:05.298] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:67:0)
[05:40:05.319] Build status: SCHEDULED
[05:40:05.319] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:68:0) after 9m52s
[05:49:57.383] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:68:0)
[05:49:57.400] Build status: SCHEDULED
[05:49:57.400] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:69:0) after 4m13s
[05:54:10.424] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:69:0)
[05:54:10.443] Build status: SCHEDULED
[05:54:10.443] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:70:0) after 9m16s
[06:03:26.474] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:70:0)
[06:03:26.522] Build status: SCHEDULED
[06:03:26.523] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:71:0) after 7m29s
[06:10:55.546] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:71:0)
[06:10:55.579] Build status: SCHEDULED
[06:10:55.579] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:72:0) after 3m40s
[06:14:35.601] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:72:0)
[06:14:35.626] Build status: SCHEDULED
[06:14:35.626] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:73:0) after 6m19s
[06:20:54.693] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:73:0)
[06:20:54.739] Build status: SCHEDULED
[06:20:54.739] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:74:0) after 7m8s
[06:28:02.824] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:74:0)
[06:28:02.842] Build status: SCHEDULED
[06:28:02.842] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:75:0) after 9m34s
[06:37:36.870] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:75:0)
[06:37:36.885] Build status: SCHEDULED
[06:37:36.885] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:76:0) after 4m38s
[06:42:14.914] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:76:0)
[06:42:14.940] Build status: SCHEDULED
[06:42:14.940] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:77:0) after 1m9s
[06:43:23.969] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:77:0)
[06:43:24.016] Build status: SCHEDULED
[06:43:24.016] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:78:0) after 9m5s
[06:52:29.068] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:78:0)
[06:52:29.088] Build status: SCHEDULED
[06:52:29.088] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:79:0) after 1m50s
[06:54:19.103] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:79:0)
[06:54:19.119] Build status: SCHEDULED
[06:54:19.119] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:80:0) after 2m58s
[06:57:17.332] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:80:0)
[06:57:17.352] Build status: SCHEDULED
[06:57:17.352] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:81:0) after 5m27s
[07:02:44.384] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:81:0)
[07:02:44.413] Build status: SCHEDULED
[07:02:44.413] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:82:0) after 1m23s
[07:04:07.453] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:82:0)
[07:04:07.484] Build status: SCHEDULED
[07:04:07.484] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:83:0) after 6m30s
[07:10:37.508] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:83:0)
[07:10:37.525] Build status: SCHEDULED
[07:10:37.525] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:84:0) after 5m57s
[07:16:34.593] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:84:0)
[07:16:34.613] Build status: SCHEDULED
[07:16:34.613] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:85:0) after 5m47s
[07:22:21.639] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:85:0)
[07:22:21.688] Build status: SCHEDULED
[07:22:21.688] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:86:0) after 2m5s
[07:24:26.758] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:86:0)
[07:24:26.779] Build status: SCHEDULED
[07:24:26.779] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:87:0) after 6m3s
[07:30:29.837] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:87:0)
[07:30:29.907] Build status: SCHEDULED
[07:30:29.907] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:88:0) after 2m6s
[07:32:35.928] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:88:0)
[07:32:35.949] Build status: SCHEDULED
[07:32:35.949] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:89:0) after 5m28s
[07:38:03.984] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:89:0)
[07:38:04.046] Build status: SCHEDULED
[07:38:04.046] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:90:0) after 2m15s
[07:40:19.180] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:90:0)
[07:40:19.207] Build status: SCHEDULED
[07:40:19.207] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:91:0) after 7m24s
[07:47:43.258] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:91:0)
[07:47:43.351] Build status: SCHEDULED
[07:47:43.351] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:92:0) after 1m40s
[07:49:23.359] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:92:0)
[07:49:23.387] Build status: SCHEDULED
[07:49:23.387] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:93:0) after 2m50s
[07:52:13.512] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:93:0)
[07:52:13.535] Build status: SCHEDULED
[07:52:13.535] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:94:0) after 1m11s
[07:53:24.611] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:94:0)
[07:53:24.634] Build status: SCHEDULED
[07:53:24.634] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:95:0) after 3m14s
[07:56:38.601] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:95:0)
[07:56:38.651] Build status: SCHEDULED
[07:56:38.651] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875117251207465072:96:0) after 6m31s
[08:00:52.468] Received PubSub notification, asking Buildbucket for the build status
[08:00:52.485] Build:
{
  "id": "8709687189108268481",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-07-11T00:00:46.022252863Z",
  "endTime": "2025-07-11T08:00:50.784192969Z",
  "updateTime": "2025-07-11T08:00:50.784192969Z",
  "status": "INFRA_FAILURE",
  "statusDetails": {
    "resourceExhaustion": {

    },
    "timeout": {

    }
  },
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "9adb198099f30a18e219fb8b16990c375a4b7c84",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[08:00:52.485] Invocation finished in 8h0m7.848155149s with status FAILED