1. All projects
  2. pigweed
  3. pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan
  4. 8872553960699786448
Status: FAILED
Triggered by: -
Duration: 8 hours
Actions:

Definition (rev d2b949be221fc8dd3acbfdb77921a85f7b1b8e9d) 

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-x86-bazel-san-asan"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "2b23689560196ea206283712f65093f3d48ab148"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/2b23689560196ea206283712f65093f3d48ab148
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[07:03:08.975] New invocation is queued and will start shortly
[07:03:10.142] Starting the invocation (attempt 1)
[07:03:10.171] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/2b23689560196ea206283712f65093f3d48ab148"
[07:03:10.171] Popped gitiles commit info from properties and tags
[07:03:10.171] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[07:03:10.172] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[07:03:10.172] Buildbucket request:
{
  "requestId": "8872553960699786448",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-asan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8872553960699786448",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "9b9b629a122fd9ea9ce26c1819f7eed811fa8b60"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@9b9b629a122fd9ea9ce26c1819f7eed811fa8b60",
                        "title": "9b9b629a122fd9ea9ce26c1819f7eed811fa8b60",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/9b9b629a122fd9ea9ce26c1819f7eed811fa8b60"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "2b23689560196ea206283712f65093f3d48ab148"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@2b23689560196ea206283712f65093f3d48ab148",
                        "title": "2b23689560196ea206283712f65093f3d48ab148",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/2b23689560196ea206283712f65093f3d48ab148"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "2b23689560196ea206283712f65093f3d48ab148",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8872553960699786448"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[07:03:10.536] Scheduled build:
{
  "id": "8707123898727471185",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-asan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-08-08T07:03:10.224289147Z",
  "updateTime": "2025-08-08T07:03:10.224289147Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "2b23689560196ea206283712f65093f3d48ab148",
      "ref": "refs/heads/main"
    }
  }
}
[07:03:10.536] Task URL: https://cr-buildbucket.appspot.com/build/8707123898727471185
[07:03:10.536] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:2:0) after 1m36s
[07:03:13.568] Received PubSub notification, asking Buildbucket for the build status
[07:03:13.679] Build status: SCHEDULED
[07:04:46.550] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:2:0)
[07:04:46.572] Build status: SCHEDULED
[07:04:46.572] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:4:0) after 7m49s
[07:12:35.767] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:4:0)
[07:12:35.793] Build status: SCHEDULED
[07:12:35.793] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:5:0) after 8m19s
[07:20:54.860] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:5:0)
[07:20:54.886] Build status: SCHEDULED
[07:20:54.886] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:6:0) after 8m58s
[07:29:53.937] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:6:0)
[07:29:53.963] Build status: SCHEDULED
[07:29:53.963] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:7:0) after 5m59s
[07:35:52.973] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:7:0)
[07:35:53.321] Build status: SCHEDULED
[07:35:53.321] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:8:0) after 2m28s
[07:38:21.374] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:8:0)
[07:38:21.392] Build status: SCHEDULED
[07:38:21.392] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:9:0) after 5m26s
[07:43:47.410] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:9:0)
[07:43:47.435] Build status: SCHEDULED
[07:43:47.435] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:10:0) after 6m35s
[07:50:22.455] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:10:0)
[07:50:22.517] Build status: SCHEDULED
[07:50:22.517] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:11:0) after 3m32s
[07:53:54.604] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:11:0)
[07:53:54.620] Build status: SCHEDULED
[07:53:54.620] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:12:0) after 5m8s
[07:59:02.784] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:12:0)
[07:59:02.805] Build status: SCHEDULED
[07:59:02.805] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:13:0) after 3m7s
[08:02:09.817] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:13:0)
[08:02:09.842] Build status: SCHEDULED
[08:02:09.842] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:14:0) after 9m6s
[08:11:15.859] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:14:0)
[08:11:15.878] Build status: SCHEDULED
[08:11:15.878] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:15:0) after 5m53s
[08:17:08.963] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:15:0)
[08:17:09.028] Build status: SCHEDULED
[08:17:09.028] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:16:0) after 7m4s
[08:24:13.089] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:16:0)
[08:24:13.130] Build status: SCHEDULED
[08:24:13.130] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:17:0) after 6m15s
[08:30:28.276] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:17:0)
[08:30:28.296] Build status: SCHEDULED
[08:30:28.296] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:18:0) after 3m1s
[08:33:29.369] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:18:0)
[08:33:29.387] Build status: SCHEDULED
[08:33:29.387] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:19:0) after 4m9s
[08:37:38.402] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:19:0)
[08:37:38.423] Build status: SCHEDULED
[08:37:38.423] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:20:0) after 7m55s
[08:45:33.421] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:20:0)
[08:45:33.445] Build status: SCHEDULED
[08:45:33.445] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:21:0) after 4m38s
[08:50:11.461] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:21:0)
[08:50:11.524] Build status: SCHEDULED
[08:50:11.524] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:22:0) after 3m30s
[08:53:41.633] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:22:0)
[08:53:41.650] Build status: SCHEDULED
[08:53:41.651] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:23:0) after 9m23s
[09:03:04.713] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:23:0)
[09:03:04.743] Build status: SCHEDULED
[09:03:04.744] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:24:0) after 3m33s
[09:06:37.758] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:24:0)
[09:06:37.778] Build status: SCHEDULED
[09:06:37.778] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:25:0) after 8m59s
[09:15:36.792] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:25:0)
[09:15:36.812] Build status: SCHEDULED
[09:15:36.812] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:26:0) after 6m27s
[09:22:03.831] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:26:0)
[09:22:03.854] Build status: SCHEDULED
[09:22:03.855] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:27:0) after 2m49s
[09:24:52.858] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:27:0)
[09:24:52.877] Build status: SCHEDULED
[09:24:52.877] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:28:0) after 8m48s
[09:33:40.893] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:28:0)
[09:33:40.925] Build status: SCHEDULED
[09:33:40.925] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:29:0) after 3m12s
[09:36:53.040] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:29:0)
[09:36:53.062] Build status: SCHEDULED
[09:36:53.062] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:30:0) after 1m6s
[09:37:59.182] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:30:0)
[09:37:59.218] Build status: SCHEDULED
[09:37:59.218] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:31:0) after 6m18s
[09:44:17.357] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:31:0)
[09:44:17.380] Build status: SCHEDULED
[09:44:17.380] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:32:0) after 1m27s
[09:45:44.475] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:32:0)
[09:45:44.494] Build status: SCHEDULED
[09:45:44.494] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:33:0) after 2m52s
[09:48:36.665] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:33:0)
[09:48:36.685] Build status: SCHEDULED
[09:48:36.685] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:34:0) after 9m59s
[09:58:35.745] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:34:0)
[09:58:35.763] Build status: SCHEDULED
[09:58:35.763] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:35:0) after 5m21s
[10:03:56.747] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:35:0)
[10:03:56.764] Build status: SCHEDULED
[10:03:56.764] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:36:0) after 2m49s
[10:06:45.746] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:36:0)
[10:06:45.788] Build status: SCHEDULED
[10:06:45.788] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:37:0) after 3m28s
[10:10:13.903] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:37:0)
[10:10:13.927] Build status: SCHEDULED
[10:10:13.927] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:38:0) after 9m32s
[10:19:46.004] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:38:0)
[10:19:46.021] Build status: SCHEDULED
[10:19:46.021] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:39:0) after 4m43s
[10:24:29.345] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:39:0)
[10:24:29.361] Build status: SCHEDULED
[10:24:29.361] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:40:0) after 9m15s
[10:33:44.839] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:40:0)
[10:33:44.855] Build status: SCHEDULED
[10:33:44.855] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:41:0) after 8m25s
[10:42:09.849] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:41:0)
[10:42:09.867] Build status: SCHEDULED
[10:42:09.867] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:42:0) after 4m38s
[10:46:47.960] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:42:0)
[10:46:47.979] Build status: SCHEDULED
[10:46:47.979] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:43:0) after 5m17s
[10:52:05.234] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:43:0)
[10:52:05.306] Build status: SCHEDULED
[10:52:05.306] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:44:0) after 3m17s
[10:55:22.773] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:44:0)
[10:55:22.809] Build status: SCHEDULED
[10:55:22.809] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:45:0) after 5m21s
[11:00:43.827] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:45:0)
[11:00:43.871] Build status: SCHEDULED
[11:00:43.871] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:46:0) after 3m49s
[11:04:33.134] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:46:0)
[11:04:33.153] Build status: SCHEDULED
[11:04:33.153] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:47:0) after 8m16s
[11:12:49.257] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:47:0)
[11:12:49.276] Build status: SCHEDULED
[11:12:49.276] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:48:0) after 3m26s
[11:16:15.335] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:48:0)
[11:16:15.416] Build status: SCHEDULED
[11:16:15.417] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:49:0) after 7m33s
[11:23:48.827] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:49:0)
[11:23:49.043] Build status: SCHEDULED
[11:23:49.043] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:50:0) after 2m5s
[11:25:54.431] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:50:0)
[11:25:54.450] Build status: SCHEDULED
[11:25:54.450] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:51:0) after 2m26s
[11:28:20.527] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:51:0)
[11:28:20.619] Build status: SCHEDULED
[11:28:20.619] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:52:0) after 2m43s
[11:31:03.638] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:52:0)
[11:31:03.653] Build status: SCHEDULED
[11:31:03.653] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:53:0) after 2m55s
[11:33:58.681] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:53:0)
[11:33:58.740] Build status: SCHEDULED
[11:33:58.740] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:54:0) after 7m18s
[11:41:16.760] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:54:0)
[11:41:16.780] Build status: SCHEDULED
[11:41:16.780] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:55:0) after 5m20s
[11:46:36.817] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:55:0)
[11:46:36.852] Build status: SCHEDULED
[11:46:36.852] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:56:0) after 3m4s
[11:49:41.011] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:56:0)
[11:49:41.032] Build status: SCHEDULED
[11:49:41.032] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:57:0) after 8m0s
[11:57:41.248] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:57:0)
[11:57:41.262] Build status: SCHEDULED
[11:57:41.262] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:58:0) after 1m10s
[11:58:51.386] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:58:0)
[11:58:51.410] Build status: SCHEDULED
[11:58:51.410] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:59:0) after 3m57s
[12:02:48.509] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:59:0)
[12:02:48.530] Build status: SCHEDULED
[12:02:48.530] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:60:0) after 5m44s
[12:08:32.989] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:60:0)
[12:08:33.075] Build status: SCHEDULED
[12:08:33.076] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:61:0) after 9m9s
[12:17:42.145] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:61:0)
[12:17:42.165] Build status: SCHEDULED
[12:17:42.165] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:62:0) after 8m26s
[12:26:08.359] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:62:0)
[12:26:08.433] Build status: SCHEDULED
[12:26:08.433] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:63:0) after 1m29s
[12:27:37.454] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:63:0)
[12:27:37.473] Build status: SCHEDULED
[12:27:37.473] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:64:0) after 6m56s
[12:34:33.493] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:64:0)
[12:34:33.519] Build status: SCHEDULED
[12:34:33.519] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:65:0) after 8m43s
[12:43:16.585] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:65:0)
[12:43:16.609] Build status: SCHEDULED
[12:43:16.609] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:66:0) after 5m27s
[12:48:43.726] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:66:0)
[12:48:43.745] Build status: SCHEDULED
[12:48:43.745] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:67:0) after 3m50s
[12:52:33.802] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:67:0)
[12:52:33.819] Build status: SCHEDULED
[12:52:33.819] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:68:0) after 6m54s
[12:59:27.839] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:68:0)
[12:59:27.862] Build status: SCHEDULED
[12:59:27.863] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:69:0) after 1m30s
[13:00:57.885] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:69:0)
[13:00:57.906] Build status: SCHEDULED
[13:00:57.906] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:70:0) after 3m4s
[13:04:01.932] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:70:0)
[13:04:01.948] Build status: SCHEDULED
[13:04:01.948] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:71:0) after 6m50s
[13:10:51.955] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:71:0)
[13:10:52.056] Build status: SCHEDULED
[13:10:52.056] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:72:0) after 5m30s
[13:16:22.074] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:72:0)
[13:16:22.096] Build status: SCHEDULED
[13:16:22.096] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:73:0) after 5m18s
[13:21:40.172] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:73:0)
[13:21:40.195] Build status: SCHEDULED
[13:21:40.195] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:74:0) after 3m31s
[13:25:11.318] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:74:0)
[13:25:11.426] Build status: SCHEDULED
[13:25:11.426] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:75:0) after 5m17s
[13:30:28.445] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:75:0)
[13:30:28.477] Build status: SCHEDULED
[13:30:28.477] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:76:0) after 3m51s
[13:34:19.463] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:76:0)
[13:34:19.480] Build status: SCHEDULED
[13:34:19.480] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:77:0) after 3m27s
[13:37:46.500] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:77:0)
[13:37:46.547] Build status: SCHEDULED
[13:37:46.547] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:78:0) after 5m37s
[13:43:23.590] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:78:0)
[13:43:23.608] Build status: SCHEDULED
[13:43:23.608] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:79:0) after 8m58s
[13:52:21.627] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:79:0)
[13:52:21.664] Build status: SCHEDULED
[13:52:21.664] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:80:0) after 2m50s
[13:55:11.790] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:80:0)
[13:55:11.807] Build status: SCHEDULED
[13:55:11.807] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:81:0) after 9m12s
[14:04:23.887] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:81:0)
[14:04:24.022] Build status: SCHEDULED
[14:04:24.022] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:82:0) after 9m15s
[14:13:39.088] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:82:0)
[14:13:39.108] Build status: SCHEDULED
[14:13:39.108] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:83:0) after 7m43s
[14:21:22.138] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:83:0)
[14:21:22.178] Build status: SCHEDULED
[14:21:22.178] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:84:0) after 6m35s
[14:27:57.233] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:84:0)
[14:27:57.253] Build status: SCHEDULED
[14:27:57.253] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:85:0) after 8m27s
[14:36:24.333] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:85:0)
[14:36:24.369] Build status: SCHEDULED
[14:36:24.369] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:86:0) after 3m57s
[14:40:21.342] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:86:0)
[14:40:21.358] Build status: SCHEDULED
[14:40:21.358] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:87:0) after 4m35s
[14:44:56.373] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:87:0)
[14:44:56.414] Build status: SCHEDULED
[14:44:56.414] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:88:0) after 3m3s
[14:47:59.434] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:88:0)
[14:47:59.455] Build status: SCHEDULED
[14:47:59.455] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:89:0) after 8m57s
[14:56:56.473] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:89:0)
[14:56:56.496] Build status: SCHEDULED
[14:56:56.496] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:90:0) after 1m11s
[14:58:07.528] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:90:0)
[14:58:07.606] Build status: SCHEDULED
[14:58:07.606] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8872553960699786448:91:0) after 6m7s
[15:03:12.513] Received PubSub notification, asking Buildbucket for the build status
[15:03:12.541] Build:
{
  "id": "8707123898727471185",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-asan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-08-08T07:03:10.224289147Z",
  "endTime": "2025-08-08T15:03:12.030810510Z",
  "updateTime": "2025-08-08T15:03:12.030810510Z",
  "status": "INFRA_FAILURE",
  "statusDetails": {
    "resourceExhaustion": {

    },
    "timeout": {

    }
  },
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "2b23689560196ea206283712f65093f3d48ab148",
      "ref": "refs/heads/main"
    }
  }
}
[15:03:12.541] Invocation finished in 8h0m3.580389453s with status FAILED