Status: FAILED
Triggered by: -
Duration: 3 hours
Actions:

Definition (rev d2b949be221fc8dd3acbfdb77921a85f7b1b8e9d)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-arm-bazel-san-tsan"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[20:56:54.336] New invocation is queued and will start shortly
[20:56:55.687] Starting the invocation (attempt 1)
[20:56:55.713] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96"
[20:56:55.713] Popped gitiles commit info from properties and tags
[20:56:55.714] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[20:56:55.714] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[20:56:55.714] Buildbucket request:
{
  "requestId": "8881470604992819248",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-san-tsan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8881470604992819248",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "5800f3c1f95521823a7a5e60eabe8c6f82644872"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@5800f3c1f95521823a7a5e60eabe8c6f82644872",
                        "title": "5800f3c1f95521823a7a5e60eabe8c6f82644872",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/5800f3c1f95521823a7a5e60eabe8c6f82644872"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "aac8b0e10b7c5b83ba1e98a059836364d733bff8"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@aac8b0e10b7c5b83ba1e98a059836364d733bff8",
                        "title": "aac8b0e10b7c5b83ba1e98a059836364d733bff8",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/aac8b0e10b7c5b83ba1e98a059836364d733bff8"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "7a7eaa8afb31013d8faaed44b523abcaa7eeb0e6"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@7a7eaa8afb31013d8faaed44b523abcaa7eeb0e6",
                        "title": "7a7eaa8afb31013d8faaed44b523abcaa7eeb0e6",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/7a7eaa8afb31013d8faaed44b523abcaa7eeb0e6"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "050cd228203dacc5609a63e5a951a6157d1ac494"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@050cd228203dacc5609a63e5a951a6157d1ac494",
                        "title": "050cd228203dacc5609a63e5a951a6157d1ac494",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/050cd228203dacc5609a63e5a951a6157d1ac494"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96",
                        "title": "ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8881470604992819248"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[20:56:56.185] Scheduled build:
{
  "id": "8716040542817335969",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-san-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-05-01T20:56:55.767817352Z",
  "updateTime": "2025-05-01T20:56:55.767817352Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96",
      "ref": "refs/heads/main"
    }
  }
}
[20:56:56.185] Task URL: https://cr-buildbucket.appspot.com/build/8716040542817335969
[20:56:56.185] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:2:0) after 3m37s
[20:56:56.502] Received PubSub notification, asking Buildbucket for the build status
[20:56:56.585] Build status: SCHEDULED
[21:00:33.206] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:2:0)
[21:00:33.238] Build status: SCHEDULED
[21:00:33.238] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:4:0) after 1m5s
[21:01:38.261] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:4:0)
[21:01:38.291] Build status: SCHEDULED
[21:01:38.291] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:5:0) after 4m10s
[21:05:48.355] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:5:0)
[21:05:48.443] Build status: SCHEDULED
[21:05:48.443] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:6:0) after 3m47s
[21:09:35.516] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:6:0)
[21:09:35.543] Build status: SCHEDULED
[21:09:35.543] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:7:0) after 5m13s
[21:14:48.639] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:7:0)
[21:14:48.707] Build status: SCHEDULED
[21:14:48.707] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:8:0) after 3m52s
[21:18:41.085] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:8:0)
[21:18:41.110] Build status: SCHEDULED
[21:18:41.110] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:9:0) after 4m34s
[21:23:15.147] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:9:0)
[21:23:15.201] Build status: SCHEDULED
[21:23:15.201] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:10:0) after 1m6s
[21:24:21.238] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:10:0)
[21:24:21.264] Build status: SCHEDULED
[21:24:21.264] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:11:0) after 7m42s
[21:32:03.331] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:11:0)
[21:32:03.405] Build status: SCHEDULED
[21:32:03.405] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:12:0) after 5m12s
[21:37:15.502] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:12:0)
[21:37:15.544] Build status: SCHEDULED
[21:37:15.544] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:13:0) after 7m3s
[21:44:18.609] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:13:0)
[21:44:18.635] Build status: SCHEDULED
[21:44:18.635] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:14:0) after 5m59s
[21:50:17.696] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:14:0)
[21:50:17.720] Build status: SCHEDULED
[21:50:17.720] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:15:0) after 1m5s
[21:51:22.743] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:15:0)
[21:51:22.757] Build status: SCHEDULED
[21:51:22.757] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:16:0) after 2m46s
[21:54:08.738] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:16:0)
[21:54:08.769] Build status: SCHEDULED
[21:54:08.769] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:17:0) after 3m24s
[21:57:32.793] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:17:0)
[21:57:32.836] Build status: SCHEDULED
[21:57:32.836] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:18:0) after 7m25s
[22:04:57.856] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:18:0)
[22:04:57.878] Build status: SCHEDULED
[22:04:57.878] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:19:0) after 4m23s
[22:09:20.997] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:19:0)
[22:09:21.015] Build status: SCHEDULED
[22:09:21.015] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:20:0) after 7m3s
[22:16:24.086] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:20:0)
[22:16:24.117] Build status: SCHEDULED
[22:16:24.117] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:21:0) after 2m9s
[22:18:33.327] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:21:0)
[22:18:33.348] Build status: SCHEDULED
[22:18:33.348] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:22:0) after 4m36s
[22:23:09.406] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:22:0)
[22:23:09.427] Build status: SCHEDULED
[22:23:09.427] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:23:0) after 1m14s
[22:24:23.454] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:23:0)
[22:24:23.478] Build status: SCHEDULED
[22:24:23.478] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:24:0) after 7m35s
[22:31:58.501] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:24:0)
[22:31:58.518] Build status: SCHEDULED
[22:31:58.518] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:25:0) after 1m14s
[22:33:12.608] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:25:0)
[22:33:12.627] Build status: SCHEDULED
[22:33:12.627] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:26:0) after 9m22s
[22:42:34.717] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:26:0)
[22:42:34.737] Build status: SCHEDULED
[22:42:34.737] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:27:0) after 2m21s
[22:44:55.757] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:27:0)
[22:44:55.777] Build status: SCHEDULED
[22:44:55.777] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:28:0) after 5m30s
[22:50:25.852] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:28:0)
[22:50:25.866] Build status: SCHEDULED
[22:50:25.866] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:29:0) after 7m58s
[22:58:23.883] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:29:0)
[22:58:23.903] Build status: SCHEDULED
[22:58:23.903] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:30:0) after 2m25s
[23:00:49.022] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:30:0)
[23:00:49.040] Build status: SCHEDULED
[23:00:49.041] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:31:0) after 7m11s
[23:08:00.063] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:31:0)
[23:08:00.081] Build status: SCHEDULED
[23:08:00.081] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:32:0) after 3m11s
[23:11:11.099] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:32:0)
[23:11:11.128] Build status: SCHEDULED
[23:11:11.128] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:33:0) after 1m3s
[23:12:14.150] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:33:0)
[23:12:14.178] Build status: SCHEDULED
[23:12:14.178] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:34:0) after 2m36s
[23:14:50.237] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:34:0)
[23:14:50.258] Build status: SCHEDULED
[23:14:50.258] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:35:0) after 6m25s
[23:21:15.329] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:35:0)
[23:21:15.349] Build status: SCHEDULED
[23:21:15.349] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:36:0) after 1m26s
[23:22:41.364] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:36:0)
[23:22:41.384] Build status: SCHEDULED
[23:22:41.384] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:37:0) after 7m20s
[23:30:01.401] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:37:0)
[23:30:01.418] Build status: SCHEDULED
[23:30:01.418] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:38:0) after 2m46s
[23:32:47.436] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:38:0)
[23:32:47.465] Build status: SCHEDULED
[23:32:47.465] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:39:0) after 9m22s
[23:42:09.473] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:39:0)
[23:42:10.249] Build status: SCHEDULED
[23:42:10.249] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:40:0) after 3m20s
[23:45:30.367] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:40:0)
[23:45:30.385] Build status: SCHEDULED
[23:45:30.385] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:41:0) after 8m10s
[23:53:40.455] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:41:0)
[23:53:40.484] Build status: SCHEDULED
[23:53:40.484] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:42:0) after 8m36s
[00:02:16.540] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:42:0)
[00:02:16.556] Build status: SCHEDULED
[00:02:16.556] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:43:0) after 9m8s
[00:11:24.574] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:43:0)
[00:11:24.595] Build status: SCHEDULED
[00:11:24.595] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:44:0) after 7m22s
[00:18:46.616] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:44:0)
[00:18:46.637] Build status: SCHEDULED
[00:18:46.637] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:45:0) after 2m58s
[00:21:44.664] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:45:0)
[00:21:44.685] Build status: SCHEDULED
[00:21:44.685] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8881470604992819248:46:0) after 5m51s
[00:25:52.433] Received PubSub notification, asking Buildbucket for the build status
[00:25:52.451] Build:
{
  "id": "8716040542817335969",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-san-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-05-01T20:56:55.767817352Z",
  "endTime": "2025-05-02T00:25:51.407844054Z",
  "updateTime": "2025-05-02T00:25:51.407844054Z",
  "status": "INFRA_FAILURE",
  "statusDetails": {
    "resourceExhaustion": {

    }
  },
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96",
      "ref": "refs/heads/main"
    }
  }
}
[00:25:52.451] Invocation finished in 3h28m58.141965886s with status FAILED