LUCI Scheduler :: pigweed :: pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox

Status: FAILED

Triggered by: -

Duration: 7 hours

Actions:

Definition (rev 70524db3852cf27d2636eedee03dcebdf74f19f6)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.sandbox.dev.ci"
  builder: "pigweed-mac-arm-sandbox"
>

Incoming triggers

8b196f5315423e39acf360c3f8f73dd7559a973b (2 weeks ago)

Incoming properties

{
  "branch": "refs/heads/upstream/main",
  "repository": "https://pigweed.googlesource.com/pigweed/sandbox",
  "revision": "8b196f5315423e39acf360c3f8f73dd7559a973b"
}

Incoming tags

buildset:commit/gitiles/pigweed.googlesource.com/pigweed/sandbox/+/8b196f5315423e39acf360c3f8f73dd7559a973b
gitiles_ref:refs/heads/upstream/main

Debug log (UTC)

[11:14:55.154] New invocation is queued and will start shortly
[11:14:56.376] Starting the invocation (attempt 1)
[11:14:56.403] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/sandbox/+/8b196f5315423e39acf360c3f8f73dd7559a973b"
[11:14:56.403] Popped gitiles commit info from properties and tags
[11:14:56.403] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[11:14:56.403] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[11:14:56.403] Buildbucket request:
{
  "requestId": "8837748885616063328",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.sandbox.dev.ci",
    "builder": "pigweed-mac-arm-sandbox"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8837748885616063328",
            "job": "pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/upstream/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/sandbox",
                              "revision": "8b196f5315423e39acf360c3f8f73dd7559a973b"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/sandbox/+/refs/heads/upstream/main@8b196f5315423e39acf360c3f8f73dd7559a973b",
                        "title": "8b196f5315423e39acf360c3f8f73dd7559a973b",
                        "url": "https://pigweed.googlesource.com/pigweed/sandbox/+/8b196f5315423e39acf360c3f8f73dd7559a973b"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/sandbox",
    "id": "8b196f5315423e39acf360c3f8f73dd7559a973b",
    "ref": "refs/heads/upstream/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8837748885616063328"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[11:14:56.815] Scheduled build:
{
  "id": "8672318823580108929",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.sandbox.dev.ci",
    "builder": "pigweed-mac-arm-sandbox"
  },
  "createdBy": "project:pigweed",
  "createTime": "2026-08-27T11:14:56.473600649Z",
  "updateTime": "2026-08-27T11:14:56.473600649Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/sandbox",
      "id": "8b196f5315423e39acf360c3f8f73dd7559a973b",
      "ref": "refs/heads/upstream/main"
    }
  }
}
[11:14:56.815] Task URL: https://cr-buildbucket.appspot.com/build/8672318823580108929
[11:14:56.815] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:2:0) after 2m41s
[11:14:57.776] Received PubSub notification, asking Buildbucket for the build status
[11:14:57.801] Build status: SCHEDULED
[11:17:37.854] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:2:0)
[11:17:37.875] Build status: SCHEDULED
[11:17:37.875] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:4:0) after 6m46s
[11:24:23.997] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:4:0)
[11:24:24.023] Build status: SCHEDULED
[11:24:24.023] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:5:0) after 7m13s
[11:31:37.205] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:5:0)
[11:31:37.327] Build status: SCHEDULED
[11:31:37.327] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:6:0) after 6m19s
[11:37:56.341] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:6:0)
[11:37:56.367] Build status: SCHEDULED
[11:37:56.367] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:7:0) after 7m41s
[11:45:37.400] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:7:0)
[11:45:37.435] Build status: SCHEDULED
[11:45:37.435] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:8:0) after 7m8s
[11:52:45.528] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:8:0)
[11:52:45.548] Build status: SCHEDULED
[11:52:45.548] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:9:0) after 3m23s
[11:56:08.747] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:9:0)
[11:56:08.763] Build status: SCHEDULED
[11:56:08.763] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:10:0) after 7m40s
[12:03:48.777] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:10:0)
[12:03:48.798] Build status: SCHEDULED
[12:03:48.798] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:11:0) after 4m42s
[12:08:30.814] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:11:0)
[12:08:30.848] Build status: SCHEDULED
[12:08:30.848] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:12:0) after 6m28s
[12:14:58.915] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:12:0)
[12:14:58.934] Build status: SCHEDULED
[12:14:58.934] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:13:0) after 3m13s
[12:18:12.002] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:13:0)
[12:18:12.052] Build status: SCHEDULED
[12:18:12.052] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:14:0) after 2m44s
[12:20:56.116] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:14:0)
[12:20:56.138] Build status: SCHEDULED
[12:20:56.138] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:15:0) after 7m50s
[12:28:46.195] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:15:0)
[12:28:46.236] Build status: SCHEDULED
[12:28:46.236] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:16:0) after 7m23s
[12:36:09.303] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:16:0)
[12:36:09.327] Build status: SCHEDULED
[12:36:09.327] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:17:0) after 7m30s
[12:43:39.457] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:17:0)
[12:43:39.482] Build status: SCHEDULED
[12:43:39.482] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:18:0) after 6m14s
[12:49:53.552] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:18:0)
[12:49:53.572] Build status: SCHEDULED
[12:49:53.572] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:19:0) after 3m31s
[12:53:24.600] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:19:0)
[12:53:24.620] Build status: SCHEDULED
[12:53:24.620] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:20:0) after 4m34s
[12:57:58.680] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:20:0)
[12:57:58.707] Build status: SCHEDULED
[12:57:58.707] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:21:0) after 8m43s
[13:06:41.723] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:21:0)
[13:06:41.742] Build status: SCHEDULED
[13:06:41.743] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:22:0) after 4m29s
[13:11:10.877] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:22:0)
[13:11:10.911] Build status: SCHEDULED
[13:11:10.911] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:23:0) after 3m32s
[13:14:43.029] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:23:0)
[13:14:43.050] Build status: SCHEDULED
[13:14:43.050] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:24:0) after 1m59s
[13:16:42.199] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:24:0)
[13:16:42.217] Build status: SCHEDULED
[13:16:42.217] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:25:0) after 9m58s
[13:26:40.244] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:25:0)
[13:26:40.272] Build status: SCHEDULED
[13:26:40.272] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:26:0) after 4m20s
[13:31:00.396] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:26:0)
[13:31:00.433] Build status: SCHEDULED
[13:31:00.433] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:27:0) after 4m3s
[13:35:03.494] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:27:0)
[13:35:03.515] Build status: SCHEDULED
[13:35:03.515] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:28:0) after 3m31s
[13:38:34.532] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:28:0)
[13:38:34.556] Build status: SCHEDULED
[13:38:34.556] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:29:0) after 4m40s
[13:43:14.788] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:29:0)
[13:43:14.965] Build status: SCHEDULED
[13:43:14.965] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:30:0) after 4m2s
[13:47:16.976] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:30:0)
[13:47:16.996] Build status: SCHEDULED
[13:47:16.997] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:31:0) after 8m11s
[13:55:28.089] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:31:0)
[13:55:28.109] Build status: SCHEDULED
[13:55:28.109] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:32:0) after 4m5s
[13:59:33.125] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:32:0)
[13:59:33.154] Build status: SCHEDULED
[13:59:33.154] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:33:0) after 4m41s
[14:04:14.355] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:33:0)
[14:04:14.377] Build status: SCHEDULED
[14:04:14.377] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:34:0) after 9m38s
[14:13:52.431] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:34:0)
[14:13:52.453] Build status: SCHEDULED
[14:13:52.453] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:35:0) after 6m13s
[14:20:05.482] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:35:0)
[14:20:05.503] Build status: SCHEDULED
[14:20:05.503] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:36:0) after 1m45s
[14:21:50.539] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:36:0)
[14:21:50.557] Build status: SCHEDULED
[14:21:50.557] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:37:0) after 9m39s
[14:31:29.673] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:37:0)
[14:31:29.708] Build status: SCHEDULED
[14:31:29.708] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:38:0) after 8m49s
[14:40:18.727] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:38:0)
[14:40:18.748] Build status: SCHEDULED
[14:40:18.748] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:39:0) after 8m57s
[14:49:15.766] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:39:0)
[14:49:15.787] Build status: SCHEDULED
[14:49:15.787] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:40:0) after 1m46s
[14:51:01.796] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:40:0)
[14:51:01.816] Build status: SCHEDULED
[14:51:01.816] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:41:0) after 3m51s
[14:54:53.007] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:41:0)
[14:54:53.082] Build status: SCHEDULED
[14:54:53.082] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:42:0) after 8m39s
[15:03:32.131] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:42:0)
[15:03:32.154] Build status: SCHEDULED
[15:03:32.154] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:43:0) after 4m25s
[15:07:57.173] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:43:0)
[15:07:57.197] Build status: SCHEDULED
[15:07:57.197] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:44:0) after 3m2s
[15:10:59.215] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:44:0)
[15:10:59.245] Build status: SCHEDULED
[15:10:59.245] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:45:0) after 3m40s
[15:14:39.418] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:45:0)
[15:14:39.439] Build status: SCHEDULED
[15:14:39.439] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:46:0) after 1m49s
[15:16:28.456] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:46:0)
[15:16:28.624] Build status: SCHEDULED
[15:16:28.624] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:47:0) after 9m2s
[15:25:30.908] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:47:0)
[15:25:30.927] Build status: SCHEDULED
[15:25:30.927] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:48:0) after 5m17s
[15:30:47.948] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:48:0)
[15:30:47.981] Build status: SCHEDULED
[15:30:47.981] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:49:0) after 8m7s
[15:38:55.052] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:49:0)
[15:38:55.080] Build status: SCHEDULED
[15:38:55.080] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:50:0) after 4m7s
[15:43:02.150] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:50:0)
[15:43:02.172] Build status: SCHEDULED
[15:43:02.172] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:51:0) after 2m35s
[15:45:37.190] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:51:0)
[15:45:37.227] Build status: SCHEDULED
[15:45:37.227] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:52:0) after 4m59s
[15:50:36.256] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:52:0)
[15:50:36.409] Build status: SCHEDULED
[15:50:36.409] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:53:0) after 8m10s
[15:58:46.586] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:53:0)
[15:58:46.611] Build status: SCHEDULED
[15:58:46.611] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:54:0) after 3m13s
[16:01:59.667] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:54:0)
[16:01:59.694] Build status: SCHEDULED
[16:01:59.694] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:55:0) after 7m56s
[16:09:55.759] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:55:0)
[16:09:55.780] Build status: SCHEDULED
[16:09:55.780] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:56:0) after 4m3s
[16:13:59.022] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:56:0)
[16:13:59.100] Build status: SCHEDULED
[16:13:59.100] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:57:0) after 1m58s
[16:15:57.259] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:57:0)
[16:15:57.280] Build status: SCHEDULED
[16:15:57.280] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:58:0) after 1m58s
[16:17:55.423] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:58:0)
[16:17:55.460] Build status: SCHEDULED
[16:17:55.460] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:59:0) after 1m36s
[16:19:31.519] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:59:0)
[16:19:31.555] Build status: SCHEDULED
[16:19:31.555] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:60:0) after 3m38s
[16:23:09.623] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:60:0)
[16:23:09.641] Build status: SCHEDULED
[16:23:09.641] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:61:0) after 9m24s
[16:32:33.678] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:61:0)
[16:32:33.699] Build status: SCHEDULED
[16:32:33.699] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:62:0) after 7m59s
[16:40:32.988] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:62:0)
[16:40:33.023] Build status: SCHEDULED
[16:40:33.023] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:63:0) after 7m53s
[16:48:26.285] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:63:0)
[16:48:26.320] Build status: SCHEDULED
[16:48:26.320] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:64:0) after 9m46s
[16:58:12.339] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:64:0)
[16:58:12.365] Build status: SCHEDULED
[16:58:12.365] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:65:0) after 1m55s
[17:00:07.428] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:65:0)
[17:00:07.459] Build status: SCHEDULED
[17:00:07.459] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:66:0) after 5m33s
[17:05:40.623] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:66:0)
[17:05:40.647] Build status: SCHEDULED
[17:05:40.648] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:67:0) after 9m26s
[17:15:06.716] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:67:0)
[17:15:06.736] Build status: SCHEDULED
[17:15:06.736] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:68:0) after 9m17s
[17:24:24.166] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:68:0)
[17:24:24.247] Build status: SCHEDULED
[17:24:24.247] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:69:0) after 7m57s
[17:32:21.457] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:69:0)
[17:32:21.484] Build status: SCHEDULED
[17:32:21.484] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:70:0) after 5m14s
[17:37:35.553] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:70:0)
[17:37:35.575] Build status: SCHEDULED
[17:37:35.575] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:71:0) after 2m57s
[17:40:32.610] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:71:0)
[17:40:32.660] Build status: SCHEDULED
[17:40:32.660] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:72:0) after 4m44s
[17:45:16.679] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:72:0)
[17:45:16.721] Build status: SCHEDULED
[17:45:16.721] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:73:0) after 4m50s
[17:50:06.694] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:73:0)
[17:50:06.716] Build status: SCHEDULED
[17:50:06.716] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:74:0) after 2m35s
[17:52:41.731] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:74:0)
[17:52:41.788] Build status: SCHEDULED
[17:52:41.788] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:75:0) after 9m1s
[18:01:42.947] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:75:0)
[18:01:42.963] Build status: SCHEDULED
[18:01:42.963] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:76:0) after 5m21s
[18:07:03.998] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:76:0)
[18:07:04.025] Build status: SCHEDULED
[18:07:04.025] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:77:0) after 1m25s
[18:08:29.138] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:77:0)
[18:08:29.170] Build status: SCHEDULED
[18:08:29.170] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:78:0) after 8m16s
[18:16:45.255] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:78:0)
[18:16:45.320] Build status: SCHEDULED
[18:16:45.320] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:79:0) after 9m2s
[18:25:47.336] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:79:0)
[18:25:47.356] Build status: SCHEDULED
[18:25:47.356] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:80:0) after 9m34s
[18:35:21.386] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:80:0)
[18:35:21.407] Build status: SCHEDULED
[18:35:21.407] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:81:0) after 4m28s
[18:39:49.454] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:81:0)
[18:39:49.485] Build status: SCHEDULED
[18:39:49.485] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:82:0) after 7m29s
[18:40:31.295] Received PubSub notification, asking Buildbucket for the build status
[18:40:31.324] Build status: STARTED
[18:47:18.465] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:82:0)
[18:47:18.486] Build status: STARTED
[18:47:18.486] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837748885616063328:84:0) after 5m33s
[18:49:41.565] Received PubSub notification, asking Buildbucket for the build status
[18:49:41.592] Build:
{
  "id": "8672318823580108929",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.sandbox.dev.ci",
    "builder": "pigweed-mac-arm-sandbox"
  },
  "createdBy": "project:pigweed",
  "createTime": "2026-08-27T11:14:56.473600649Z",
  "startTime": "2026-08-27T18:40:21.298011637Z",
  "endTime": "2026-08-27T18:49:41.260365848Z",
  "updateTime": "2026-08-27T18:49:41.260365848Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/sandbox",
      "id": "8b196f5315423e39acf360c3f8f73dd7559a973b",
      "ref": "refs/heads/upstream/main"
    }
  }
}
[18:49:41.592] Invocation finished in 7h34m46.443699015s with status FAILED