1. All projects
  2. pigweed
  3. pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox
  4. 8840973411042726976
Status: FAILED
Triggered by: -
Duration: 8 hours
Actions:

Definition (rev 70524db3852cf27d2636eedee03dcebdf74f19f6) 

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.sandbox.dev.ci"
  builder: "pigweed-mac-arm-sandbox"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/upstream/main",
  "repository": "https://pigweed.googlesource.com/pigweed/sandbox",
  "revision": "9ae22d958cd9149dd2e7bdfeb24d0be4fca4154e"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/sandbox/+/9ae22d958cd9149dd2e7bdfeb24d0be4fca4154e
  • gitiles_ref:refs/heads/upstream/main

Debug log (UTC)

[21:02:28.069] New invocation is queued and will start shortly
[21:02:29.168] Starting the invocation (attempt 1)
[21:02:29.199] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/sandbox/+/9ae22d958cd9149dd2e7bdfeb24d0be4fca4154e"
[21:02:29.199] Popped gitiles commit info from properties and tags
[21:02:29.199] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[21:02:29.199] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[21:02:29.199] Buildbucket request:
{
  "requestId": "8840973411042726976",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.sandbox.dev.ci",
    "builder": "pigweed-mac-arm-sandbox"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8840973411042726976",
            "job": "pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/upstream/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/sandbox",
                              "revision": "9ae22d958cd9149dd2e7bdfeb24d0be4fca4154e"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/sandbox/+/refs/heads/upstream/main@9ae22d958cd9149dd2e7bdfeb24d0be4fca4154e",
                        "title": "9ae22d958cd9149dd2e7bdfeb24d0be4fca4154e",
                        "url": "https://pigweed.googlesource.com/pigweed/sandbox/+/9ae22d958cd9149dd2e7bdfeb24d0be4fca4154e"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/sandbox",
    "id": "9ae22d958cd9149dd2e7bdfeb24d0be4fca4154e",
    "ref": "refs/heads/upstream/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8840973411042726976"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[21:02:29.722] Scheduled build:
{
  "id": "8675543349114804801",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.sandbox.dev.ci",
    "builder": "pigweed-mac-arm-sandbox"
  },
  "createdBy": "project:pigweed",
  "createTime": "2026-07-22T21:02:29.288956424Z",
  "updateTime": "2026-07-22T21:02:29.288956424Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/sandbox",
      "id": "9ae22d958cd9149dd2e7bdfeb24d0be4fca4154e",
      "ref": "refs/heads/upstream/main"
    }
  }
}
[21:02:29.722] Task URL: https://cr-buildbucket.appspot.com/build/8675543349114804801
[21:02:29.722] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:2:0) after 1m15s
[21:02:30.064] Received PubSub notification, asking Buildbucket for the build status
[21:02:30.093] Build status: SCHEDULED
[21:03:44.735] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:2:0)
[21:03:44.786] Build status: SCHEDULED
[21:03:44.786] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:4:0) after 3m26s
[21:07:10.937] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:4:0)
[21:07:10.972] Build status: SCHEDULED
[21:07:10.972] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:5:0) after 1m29s
[21:08:40.043] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:5:0)
[21:08:40.078] Build status: SCHEDULED
[21:08:40.078] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:6:0) after 8m56s
[21:17:36.089] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:6:0)
[21:17:36.126] Build status: SCHEDULED
[21:17:36.126] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:7:0) after 3m46s
[21:21:22.293] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:7:0)
[21:21:22.327] Build status: SCHEDULED
[21:21:22.327] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:8:0) after 5m45s
[21:27:07.338] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:8:0)
[21:27:07.367] Build status: SCHEDULED
[21:27:07.367] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:9:0) after 5m39s
[21:32:46.475] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:9:0)
[21:32:46.581] Build status: SCHEDULED
[21:32:46.581] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:10:0) after 1m15s
[21:34:01.576] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:10:0)
[21:34:01.605] Build status: SCHEDULED
[21:34:01.605] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:11:0) after 1m50s
[21:35:51.616] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:11:0)
[21:35:51.636] Build status: SCHEDULED
[21:35:51.636] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:12:0) after 8m32s
[21:44:23.749] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:12:0)
[21:44:23.770] Build status: SCHEDULED
[21:44:23.770] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:13:0) after 1m51s
[21:46:14.885] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:13:0)
[21:46:14.922] Build status: SCHEDULED
[21:46:14.922] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:14:0) after 9m13s
[21:55:28.210] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:14:0)
[21:55:28.230] Build status: SCHEDULED
[21:55:28.230] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:15:0) after 9m10s
[22:04:38.223] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:15:0)
[22:04:38.251] Build status: SCHEDULED
[22:04:38.251] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:16:0) after 2m48s
[22:07:26.313] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:16:0)
[22:07:26.349] Build status: SCHEDULED
[22:07:26.349] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:17:0) after 7m29s
[22:14:55.474] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:17:0)
[22:14:55.509] Build status: SCHEDULED
[22:14:55.509] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:18:0) after 1m40s
[22:16:35.749] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:18:0)
[22:16:35.767] Build status: SCHEDULED
[22:16:35.767] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:19:0) after 3m19s
[22:19:54.928] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:19:0)
[22:19:54.946] Build status: SCHEDULED
[22:19:54.946] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:20:0) after 1m23s
[22:21:17.957] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:20:0)
[22:21:17.979] Build status: SCHEDULED
[22:21:17.979] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:21:0) after 5m37s
[22:26:55.098] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:21:0)
[22:26:55.142] Build status: SCHEDULED
[22:26:55.142] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:22:0) after 7m59s
[22:34:54.218] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:22:0)
[22:34:54.249] Build status: SCHEDULED
[22:34:54.249] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:23:0) after 4m41s
[22:39:35.264] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:23:0)
[22:39:35.290] Build status: SCHEDULED
[22:39:35.290] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:24:0) after 4m21s
[22:43:56.469] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:24:0)
[22:43:56.528] Build status: SCHEDULED
[22:43:56.528] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:25:0) after 2m35s
[22:46:31.558] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:25:0)
[22:46:31.581] Build status: SCHEDULED
[22:46:31.581] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:26:0) after 6m44s
[22:53:15.597] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:26:0)
[22:53:15.623] Build status: SCHEDULED
[22:53:15.623] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:27:0) after 7m14s
[23:00:29.606] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:27:0)
[23:00:29.627] Build status: SCHEDULED
[23:00:29.627] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:28:0) after 2m2s
[23:02:31.620] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:28:0)
[23:02:31.646] Build status: SCHEDULED
[23:02:31.646] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:29:0) after 5m52s
[23:08:23.684] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:29:0)
[23:08:23.756] Build status: SCHEDULED
[23:08:23.756] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:30:0) after 2m54s
[23:11:17.770] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:30:0)
[23:11:17.787] Build status: SCHEDULED
[23:11:17.787] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:31:0) after 5m7s
[23:16:24.868] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:31:0)
[23:16:24.917] Build status: SCHEDULED
[23:16:24.917] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:32:0) after 1m36s
[23:18:00.930] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:32:0)
[23:18:00.950] Build status: SCHEDULED
[23:18:00.950] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:33:0) after 8m12s
[23:26:13.092] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:33:0)
[23:26:13.109] Build status: SCHEDULED
[23:26:13.109] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:34:0) after 8m51s
[23:35:04.124] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:34:0)
[23:35:04.148] Build status: SCHEDULED
[23:35:04.148] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:35:0) after 5m11s
[23:40:15.197] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:35:0)
[23:40:15.217] Build status: SCHEDULED
[23:40:15.217] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:36:0) after 2m55s
[23:43:10.232] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:36:0)
[23:43:10.262] Build status: SCHEDULED
[23:43:10.262] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:37:0) after 3m3s
[23:46:13.279] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:37:0)
[23:46:13.303] Build status: SCHEDULED
[23:46:13.303] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:38:0) after 8m56s
[23:55:09.320] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:38:0)
[23:55:09.350] Build status: SCHEDULED
[23:55:09.350] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:39:0) after 9m42s
[00:04:51.584] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:39:0)
[00:04:51.652] Build status: SCHEDULED
[00:04:51.652] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:40:0) after 2m40s
[00:07:31.715] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:40:0)
[00:07:31.741] Build status: SCHEDULED
[00:07:31.741] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:41:0) after 1m36s
[00:09:07.806] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:41:0)
[00:09:07.829] Build status: SCHEDULED
[00:09:07.829] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:42:0) after 8m1s
[00:17:08.850] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:42:0)
[00:17:08.878] Build status: SCHEDULED
[00:17:08.878] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:43:0) after 6m21s
[00:23:29.930] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:43:0)
[00:23:29.952] Build status: SCHEDULED
[00:23:29.952] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:44:0) after 5m39s
[00:29:08.971] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:44:0)
[00:29:08.990] Build status: SCHEDULED
[00:29:08.990] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:45:0) after 9m30s
[00:38:39.120] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:45:0)
[00:38:39.167] Build status: SCHEDULED
[00:38:39.167] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:46:0) after 7m15s
[00:45:54.229] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:46:0)
[00:45:54.253] Build status: SCHEDULED
[00:45:54.253] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:47:0) after 3m49s
[00:49:43.451] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:47:0)
[00:49:43.471] Build status: SCHEDULED
[00:49:43.471] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:48:0) after 2m10s
[00:51:53.479] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:48:0)
[00:51:53.573] Build status: SCHEDULED
[00:51:53.573] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:49:0) after 2m55s
[00:54:48.578] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:49:0)
[00:54:48.599] Build status: SCHEDULED
[00:54:48.599] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:50:0) after 5m54s
[01:00:42.666] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:50:0)
[01:00:42.698] Build status: SCHEDULED
[01:00:42.698] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:51:0) after 4m44s
[01:05:26.749] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:51:0)
[01:05:26.768] Build status: SCHEDULED
[01:05:26.768] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:52:0) after 1m14s
[01:06:40.862] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:52:0)
[01:06:40.879] Build status: SCHEDULED
[01:06:40.879] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:53:0) after 7m29s
[01:14:09.947] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:53:0)
[01:14:09.968] Build status: SCHEDULED
[01:14:09.968] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:54:0) after 9m23s
[01:23:32.988] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:54:0)
[01:23:33.009] Build status: SCHEDULED
[01:23:33.009] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:55:0) after 5m46s
[01:29:19.037] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:55:0)
[01:29:19.064] Build status: SCHEDULED
[01:29:19.064] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:56:0) after 4m18s
[01:33:37.053] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:56:0)
[01:33:37.072] Build status: SCHEDULED
[01:33:37.072] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:57:0) after 4m30s
[01:38:07.142] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:57:0)
[01:38:07.166] Build status: SCHEDULED
[01:38:07.166] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:58:0) after 8m13s
[01:46:20.222] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:58:0)
[01:46:20.246] Build status: SCHEDULED
[01:46:20.246] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:59:0) after 1m25s
[01:47:45.268] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:59:0)
[01:47:45.321] Build status: SCHEDULED
[01:47:45.321] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:60:0) after 6m47s
[01:54:32.307] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:60:0)
[01:54:32.323] Build status: SCHEDULED
[01:54:32.323] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:61:0) after 9m22s
[02:03:54.395] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:61:0)
[02:03:54.414] Build status: SCHEDULED
[02:03:54.414] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:62:0) after 2m18s
[02:06:12.453] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:62:0)
[02:06:12.537] Build status: SCHEDULED
[02:06:12.537] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:63:0) after 5m20s
[02:11:32.532] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:63:0)
[02:11:32.555] Build status: SCHEDULED
[02:11:32.555] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:64:0) after 9m8s
[02:20:40.725] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:64:0)
[02:20:40.747] Build status: SCHEDULED
[02:20:40.747] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:65:0) after 7m37s
[02:28:17.762] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:65:0)
[02:28:17.784] Build status: SCHEDULED
[02:28:17.784] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:66:0) after 4m48s
[02:33:05.961] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:66:0)
[02:33:05.977] Build status: SCHEDULED
[02:33:05.977] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:67:0) after 7m54s
[02:40:59.993] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:67:0)
[02:41:00.018] Build status: SCHEDULED
[02:41:00.018] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:68:0) after 1m37s
[02:42:37.384] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:68:0)
[02:42:37.420] Build status: SCHEDULED
[02:42:37.420] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:69:0) after 4m6s
[02:46:43.520] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:69:0)
[02:46:43.543] Build status: SCHEDULED
[02:46:43.543] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:70:0) after 7m43s
[02:54:26.664] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:70:0)
[02:54:26.688] Build status: SCHEDULED
[02:54:26.688] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:71:0) after 5m43s
[03:00:09.917] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:71:0)
[03:00:09.938] Build status: SCHEDULED
[03:00:09.938] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:72:0) after 8m31s
[03:08:41.003] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:72:0)
[03:08:41.025] Build status: SCHEDULED
[03:08:41.025] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:73:0) after 5m22s
[03:14:03.061] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:73:0)
[03:14:03.079] Build status: SCHEDULED
[03:14:03.079] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:74:0) after 4m37s
[03:18:40.178] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:74:0)
[03:18:40.213] Build status: SCHEDULED
[03:18:40.213] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:75:0) after 2m7s
[03:20:47.229] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:75:0)
[03:20:47.259] Build status: SCHEDULED
[03:20:47.259] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:76:0) after 3m23s
[03:24:10.264] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:76:0)
[03:24:10.285] Build status: SCHEDULED
[03:24:10.286] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:77:0) after 4m42s
[03:28:52.385] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:77:0)
[03:28:52.404] Build status: SCHEDULED
[03:28:52.404] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:78:0) after 2m27s
[03:31:19.468] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:78:0)
[03:31:19.485] Build status: SCHEDULED
[03:31:19.485] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:79:0) after 1m57s
[03:33:16.498] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:79:0)
[03:33:16.670] Build status: SCHEDULED
[03:33:16.670] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:80:0) after 1m8s
[03:34:24.686] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:80:0)
[03:34:24.705] Build status: SCHEDULED
[03:34:24.705] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:81:0) after 5m4s
[03:39:28.812] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:81:0)
[03:39:28.841] Build status: SCHEDULED
[03:39:28.841] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:82:0) after 4m40s
[03:44:08.877] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:82:0)
[03:44:08.903] Build status: SCHEDULED
[03:44:08.903] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:83:0) after 1m31s
[03:45:39.920] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:83:0)
[03:45:39.944] Build status: SCHEDULED
[03:45:39.944] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:84:0) after 8m21s
[03:54:01.062] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:84:0)
[03:54:01.084] Build status: SCHEDULED
[03:54:01.084] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:85:0) after 4m47s
[03:58:48.100] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:85:0)
[03:58:48.115] Build status: SCHEDULED
[03:58:48.116] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:86:0) after 7m18s
[04:06:06.155] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:86:0)
[04:06:09.398] Build status: SCHEDULED
[04:06:09.398] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:87:0) after 9m40s
[04:15:49.481] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:87:0)
[04:15:49.507] Build status: SCHEDULED
[04:15:49.507] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:88:0) after 6m0s
[04:21:49.512] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:88:0)
[04:21:49.549] Build status: SCHEDULED
[04:21:49.549] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:89:0) after 1m47s
[04:23:36.587] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:89:0)
[04:23:36.604] Build status: SCHEDULED
[04:23:36.604] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:90:0) after 5m59s
[04:29:35.676] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:90:0)
[04:29:35.695] Build status: SCHEDULED
[04:29:35.695] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:91:0) after 7m42s
[04:37:17.671] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:91:0)
[04:37:17.912] Build status: SCHEDULED
[04:37:17.912] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:92:0) after 4m22s
[04:41:39.995] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:92:0)
[04:41:40.043] Build status: SCHEDULED
[04:41:40.043] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:93:0) after 8m31s
[04:50:11.180] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:93:0)
[04:50:11.203] Build status: SCHEDULED
[04:50:11.203] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:94:0) after 7m41s
[04:57:52.186] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:94:0)
[04:57:52.206] Build status: SCHEDULED
[04:57:52.206] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8840973411042726976:95:0) after 9m41s
[05:02:39.167] Received PubSub notification, asking Buildbucket for the build status
[05:02:39.217] Build:
{
  "id": "8675543349114804801",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.sandbox.dev.ci",
    "builder": "pigweed-mac-arm-sandbox"
  },
  "createdBy": "project:pigweed",
  "createTime": "2026-07-22T21:02:29.288956424Z",
  "endTime": "2026-07-23T05:02:38.910765289Z",
  "updateTime": "2026-07-23T05:02:38.910765289Z",
  "status": "INFRA_FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/sandbox",
      "id": "9ae22d958cd9149dd2e7bdfeb24d0be4fca4154e",
      "ref": "refs/heads/upstream/main"
    }
  }
}
[05:02:39.217] Invocation finished in 8h0m11.152502249s with status FAILED