[00:09:08.285] New invocation is queued and will start shortly
[00:09:09.556] Starting the invocation (attempt 1)
[00:09:09.582] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/sandbox/+/b50aeebf1e99973396b609773b2d940da59dbd59"
[00:09:09.582] Popped gitiles commit info from properties and tags
[00:09:09.582] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[00:09:09.582] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[00:09:09.582] Buildbucket request:
{
"requestId": "8837971966874633408",
"builder": {
"project": "pigweed",
"bucket": "pigweed.sandbox.dev.ci",
"builder": "pigweed-mac-arm-sandbox"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8837971966874633408",
"job": "pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/upstream/main",
"repo": "https://pigweed.googlesource.com/pigweed/sandbox",
"revision": "84928195de0849553cff0369235fd1c0cfee9940"
},
"id": "https://pigweed.googlesource.com/pigweed/sandbox/+/refs/heads/upstream/main@84928195de0849553cff0369235fd1c0cfee9940",
"title": "84928195de0849553cff0369235fd1c0cfee9940",
"url": "https://pigweed.googlesource.com/pigweed/sandbox/+/84928195de0849553cff0369235fd1c0cfee9940"
},
{
"gitiles": {
"ref": "refs/heads/upstream/main",
"repo": "https://pigweed.googlesource.com/pigweed/sandbox",
"revision": "d48c1f5ab6b568d2144e45f8139a494ddde06c3d"
},
"id": "https://pigweed.googlesource.com/pigweed/sandbox/+/refs/heads/upstream/main@d48c1f5ab6b568d2144e45f8139a494ddde06c3d",
"title": "d48c1f5ab6b568d2144e45f8139a494ddde06c3d",
"url": "https://pigweed.googlesource.com/pigweed/sandbox/+/d48c1f5ab6b568d2144e45f8139a494ddde06c3d"
},
{
"gitiles": {
"ref": "refs/heads/upstream/main",
"repo": "https://pigweed.googlesource.com/pigweed/sandbox",
"revision": "769b8bb26eed127d39a33e76bc921d12126cafb0"
},
"id": "https://pigweed.googlesource.com/pigweed/sandbox/+/refs/heads/upstream/main@769b8bb26eed127d39a33e76bc921d12126cafb0",
"title": "769b8bb26eed127d39a33e76bc921d12126cafb0",
"url": "https://pigweed.googlesource.com/pigweed/sandbox/+/769b8bb26eed127d39a33e76bc921d12126cafb0"
},
{
"gitiles": {
"ref": "refs/heads/upstream/main",
"repo": "https://pigweed.googlesource.com/pigweed/sandbox",
"revision": "7666f7d35812186157fbeb804de4db09a0cf1d05"
},
"id": "https://pigweed.googlesource.com/pigweed/sandbox/+/refs/heads/upstream/main@7666f7d35812186157fbeb804de4db09a0cf1d05",
"title": "7666f7d35812186157fbeb804de4db09a0cf1d05",
"url": "https://pigweed.googlesource.com/pigweed/sandbox/+/7666f7d35812186157fbeb804de4db09a0cf1d05"
},
{
"gitiles": {
"ref": "refs/heads/upstream/main",
"repo": "https://pigweed.googlesource.com/pigweed/sandbox",
"revision": "d5fef7a0aaf91239ef82aa5adc83fd7ff63870a4"
},
"id": "https://pigweed.googlesource.com/pigweed/sandbox/+/refs/heads/upstream/main@d5fef7a0aaf91239ef82aa5adc83fd7ff63870a4",
"title": "d5fef7a0aaf91239ef82aa5adc83fd7ff63870a4",
"url": "https://pigweed.googlesource.com/pigweed/sandbox/+/d5fef7a0aaf91239ef82aa5adc83fd7ff63870a4"
},
{
"gitiles": {
"ref": "refs/heads/upstream/main",
"repo": "https://pigweed.googlesource.com/pigweed/sandbox",
"revision": "b50aeebf1e99973396b609773b2d940da59dbd59"
},
"id": "https://pigweed.googlesource.com/pigweed/sandbox/+/refs/heads/upstream/main@b50aeebf1e99973396b609773b2d940da59dbd59",
"title": "b50aeebf1e99973396b609773b2d940da59dbd59",
"url": "https://pigweed.googlesource.com/pigweed/sandbox/+/b50aeebf1e99973396b609773b2d940da59dbd59"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/sandbox",
"id": "b50aeebf1e99973396b609773b2d940da59dbd59",
"ref": "refs/heads/upstream/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8837971966874633408"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[00:09:09.984] Scheduled build:
{
"id": "8672541904792033793",
"builder": {
"project": "pigweed",
"bucket": "pigweed.sandbox.dev.ci",
"builder": "pigweed-mac-arm-sandbox"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-25T00:09:09.650307371Z",
"updateTime": "2026-08-25T00:09:09.650307371Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/sandbox",
"id": "b50aeebf1e99973396b609773b2d940da59dbd59",
"ref": "refs/heads/upstream/main"
}
}
}
[00:09:09.984] Task URL: https://cr-buildbucket.appspot.com/build/8672541904792033793
[00:09:09.984] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:2:0) after 2m18s
[00:09:10.918] Received PubSub notification, asking Buildbucket for the build status
[00:09:11.258] Build status: SCHEDULED
[00:11:27.999] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:2:0)
[00:11:28.034] Build status: SCHEDULED
[00:11:28.034] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:4:0) after 9m8s
[00:20:36.053] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:4:0)
[00:20:36.082] Build status: SCHEDULED
[00:20:36.082] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:5:0) after 3m30s
[00:24:06.150] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:5:0)
[00:24:06.171] Build status: SCHEDULED
[00:24:06.171] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:6:0) after 8m3s
[00:32:09.236] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:6:0)
[00:32:09.255] Build status: SCHEDULED
[00:32:09.255] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:7:0) after 8m59s
[00:41:08.292] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:7:0)
[00:41:08.316] Build status: SCHEDULED
[00:41:08.316] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:8:0) after 5m28s
[00:46:36.339] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:8:0)
[00:46:36.370] Build status: SCHEDULED
[00:46:36.370] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:9:0) after 5m12s
[00:51:48.385] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:9:0)
[00:51:48.416] Build status: SCHEDULED
[00:51:48.416] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:10:0) after 1m35s
[00:53:23.490] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:10:0)
[00:53:23.513] Build status: SCHEDULED
[00:53:23.513] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:11:0) after 9m45s
[01:03:08.573] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:11:0)
[01:03:08.593] Build status: SCHEDULED
[01:03:08.593] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:12:0) after 5m3s
[01:08:11.635] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:12:0)
[01:08:11.666] Build status: SCHEDULED
[01:08:11.666] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:13:0) after 8m10s
[01:16:21.746] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:13:0)
[01:16:21.841] Build status: SCHEDULED
[01:16:21.841] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:14:0) after 2m47s
[01:19:08.843] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:14:0)
[01:19:08.861] Build status: SCHEDULED
[01:19:08.861] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:15:0) after 7m58s
[01:27:06.883] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:15:0)
[01:27:06.963] Build status: SCHEDULED
[01:27:06.964] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:16:0) after 3m57s
[01:31:03.959] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:16:0)
[01:31:03.979] Build status: SCHEDULED
[01:31:03.980] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:17:0) after 1m55s
[01:32:59.045] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:17:0)
[01:32:59.080] Build status: SCHEDULED
[01:32:59.081] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:18:0) after 8m0s
[01:40:59.149] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:18:0)
[01:40:59.163] Build status: SCHEDULED
[01:40:59.163] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:19:0) after 1m6s
[01:42:05.237] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:19:0)
[01:42:05.258] Build status: SCHEDULED
[01:42:05.258] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:20:0) after 2m54s
[01:44:59.330] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:20:0)
[01:44:59.354] Build status: SCHEDULED
[01:44:59.354] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:21:0) after 7m13s
[01:52:12.377] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:21:0)
[01:52:12.397] Build status: SCHEDULED
[01:52:12.397] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:22:0) after 6m38s
[01:58:50.700] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:22:0)
[01:58:50.717] Build status: SCHEDULED
[01:58:50.717] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:23:0) after 7m13s
[02:06:03.786] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:23:0)
[02:06:03.813] Build status: SCHEDULED
[02:06:03.813] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:24:0) after 9m21s
[02:15:24.864] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:24:0)
[02:15:24.889] Build status: SCHEDULED
[02:15:24.889] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:25:0) after 2m59s
[02:18:23.912] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:25:0)
[02:18:23.944] Build status: SCHEDULED
[02:18:23.944] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:26:0) after 6m4s
[02:24:27.957] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:26:0)
[02:24:27.994] Build status: SCHEDULED
[02:24:27.994] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:27:0) after 3m38s
[02:28:06.105] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:27:0)
[02:28:06.128] Build status: SCHEDULED
[02:28:06.128] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:28:0) after 4m18s
[02:32:24.161] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:28:0)
[02:32:24.203] Build status: SCHEDULED
[02:32:24.203] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:29:0) after 5m13s
[02:37:37.173] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:29:0)
[02:37:37.192] Build status: SCHEDULED
[02:37:37.192] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:30:0) after 2m51s
[02:40:28.209] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:30:0)
[02:40:28.227] Build status: SCHEDULED
[02:40:28.227] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:31:0) after 1m21s
[02:41:49.322] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:31:0)
[02:41:49.398] Build status: SCHEDULED
[02:41:49.398] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:32:0) after 3m54s
[02:45:43.496] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:32:0)
[02:45:43.524] Build status: SCHEDULED
[02:45:43.524] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:33:0) after 4m50s
[02:50:33.759] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:33:0)
[02:50:33.783] Build status: SCHEDULED
[02:50:33.783] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:34:0) after 7m26s
[02:57:59.849] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:34:0)
[02:58:00.176] Build status: SCHEDULED
[02:58:00.176] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:35:0) after 7m12s
[03:05:12.273] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:35:0)
[03:05:12.318] Build status: SCHEDULED
[03:05:12.319] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:36:0) after 1m7s
[03:06:19.339] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:36:0)
[03:06:19.391] Build status: SCHEDULED
[03:06:19.391] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:37:0) after 5m14s
[03:11:33.394] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:37:0)
[03:11:33.450] Build status: SCHEDULED
[03:11:33.450] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:38:0) after 2m3s
[03:13:36.516] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:38:0)
[03:13:36.546] Build status: SCHEDULED
[03:13:36.546] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:39:0) after 7m2s
[03:20:38.568] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:39:0)
[03:20:38.592] Build status: SCHEDULED
[03:20:38.592] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:40:0) after 9m18s
[03:26:10.427] Received PubSub notification, asking Buildbucket for the build status
[03:26:10.445] Build status: STARTED
[03:29:56.663] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:40:0)
[03:29:56.724] Build status: STARTED
[03:29:56.724] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8837971966874633408:42:0) after 9m45s
[03:35:39.471] Received PubSub notification, asking Buildbucket for the build status
[03:35:39.491] Build:
{
"id": "8672541904792033793",
"builder": {
"project": "pigweed",
"bucket": "pigweed.sandbox.dev.ci",
"builder": "pigweed-mac-arm-sandbox"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-25T00:09:09.650307371Z",
"startTime": "2026-08-25T03:26:10.105607883Z",
"endTime": "2026-08-25T03:35:38.702002628Z",
"updateTime": "2026-08-25T03:35:38.702002628Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/sandbox",
"id": "b50aeebf1e99973396b609773b2d940da59dbd59",
"ref": "refs/heads/upstream/main"
}
}
}
[03:35:39.491] Invocation finished in 3h26m31.210468433s with status FAILED