[20:56:07.009] New invocation is queued and will start shortly
[20:56:09.758] Starting the invocation (attempt 1)
[20:56:09.785] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/sandbox/+/36301830c018cf91f5d17aa19d1428c32b411af4"
[20:56:09.785] Popped gitiles commit info from properties and tags
[20:56:09.785] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[20:56:09.786] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[20:56:09.786] Buildbucket request:
{
"requestId": "8843419928709863664",
"builder": {
"project": "pigweed",
"bucket": "pigweed.sandbox.dev.ci",
"builder": "pigweed-mac-arm-sandbox"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8843419928709863664",
"job": "pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/upstream/main",
"repo": "https://pigweed.googlesource.com/pigweed/sandbox",
"revision": "71edf35190d9b7fe4f1453941a1a255f2012ff49"
},
"id": "https://pigweed.googlesource.com/pigweed/sandbox/+/refs/heads/upstream/main@71edf35190d9b7fe4f1453941a1a255f2012ff49",
"title": "71edf35190d9b7fe4f1453941a1a255f2012ff49",
"url": "https://pigweed.googlesource.com/pigweed/sandbox/+/71edf35190d9b7fe4f1453941a1a255f2012ff49"
},
{
"gitiles": {
"ref": "refs/heads/upstream/main",
"repo": "https://pigweed.googlesource.com/pigweed/sandbox",
"revision": "ea91b2e6e426f9ff29fea0a16920261498f87b93"
},
"id": "https://pigweed.googlesource.com/pigweed/sandbox/+/refs/heads/upstream/main@ea91b2e6e426f9ff29fea0a16920261498f87b93",
"title": "ea91b2e6e426f9ff29fea0a16920261498f87b93",
"url": "https://pigweed.googlesource.com/pigweed/sandbox/+/ea91b2e6e426f9ff29fea0a16920261498f87b93"
},
{
"gitiles": {
"ref": "refs/heads/upstream/main",
"repo": "https://pigweed.googlesource.com/pigweed/sandbox",
"revision": "51da2bb6551d0dadf63650916b116578f0ffef0e"
},
"id": "https://pigweed.googlesource.com/pigweed/sandbox/+/refs/heads/upstream/main@51da2bb6551d0dadf63650916b116578f0ffef0e",
"title": "51da2bb6551d0dadf63650916b116578f0ffef0e",
"url": "https://pigweed.googlesource.com/pigweed/sandbox/+/51da2bb6551d0dadf63650916b116578f0ffef0e"
},
{
"gitiles": {
"ref": "refs/heads/upstream/main",
"repo": "https://pigweed.googlesource.com/pigweed/sandbox",
"revision": "36301830c018cf91f5d17aa19d1428c32b411af4"
},
"id": "https://pigweed.googlesource.com/pigweed/sandbox/+/refs/heads/upstream/main@36301830c018cf91f5d17aa19d1428c32b411af4",
"title": "36301830c018cf91f5d17aa19d1428c32b411af4",
"url": "https://pigweed.googlesource.com/pigweed/sandbox/+/36301830c018cf91f5d17aa19d1428c32b411af4"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/sandbox",
"id": "36301830c018cf91f5d17aa19d1428c32b411af4",
"ref": "refs/heads/upstream/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8843419928709863664"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[20:56:10.519] Scheduled build:
{
"id": "8677989865019911601",
"builder": {
"project": "pigweed",
"bucket": "pigweed.sandbox.dev.ci",
"builder": "pigweed-mac-arm-sandbox"
},
"createdBy": "project:pigweed",
"createTime": "2026-06-25T20:56:09.904835566Z",
"updateTime": "2026-06-25T20:56:09.904835566Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/sandbox",
"id": "36301830c018cf91f5d17aa19d1428c32b411af4",
"ref": "refs/heads/upstream/main"
}
}
}
[20:56:10.519] Task URL: https://cr-buildbucket.appspot.com/build/8677989865019911601
[20:56:10.519] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:2:0) after 6m48s
[20:56:11.220] Received PubSub notification, asking Buildbucket for the build status
[20:56:11.242] Build status: SCHEDULED
[21:02:58.587] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:2:0)
[21:02:58.606] Build status: SCHEDULED
[21:02:58.606] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:4:0) after 9m9s
[21:12:07.811] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:4:0)
[21:12:07.828] Build status: SCHEDULED
[21:12:07.828] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:5:0) after 1m29s
[21:13:36.890] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:5:0)
[21:13:36.913] Build status: SCHEDULED
[21:13:36.913] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:6:0) after 9m24s
[21:23:00.909] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:6:0)
[21:23:00.927] Build status: SCHEDULED
[21:23:00.927] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:7:0) after 8m24s
[21:31:25.097] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:7:0)
[21:31:25.114] Build status: SCHEDULED
[21:31:25.114] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:8:0) after 2m0s
[21:33:25.264] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:8:0)
[21:33:25.284] Build status: SCHEDULED
[21:33:25.284] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:9:0) after 3m43s
[21:37:08.271] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:9:0)
[21:37:08.676] Build status: SCHEDULED
[21:37:08.676] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:10:0) after 7m10s
[21:44:18.693] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:10:0)
[21:44:18.715] Build status: SCHEDULED
[21:44:18.715] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:11:0) after 6m58s
[21:51:16.739] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:11:0)
[21:51:16.786] Build status: SCHEDULED
[21:51:16.786] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:12:0) after 1m59s
[21:53:15.852] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:12:0)
[21:53:15.890] Build status: SCHEDULED
[21:53:15.890] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:13:0) after 2m25s
[21:55:40.906] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:13:0)
[21:55:40.967] Build status: SCHEDULED
[21:55:40.967] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:14:0) after 1m11s
[21:56:51.995] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:14:0)
[21:56:52.015] Build status: SCHEDULED
[21:56:52.016] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:15:0) after 6m43s
[22:03:35.030] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:15:0)
[22:03:35.066] Build status: SCHEDULED
[22:03:35.066] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:16:0) after 1m44s
[22:05:19.175] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:16:0)
[22:05:19.195] Build status: SCHEDULED
[22:05:19.195] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:17:0) after 5m57s
[22:11:16.235] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:17:0)
[22:11:16.453] Build status: SCHEDULED
[22:11:16.453] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:18:0) after 5m7s
[22:16:23.466] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:18:0)
[22:16:23.504] Build status: SCHEDULED
[22:16:23.504] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:19:0) after 7m47s
[22:24:10.572] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:19:0)
[22:24:10.591] Build status: SCHEDULED
[22:24:10.592] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:20:0) after 1m14s
[22:25:24.612] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:20:0)
[22:25:24.631] Build status: SCHEDULED
[22:25:24.631] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:21:0) after 9m28s
[22:34:52.652] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:21:0)
[22:34:52.672] Build status: SCHEDULED
[22:34:52.672] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:22:0) after 2m3s
[22:36:55.840] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:22:0)
[22:36:55.860] Build status: SCHEDULED
[22:36:55.860] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:23:0) after 5m19s
[22:42:14.874] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:23:0)
[22:42:14.896] Build status: SCHEDULED
[22:42:14.896] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:24:0) after 6m21s
[22:48:35.987] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:24:0)
[22:48:36.028] Build status: SCHEDULED
[22:48:36.028] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:25:0) after 4m59s
[22:53:35.179] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:25:0)
[22:53:35.282] Build status: SCHEDULED
[22:53:35.282] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:26:0) after 7m33s
[23:01:08.289] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:26:0)
[23:01:08.309] Build status: SCHEDULED
[23:01:08.309] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:27:0) after 4m1s
[23:05:09.458] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:27:0)
[23:05:09.488] Build status: SCHEDULED
[23:05:09.488] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:28:0) after 5m54s
[23:11:03.657] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:28:0)
[23:11:03.689] Build status: SCHEDULED
[23:11:03.689] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:29:0) after 5m13s
[23:16:16.716] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:29:0)
[23:16:16.737] Build status: SCHEDULED
[23:16:16.737] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:30:0) after 7m38s
[23:23:54.755] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:30:0)
[23:23:54.777] Build status: SCHEDULED
[23:23:54.777] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:31:0) after 2m21s
[23:26:15.839] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:31:0)
[23:26:15.874] Build status: SCHEDULED
[23:26:15.874] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:32:0) after 8m13s
[23:34:28.990] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:32:0)
[23:34:29.010] Build status: SCHEDULED
[23:34:29.010] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:33:0) after 2m38s
[23:37:07.027] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:33:0)
[23:37:07.055] Build status: SCHEDULED
[23:37:07.056] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:34:0) after 4m7s
[23:41:14.121] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:34:0)
[23:41:14.141] Build status: SCHEDULED
[23:41:14.141] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:35:0) after 5m36s
[23:46:50.224] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:35:0)
[23:46:50.244] Build status: SCHEDULED
[23:46:50.244] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.sandbox.dev.ci-pigweed-mac-arm-sandbox:8843419928709863664:36:0) after 8m52s
[23:50:37.694] Received PubSub notification, asking Buildbucket for the build status
[23:50:37.720] Build status: STARTED
[23:54:14.950] Received PubSub notification, asking Buildbucket for the build status
[23:54:15.079] Build:
{
"id": "8677989865019911601",
"builder": {
"project": "pigweed",
"bucket": "pigweed.sandbox.dev.ci",
"builder": "pigweed-mac-arm-sandbox"
},
"createdBy": "project:pigweed",
"createTime": "2026-06-25T20:56:09.904835566Z",
"startTime": "2026-06-25T23:50:37.528783191Z",
"endTime": "2026-06-25T23:54:14.479498207Z",
"updateTime": "2026-06-25T23:54:14.479498207Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/sandbox",
"id": "36301830c018cf91f5d17aa19d1428c32b411af4",
"ref": "refs/heads/upstream/main"
}
}
}
[23:54:15.079] Invocation finished in 2h58m8.077326874s with status FAILED