[14:04:00.272] New invocation is queued and will start shortly
[14:04:00.272] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[14:04:01.399] Starting the invocation (attempt 1)
[14:04:01.430] Ignoring gitiles_ref tag without the buildset tag
[14:04:01.430] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[14:04:01.430] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[14:04:01.430] Buildbucket request:
{
"requestId": "8837738247685531424",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-host-cpp23"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8837738247685531424",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "6d247e90-cc99-4091-8755-d2b381b9ab84",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8672308548239036577"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8837738247685531424"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[14:04:01.853] Scheduled build:
{
"id": "8672308185734069201",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-host-cpp23"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-27T14:04:01.513861642Z",
"updateTime": "2026-08-27T14:04:01.513861642Z",
"status": "SCHEDULED",
"input": {
},
"canary": true
}
[14:04:01.853] Task URL: https://cr-buildbucket.appspot.com/build/8672308185734069201
[14:04:01.853] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:2:0) after 1m10s
[14:04:02.755] Received PubSub notification, asking Buildbucket for the build status
[14:04:02.787] Build status: SCHEDULED
[14:05:11.828] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:2:0)
[14:05:11.844] Build status: SCHEDULED
[14:05:11.845] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:4:0) after 9m57s
[14:15:08.887] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:4:0)
[14:15:08.948] Build status: SCHEDULED
[14:15:08.948] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:5:0) after 1m47s
[14:16:56.236] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:5:0)
[14:16:56.278] Build status: SCHEDULED
[14:16:56.278] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:6:0) after 3m30s
[14:20:26.293] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:6:0)
[14:20:26.322] Build status: SCHEDULED
[14:20:26.322] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:7:0) after 8m39s
[14:29:05.502] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:7:0)
[14:29:05.524] Build status: SCHEDULED
[14:29:05.524] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:8:0) after 5m15s
[14:34:20.538] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:8:0)
[14:34:20.591] Build status: SCHEDULED
[14:34:20.591] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:9:0) after 1m14s
[14:35:34.679] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:9:0)
[14:35:34.705] Build status: SCHEDULED
[14:35:34.705] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:10:0) after 1m43s
[14:37:17.776] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:10:0)
[14:37:17.815] Build status: SCHEDULED
[14:37:17.815] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:11:0) after 4m19s
[14:41:37.135] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:11:0)
[14:41:37.219] Build status: SCHEDULED
[14:41:37.219] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:12:0) after 4m9s
[14:45:46.448] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:12:0)
[14:45:46.491] Build status: SCHEDULED
[14:45:46.491] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:13:0) after 5m52s
[14:51:38.509] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:13:0)
[14:51:38.530] Build status: SCHEDULED
[14:51:38.530] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:14:0) after 5m42s
[14:57:20.636] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:14:0)
[14:57:20.663] Build status: SCHEDULED
[14:57:20.663] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:15:0) after 2m47s
[15:00:07.709] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:15:0)
[15:00:07.775] Build status: SCHEDULED
[15:00:07.775] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:16:0) after 4m47s
[15:04:54.882] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:16:0)
[15:04:54.907] Build status: SCHEDULED
[15:04:54.907] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:17:0) after 9m43s
[15:14:37.933] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:17:0)
[15:14:37.954] Build status: SCHEDULED
[15:14:37.954] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:18:0) after 3m34s
[15:18:12.161] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:18:0)
[15:18:12.184] Build status: SCHEDULED
[15:18:12.184] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:19:0) after 2m33s
[15:20:45.285] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:19:0)
[15:20:45.307] Build status: SCHEDULED
[15:20:45.307] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:20:0) after 9m42s
[15:30:27.322] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:20:0)
[15:30:27.344] Build status: SCHEDULED
[15:30:27.344] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:21:0) after 1m54s
[15:32:21.587] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:21:0)
[15:32:21.607] Build status: SCHEDULED
[15:32:21.607] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:22:0) after 6m13s
[15:38:34.751] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:22:0)
[15:38:34.838] Build status: SCHEDULED
[15:38:34.838] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:23:0) after 8m27s
[15:47:01.867] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:23:0)
[15:47:01.885] Build status: SCHEDULED
[15:47:01.885] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:24:0) after 6m3s
[15:53:05.155] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:24:0)
[15:53:05.179] Build status: SCHEDULED
[15:53:05.179] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:25:0) after 8m55s
[16:02:00.193] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:25:0)
[16:02:00.212] Build status: SCHEDULED
[16:02:00.212] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:26:0) after 2m38s
[16:04:38.603] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:26:0)
[16:04:38.627] Build status: SCHEDULED
[16:04:38.627] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:27:0) after 4m30s
[16:09:08.639] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:27:0)
[16:09:08.668] Build status: SCHEDULED
[16:09:08.668] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:28:0) after 2m28s
[16:11:36.778] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:28:0)
[16:11:36.795] Build status: SCHEDULED
[16:11:36.795] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:29:0) after 3m43s
[16:15:19.819] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:29:0)
[16:15:19.839] Build status: SCHEDULED
[16:15:19.839] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:30:0) after 2m20s
[16:17:40.114] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:30:0)
[16:17:40.137] Build status: SCHEDULED
[16:17:40.137] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:31:0) after 3m51s
[16:21:31.272] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:31:0)
[16:21:31.294] Build status: SCHEDULED
[16:21:31.294] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:32:0) after 5m37s
[16:27:08.389] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:32:0)
[16:27:08.469] Build status: SCHEDULED
[16:27:08.470] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:33:0) after 7m42s
[16:34:50.488] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:33:0)
[16:34:50.507] Build status: SCHEDULED
[16:34:50.507] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:34:0) after 7m22s
[16:42:12.567] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:34:0)
[16:42:12.594] Build status: SCHEDULED
[16:42:12.594] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:35:0) after 4m23s
[16:46:35.658] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:35:0)
[16:46:35.758] Build status: SCHEDULED
[16:46:35.758] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:36:0) after 3m46s
[16:50:21.793] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:36:0)
[16:50:21.822] Build status: SCHEDULED
[16:50:21.822] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:37:0) after 3m18s
[16:53:39.869] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:37:0)
[16:53:39.886] Build status: SCHEDULED
[16:53:39.887] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:38:0) after 7m43s
[17:01:22.908] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:38:0)
[17:01:22.948] Build status: SCHEDULED
[17:01:22.948] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:39:0) after 2m0s
[17:03:19.302] Received PubSub notification, asking Buildbucket for the build status
[17:03:19.329] Build status: STARTED
[17:03:23.064] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:39:0)
[17:03:23.093] Build status: STARTED
[17:03:23.093] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:41:0) after 1m2s
[17:04:25.261] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:41:0)
[17:04:25.289] Build status: STARTED
[17:04:25.289] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:42:0) after 4m48s
[17:09:13.336] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:42:0)
[17:09:13.459] Build status: STARTED
[17:09:13.459] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-host-cpp23:8837738247685531424:43:0) after 5m24s
[17:10:37.364] Received PubSub notification, asking Buildbucket for the build status
[17:10:37.383] Build:
{
"id": "8672308185734069201",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-host-cpp23"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-27T14:04:01.513861642Z",
"startTime": "2026-08-27T17:03:18.854243151Z",
"endTime": "2026-08-27T17:10:37.034453066Z",
"updateTime": "2026-08-27T17:10:37.034453066Z",
"status": "FAILURE",
"input": {
},
"canary": true
}
[17:10:37.383] Invocation finished in 3h6m37.113985986s with status FAILED