[07:37:26.242] New invocation is queued and will start shortly
[07:37:26.242] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[07:37:27.400] Starting the invocation (attempt 1)
[07:37:27.488] Ignoring gitiles_ref tag without the buildset tag
[07:37:27.488] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[07:37:27.488] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[07:37:27.488] Buildbucket request:
{
"requestId": "8832779735238912192",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "python-packages"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8832779735238912192",
"job": "pigweed/pigweed.dev.ci-python-packages",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "30818e03-1840-480a-be0a-32e156c92ca2",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8667350118906280849"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8832779735238912192"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-python-packages"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[07:37:28.295] Scheduled build:
{
"id": "8667349672817459073",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "python-packages"
},
"createdBy": "project:pigweed",
"createTime": "2026-10-21T07:37:27.929931257Z",
"updateTime": "2026-10-21T07:37:27.929931257Z",
"status": "SCHEDULED",
"input": {
}
}
[07:37:28.295] Task URL: https://cr-buildbucket.appspot.com/build/8667349672817459073
[07:37:28.295] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:2:0) after 2m16s
[07:37:30.313] Received PubSub notification, asking Buildbucket for the build status
[07:37:30.342] Build status: SCHEDULED
[07:37:33.928] Received PubSub notification, asking Buildbucket for the build status
[07:37:33.959] Build status: STARTED
[07:39:44.322] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:2:0)
[07:39:44.346] Build status: STARTED
[07:39:44.346] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:5:0) after 2m26s
[07:42:10.432] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:5:0)
[07:42:10.459] Build status: STARTED
[07:42:10.459] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:6:0) after 7m14s
[07:49:24.494] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:6:0)
[07:49:24.768] Build status: STARTED
[07:49:24.768] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:7:0) after 5m31s
[07:54:55.803] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:7:0)
[07:54:55.823] Build status: STARTED
[07:54:55.823] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:8:0) after 2m39s
[07:57:34.933] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:8:0)
[07:57:34.966] Build status: STARTED
[07:57:34.966] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:9:0) after 2m2s
[07:59:36.965] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:9:0)
[07:59:36.989] Build status: STARTED
[07:59:36.989] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:10:0) after 4m59s
[08:04:36.111] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:10:0)
[08:04:36.140] Build status: STARTED
[08:04:36.140] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:11:0) after 3m42s
[08:08:18.189] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:11:0)
[08:08:18.213] Build status: STARTED
[08:08:18.213] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:12:0) after 9m38s
[08:17:56.233] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:12:0)
[08:17:56.254] Build status: STARTED
[08:17:56.255] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:13:0) after 3m49s
[08:21:45.265] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:13:0)
[08:21:45.283] Build status: STARTED
[08:21:45.283] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:14:0) after 5m58s
[08:27:43.366] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:14:0)
[08:27:43.412] Build status: STARTED
[08:27:43.412] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:15:0) after 5m15s
[08:32:58.506] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:15:0)
[08:32:58.535] Build status: STARTED
[08:32:58.535] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:16:0) after 8m42s
[08:41:40.664] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:16:0)
[08:41:40.696] Build status: STARTED
[08:41:40.696] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:17:0) after 7m36s
[08:49:16.923] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:17:0)
[08:49:17.034] Build status: STARTED
[08:49:17.034] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:18:0) after 3m52s
[08:53:09.128] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:18:0)
[08:53:09.151] Build status: STARTED
[08:53:09.151] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:19:0) after 3m50s
[08:56:59.293] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:19:0)
[08:56:59.312] Build status: STARTED
[08:56:59.312] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:20:0) after 6m57s
[09:03:56.287] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:20:0)
[09:03:56.365] Build status: STARTED
[09:03:56.366] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:21:0) after 5m3s
[09:08:59.487] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:21:0)
[09:08:59.506] Build status: STARTED
[09:08:59.507] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:22:0) after 3m23s
[09:12:22.533] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:22:0)
[09:12:22.551] Build status: STARTED
[09:12:22.551] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:23:0) after 3m2s
[09:15:24.619] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:23:0)
[09:15:24.644] Build status: STARTED
[09:15:24.644] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:24:0) after 6m26s
[09:21:50.643] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:24:0)
[09:21:50.701] Build status: STARTED
[09:21:50.701] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:25:0) after 1m44s
[09:23:34.788] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:25:0)
[09:23:34.809] Build status: STARTED
[09:23:34.809] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:26:0) after 4m49s
[09:28:23.829] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:26:0)
[09:28:23.850] Build status: STARTED
[09:28:23.850] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:27:0) after 4m19s
[09:32:43.064] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:27:0)
[09:32:43.086] Build status: STARTED
[09:32:43.086] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:28:0) after 2m56s
[09:35:39.176] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:28:0)
[09:35:39.217] Build status: STARTED
[09:35:39.217] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:29:0) after 7m32s
[09:43:11.258] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:29:0)
[09:43:11.273] Build status: STARTED
[09:43:11.273] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:30:0) after 8m55s
[09:52:06.292] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:30:0)
[09:52:06.337] Build status: STARTED
[09:52:06.337] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:31:0) after 1m24s
[09:53:30.352] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:31:0)
[09:53:30.391] Build status: STARTED
[09:53:30.391] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:32:0) after 1m36s
[09:55:06.424] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:32:0)
[09:55:06.441] Build status: STARTED
[09:55:06.441] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:33:0) after 3m24s
[09:58:30.471] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:33:0)
[09:58:30.644] Build status: STARTED
[09:58:30.644] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:34:0) after 4m43s
[10:03:13.671] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:34:0)
[10:03:13.690] Build status: STARTED
[10:03:13.690] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:35:0) after 6m10s
[10:09:23.789] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:35:0)
[10:09:23.813] Build status: STARTED
[10:09:23.813] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:36:0) after 5m43s
[10:15:06.886] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:36:0)
[10:15:06.906] Build status: STARTED
[10:15:06.906] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:37:0) after 7m17s
[10:22:23.923] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:37:0)
[10:22:24.103] Build status: STARTED
[10:22:24.103] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:38:0) after 8m47s
[10:31:11.173] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:38:0)
[10:31:11.203] Build status: STARTED
[10:31:11.203] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:39:0) after 4m15s
[10:35:26.225] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:39:0)
[10:35:26.246] Build status: STARTED
[10:35:26.246] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:40:0) after 2m31s
[10:37:57.263] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:40:0)
[10:37:57.283] Build status: STARTED
[10:37:57.283] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:41:0) after 1m18s
[10:39:15.319] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:41:0)
[10:39:15.336] Build status: STARTED
[10:39:15.336] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:42:0) after 8m49s
[10:48:04.449] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:42:0)
[10:48:04.470] Build status: STARTED
[10:48:04.470] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:43:0) after 6m11s
[10:54:15.513] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:43:0)
[10:54:15.556] Build status: STARTED
[10:54:15.556] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-python-packages:8832779735238912192:44:0) after 9m19s
[10:59:50.745] Received PubSub notification, asking Buildbucket for the build status
[10:59:50.848] Build:
{
"id": "8667349672817459073",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "python-packages"
},
"createdBy": "project:pigweed",
"createTime": "2026-10-21T07:37:27.929931257Z",
"startTime": "2026-10-21T07:37:31.246268849Z",
"endTime": "2026-10-21T10:59:40.303776696Z",
"updateTime": "2026-10-21T10:59:40.303776696Z",
"status": "INFRA_FAILURE",
"statusDetails": {
"timeout": {
}
},
"input": {
}
}
[10:59:50.848] Invocation finished in 3h22m24.610460727s with status FAILED