[02:05:28.147] New invocation is queued and will start shortly
[02:05:29.245] Starting the invocation (attempt 1)
[02:05:29.267] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/experimental/+/c18d393383fe06190941e5fedbb66dfb776b3924"
[02:05:29.267] Popped gitiles commit info from properties and tags
[02:05:29.267] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[02:05:29.267] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[02:05:29.267] Buildbucket request:
{
"requestId": "8861701053159047808",
"builder": {
"project": "pigweed",
"bucket": "experimental.dev.ci",
"builder": "experimental-full"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8861701053159047808",
"job": "pigweed/experimental.dev.ci-experimental-full",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/experimental",
"revision": "c18d393383fe06190941e5fedbb66dfb776b3924"
},
"id": "https://pigweed.googlesource.com/pigweed/experimental/+/refs/heads/main@c18d393383fe06190941e5fedbb66dfb776b3924",
"title": "c18d393383fe06190941e5fedbb66dfb776b3924",
"url": "https://pigweed.googlesource.com/pigweed/experimental/+/c18d393383fe06190941e5fedbb66dfb776b3924"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/experimental",
"id": "c18d393383fe06190941e5fedbb66dfb776b3924",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8861701053159047808"
},
{
"key": "scheduler_job_id",
"value": "pigweed/experimental.dev.ci-experimental-full"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[02:05:29.582] Scheduled build:
{
"id": "8696270991277042561",
"builder": {
"project": "pigweed",
"bucket": "experimental.dev.ci",
"builder": "experimental-full"
},
"createdBy": "project:pigweed",
"createTime": "2025-12-06T02:05:29.318183926Z",
"updateTime": "2025-12-06T02:05:29.318183926Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/experimental",
"id": "c18d393383fe06190941e5fedbb66dfb776b3924",
"ref": "refs/heads/main"
}
},
"canary": true
}
[02:05:29.582] Task URL: https://cr-buildbucket.appspot.com/build/8696270991277042561
[02:05:29.582] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:2:0) after 1m25s
[02:05:30.595] Received PubSub notification, asking Buildbucket for the build status
[02:05:30.650] Build status: SCHEDULED
[02:06:54.708] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:2:0)
[02:06:54.728] Build status: SCHEDULED
[02:06:54.728] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:4:0) after 9m8s
[02:16:02.999] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:4:0)
[02:16:03.110] Build status: SCHEDULED
[02:16:03.110] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:5:0) after 9m12s
[02:25:15.121] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:5:0)
[02:25:15.143] Build status: SCHEDULED
[02:25:15.143] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:6:0) after 6m41s
[02:31:56.153] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:6:0)
[02:31:56.169] Build status: SCHEDULED
[02:31:56.169] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:7:0) after 6m35s
[02:38:31.148] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:7:0)
[02:38:31.168] Build status: SCHEDULED
[02:38:31.168] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:8:0) after 4m4s
[02:42:35.181] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:8:0)
[02:42:35.213] Build status: SCHEDULED
[02:42:35.213] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:9:0) after 4m33s
[02:47:08.200] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:9:0)
[02:47:08.237] Build status: SCHEDULED
[02:47:08.237] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:10:0) after 4m34s
[02:51:42.249] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:10:0)
[02:51:42.268] Build status: SCHEDULED
[02:51:42.268] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:11:0) after 9m21s
[03:01:03.281] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:11:0)
[03:01:03.404] Build status: SCHEDULED
[03:01:03.404] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:12:0) after 5m4s
[03:06:07.419] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:12:0)
[03:06:07.441] Build status: SCHEDULED
[03:06:07.441] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:13:0) after 7m26s
[03:13:33.456] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:13:0)
[03:13:33.481] Build status: SCHEDULED
[03:13:33.481] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:14:0) after 2m45s
[03:16:18.496] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:14:0)
[03:16:18.519] Build status: SCHEDULED
[03:16:18.519] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:15:0) after 6m13s
[03:22:31.536] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:15:0)
[03:22:31.552] Build status: SCHEDULED
[03:22:31.552] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:16:0) after 1m35s
[03:24:06.614] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:16:0)
[03:24:06.654] Build status: SCHEDULED
[03:24:06.655] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:17:0) after 4m21s
[03:28:27.650] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:17:0)
[03:28:27.667] Build status: SCHEDULED
[03:28:27.667] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:18:0) after 8m44s
[03:37:11.681] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:18:0)
[03:37:11.696] Build status: SCHEDULED
[03:37:11.696] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:19:0) after 9m30s
[03:46:41.710] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:19:0)
[03:46:41.732] Build status: SCHEDULED
[03:46:41.732] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:20:0) after 2m10s
[03:48:51.743] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:20:0)
[03:48:51.758] Build status: SCHEDULED
[03:48:51.758] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:21:0) after 8m37s
[03:57:28.756] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:21:0)
[03:57:28.778] Build status: SCHEDULED
[03:57:28.778] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:22:0) after 4m54s
[04:02:22.846] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:22:0)
[04:02:22.876] Build status: SCHEDULED
[04:02:22.876] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:23:0) after 2m18s
[04:04:40.889] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:23:0)
[04:04:40.955] Build status: SCHEDULED
[04:04:40.955] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:24:0) after 2m9s
[04:06:49.965] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:24:0)
[04:06:49.982] Build status: SCHEDULED
[04:06:49.983] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:25:0) after 3m30s
[04:10:19.997] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:25:0)
[04:10:20.019] Build status: SCHEDULED
[04:10:20.019] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:26:0) after 9m53s
[04:20:13.038] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:26:0)
[04:20:13.054] Build status: SCHEDULED
[04:20:13.054] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:27:0) after 4m1s
[04:24:14.117] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:27:0)
[04:24:14.135] Build status: SCHEDULED
[04:24:14.135] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:28:0) after 9m4s
[04:33:18.153] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:28:0)
[04:33:18.170] Build status: SCHEDULED
[04:33:18.170] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:29:0) after 3m37s
[04:36:55.183] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:29:0)
[04:36:55.433] Build status: SCHEDULED
[04:36:55.433] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:30:0) after 9m30s
[04:46:25.466] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:30:0)
[04:46:25.485] Build status: SCHEDULED
[04:46:25.485] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:31:0) after 5m20s
[04:51:45.564] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:31:0)
[04:51:45.577] Build status: SCHEDULED
[04:51:45.578] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:32:0) after 5m9s
[04:56:54.594] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:32:0)
[04:56:54.612] Build status: SCHEDULED
[04:56:54.612] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:33:0) after 8m57s
[05:05:51.589] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:33:0)
[05:05:51.614] Build status: SCHEDULED
[05:05:51.614] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:34:0) after 3m52s
[05:09:43.632] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:34:0)
[05:09:43.653] Build status: SCHEDULED
[05:09:43.653] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:35:0) after 2m51s
[05:12:34.669] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:35:0)
[05:12:34.684] Build status: SCHEDULED
[05:12:34.684] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:36:0) after 2m47s
[05:15:21.746] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:36:0)
[05:15:21.761] Build status: SCHEDULED
[05:15:21.761] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:37:0) after 4m22s
[05:19:43.825] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:37:0)
[05:19:43.842] Build status: SCHEDULED
[05:19:43.842] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:38:0) after 1m33s
[05:21:16.884] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:38:0)
[05:21:16.929] Build status: SCHEDULED
[05:21:16.929] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:39:0) after 1m56s
[05:23:12.949] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:39:0)
[05:23:12.966] Build status: SCHEDULED
[05:23:12.966] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:40:0) after 6m1s
[05:29:14.178] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:40:0)
[05:29:14.199] Build status: SCHEDULED
[05:29:14.199] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:41:0) after 4m50s
[05:34:04.217] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:41:0)
[05:34:04.565] Build status: SCHEDULED
[05:34:04.565] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:42:0) after 2m52s
[05:36:56.577] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:42:0)
[05:36:56.595] Build status: SCHEDULED
[05:36:56.595] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:43:0) after 5m2s
[05:41:58.609] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:43:0)
[05:41:58.627] Build status: SCHEDULED
[05:41:58.627] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:44:0) after 1m46s
[05:43:44.643] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:44:0)
[05:43:44.666] Build status: SCHEDULED
[05:43:44.666] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:45:0) after 1m31s
[05:45:15.688] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:45:0)
[05:45:15.705] Build status: SCHEDULED
[05:45:15.705] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:46:0) after 3m48s
[05:49:03.721] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:46:0)
[05:49:03.739] Build status: SCHEDULED
[05:49:03.739] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8861701053159047808:47:0) after 3m39s
[05:50:44.686] Received PubSub notification, asking Buildbucket for the build status
[05:50:44.703] Build status: STARTED
[05:51:13.069] Received PubSub notification, asking Buildbucket for the build status
[05:51:13.135] Build:
{
"id": "8696270991277042561",
"builder": {
"project": "pigweed",
"bucket": "experimental.dev.ci",
"builder": "experimental-full"
},
"createdBy": "project:pigweed",
"createTime": "2025-12-06T02:05:29.318183926Z",
"startTime": "2025-12-06T05:50:44.547629584Z",
"endTime": "2025-12-06T05:51:12.545117657Z",
"updateTime": "2025-12-06T05:51:12.545117657Z",
"status": "INFRA_FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/experimental",
"id": "c18d393383fe06190941e5fedbb66dfb776b3924",
"ref": "refs/heads/main"
}
},
"canary": true
}
[05:51:13.135] Invocation finished in 3h45m44.996388836s with status FAILED