[19:21:36.585] New invocation is queued and will start shortly
[19:21:37.721] Starting the invocation (attempt 1)
[19:21:37.753] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/experimental/+/aee184726120415f98c9539570a0aa17250f6fda"
[19:21:37.753] Popped gitiles commit info from properties and tags
[19:21:37.753] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[19:21:37.753] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[19:21:37.754] Buildbucket request:
{
"requestId": "8851670198524842944",
"builder": {
"project": "pigweed",
"bucket": "experimental.dev.ci",
"builder": "experimental-full"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8851670198524842944",
"job": "pigweed/experimental.dev.ci-experimental-full",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/experimental",
"revision": "aee184726120415f98c9539570a0aa17250f6fda"
},
"id": "https://pigweed.googlesource.com/pigweed/experimental/+/refs/heads/main@aee184726120415f98c9539570a0aa17250f6fda",
"title": "aee184726120415f98c9539570a0aa17250f6fda",
"url": "https://pigweed.googlesource.com/pigweed/experimental/+/aee184726120415f98c9539570a0aa17250f6fda"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/experimental",
"id": "aee184726120415f98c9539570a0aa17250f6fda",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8851670198524842944"
},
{
"key": "scheduler_job_id",
"value": "pigweed/experimental.dev.ci-experimental-full"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[19:21:38.265] Scheduled build:
{
"id": "8686240136581038945",
"builder": {
"project": "pigweed",
"bucket": "experimental.dev.ci",
"builder": "experimental-full"
},
"createdBy": "project:pigweed",
"createTime": "2026-03-26T19:21:37.813721291Z",
"updateTime": "2026-03-26T19:21:37.813721291Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/experimental",
"id": "aee184726120415f98c9539570a0aa17250f6fda",
"ref": "refs/heads/main"
}
}
}
[19:21:38.265] Task URL: https://cr-buildbucket.appspot.com/build/8686240136581038945
[19:21:38.265] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:2:0) after 2m38s
[19:21:38.755] Received PubSub notification, asking Buildbucket for the build status
[19:21:38.775] Build status: SCHEDULED
[19:24:16.349] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:2:0)
[19:24:16.379] Build status: SCHEDULED
[19:24:16.379] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:4:0) after 8m40s
[19:32:56.394] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:4:0)
[19:32:56.421] Build status: SCHEDULED
[19:32:56.421] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:5:0) after 6m32s
[19:39:28.918] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:5:0)
[19:39:28.934] Build status: SCHEDULED
[19:39:28.934] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:6:0) after 6m27s
[19:45:56.126] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:6:0)
[19:45:56.151] Build status: SCHEDULED
[19:45:56.151] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:7:0) after 7m38s
[19:53:34.187] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:7:0)
[19:53:34.208] Build status: SCHEDULED
[19:53:34.208] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:8:0) after 2m57s
[19:56:31.273] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:8:0)
[19:56:31.343] Build status: SCHEDULED
[19:56:31.343] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:9:0) after 1m49s
[19:58:20.372] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:9:0)
[19:58:20.428] Build status: SCHEDULED
[19:58:20.428] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:10:0) after 1m58s
[20:00:18.511] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:10:0)
[20:00:18.608] Build status: SCHEDULED
[20:00:18.608] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:11:0) after 5m57s
[20:06:15.625] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:11:0)
[20:06:15.644] Build status: SCHEDULED
[20:06:15.644] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:12:0) after 8m24s
[20:14:39.660] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:12:0)
[20:14:39.678] Build status: SCHEDULED
[20:14:39.678] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:13:0) after 2m7s
[20:16:46.695] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:13:0)
[20:16:46.734] Build status: SCHEDULED
[20:16:46.734] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:14:0) after 7m20s
[20:24:06.789] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:14:0)
[20:24:06.808] Build status: SCHEDULED
[20:24:06.808] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:15:0) after 6m41s
[20:30:47.779] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:15:0)
[20:30:47.796] Build status: SCHEDULED
[20:30:47.797] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:16:0) after 8m8s
[20:38:55.816] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:16:0)
[20:38:55.834] Build status: SCHEDULED
[20:38:55.834] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:17:0) after 2m6s
[20:41:01.907] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:17:0)
[20:41:01.926] Build status: SCHEDULED
[20:41:01.926] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:18:0) after 1m39s
[20:42:40.977] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:18:0)
[20:42:41.052] Build status: SCHEDULED
[20:42:41.052] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:19:0) after 4m52s
[20:47:33.073] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:19:0)
[20:47:33.098] Build status: SCHEDULED
[20:47:33.098] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:20:0) after 5m37s
[20:53:10.103] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:20:0)
[20:53:10.121] Build status: SCHEDULED
[20:53:10.121] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:21:0) after 9m51s
[21:03:01.140] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:21:0)
[21:03:01.160] Build status: SCHEDULED
[21:03:01.160] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:22:0) after 1m23s
[21:04:24.178] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:22:0)
[21:04:24.200] Build status: SCHEDULED
[21:04:24.200] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:23:0) after 7m24s
[21:11:48.222] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:23:0)
[21:11:48.239] Build status: SCHEDULED
[21:11:48.239] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:24:0) after 8m41s
[21:20:29.336] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:24:0)
[21:20:29.353] Build status: SCHEDULED
[21:20:29.353] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:25:0) after 4m9s
[21:24:38.430] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:25:0)
[21:24:38.452] Build status: SCHEDULED
[21:24:38.452] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:26:0) after 3m13s
[21:27:51.469] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:26:0)
[21:27:51.494] Build status: SCHEDULED
[21:27:51.494] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:27:0) after 8m50s
[21:36:41.548] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:27:0)
[21:36:41.699] Build status: SCHEDULED
[21:36:41.699] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:28:0) after 6m19s
[21:43:00.710] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:28:0)
[21:43:00.729] Build status: SCHEDULED
[21:43:00.729] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:29:0) after 3m55s
[21:46:55.845] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:29:0)
[21:46:55.959] Build status: SCHEDULED
[21:46:55.959] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:30:0) after 6m47s
[21:53:42.976] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:30:0)
[21:53:43.066] Build status: SCHEDULED
[21:53:43.066] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:31:0) after 3m42s
[21:57:25.129] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:31:0)
[21:57:25.159] Build status: SCHEDULED
[21:57:25.159] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:32:0) after 9m28s
[22:06:53.493] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:32:0)
[22:06:53.509] Build status: SCHEDULED
[22:06:53.509] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:33:0) after 4m23s
[22:11:16.630] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:33:0)
[22:11:16.658] Build status: SCHEDULED
[22:11:16.659] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:34:0) after 8m34s
[22:19:50.776] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:34:0)
[22:19:50.799] Build status: SCHEDULED
[22:19:50.799] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:35:0) after 7m37s
[22:27:28.120] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:35:0)
[22:27:28.156] Build status: SCHEDULED
[22:27:28.156] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:36:0) after 7m38s
[22:35:06.162] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:36:0)
[22:35:06.180] Build status: SCHEDULED
[22:35:06.180] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:37:0) after 6m3s
[22:41:09.299] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:37:0)
[22:41:09.320] Build status: SCHEDULED
[22:41:09.320] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:38:0) after 9m36s
[22:50:45.336] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:38:0)
[22:50:45.372] Build status: SCHEDULED
[22:50:45.372] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:39:0) after 1m44s
[22:52:29.387] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:39:0)
[22:52:29.412] Build status: SCHEDULED
[22:52:29.412] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:40:0) after 1m5s
[22:53:34.451] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:40:0)
[22:53:34.475] Build status: SCHEDULED
[22:53:34.475] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:41:0) after 3m25s
[22:56:59.682] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:41:0)
[22:56:59.705] Build status: SCHEDULED
[22:56:59.706] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:42:0) after 8m4s
[23:05:03.808] Handling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:42:0)
[23:05:03.827] Build status: SCHEDULED
[23:05:03.827] Scheduling timer "check-buildbucket-build-status" (pigweed/experimental.dev.ci-experimental-full:8851670198524842944:43:0) after 3m57s
[23:06:08.778] Received PubSub notification, asking Buildbucket for the build status
[23:06:08.797] Build status: STARTED
[23:06:39.357] Received PubSub notification, asking Buildbucket for the build status
[23:06:39.373] Build:
{
"id": "8686240136581038945",
"builder": {
"project": "pigweed",
"bucket": "experimental.dev.ci",
"builder": "experimental-full"
},
"createdBy": "project:pigweed",
"createTime": "2026-03-26T19:21:37.813721291Z",
"startTime": "2026-03-26T23:06:08.616983453Z",
"endTime": "2026-03-26T23:06:39.092473794Z",
"updateTime": "2026-03-26T23:06:39.092473794Z",
"status": "INFRA_FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/experimental",
"id": "aee184726120415f98c9539570a0aa17250f6fda",
"ref": "refs/heads/main"
}
}
}
[23:06:39.373] Invocation finished in 3h45m2.796615758s with status FAILED