[08:17:24.923] New invocation is queued and will start shortly
[08:17:24.923] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[08:17:26.008] Starting the invocation (attempt 1)
[08:17:26.038] Ignoring gitiles_ref tag without the buildset tag
[08:17:26.038] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[08:17:26.038] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[08:17:26.038] Buildbucket request:
{
"requestId": "8853433328378310432",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-asan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8853433328378310432",
"job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "2ed0b9e3-f10a-49a2-96a5-0d1d692977a6",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8688003436759648433"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8853433328378310432"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[08:17:26.661] Scheduled build:
{
"id": "8688003266492722321",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2026-03-07T08:17:26.100403739Z",
"updateTime": "2026-03-07T08:17:26.100403739Z",
"status": "SCHEDULED",
"input": {
}
}
[08:17:26.661] Task URL: https://cr-buildbucket.appspot.com/build/8688003266492722321
[08:17:26.661] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:2:0) after 9m34s
[08:17:27.474] Received PubSub notification, asking Buildbucket for the build status
[08:17:27.489] Build status: SCHEDULED
[08:27:00.726] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:2:0)
[08:27:00.783] Build status: SCHEDULED
[08:27:00.783] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:4:0) after 8m52s
[08:35:52.882] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:4:0)
[08:35:52.942] Build status: SCHEDULED
[08:35:52.942] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:5:0) after 2m34s
[08:38:26.967] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:5:0)
[08:38:26.999] Build status: SCHEDULED
[08:38:26.999] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:6:0) after 1m23s
[08:39:50.083] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:6:0)
[08:39:50.216] Build status: SCHEDULED
[08:39:50.216] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:7:0) after 3m14s
[08:43:04.404] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:7:0)
[08:43:04.420] Build status: SCHEDULED
[08:43:04.420] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:8:0) after 9m7s
[08:52:11.483] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:8:0)
[08:52:11.503] Build status: SCHEDULED
[08:52:11.503] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:9:0) after 6m19s
[08:58:30.653] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:9:0)
[08:58:30.673] Build status: SCHEDULED
[08:58:30.673] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:10:0) after 1m15s
[08:59:45.822] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:10:0)
[08:59:45.840] Build status: SCHEDULED
[08:59:45.841] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:11:0) after 5m8s
[09:04:53.863] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:11:0)
[09:04:53.889] Build status: SCHEDULED
[09:04:53.889] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:12:0) after 6m52s
[09:11:45.902] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:12:0)
[09:11:45.918] Build status: SCHEDULED
[09:11:45.918] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:13:0) after 2m53s
[09:14:38.932] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:13:0)
[09:14:38.956] Build status: SCHEDULED
[09:14:38.956] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:14:0) after 8m59s
[09:23:37.974] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:14:0)
[09:23:38.157] Build status: SCHEDULED
[09:23:38.158] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:15:0) after 6m51s
[09:30:29.169] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:15:0)
[09:30:29.219] Build status: SCHEDULED
[09:30:29.219] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:16:0) after 9m42s
[09:40:11.369] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:16:0)
[09:40:11.440] Build status: SCHEDULED
[09:40:11.440] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:17:0) after 8m42s
[09:48:53.746] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:17:0)
[09:48:53.765] Build status: SCHEDULED
[09:48:53.765] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:18:0) after 4m58s
[09:53:51.782] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:18:0)
[09:53:51.806] Build status: SCHEDULED
[09:53:51.806] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:19:0) after 7m11s
[10:01:02.820] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:19:0)
[10:01:02.838] Build status: SCHEDULED
[10:01:02.838] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:20:0) after 2m53s
[10:03:56.325] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:20:0)
[10:03:56.352] Build status: SCHEDULED
[10:03:56.352] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:21:0) after 3m2s
[10:06:58.367] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:21:0)
[10:06:58.423] Build status: SCHEDULED
[10:06:58.423] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:22:0) after 1m2s
[10:08:00.857] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:22:0)
[10:08:00.873] Build status: SCHEDULED
[10:08:00.873] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:23:0) after 8m7s
[10:16:08.138] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:23:0)
[10:16:08.188] Build status: SCHEDULED
[10:16:08.188] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:24:0) after 5m7s
[10:21:15.199] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:24:0)
[10:21:15.216] Build status: SCHEDULED
[10:21:15.216] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:25:0) after 1m44s
[10:22:59.434] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:25:0)
[10:22:59.456] Build status: SCHEDULED
[10:22:59.456] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:26:0) after 6m24s
[10:29:23.524] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:26:0)
[10:29:23.547] Build status: SCHEDULED
[10:29:23.547] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:27:0) after 6m35s
[10:35:58.862] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:27:0)
[10:35:58.877] Build status: SCHEDULED
[10:35:58.877] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:28:0) after 9m43s
[10:45:41.895] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:28:0)
[10:45:41.933] Build status: SCHEDULED
[10:45:41.934] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:29:0) after 6m39s
[10:52:21.033] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:29:0)
[10:52:21.052] Build status: SCHEDULED
[10:52:21.052] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:30:0) after 1m30s
[10:53:51.105] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:30:0)
[10:53:51.126] Build status: SCHEDULED
[10:53:51.126] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:31:0) after 6m37s
[11:00:28.106] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:31:0)
[11:00:28.145] Build status: SCHEDULED
[11:00:28.145] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:32:0) after 6m10s
[11:06:38.172] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:32:0)
[11:06:38.194] Build status: SCHEDULED
[11:06:38.194] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:33:0) after 2m13s
[11:08:51.219] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:33:0)
[11:08:51.235] Build status: SCHEDULED
[11:08:51.235] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:34:0) after 3m21s
[11:12:12.321] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:34:0)
[11:12:12.394] Build status: SCHEDULED
[11:12:12.394] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:35:0) after 2m40s
[11:14:52.367] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:35:0)
[11:14:52.450] Build status: SCHEDULED
[11:14:52.450] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:36:0) after 8m44s
[11:23:36.563] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:36:0)
[11:23:36.591] Build status: SCHEDULED
[11:23:36.591] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:37:0) after 9m41s
[11:33:17.784] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:37:0)
[11:33:17.853] Build status: SCHEDULED
[11:33:17.853] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:38:0) after 6m32s
[11:39:49.974] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:38:0)
[11:39:50.051] Build status: SCHEDULED
[11:39:50.051] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:39:0) after 6m32s
[11:46:22.067] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:39:0)
[11:46:22.109] Build status: SCHEDULED
[11:46:22.110] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:40:0) after 6m14s
[11:52:36.261] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:40:0)
[11:52:36.279] Build status: SCHEDULED
[11:52:36.279] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:41:0) after 4m5s
[11:56:41.463] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:41:0)
[11:56:41.480] Build status: SCHEDULED
[11:56:41.480] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:42:0) after 2m37s
[11:59:18.537] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:42:0)
[11:59:18.567] Build status: SCHEDULED
[11:59:18.568] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:43:0) after 9m41s
[12:08:59.961] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:43:0)
[12:08:59.977] Build status: SCHEDULED
[12:08:59.977] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:44:0) after 1m19s
[12:10:19.026] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:44:0)
[12:10:19.047] Build status: SCHEDULED
[12:10:19.047] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8853433328378310432:45:0) after 7m8s
[12:13:19.675] Received PubSub notification, asking Buildbucket for the build status
[12:13:19.699] Build status: STARTED
[12:17:02.775] Received PubSub notification, asking Buildbucket for the build status
[12:17:02.801] Build:
{
"id": "8688003266492722321",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2026-03-07T08:17:26.100403739Z",
"startTime": "2026-03-07T12:13:19.337955056Z",
"endTime": "2026-03-07T12:17:02.453125843Z",
"updateTime": "2026-03-07T12:17:02.453125843Z",
"status": "FAILURE",
"input": {
}
}
[12:17:02.801] Invocation finished in 3h59m37.882312178s with status FAILED