[08:38:34.194] New invocation is queued and will start shortly
[08:38:34.194] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[08:38:35.246] Starting the invocation (attempt 1)
[08:38:35.275] Ignoring gitiles_ref tag without the buildset tag
[08:38:35.275] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[08:38:35.275] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[08:38:35.275] Buildbucket request:
{
"requestId": "8840929616097373712",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-asan-build"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8840929616097373712",
"job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "7bc08e7e-d512-46c6-b4a4-0c45cb480a24",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8675499914418045137"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8840929616097373712"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[08:38:35.661] Scheduled build:
{
"id": "8675499554213243265",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-asan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-07-23T08:38:35.360163609Z",
"updateTime": "2026-07-23T08:38:35.360163609Z",
"status": "SCHEDULED",
"input": {
},
"canary": true
}
[08:38:35.661] Task URL: https://cr-buildbucket.appspot.com/build/8675499554213243265
[08:38:35.661] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:2:0) after 7m48s
[08:38:36.710] Received PubSub notification, asking Buildbucket for the build status
[08:38:36.730] Build status: SCHEDULED
[08:46:23.674] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:2:0)
[08:46:23.696] Build status: SCHEDULED
[08:46:23.696] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:4:0) after 8m21s
[08:54:44.812] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:4:0)
[08:54:44.831] Build status: SCHEDULED
[08:54:44.831] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:5:0) after 5m57s
[09:00:41.844] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:5:0)
[09:00:41.874] Build status: SCHEDULED
[09:00:41.874] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:6:0) after 9m40s
[09:10:21.887] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:6:0)
[09:10:21.905] Build status: SCHEDULED
[09:10:21.905] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:7:0) after 7m22s
[09:17:43.940] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:7:0)
[09:17:43.970] Build status: SCHEDULED
[09:17:43.970] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:8:0) after 1m8s
[09:18:52.029] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:8:0)
[09:18:52.047] Build status: SCHEDULED
[09:18:52.047] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:9:0) after 1m10s
[09:20:02.060] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:9:0)
[09:20:02.079] Build status: SCHEDULED
[09:20:02.079] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:10:0) after 7m10s
[09:27:12.093] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:10:0)
[09:27:12.109] Build status: SCHEDULED
[09:27:12.109] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:11:0) after 5m5s
[09:32:17.316] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:11:0)
[09:32:17.337] Build status: SCHEDULED
[09:32:17.337] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:12:0) after 3m53s
[09:36:10.351] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:12:0)
[09:36:10.371] Build status: SCHEDULED
[09:36:10.371] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:13:0) after 8m17s
[09:44:27.383] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:13:0)
[09:44:27.408] Build status: SCHEDULED
[09:44:27.408] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:14:0) after 9m5s
[09:53:32.507] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:14:0)
[09:53:32.579] Build status: SCHEDULED
[09:53:32.579] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:15:0) after 9m2s
[10:02:34.593] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:15:0)
[10:02:34.630] Build status: SCHEDULED
[10:02:34.630] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:16:0) after 7m26s
[10:10:00.642] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:16:0)
[10:10:00.789] Build status: SCHEDULED
[10:10:00.789] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:17:0) after 2m49s
[10:12:49.805] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:17:0)
[10:12:49.858] Build status: SCHEDULED
[10:12:49.858] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:18:0) after 5m42s
[10:18:31.825] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:18:0)
[10:18:31.843] Build status: SCHEDULED
[10:18:31.843] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:19:0) after 1m23s
[10:19:54.853] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:19:0)
[10:19:54.876] Build status: SCHEDULED
[10:19:54.876] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:20:0) after 5m4s
[10:24:58.918] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:20:0)
[10:24:58.935] Build status: SCHEDULED
[10:24:58.935] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:21:0) after 7m23s
[10:32:21.950] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:21:0)
[10:32:21.976] Build status: SCHEDULED
[10:32:21.976] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:22:0) after 5m2s
[10:37:24.071] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:22:0)
[10:37:24.092] Build status: SCHEDULED
[10:37:24.092] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:23:0) after 1m10s
[10:38:34.446] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:23:0)
[10:38:34.475] Build status: SCHEDULED
[10:38:34.475] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:24:0) after 3m6s
[10:41:40.560] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:24:0)
[10:41:40.579] Build status: SCHEDULED
[10:41:40.579] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:25:0) after 9m57s
[10:51:37.662] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:25:0)
[10:51:37.692] Build status: SCHEDULED
[10:51:37.692] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:26:0) after 5m19s
[10:56:56.872] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:26:0)
[10:56:56.891] Build status: SCHEDULED
[10:56:56.891] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:27:0) after 2m52s
[10:59:48.902] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:27:0)
[10:59:48.924] Build status: SCHEDULED
[10:59:48.924] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:28:0) after 2m19s
[11:02:07.939] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:28:0)
[11:02:07.957] Build status: SCHEDULED
[11:02:07.958] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:29:0) after 7m42s
[11:09:49.973] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:29:0)
[11:09:49.994] Build status: SCHEDULED
[11:09:49.994] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:30:0) after 4m13s
[11:14:03.022] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:30:0)
[11:14:03.043] Build status: SCHEDULED
[11:14:03.043] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:31:0) after 3m43s
[11:17:46.157] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:31:0)
[11:17:46.175] Build status: SCHEDULED
[11:17:46.175] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:32:0) after 5m0s
[11:22:46.228] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:32:0)
[11:22:46.247] Build status: SCHEDULED
[11:22:46.247] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:33:0) after 9m33s
[11:32:19.270] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:33:0)
[11:32:19.293] Build status: SCHEDULED
[11:32:19.293] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:34:0) after 9m28s
[11:41:06.062] Received PubSub notification, asking Buildbucket for the build status
[11:41:06.092] Build status: STARTED
[11:41:47.355] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:34:0)
[11:41:47.448] Build status: STARTED
[11:41:47.448] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8840929616097373712:36:0) after 5m57s
[11:45:40.225] Received PubSub notification, asking Buildbucket for the build status
[11:45:40.275] Build:
{
"id": "8675499554213243265",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-asan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-07-23T08:38:35.360163609Z",
"startTime": "2026-07-23T11:41:05.631882161Z",
"endTime": "2026-07-23T11:45:22.352283990Z",
"updateTime": "2026-07-23T11:45:22.352283990Z",
"status": "FAILURE",
"input": {
},
"canary": true
}
[11:45:40.275] Invocation finished in 3h7m6.095469007s with status FAILED