[07:38:29.011] New invocation is queued and will start shortly
[07:38:29.011] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[07:38:30.568] Starting the invocation (attempt 1)
[07:38:30.590] Ignoring gitiles_ref tag without the buildset tag
[07:38:30.590] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[07:38:30.590] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[07:38:30.590] Buildbucket request:
{
"requestId": "8843017126625091072",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-asan-build"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8843017126625091072",
"job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "c2c448d5-7a7c-4b77-b034-18904ccdc7e8",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8677587413916304641"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8843017126625091072"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[07:38:30.966] Scheduled build:
{
"id": "8677587064253533441",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-asan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-06-30T07:38:30.650032926Z",
"updateTime": "2026-06-30T07:38:30.650032926Z",
"status": "SCHEDULED",
"input": {
},
"canary": true
}
[07:38:30.966] Task URL: https://cr-buildbucket.appspot.com/build/8677587064253533441
[07:38:30.966] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:2:0) after 1m21s
[07:38:31.329] Received PubSub notification, asking Buildbucket for the build status
[07:38:31.366] Build status: SCHEDULED
[07:39:51.978] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:2:0)
[07:39:51.995] Build status: SCHEDULED
[07:39:51.995] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:4:0) after 6m47s
[07:46:39.008] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:4:0)
[07:46:39.030] Build status: SCHEDULED
[07:46:39.030] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:5:0) after 9m49s
[07:56:28.091] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:5:0)
[07:56:28.108] Build status: SCHEDULED
[07:56:28.108] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:6:0) after 8m56s
[08:05:24.246] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:6:0)
[08:05:24.261] Build status: SCHEDULED
[08:05:24.261] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:7:0) after 6m36s
[08:12:00.260] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:7:0)
[08:12:00.325] Build status: SCHEDULED
[08:12:00.326] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:8:0) after 3m35s
[08:15:35.417] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:8:0)
[08:15:35.439] Build status: SCHEDULED
[08:15:35.439] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:9:0) after 6m32s
[08:22:07.434] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:9:0)
[08:22:07.632] Build status: SCHEDULED
[08:22:07.632] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:10:0) after 7m47s
[08:29:54.693] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:10:0)
[08:29:54.718] Build status: SCHEDULED
[08:29:54.718] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:11:0) after 1m46s
[08:31:40.749] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:11:0)
[08:31:40.767] Build status: SCHEDULED
[08:31:40.767] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:12:0) after 6m49s
[08:38:29.781] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:12:0)
[08:38:29.802] Build status: SCHEDULED
[08:38:29.802] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:13:0) after 6m22s
[08:44:51.958] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:13:0)
[08:44:51.996] Build status: SCHEDULED
[08:44:51.996] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:14:0) after 5m53s
[08:50:45.062] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:14:0)
[08:50:45.131] Build status: SCHEDULED
[08:50:45.131] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:15:0) after 2m54s
[08:53:39.157] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:15:0)
[08:53:39.175] Build status: SCHEDULED
[08:53:39.175] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:16:0) after 1m12s
[08:54:51.188] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:16:0)
[08:54:51.211] Build status: SCHEDULED
[08:54:51.211] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:17:0) after 6m27s
[09:01:18.282] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:17:0)
[09:01:18.312] Build status: SCHEDULED
[09:01:18.312] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:18:0) after 3m21s
[09:04:39.330] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:18:0)
[09:04:39.348] Build status: SCHEDULED
[09:04:39.348] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:19:0) after 5m34s
[09:10:13.476] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:19:0)
[09:10:13.499] Build status: SCHEDULED
[09:10:13.499] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:20:0) after 4m55s
[09:15:08.567] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:20:0)
[09:15:08.599] Build status: SCHEDULED
[09:15:08.599] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:21:0) after 8m4s
[09:23:12.613] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:21:0)
[09:23:12.630] Build status: SCHEDULED
[09:23:12.631] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:22:0) after 3m25s
[09:26:37.684] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:22:0)
[09:26:37.706] Build status: SCHEDULED
[09:26:37.706] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:23:0) after 9m1s
[09:35:38.801] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:23:0)
[09:35:38.823] Build status: SCHEDULED
[09:35:38.823] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:24:0) after 7m56s
[09:43:34.908] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:24:0)
[09:43:34.928] Build status: SCHEDULED
[09:43:34.928] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:25:0) after 9m40s
[09:53:14.915] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:25:0)
[09:53:14.964] Build status: SCHEDULED
[09:53:14.964] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:26:0) after 8m40s
[10:01:54.949] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:26:0)
[10:01:54.979] Build status: SCHEDULED
[10:01:54.979] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:27:0) after 7m29s
[10:09:23.998] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:27:0)
[10:09:24.031] Build status: SCHEDULED
[10:09:24.031] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:28:0) after 3m35s
[10:12:59.096] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:28:0)
[10:12:59.120] Build status: SCHEDULED
[10:12:59.120] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:29:0) after 9m58s
[10:22:57.136] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:29:0)
[10:22:57.154] Build status: SCHEDULED
[10:22:57.154] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:30:0) after 4m4s
[10:27:01.172] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:30:0)
[10:27:01.190] Build status: SCHEDULED
[10:27:01.190] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:31:0) after 9m34s
[10:36:35.206] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:31:0)
[10:36:35.265] Build status: SCHEDULED
[10:36:35.265] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:32:0) after 8m40s
[10:45:15.297] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:32:0)
[10:45:15.372] Build status: SCHEDULED
[10:45:15.372] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:33:0) after 3m50s
[10:48:17.398] Received PubSub notification, asking Buildbucket for the build status
[10:48:17.415] Build status: STARTED
[10:49:05.389] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:33:0)
[10:49:05.411] Build status: STARTED
[10:49:05.411] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:35:0) after 3m9s
[10:52:14.420] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:35:0)
[10:52:14.443] Build status: STARTED
[10:52:14.443] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:36:0) after 3m18s
[10:55:32.486] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:36:0)
[10:55:32.563] Build status: STARTED
[10:55:32.563] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan-build:8843017126625091072:37:0) after 9m5s
[11:00:49.119] Received PubSub notification, asking Buildbucket for the build status
[11:00:49.154] Build:
{
"id": "8677587064253533441",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-asan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-06-30T07:38:30.650032926Z",
"startTime": "2026-06-30T10:48:17.216376251Z",
"endTime": "2026-06-30T11:00:47.878298718Z",
"updateTime": "2026-06-30T11:00:47.878298718Z",
"status": "FAILURE",
"input": {
},
"canary": true
}
[11:00:49.154] Invocation finished in 3h22m20.148930406s with status FAILED