[20:36:02.841] New invocation is queued and will start shortly
[20:36:02.841] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[20:36:04.631] Starting the invocation (attempt 1)
[20:36:04.654] Ignoring gitiles_ref tag without the buildset tag
[20:36:04.654] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[20:36:04.654] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[20:36:04.654] Buildbucket request:
{
"requestId": "8840250297544295776",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8840250297544295776",
"job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "51447208-caf5-4774-819a-83a620e9eee7",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8674820602026291313"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8840250297544295776"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[20:36:05.151] Scheduled build:
{
"id": "8674820234885511425",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-07-30T20:36:04.756132589Z",
"updateTime": "2026-07-30T20:36:04.756132589Z",
"status": "SCHEDULED",
"input": {
}
}
[20:36:05.151] Task URL: https://cr-buildbucket.appspot.com/build/8674820234885511425
[20:36:05.151] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:2:0) after 3m24s
[20:36:06.058] Received PubSub notification, asking Buildbucket for the build status
[20:36:06.088] Build status: SCHEDULED
[20:39:29.237] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:2:0)
[20:39:29.260] Build status: SCHEDULED
[20:39:29.260] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:4:0) after 6m41s
[20:46:10.348] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:4:0)
[20:46:10.392] Build status: SCHEDULED
[20:46:10.392] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:5:0) after 3m27s
[20:49:37.405] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:5:0)
[20:49:37.435] Build status: SCHEDULED
[20:49:37.435] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:6:0) after 8m23s
[20:58:00.479] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:6:0)
[20:58:00.500] Build status: SCHEDULED
[20:58:00.500] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:7:0) after 1m11s
[20:59:11.557] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:7:0)
[20:59:11.582] Build status: SCHEDULED
[20:59:11.582] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:8:0) after 2m15s
[21:01:26.697] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:8:0)
[21:01:26.784] Build status: SCHEDULED
[21:01:26.784] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:9:0) after 2m37s
[21:04:03.894] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:9:0)
[21:04:03.909] Build status: SCHEDULED
[21:04:03.909] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:10:0) after 5m8s
[21:09:12.013] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:10:0)
[21:09:12.030] Build status: SCHEDULED
[21:09:12.030] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:11:0) after 9m47s
[21:18:59.057] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:11:0)
[21:18:59.084] Build status: SCHEDULED
[21:18:59.084] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:12:0) after 8m44s
[21:27:43.097] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:12:0)
[21:27:43.137] Build status: SCHEDULED
[21:27:43.137] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:13:0) after 5m8s
[21:32:51.213] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:13:0)
[21:32:51.243] Build status: SCHEDULED
[21:32:51.243] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:14:0) after 3m4s
[21:35:55.349] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:14:0)
[21:35:55.400] Build status: SCHEDULED
[21:35:55.400] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:15:0) after 5m43s
[21:41:38.374] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:15:0)
[21:41:38.402] Build status: SCHEDULED
[21:41:38.402] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:16:0) after 1m37s
[21:43:15.477] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:16:0)
[21:43:15.510] Build status: SCHEDULED
[21:43:15.510] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:17:0) after 5m5s
[21:48:20.500] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:17:0)
[21:48:20.520] Build status: SCHEDULED
[21:48:20.520] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:18:0) after 9m15s
[21:57:35.719] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:18:0)
[21:57:35.740] Build status: SCHEDULED
[21:57:35.740] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:19:0) after 7m58s
[22:05:33.769] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:19:0)
[22:05:33.862] Build status: SCHEDULED
[22:05:33.862] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:20:0) after 9m10s
[22:14:44.442] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:20:0)
[22:14:44.458] Build status: SCHEDULED
[22:14:44.458] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:21:0) after 8m21s
[22:23:05.492] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:21:0)
[22:23:05.531] Build status: SCHEDULED
[22:23:05.531] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:22:0) after 5m32s
[22:28:37.548] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:22:0)
[22:28:37.568] Build status: SCHEDULED
[22:28:37.568] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:23:0) after 6m33s
[22:35:10.656] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:23:0)
[22:35:10.743] Build status: SCHEDULED
[22:35:10.743] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:24:0) after 5m12s
[22:40:22.756] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:24:0)
[22:40:22.777] Build status: SCHEDULED
[22:40:22.777] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:25:0) after 3m47s
[22:44:09.879] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:25:0)
[22:44:09.896] Build status: SCHEDULED
[22:44:09.896] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:26:0) after 3m34s
[22:47:44.076] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:26:0)
[22:47:44.096] Build status: SCHEDULED
[22:47:44.096] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:27:0) after 9m45s
[22:57:29.189] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:27:0)
[22:57:29.212] Build status: SCHEDULED
[22:57:29.212] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:28:0) after 4m46s
[23:02:15.333] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:28:0)
[23:02:15.427] Build status: SCHEDULED
[23:02:15.427] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:29:0) after 5m52s
[23:08:07.480] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:29:0)
[23:08:07.504] Build status: SCHEDULED
[23:08:07.504] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:30:0) after 1m55s
[23:10:02.654] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:30:0)
[23:10:02.669] Build status: SCHEDULED
[23:10:02.669] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:31:0) after 4m13s
[23:14:15.736] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:31:0)
[23:14:15.768] Build status: SCHEDULED
[23:14:15.768] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:32:0) after 3m5s
[23:17:20.783] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:32:0)
[23:17:20.802] Build status: SCHEDULED
[23:17:20.802] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:33:0) after 1m50s
[23:19:10.857] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:33:0)
[23:19:10.873] Build status: SCHEDULED
[23:19:10.873] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:34:0) after 6m30s
[23:25:40.911] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:34:0)
[23:25:40.935] Build status: SCHEDULED
[23:25:40.935] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:35:0) after 8m16s
[23:33:56.951] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:35:0)
[23:33:56.970] Build status: SCHEDULED
[23:33:56.970] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:36:0) after 1m42s
[23:35:38.986] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:36:0)
[23:35:39.016] Build status: SCHEDULED
[23:35:39.016] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:37:0) after 9m54s
[23:45:33.208] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:37:0)
[23:45:33.254] Build status: SCHEDULED
[23:45:33.254] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:38:0) after 1m5s
[23:46:38.478] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:38:0)
[23:46:38.547] Build status: SCHEDULED
[23:46:38.547] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:39:0) after 1m58s
[23:47:43.669] Received PubSub notification, asking Buildbucket for the build status
[23:47:43.685] Build status: STARTED
[23:48:36.523] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:39:0)
[23:48:36.546] Build status: STARTED
[23:48:36.546] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8840250297544295776:41:0) after 6m12s
[23:51:52.927] Received PubSub notification, asking Buildbucket for the build status
[23:51:52.991] Build:
{
"id": "8674820234885511425",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-07-30T20:36:04.756132589Z",
"startTime": "2026-07-30T23:47:43.488746818Z",
"endTime": "2026-07-30T23:51:52.482058336Z",
"updateTime": "2026-07-30T23:51:52.482058336Z",
"status": "FAILURE",
"input": {
}
}
[23:51:52.991] Invocation finished in 3h15m50.154914229s with status FAILED