[23:22:52.055] New invocation is queued and will start shortly
[23:22:52.055] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[23:22:54.480] Starting the invocation (attempt 1)
[23:22:54.517] Ignoring gitiles_ref tag without the buildset tag
[23:22:54.517] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[23:22:54.517] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[23:22:54.518] Buildbucket request:
{
"requestId": "8836072341668677760",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8836072341668677760",
"job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "262e567d-7883-47d1-b73a-db3de74e5e75",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8670642721127439985"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8836072341668677760"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[23:22:54.876] Scheduled build:
{
"id": "8670642278359838993",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-09-14T23:22:54.590416410Z",
"updateTime": "2026-09-14T23:22:54.590416410Z",
"status": "SCHEDULED",
"input": {
},
"canary": true
}
[23:22:54.876] Task URL: https://cr-buildbucket.appspot.com/build/8670642278359838993
[23:22:54.876] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:2:0) after 1m23s
[23:22:55.823] Received PubSub notification, asking Buildbucket for the build status
[23:22:55.844] Build status: SCHEDULED
[23:24:17.846] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:2:0)
[23:24:17.873] Build status: SCHEDULED
[23:24:17.873] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:4:0) after 6m34s
[23:30:51.888] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:4:0)
[23:30:51.911] Build status: SCHEDULED
[23:30:51.911] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:5:0) after 4m17s
[23:35:08.926] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:5:0)
[23:35:08.950] Build status: SCHEDULED
[23:35:08.950] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:6:0) after 4m37s
[23:39:46.038] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:6:0)
[23:39:46.062] Build status: SCHEDULED
[23:39:46.062] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:7:0) after 2m47s
[23:42:33.215] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:7:0)
[23:42:33.260] Build status: SCHEDULED
[23:42:33.260] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:8:0) after 3m22s
[23:45:55.327] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:8:0)
[23:45:55.343] Build status: SCHEDULED
[23:45:55.343] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:9:0) after 2m0s
[23:47:55.434] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:9:0)
[23:47:55.455] Build status: SCHEDULED
[23:47:55.455] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:10:0) after 2m24s
[23:50:19.469] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:10:0)
[23:50:19.514] Build status: SCHEDULED
[23:50:19.514] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:11:0) after 1m25s
[23:51:44.556] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:11:0)
[23:51:44.578] Build status: SCHEDULED
[23:51:44.578] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:12:0) after 4m6s
[23:55:50.645] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:12:0)
[23:55:50.673] Build status: SCHEDULED
[23:55:50.673] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:13:0) after 3m3s
[23:58:53.692] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:13:0)
[23:58:53.710] Build status: SCHEDULED
[23:58:53.711] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:14:0) after 4m58s
[00:03:51.923] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:14:0)
[00:03:51.940] Build status: SCHEDULED
[00:03:51.940] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:15:0) after 7m31s
[00:11:23.039] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:15:0)
[00:11:23.062] Build status: SCHEDULED
[00:11:23.062] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:16:0) after 4m37s
[00:16:00.080] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:16:0)
[00:16:00.100] Build status: SCHEDULED
[00:16:00.100] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:17:0) after 3m19s
[00:19:19.470] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:17:0)
[00:19:19.492] Build status: SCHEDULED
[00:19:19.492] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:18:0) after 9m47s
[00:29:06.567] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:18:0)
[00:29:06.629] Build status: SCHEDULED
[00:29:06.629] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:19:0) after 7m20s
[00:36:26.733] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:19:0)
[00:36:26.754] Build status: SCHEDULED
[00:36:26.754] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:20:0) after 1m28s
[00:37:54.769] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:20:0)
[00:37:54.942] Build status: SCHEDULED
[00:37:54.942] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:21:0) after 3m5s
[00:40:59.955] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:21:0)
[00:40:59.988] Build status: SCHEDULED
[00:40:59.988] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:22:0) after 4m1s
[00:45:01.023] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:22:0)
[00:45:01.049] Build status: SCHEDULED
[00:45:01.049] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:23:0) after 5m32s
[00:50:33.246] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:23:0)
[00:50:33.273] Build status: SCHEDULED
[00:50:33.273] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:24:0) after 7m44s
[00:58:17.380] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:24:0)
[00:58:17.449] Build status: SCHEDULED
[00:58:17.449] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:25:0) after 6m50s
[01:05:07.493] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:25:0)
[01:05:07.513] Build status: SCHEDULED
[01:05:07.513] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:26:0) after 3m11s
[01:08:18.731] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:26:0)
[01:08:18.755] Build status: SCHEDULED
[01:08:18.755] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:27:0) after 4m45s
[01:13:03.731] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:27:0)
[01:13:03.751] Build status: SCHEDULED
[01:13:03.751] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:28:0) after 4m14s
[01:17:17.818] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:28:0)
[01:17:17.892] Build status: SCHEDULED
[01:17:17.892] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:29:0) after 9m54s
[01:27:12.055] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:29:0)
[01:27:12.077] Build status: SCHEDULED
[01:27:12.077] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:30:0) after 6m37s
[01:33:49.124] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:30:0)
[01:33:49.150] Build status: SCHEDULED
[01:33:49.150] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:31:0) after 3m1s
[01:36:50.205] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:31:0)
[01:36:50.226] Build status: SCHEDULED
[01:36:50.226] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:32:0) after 1m34s
[01:38:24.253] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:32:0)
[01:38:24.313] Build status: SCHEDULED
[01:38:24.313] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:33:0) after 1m19s
[01:39:43.455] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:33:0)
[01:39:43.479] Build status: SCHEDULED
[01:39:43.479] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:34:0) after 1m46s
[01:41:29.545] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:34:0)
[01:41:29.714] Build status: SCHEDULED
[01:41:29.714] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:35:0) after 6m33s
[01:48:02.753] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:35:0)
[01:48:02.769] Build status: SCHEDULED
[01:48:02.770] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:36:0) after 4m57s
[01:52:59.787] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:36:0)
[01:52:59.849] Build status: SCHEDULED
[01:52:59.849] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:37:0) after 7m20s
[02:00:19.921] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:37:0)
[02:00:19.944] Build status: SCHEDULED
[02:00:19.944] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:38:0) after 9m24s
[02:09:43.962] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:38:0)
[02:09:44.000] Build status: SCHEDULED
[02:09:44.000] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:39:0) after 9m1s
[02:18:45.049] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:39:0)
[02:18:45.083] Build status: SCHEDULED
[02:18:45.083] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:40:0) after 9m44s
[02:28:29.109] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:40:0)
[02:28:29.301] Build status: SCHEDULED
[02:28:29.301] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:41:0) after 1m9s
[02:29:38.367] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:41:0)
[02:29:38.414] Build status: SCHEDULED
[02:29:38.414] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:42:0) after 2m55s
[02:30:37.799] Received PubSub notification, asking Buildbucket for the build status
[02:30:37.819] Build status: STARTED
[02:32:33.463] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:42:0)
[02:32:33.505] Build status: STARTED
[02:32:33.505] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:44:0) after 3m45s
[02:36:18.553] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:44:0)
[02:36:18.579] Build status: STARTED
[02:36:18.579] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:45:0) after 5m45s
[02:42:03.770] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:45:0)
[02:42:03.810] Build status: STARTED
[02:42:03.810] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:46:0) after 6m56s
[02:48:59.785] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:46:0)
[02:48:59.809] Build status: STARTED
[02:48:59.809] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8836072341668677760:47:0) after 4m52s
[02:52:42.574] Received PubSub notification, asking Buildbucket for the build status
[02:52:42.595] Build:
{
"id": "8670642278359838993",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-09-14T23:22:54.590416410Z",
"startTime": "2026-09-15T02:30:37.604667440Z",
"endTime": "2026-09-15T02:52:42.322753521Z",
"updateTime": "2026-09-15T02:52:42.322753521Z",
"status": "SUCCESS",
"input": {
},
"canary": true
}
[02:52:42.595] Invocation finished in 3h29m50.545074626s with status SUCCEEDED