[23:18:22.196] New invocation is queued and will start shortly
[23:18:22.196] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[23:18:23.312] Starting the invocation (attempt 1)
[23:18:23.337] Ignoring gitiles_ref tag without the buildset tag
[23:18:23.337] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[23:18:23.337] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[23:18:23.337] Buildbucket request:
{
"requestId": "8838337548796403232",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8838337548796403232",
"job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "03b42de6-0e19-4ed7-9600-b6003830c0a2",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8672907886639880897"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8838337548796403232"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[23:18:23.784] Scheduled build:
{
"id": "8672907486858566241",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-20T23:18:23.423402286Z",
"updateTime": "2026-08-20T23:18:23.423402286Z",
"status": "SCHEDULED",
"input": {
}
}
[23:18:23.784] Task URL: https://cr-buildbucket.appspot.com/build/8672907486858566241
[23:18:23.784] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:2:0) after 8m56s
[23:18:24.715] Received PubSub notification, asking Buildbucket for the build status
[23:18:24.732] Build status: SCHEDULED
[23:27:19.801] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:2:0)
[23:27:19.827] Build status: SCHEDULED
[23:27:19.827] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:4:0) after 3m13s
[23:30:32.877] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:4:0)
[23:30:32.900] Build status: SCHEDULED
[23:30:32.900] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:5:0) after 8m8s
[23:38:40.968] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:5:0)
[23:38:41.003] Build status: SCHEDULED
[23:38:41.003] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:6:0) after 4m51s
[23:43:32.068] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:6:0)
[23:43:32.088] Build status: SCHEDULED
[23:43:32.088] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:7:0) after 9m0s
[23:52:32.350] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:7:0)
[23:52:32.368] Build status: SCHEDULED
[23:52:32.368] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:8:0) after 9m17s
[00:01:49.443] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:8:0)
[00:01:49.466] Build status: SCHEDULED
[00:01:49.466] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:9:0) after 6m57s
[00:08:46.500] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:9:0)
[00:08:46.524] Build status: SCHEDULED
[00:08:46.524] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:10:0) after 4m49s
[00:13:35.752] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:10:0)
[00:13:35.801] Build status: SCHEDULED
[00:13:35.801] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:11:0) after 8m39s
[00:22:14.868] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:11:0)
[00:22:14.889] Build status: SCHEDULED
[00:22:14.889] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:12:0) after 8m14s
[00:30:28.902] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:12:0)
[00:30:28.925] Build status: SCHEDULED
[00:30:28.925] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:13:0) after 7m0s
[00:37:29.020] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:13:0)
[00:37:29.050] Build status: SCHEDULED
[00:37:29.050] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:14:0) after 6m18s
[00:43:47.115] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:14:0)
[00:43:47.140] Build status: SCHEDULED
[00:43:47.140] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:15:0) after 1m0s
[00:44:47.298] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:15:0)
[00:44:47.325] Build status: SCHEDULED
[00:44:47.325] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:16:0) after 5m48s
[00:50:35.344] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:16:0)
[00:50:35.372] Build status: SCHEDULED
[00:50:35.372] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:17:0) after 6m42s
[00:57:17.468] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:17:0)
[00:57:17.490] Build status: SCHEDULED
[00:57:17.490] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:18:0) after 7m57s
[01:05:14.508] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:18:0)
[01:05:14.532] Build status: SCHEDULED
[01:05:14.532] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:19:0) after 2m54s
[01:08:08.545] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:19:0)
[01:08:08.606] Build status: SCHEDULED
[01:08:08.606] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:20:0) after 7m58s
[01:16:06.759] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:20:0)
[01:16:06.780] Build status: SCHEDULED
[01:16:06.780] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:21:0) after 1m28s
[01:17:34.794] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:21:0)
[01:17:34.815] Build status: SCHEDULED
[01:17:34.815] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:22:0) after 1m12s
[01:18:46.913] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:22:0)
[01:18:46.934] Build status: SCHEDULED
[01:18:46.934] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:23:0) after 3m43s
[01:22:29.960] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:23:0)
[01:22:29.990] Build status: SCHEDULED
[01:22:29.990] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:24:0) after 2m17s
[01:24:47.003] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:24:0)
[01:24:47.027] Build status: SCHEDULED
[01:24:47.027] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:25:0) after 3m19s
[01:28:06.083] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:25:0)
[01:28:06.098] Build status: SCHEDULED
[01:28:06.098] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:26:0) after 9m36s
[01:37:42.114] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:26:0)
[01:37:42.136] Build status: SCHEDULED
[01:37:42.136] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:27:0) after 2m53s
[01:40:35.277] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:27:0)
[01:40:35.300] Build status: SCHEDULED
[01:40:35.300] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:28:0) after 4m30s
[01:45:05.353] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:28:0)
[01:45:05.381] Build status: SCHEDULED
[01:45:05.381] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:29:0) after 1m56s
[01:47:01.397] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:29:0)
[01:47:01.649] Build status: SCHEDULED
[01:47:01.649] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:30:0) after 9m53s
[01:56:54.676] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:30:0)
[01:56:54.709] Build status: SCHEDULED
[01:56:54.709] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:31:0) after 9m31s
[02:06:25.822] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:31:0)
[02:06:25.852] Build status: SCHEDULED
[02:06:25.852] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:32:0) after 1m41s
[02:08:06.919] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:32:0)
[02:08:06.941] Build status: SCHEDULED
[02:08:06.941] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:33:0) after 1m41s
[02:09:47.988] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:33:0)
[02:09:48.077] Build status: SCHEDULED
[02:09:48.078] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:34:0) after 4m8s
[02:13:56.115] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:34:0)
[02:13:56.140] Build status: SCHEDULED
[02:13:56.140] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:35:0) after 9m19s
[02:23:15.263] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:35:0)
[02:23:15.298] Build status: SCHEDULED
[02:23:15.298] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:36:0) after 2m28s
[02:25:35.407] Received PubSub notification, asking Buildbucket for the build status
[02:25:35.490] Build status: STARTED
[02:25:43.320] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:36:0)
[02:25:43.352] Build status: STARTED
[02:25:43.353] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:38:0) after 5m33s
[02:31:16.384] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:38:0)
[02:31:16.486] Build status: STARTED
[02:31:16.486] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8838337548796403232:39:0) after 1m26s
[02:31:40.946] Received PubSub notification, asking Buildbucket for the build status
[02:31:40.971] Build:
{
"id": "8672907486858566241",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-20T23:18:23.423402286Z",
"startTime": "2026-08-21T02:25:33.480139380Z",
"endTime": "2026-08-21T02:31:40.615600182Z",
"updateTime": "2026-08-21T02:31:40.615600182Z",
"status": "FAILURE",
"input": {
}
}
[02:31:40.971] Invocation finished in 3h13m18.780593498s with status FAILED