[03:32:42.608] New invocation is queued and will start shortly
[03:32:42.608] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[03:32:43.841] Starting the invocation (attempt 1)
[03:32:43.862] Ignoring gitiles_ref tag without the buildset tag
[03:32:43.862] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[03:32:43.862] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[03:32:43.862] Buildbucket request:
{
"requestId": "8849374376998825840",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8849374376998825840",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "7400076a-53b5-4487-835d-8683125be5b7",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8683944490649707441"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8849374376998825840"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[03:32:44.179] Scheduled build:
{
"id": "8683944314971188641",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-04-21T03:32:43.917733030Z",
"updateTime": "2026-04-21T03:32:43.917733030Z",
"status": "SCHEDULED",
"input": {
}
}
[03:32:44.179] Task URL: https://cr-buildbucket.appspot.com/build/8683944314971188641
[03:32:44.179] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:2:0) after 3m53s
[03:32:45.167] Received PubSub notification, asking Buildbucket for the build status
[03:32:45.244] Build status: SCHEDULED
[03:36:37.245] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:2:0)
[03:36:37.261] Build status: SCHEDULED
[03:36:37.261] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:4:0) after 2m18s
[03:38:55.275] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:4:0)
[03:38:55.315] Build status: SCHEDULED
[03:38:55.315] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:5:0) after 9m38s
[03:48:33.299] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:5:0)
[03:48:33.318] Build status: SCHEDULED
[03:48:33.318] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:6:0) after 4m45s
[03:53:18.331] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:6:0)
[03:53:18.384] Build status: SCHEDULED
[03:53:18.384] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:7:0) after 5m16s
[03:58:34.395] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:7:0)
[03:58:34.455] Build status: SCHEDULED
[03:58:34.455] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:8:0) after 1m28s
[04:00:02.467] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:8:0)
[04:00:02.490] Build status: SCHEDULED
[04:00:02.490] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:9:0) after 8m56s
[04:08:58.504] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:9:0)
[04:08:58.521] Build status: SCHEDULED
[04:08:58.522] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:10:0) after 2m53s
[04:11:51.521] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:10:0)
[04:11:51.580] Build status: SCHEDULED
[04:11:51.580] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:11:0) after 3m47s
[04:15:38.690] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:11:0)
[04:15:38.709] Build status: SCHEDULED
[04:15:38.709] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:12:0) after 7m10s
[04:22:48.732] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:12:0)
[04:22:48.749] Build status: SCHEDULED
[04:22:48.749] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:13:0) after 1m4s
[04:23:52.779] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:13:0)
[04:23:52.799] Build status: SCHEDULED
[04:23:52.799] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:14:0) after 6m31s
[04:30:23.811] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:14:0)
[04:30:23.879] Build status: SCHEDULED
[04:30:23.879] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:15:0) after 9m58s
[04:40:21.962] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:15:0)
[04:40:21.986] Build status: SCHEDULED
[04:40:21.986] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:16:0) after 4m14s
[04:44:36.002] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:16:0)
[04:44:36.067] Build status: SCHEDULED
[04:44:36.067] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:17:0) after 7m52s
[04:52:28.084] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:17:0)
[04:52:28.102] Build status: SCHEDULED
[04:52:28.103] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:18:0) after 2m41s
[04:55:09.187] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:18:0)
[04:55:09.207] Build status: SCHEDULED
[04:55:09.207] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:19:0) after 3m29s
[04:58:38.368] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:19:0)
[04:58:38.388] Build status: SCHEDULED
[04:58:38.388] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:20:0) after 8m7s
[05:06:45.411] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:20:0)
[05:06:45.429] Build status: SCHEDULED
[05:06:45.429] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:21:0) after 2m45s
[05:09:30.442] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:21:0)
[05:09:30.457] Build status: SCHEDULED
[05:09:30.457] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:22:0) after 8m42s
[05:18:12.478] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:22:0)
[05:18:12.531] Build status: SCHEDULED
[05:18:12.531] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:23:0) after 6m42s
[05:24:54.695] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:23:0)
[05:24:54.724] Build status: SCHEDULED
[05:24:54.724] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:24:0) after 4m35s
[05:29:29.731] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:24:0)
[05:29:29.755] Build status: SCHEDULED
[05:29:29.755] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:25:0) after 3m0s
[05:32:29.774] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:25:0)
[05:32:29.798] Build status: SCHEDULED
[05:32:29.798] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:26:0) after 5m37s
[05:38:06.911] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:26:0)
[05:38:06.929] Build status: SCHEDULED
[05:38:06.929] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:27:0) after 5m6s
[05:43:12.964] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:27:0)
[05:43:12.985] Build status: SCHEDULED
[05:43:12.985] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:28:0) after 7m34s
[05:50:47.002] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:28:0)
[05:50:47.021] Build status: SCHEDULED
[05:50:47.021] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:29:0) after 5m4s
[05:55:51.046] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:29:0)
[05:55:51.062] Build status: SCHEDULED
[05:55:51.062] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:30:0) after 7m42s
[06:02:18.348] Received PubSub notification, asking Buildbucket for the build status
[06:02:18.367] Build status: STARTED
[06:03:33.055] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:30:0)
[06:03:33.072] Build status: STARTED
[06:03:33.072] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:32:0) after 2m35s
[06:06:08.112] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:32:0)
[06:06:08.146] Build status: STARTED
[06:06:08.146] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8849374376998825840:33:0) after 8m5s
[06:06:13.934] Received PubSub notification, asking Buildbucket for the build status
[06:06:13.951] Build:
{
"id": "8683944314971188641",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-04-21T03:32:43.917733030Z",
"startTime": "2026-04-21T06:02:18.126536128Z",
"endTime": "2026-04-21T06:06:13.570920865Z",
"updateTime": "2026-04-21T06:06:13.570920865Z",
"status": "FAILURE",
"input": {
}
}
[06:06:13.951] Invocation finished in 2h33m31.351593826s with status FAILED