[00:19:58.429] New invocation is queued and will start shortly
[00:19:58.429] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[00:19:59.490] Starting the invocation (attempt 1)
[00:19:59.520] Ignoring gitiles_ref tag without the buildset tag
[00:19:59.520] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[00:19:59.520] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[00:19:59.521] Buildbucket request:
{
"requestId": "8837880688186709408",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8837880688186709408",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "4e7aad0f-a55b-4881-a76e-bc05439f49bd",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8672451177944947537"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8837880688186709408"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[00:19:59.973] Scheduled build:
{
"id": "8672450626303765345",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-26T00:19:59.600209589Z",
"updateTime": "2026-08-26T00:19:59.600209589Z",
"status": "SCHEDULED",
"input": {
}
}
[00:19:59.973] Task URL: https://cr-buildbucket.appspot.com/build/8672450626303765345
[00:19:59.973] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:2:0) after 5m4s
[00:20:00.881] Received PubSub notification, asking Buildbucket for the build status
[00:20:00.974] Build status: SCHEDULED
[00:25:03.976] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:2:0)
[00:25:04.001] Build status: SCHEDULED
[00:25:04.001] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:4:0) after 8m59s
[00:34:03.014] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:4:0)
[00:34:03.129] Build status: SCHEDULED
[00:34:03.129] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:5:0) after 9m26s
[00:43:29.213] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:5:0)
[00:43:29.241] Build status: SCHEDULED
[00:43:29.241] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:6:0) after 6m9s
[00:49:38.277] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:6:0)
[00:49:38.294] Build status: SCHEDULED
[00:49:38.294] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:7:0) after 6m0s
[00:55:38.311] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:7:0)
[00:55:38.344] Build status: SCHEDULED
[00:55:38.344] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:8:0) after 6m54s
[01:02:32.397] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:8:0)
[01:02:32.415] Build status: SCHEDULED
[01:02:32.415] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:9:0) after 6m2s
[01:08:34.435] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:9:0)
[01:08:34.452] Build status: SCHEDULED
[01:08:34.452] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:10:0) after 8m56s
[01:17:30.453] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:10:0)
[01:17:30.474] Build status: SCHEDULED
[01:17:30.474] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:11:0) after 1m27s
[01:18:57.519] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:11:0)
[01:18:57.540] Build status: SCHEDULED
[01:18:57.540] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:12:0) after 9m4s
[01:28:01.564] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:12:0)
[01:28:01.672] Build status: SCHEDULED
[01:28:01.672] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:13:0) after 9m7s
[01:37:08.747] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:13:0)
[01:37:08.765] Build status: SCHEDULED
[01:37:08.765] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:14:0) after 2m3s
[01:39:11.906] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:14:0)
[01:39:11.937] Build status: SCHEDULED
[01:39:11.937] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:15:0) after 5m23s
[01:44:34.962] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:15:0)
[01:44:35.036] Build status: SCHEDULED
[01:44:35.036] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:16:0) after 8m25s
[01:53:00.033] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:16:0)
[01:53:00.126] Build status: SCHEDULED
[01:53:00.126] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:17:0) after 2m51s
[01:55:51.317] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:17:0)
[01:55:51.341] Build status: SCHEDULED
[01:55:51.341] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:18:0) after 9m7s
[02:04:58.334] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:18:0)
[02:04:58.363] Build status: SCHEDULED
[02:04:58.363] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:19:0) after 4m30s
[02:09:28.461] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:19:0)
[02:09:28.490] Build status: SCHEDULED
[02:09:28.490] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:20:0) after 7m58s
[02:17:26.531] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:20:0)
[02:17:26.596] Build status: SCHEDULED
[02:17:26.596] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:21:0) after 2m57s
[02:20:23.680] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:21:0)
[02:20:23.702] Build status: SCHEDULED
[02:20:23.702] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:22:0) after 3m11s
[02:23:34.718] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:22:0)
[02:23:34.757] Build status: SCHEDULED
[02:23:34.757] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:23:0) after 9m10s
[02:32:44.779] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:23:0)
[02:32:44.810] Build status: SCHEDULED
[02:32:44.810] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:24:0) after 5m9s
[02:37:53.839] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:24:0)
[02:37:53.860] Build status: SCHEDULED
[02:37:53.860] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:25:0) after 1m23s
[02:39:16.914] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:25:0)
[02:39:16.937] Build status: SCHEDULED
[02:39:16.938] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:26:0) after 7m51s
[02:47:08.011] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:26:0)
[02:47:08.043] Build status: SCHEDULED
[02:47:08.043] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:27:0) after 4m47s
[02:51:55.119] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:27:0)
[02:51:55.138] Build status: SCHEDULED
[02:51:55.138] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:28:0) after 9m41s
[03:01:36.157] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:28:0)
[03:01:36.181] Build status: SCHEDULED
[03:01:36.181] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:29:0) after 7m55s
[03:09:31.174] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:29:0)
[03:09:31.194] Build status: SCHEDULED
[03:09:31.194] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:30:0) after 1m58s
[03:11:29.418] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:30:0)
[03:11:29.435] Build status: SCHEDULED
[03:11:29.435] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:31:0) after 6m57s
[03:18:26.575] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:31:0)
[03:18:26.616] Build status: SCHEDULED
[03:18:26.616] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:32:0) after 8m32s
[03:26:58.841] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:32:0)
[03:26:58.873] Build status: SCHEDULED
[03:26:58.873] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:33:0) after 5m40s
[03:32:38.972] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:33:0)
[03:32:38.989] Build status: SCHEDULED
[03:32:38.989] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:34:0) after 2m6s
[03:34:45.010] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:34:0)
[03:34:45.043] Build status: SCHEDULED
[03:34:45.043] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:35:0) after 2m55s
[03:37:40.116] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:35:0)
[03:37:40.133] Build status: SCHEDULED
[03:37:40.133] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:36:0) after 4m6s
[03:41:46.142] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:36:0)
[03:41:46.163] Build status: SCHEDULED
[03:41:46.163] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:37:0) after 9m16s
[03:51:02.190] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:37:0)
[03:51:02.205] Build status: SCHEDULED
[03:51:02.205] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:38:0) after 3m16s
[03:54:18.315] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:38:0)
[03:54:18.333] Build status: SCHEDULED
[03:54:18.333] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:39:0) after 8m44s
[04:03:02.373] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:39:0)
[04:03:02.394] Build status: SCHEDULED
[04:03:02.395] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:40:0) after 9m58s
[04:13:00.474] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:40:0)
[04:13:00.494] Build status: SCHEDULED
[04:13:00.494] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:41:0) after 4m52s
[04:17:52.546] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:41:0)
[04:17:52.592] Build status: SCHEDULED
[04:17:52.592] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:42:0) after 3m46s
[04:21:38.873] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:42:0)
[04:21:38.895] Build status: SCHEDULED
[04:21:38.895] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:43:0) after 1m32s
[04:23:10.913] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:43:0)
[04:23:11.071] Build status: SCHEDULED
[04:23:11.071] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:44:0) after 8m12s
[04:31:06.414] Received PubSub notification, asking Buildbucket for the build status
[04:31:06.447] Build status: STARTED
[04:31:23.114] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:44:0)
[04:31:23.167] Build status: STARTED
[04:31:23.167] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:46:0) after 3m45s
[04:35:08.182] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:46:0)
[04:35:08.227] Build status: STARTED
[04:35:08.227] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8837880688186709408:47:0) after 8m42s
[04:35:53.172] Received PubSub notification, asking Buildbucket for the build status
[04:35:53.283] Build:
{
"id": "8672450626303765345",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-26T00:19:59.600209589Z",
"startTime": "2026-08-26T04:31:05.746945661Z",
"endTime": "2026-08-26T04:35:52.744111356Z",
"updateTime": "2026-08-26T04:35:52.744111356Z",
"status": "FAILURE",
"input": {
}
}
[04:35:53.283] Invocation finished in 4h15m54.862243211s with status FAILED