[22:29:14.518] New invocation is queued and will start shortly
[22:29:14.518] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[22:29:16.346] Starting the invocation (attempt 1)
[22:29:16.378] Ignoring gitiles_ref tag without the buildset tag
[22:29:16.378] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[22:29:16.378] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[22:29:16.378] Buildbucket request:
{
"requestId": "8843323472813648288",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-asan-gtest"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8843323472813648288",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "aa367dcb-5577-4cb9-81d7-19251d98ef7d",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8677893762725455121"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8843323472813648288"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[22:29:16.762] Scheduled build:
{
"id": "8677893410134112113",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-asan-gtest"
},
"createdBy": "project:pigweed",
"createTime": "2026-06-26T22:29:16.452977271Z",
"updateTime": "2026-06-26T22:29:16.452977271Z",
"status": "SCHEDULED",
"input": {
}
}
[22:29:16.762] Task URL: https://cr-buildbucket.appspot.com/build/8677893410134112113
[22:29:16.762] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:2:0) after 5m4s
[22:29:17.680] Received PubSub notification, asking Buildbucket for the build status
[22:29:17.736] Build status: SCHEDULED
[22:34:20.805] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:2:0)
[22:34:20.823] Build status: SCHEDULED
[22:34:20.824] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:4:0) after 2m58s
[22:37:18.897] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:4:0)
[22:37:19.025] Build status: SCHEDULED
[22:37:19.025] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:5:0) after 2m26s
[22:39:45.042] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:5:0)
[22:39:45.064] Build status: SCHEDULED
[22:39:45.064] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:6:0) after 4m28s
[22:44:13.081] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:6:0)
[22:44:13.108] Build status: SCHEDULED
[22:44:13.108] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:7:0) after 9m21s
[22:53:34.196] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:7:0)
[22:53:34.219] Build status: SCHEDULED
[22:53:34.219] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:8:0) after 5m26s
[22:59:00.224] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:8:0)
[22:59:00.242] Build status: SCHEDULED
[22:59:00.242] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:9:0) after 8m46s
[23:07:46.307] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:9:0)
[23:07:46.326] Build status: SCHEDULED
[23:07:46.327] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:10:0) after 1m24s
[23:09:10.340] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:10:0)
[23:09:10.363] Build status: SCHEDULED
[23:09:10.363] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:11:0) after 2m5s
[23:11:15.381] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:11:0)
[23:11:15.407] Build status: SCHEDULED
[23:11:15.407] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:12:0) after 5m3s
[23:16:18.475] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:12:0)
[23:16:18.498] Build status: SCHEDULED
[23:16:18.498] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:13:0) after 7m44s
[23:24:02.499] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:13:0)
[23:24:02.534] Build status: SCHEDULED
[23:24:02.534] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:14:0) after 1m11s
[23:25:13.551] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:14:0)
[23:25:13.579] Build status: SCHEDULED
[23:25:13.579] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:15:0) after 4m38s
[23:29:51.642] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:15:0)
[23:29:51.802] Build status: SCHEDULED
[23:29:51.802] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:16:0) after 3m25s
[23:33:16.885] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:16:0)
[23:33:16.913] Build status: SCHEDULED
[23:33:16.913] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:17:0) after 5m49s
[23:39:05.980] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:17:0)
[23:39:06.043] Build status: SCHEDULED
[23:39:06.043] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:18:0) after 8m5s
[23:47:11.103] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:18:0)
[23:47:11.123] Build status: SCHEDULED
[23:47:11.123] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:19:0) after 2m24s
[23:49:35.150] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:19:0)
[23:49:35.170] Build status: SCHEDULED
[23:49:35.170] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:20:0) after 2m28s
[23:52:03.287] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:20:0)
[23:52:03.323] Build status: SCHEDULED
[23:52:03.323] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:21:0) after 1m33s
[23:53:36.426] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:21:0)
[23:53:36.444] Build status: SCHEDULED
[23:53:36.444] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:22:0) after 6m32s
[00:00:08.540] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:22:0)
[00:00:08.562] Build status: SCHEDULED
[00:00:08.562] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:23:0) after 2m23s
[00:02:31.552] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:23:0)
[00:02:31.569] Build status: SCHEDULED
[00:02:31.569] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:24:0) after 4m15s
[00:06:46.585] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:24:0)
[00:06:46.663] Build status: SCHEDULED
[00:06:46.663] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:25:0) after 3m26s
[00:10:12.705] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:25:0)
[00:10:12.722] Build status: SCHEDULED
[00:10:12.723] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:26:0) after 2m3s
[00:12:15.811] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:26:0)
[00:12:15.978] Build status: SCHEDULED
[00:12:15.978] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:27:0) after 3m45s
[00:16:01.044] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:27:0)
[00:16:01.075] Build status: SCHEDULED
[00:16:01.075] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:28:0) after 6m29s
[00:22:30.091] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:28:0)
[00:22:30.125] Build status: SCHEDULED
[00:22:30.125] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:29:0) after 1m14s
[00:23:44.141] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:29:0)
[00:23:44.185] Build status: SCHEDULED
[00:23:44.185] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:30:0) after 2m7s
[00:25:51.229] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:30:0)
[00:25:51.246] Build status: SCHEDULED
[00:25:51.246] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:31:0) after 3m57s
[00:29:48.263] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:31:0)
[00:29:48.282] Build status: SCHEDULED
[00:29:48.282] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:32:0) after 8m39s
[00:38:27.298] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:32:0)
[00:38:27.317] Build status: SCHEDULED
[00:38:27.317] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:33:0) after 5m19s
[00:43:46.349] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:33:0)
[00:43:46.383] Build status: SCHEDULED
[00:43:46.383] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:34:0) after 8m12s
[00:51:58.398] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:34:0)
[00:51:58.420] Build status: SCHEDULED
[00:51:58.420] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:35:0) after 3m29s
[00:55:27.503] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:35:0)
[00:55:27.532] Build status: SCHEDULED
[00:55:27.532] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:36:0) after 9m37s
[01:05:04.547] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:36:0)
[01:05:04.598] Build status: SCHEDULED
[01:05:04.598] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:37:0) after 5m15s
[01:10:19.613] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:37:0)
[01:10:19.632] Build status: SCHEDULED
[01:10:19.632] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:38:0) after 7m46s
[01:18:05.669] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:38:0)
[01:18:05.711] Build status: SCHEDULED
[01:18:05.711] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8843323472813648288:39:0) after 7m1s
[01:18:50.157] Received PubSub notification, asking Buildbucket for the build status
[01:18:50.180] Build status: STARTED
[01:22:39.860] Received PubSub notification, asking Buildbucket for the build status
[01:22:39.882] Build:
{
"id": "8677893410134112113",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-asan-gtest"
},
"createdBy": "project:pigweed",
"createTime": "2026-06-26T22:29:16.452977271Z",
"startTime": "2026-06-27T01:18:49.973811247Z",
"endTime": "2026-06-27T01:22:39.494423470Z",
"updateTime": "2026-06-27T01:22:39.494423470Z",
"status": "FAILURE",
"input": {
}
}
[01:22:39.882] Invocation finished in 2h53m25.369002226s with status FAILED