[01:40:54.567] New invocation is queued and will start shortly
[01:40:54.567] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[01:40:55.769] Starting the invocation (attempt 1)
[01:40:55.792] Ignoring gitiles_ref tag without the buildset tag
[01:40:55.792] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[01:40:55.793] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[01:40:55.793] Buildbucket request:
{
"requestId": "8839687535482917632",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-asan-build"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8839687535482917632",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "eab6bba7-8aac-4b51-a454-6b46aec021a7",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8674270341060882321"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8839687535482917632"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[01:40:56.152] Scheduled build:
{
"id": "8674257473481236961",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-asan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-06T01:40:55.853050565Z",
"updateTime": "2026-08-06T01:40:55.853050565Z",
"status": "SCHEDULED",
"input": {
},
"canary": true
}
[01:40:56.152] Task URL: https://cr-buildbucket.appspot.com/build/8674257473481236961
[01:40:56.152] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:2:0) after 2m17s
[01:40:57.707] Received PubSub notification, asking Buildbucket for the build status
[01:40:57.837] Build status: SCHEDULED
[01:43:13.354] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:2:0)
[01:43:13.374] Build status: SCHEDULED
[01:43:13.374] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:4:0) after 7m17s
[01:50:30.574] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:4:0)
[01:50:30.602] Build status: SCHEDULED
[01:50:30.602] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:5:0) after 2m52s
[01:53:22.596] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:5:0)
[01:53:22.618] Build status: SCHEDULED
[01:53:22.618] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:6:0) after 2m57s
[01:56:19.662] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:6:0)
[01:56:19.683] Build status: SCHEDULED
[01:56:19.683] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:7:0) after 9m17s
[02:05:36.705] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:7:0)
[02:05:36.727] Build status: SCHEDULED
[02:05:36.727] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:8:0) after 5m5s
[02:10:41.880] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:8:0)
[02:10:41.898] Build status: SCHEDULED
[02:10:41.898] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:9:0) after 3m58s
[02:14:40.045] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:9:0)
[02:14:40.070] Build status: SCHEDULED
[02:14:40.070] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:10:0) after 4m50s
[02:19:30.086] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:10:0)
[02:19:30.117] Build status: SCHEDULED
[02:19:30.117] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:11:0) after 2m15s
[02:21:45.133] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:11:0)
[02:21:45.153] Build status: SCHEDULED
[02:21:45.153] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:12:0) after 6m37s
[02:28:22.272] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:12:0)
[02:28:22.302] Build status: SCHEDULED
[02:28:22.302] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:13:0) after 3m57s
[02:32:19.276] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:13:0)
[02:32:19.295] Build status: SCHEDULED
[02:32:19.295] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:14:0) after 2m6s
[02:34:25.468] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:14:0)
[02:34:25.492] Build status: SCHEDULED
[02:34:25.492] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:15:0) after 9m54s
[02:44:19.464] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:15:0)
[02:44:19.530] Build status: SCHEDULED
[02:44:19.530] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:16:0) after 7m15s
[02:51:34.595] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:16:0)
[02:51:34.615] Build status: SCHEDULED
[02:51:34.615] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:17:0) after 9m25s
[03:00:59.680] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:17:0)
[03:00:59.713] Build status: SCHEDULED
[03:00:59.713] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:18:0) after 1m14s
[03:02:13.694] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:18:0)
[03:02:13.716] Build status: SCHEDULED
[03:02:13.716] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:19:0) after 5m27s
[03:07:40.685] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:19:0)
[03:07:40.702] Build status: SCHEDULED
[03:07:40.702] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:20:0) after 3m51s
[03:11:31.845] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:20:0)
[03:11:31.867] Build status: SCHEDULED
[03:11:31.867] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:21:0) after 4m25s
[03:15:57.033] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:21:0)
[03:15:57.147] Build status: SCHEDULED
[03:15:57.147] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:22:0) after 1m8s
[03:17:05.161] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:22:0)
[03:17:05.199] Build status: SCHEDULED
[03:17:05.199] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:23:0) after 1m55s
[03:19:00.285] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:23:0)
[03:19:00.333] Build status: SCHEDULED
[03:19:00.333] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:24:0) after 4m10s
[03:23:10.422] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:24:0)
[03:23:10.468] Build status: SCHEDULED
[03:23:10.468] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:25:0) after 4m2s
[03:27:12.698] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:25:0)
[03:27:12.715] Build status: SCHEDULED
[03:27:12.715] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:26:0) after 4m34s
[03:31:46.738] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:26:0)
[03:31:46.764] Build status: SCHEDULED
[03:31:46.765] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:27:0) after 5m3s
[03:36:49.781] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:27:0)
[03:36:49.815] Build status: SCHEDULED
[03:36:49.815] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:28:0) after 1m17s
[03:38:06.830] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:28:0)
[03:38:06.851] Build status: SCHEDULED
[03:38:06.851] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:29:0) after 6m45s
[03:44:51.868] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:29:0)
[03:44:51.894] Build status: SCHEDULED
[03:44:51.894] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:30:0) after 4m19s
[03:49:10.991] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:30:0)
[03:49:11.021] Build status: SCHEDULED
[03:49:11.021] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:31:0) after 5m43s
[03:54:54.078] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:31:0)
[03:54:54.099] Build status: SCHEDULED
[03:54:54.099] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:32:0) after 9m25s
[04:04:19.116] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:32:0)
[04:04:19.142] Build status: SCHEDULED
[04:04:19.142] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:33:0) after 9m25s
[04:13:44.257] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:33:0)
[04:13:44.305] Build status: SCHEDULED
[04:13:44.305] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:34:0) after 9m37s
[04:23:21.371] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:34:0)
[04:23:21.394] Build status: SCHEDULED
[04:23:21.394] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:35:0) after 2m46s
[04:24:51.888] Received PubSub notification, asking Buildbucket for the build status
[04:24:51.951] Build status: STARTED
[04:26:07.388] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:35:0)
[04:26:07.409] Build status: STARTED
[04:26:07.409] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-build:8839687535482917632:37:0) after 9m49s
[04:29:50.488] Received PubSub notification, asking Buildbucket for the build status
[04:29:50.519] Build:
{
"id": "8674257473481236961",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-asan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-06T01:40:55.853050565Z",
"startTime": "2026-08-06T04:24:34.890257861Z",
"endTime": "2026-08-06T04:29:50.223790081Z",
"updateTime": "2026-08-06T04:29:50.223790081Z",
"status": "FAILURE",
"input": {
},
"canary": true
}
[04:29:50.519] Invocation finished in 2h48m55.958011484s with status FAILED