[06:10:47.989] New invocation is queued and will start shortly
[06:10:47.990] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[06:10:49.063] Starting the invocation (attempt 1)
[06:10:49.092] Ignoring gitiles_ref tag without the buildset tag
[06:10:49.092] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[06:10:49.092] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[06:10:49.092] Buildbucket request:
{
"requestId": "8838402197917049456",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-ubsan-gtest"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8838402197917049456",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "3497b045-d2fd-4e48-9d50-ffb55c72d031",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8672972531826828513"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8838402197917049456"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[06:10:49.781] Scheduled build:
{
"id": "8672972135835447137",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-ubsan-gtest"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-20T06:10:49.354215186Z",
"updateTime": "2026-08-20T06:10:49.354215186Z",
"status": "SCHEDULED",
"input": {
}
}
[06:10:49.781] Task URL: https://cr-buildbucket.appspot.com/build/8672972135835447137
[06:10:49.781] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:2:0) after 3m13s
[06:10:50.706] Received PubSub notification, asking Buildbucket for the build status
[06:10:50.731] Build status: SCHEDULED
[06:14:02.805] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:2:0)
[06:14:02.833] Build status: SCHEDULED
[06:14:02.833] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:4:0) after 1m17s
[06:15:19.868] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:4:0)
[06:15:19.940] Build status: SCHEDULED
[06:15:19.940] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:5:0) after 5m57s
[06:21:17.043] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:5:0)
[06:21:17.062] Build status: SCHEDULED
[06:21:17.062] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:6:0) after 1m49s
[06:23:06.130] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:6:0)
[06:23:06.153] Build status: SCHEDULED
[06:23:06.154] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:7:0) after 5m31s
[06:28:37.243] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:7:0)
[06:28:37.385] Build status: SCHEDULED
[06:28:37.385] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:8:0) after 1m44s
[06:30:21.619] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:8:0)
[06:30:21.643] Build status: SCHEDULED
[06:30:21.643] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:9:0) after 2m59s
[06:33:20.658] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:9:0)
[06:33:20.790] Build status: SCHEDULED
[06:33:20.790] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:10:0) after 9m12s
[06:42:32.822] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:10:0)
[06:42:32.850] Build status: SCHEDULED
[06:42:32.850] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:11:0) after 9m44s
[06:52:16.844] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:11:0)
[06:52:16.864] Build status: SCHEDULED
[06:52:16.864] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:12:0) after 2m47s
[06:55:04.032] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:12:0)
[06:55:04.073] Build status: SCHEDULED
[06:55:04.073] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:13:0) after 9m13s
[07:04:17.203] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:13:0)
[07:04:17.231] Build status: SCHEDULED
[07:04:17.231] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:14:0) after 9m11s
[07:13:28.246] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:14:0)
[07:13:28.266] Build status: SCHEDULED
[07:13:28.266] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:15:0) after 7m34s
[07:21:02.282] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:15:0)
[07:21:02.324] Build status: SCHEDULED
[07:21:02.324] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:16:0) after 3m58s
[07:25:00.296] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:16:0)
[07:25:00.329] Build status: SCHEDULED
[07:25:00.329] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:17:0) after 5m26s
[07:30:26.395] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:17:0)
[07:30:26.418] Build status: SCHEDULED
[07:30:26.418] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:18:0) after 1m26s
[07:31:52.433] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:18:0)
[07:31:52.481] Build status: SCHEDULED
[07:31:52.481] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:19:0) after 6m59s
[07:38:51.573] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:19:0)
[07:38:51.597] Build status: SCHEDULED
[07:38:51.597] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:20:0) after 2m40s
[07:41:31.618] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:20:0)
[07:41:31.642] Build status: SCHEDULED
[07:41:31.642] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:21:0) after 4m41s
[07:46:12.693] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:21:0)
[07:46:12.713] Build status: SCHEDULED
[07:46:12.714] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:22:0) after 9m0s
[07:55:12.753] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:22:0)
[07:55:12.772] Build status: SCHEDULED
[07:55:12.772] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:23:0) after 1m8s
[07:56:20.841] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:23:0)
[07:56:20.865] Build status: SCHEDULED
[07:56:20.865] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:24:0) after 1m15s
[07:57:35.883] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:24:0)
[07:57:36.043] Build status: SCHEDULED
[07:57:36.043] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:25:0) after 7m5s
[08:04:41.123] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:25:0)
[08:04:41.158] Build status: SCHEDULED
[08:04:41.159] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:26:0) after 7m1s
[08:11:42.244] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:26:0)
[08:11:42.426] Build status: SCHEDULED
[08:11:42.427] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:27:0) after 1m27s
[08:13:09.492] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:27:0)
[08:13:09.522] Build status: SCHEDULED
[08:13:09.522] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:28:0) after 9m12s
[08:22:21.639] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:28:0)
[08:22:21.659] Build status: SCHEDULED
[08:22:21.659] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:29:0) after 1m10s
[08:23:31.725] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:29:0)
[08:23:31.745] Build status: SCHEDULED
[08:23:31.745] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:30:0) after 6m0s
[08:29:31.762] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:30:0)
[08:29:31.780] Build status: SCHEDULED
[08:29:31.780] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:31:0) after 3m56s
[08:33:27.797] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:31:0)
[08:33:27.817] Build status: SCHEDULED
[08:33:27.817] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:32:0) after 9m50s
[08:43:17.912] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:32:0)
[08:43:17.932] Build status: SCHEDULED
[08:43:17.932] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:33:0) after 4m5s
[08:47:23.008] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:33:0)
[08:47:23.032] Build status: SCHEDULED
[08:47:23.033] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:34:0) after 3m57s
[08:51:20.089] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:34:0)
[08:51:20.119] Build status: SCHEDULED
[08:51:20.119] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:35:0) after 6m1s
[08:57:21.339] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:35:0)
[08:57:21.371] Build status: SCHEDULED
[08:57:21.371] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:36:0) after 5m45s
[09:03:06.385] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:36:0)
[09:03:06.425] Build status: SCHEDULED
[09:03:06.425] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:37:0) after 1m37s
[09:04:43.563] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:37:0)
[09:04:43.596] Build status: SCHEDULED
[09:04:43.596] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:38:0) after 6m12s
[09:10:55.725] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:38:0)
[09:10:55.763] Build status: SCHEDULED
[09:10:55.763] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:39:0) after 3m44s
[09:14:39.787] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:39:0)
[09:14:39.809] Build status: SCHEDULED
[09:14:39.809] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:40:0) after 1m11s
[09:15:50.802] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:40:0)
[09:15:50.829] Build status: SCHEDULED
[09:15:50.829] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:41:0) after 4m22s
[09:20:12.870] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:41:0)
[09:20:12.892] Build status: SCHEDULED
[09:20:12.892] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:42:0) after 5m39s
[09:25:52.006] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:42:0)
[09:25:52.027] Build status: SCHEDULED
[09:25:52.027] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:43:0) after 1m22s
[09:27:14.001] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:43:0)
[09:27:14.041] Build status: SCHEDULED
[09:27:14.041] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-ubsan-gtest:8838402197917049456:44:0) after 9m52s
[09:29:53.044] Received PubSub notification, asking Buildbucket for the build status
[09:29:53.077] Build status: STARTED
[09:35:12.660] Received PubSub notification, asking Buildbucket for the build status
[09:35:12.707] Build:
{
"id": "8672972135835447137",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-ubsan-gtest"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-20T06:10:49.354215186Z",
"startTime": "2026-08-20T09:29:52.828026004Z",
"endTime": "2026-08-20T09:35:12.345134818Z",
"updateTime": "2026-08-20T09:35:12.345134818Z",
"status": "FAILURE",
"input": {
}
}
[09:35:12.707] Invocation finished in 3h24m24.722379133s with status FAILED