[09:28:49.471] New invocation is queued and will start shortly
[09:28:49.471] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[09:28:50.503] Starting the invocation (attempt 1)
[09:28:50.537] Ignoring gitiles_ref tag without the buildset tag
[09:28:50.537] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[09:28:50.537] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[09:28:50.537] Buildbucket request:
{
"requestId": "8861310771787951120",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8861310771787951120",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "aafa9462-d8bd-4972-9376-1689789f0727",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8695880883326738337"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8861310771787951120"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[09:28:50.920] Scheduled build:
{
"id": "8695880709940378369",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-12-10T09:28:50.605893100Z",
"updateTime": "2025-12-10T09:28:50.605893100Z",
"status": "SCHEDULED",
"input": {
}
}
[09:28:50.920] Task URL: https://cr-buildbucket.appspot.com/build/8695880709940378369
[09:28:50.920] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:2:0) after 7m50s
[09:28:51.903] Received PubSub notification, asking Buildbucket for the build status
[09:28:51.922] Build status: SCHEDULED
[09:36:40.932] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:2:0)
[09:36:40.953] Build status: SCHEDULED
[09:36:40.953] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:4:0) after 6m36s
[09:43:17.034] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:4:0)
[09:43:17.109] Build status: SCHEDULED
[09:43:17.109] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:5:0) after 1m52s
[09:45:09.130] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:5:0)
[09:45:09.143] Build status: SCHEDULED
[09:45:09.143] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:6:0) after 8m25s
[09:53:34.158] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:6:0)
[09:53:34.172] Build status: SCHEDULED
[09:53:34.172] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:7:0) after 8m54s
[10:02:28.212] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:7:0)
[10:02:28.238] Build status: SCHEDULED
[10:02:28.238] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:8:0) after 5m17s
[10:07:45.295] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:8:0)
[10:07:45.317] Build status: SCHEDULED
[10:07:45.317] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:9:0) after 2m53s
[10:10:38.326] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:9:0)
[10:10:38.340] Build status: SCHEDULED
[10:10:38.340] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:10:0) after 8m5s
[10:18:43.353] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:10:0)
[10:18:43.375] Build status: SCHEDULED
[10:18:43.375] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:11:0) after 9m46s
[10:28:29.462] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:11:0)
[10:28:29.477] Build status: SCHEDULED
[10:28:29.477] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:12:0) after 7m51s
[10:36:20.504] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:12:0)
[10:36:20.524] Build status: SCHEDULED
[10:36:20.524] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:13:0) after 5m27s
[10:41:47.600] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:13:0)
[10:41:47.619] Build status: SCHEDULED
[10:41:47.620] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:14:0) after 6m15s
[10:48:02.690] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:14:0)
[10:48:02.706] Build status: SCHEDULED
[10:48:02.706] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:15:0) after 3m46s
[10:51:48.717] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:15:0)
[10:51:48.740] Build status: SCHEDULED
[10:51:48.740] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:16:0) after 4m20s
[10:56:08.782] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:16:0)
[10:56:08.813] Build status: SCHEDULED
[10:56:08.813] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:17:0) after 3m39s
[10:59:47.875] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:17:0)
[10:59:47.890] Build status: SCHEDULED
[10:59:47.890] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:18:0) after 8m6s
[11:07:53.953] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:18:0)
[11:07:53.968] Build status: SCHEDULED
[11:07:53.968] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:19:0) after 3m48s
[11:11:41.949] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:19:0)
[11:11:41.966] Build status: SCHEDULED
[11:11:41.966] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:20:0) after 6m59s
[11:18:41.029] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:20:0)
[11:18:41.063] Build status: SCHEDULED
[11:18:41.063] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:21:0) after 4m1s
[11:22:42.075] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:21:0)
[11:22:42.088] Build status: SCHEDULED
[11:22:42.088] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:22:0) after 1m40s
[11:24:22.120] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:22:0)
[11:24:22.301] Build status: SCHEDULED
[11:24:22.301] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:23:0) after 5m31s
[11:29:53.311] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:23:0)
[11:29:53.327] Build status: SCHEDULED
[11:29:53.327] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:24:0) after 5m43s
[11:35:36.364] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:24:0)
[11:35:36.377] Build status: SCHEDULED
[11:35:36.377] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:25:0) after 2m45s
[11:38:21.389] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:25:0)
[11:38:21.417] Build status: SCHEDULED
[11:38:21.417] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:26:0) after 1m10s
[11:39:31.670] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:26:0)
[11:39:31.693] Build status: SCHEDULED
[11:39:31.693] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:27:0) after 5m11s
[11:44:42.714] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:27:0)
[11:44:42.772] Build status: SCHEDULED
[11:44:42.772] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:28:0) after 2m35s
[11:47:17.785] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:28:0)
[11:47:17.797] Build status: SCHEDULED
[11:47:17.797] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:29:0) after 5m55s
[11:53:12.860] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:29:0)
[11:53:12.874] Build status: SCHEDULED
[11:53:12.874] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:30:0) after 1m2s
[11:54:14.933] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:30:0)
[11:54:14.958] Build status: SCHEDULED
[11:54:14.958] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:31:0) after 9m1s
[12:03:15.929] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:31:0)
[12:03:15.949] Build status: SCHEDULED
[12:03:15.949] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:32:0) after 6m32s
[12:09:48.001] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:32:0)
[12:09:48.056] Build status: SCHEDULED
[12:09:48.056] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:33:0) after 7m50s
[12:17:38.119] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:33:0)
[12:17:38.182] Build status: SCHEDULED
[12:17:38.182] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:34:0) after 6m19s
[12:23:57.258] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:34:0)
[12:23:57.274] Build status: SCHEDULED
[12:23:57.274] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:35:0) after 2m38s
[12:26:35.305] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:35:0)
[12:26:35.323] Build status: SCHEDULED
[12:26:35.323] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:36:0) after 7m29s
[12:34:04.388] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:36:0)
[12:34:04.412] Build status: SCHEDULED
[12:34:04.412] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:37:0) after 6m55s
[12:40:59.431] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:37:0)
[12:40:59.459] Build status: SCHEDULED
[12:40:59.459] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:38:0) after 6m28s
[12:47:27.533] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:38:0)
[12:47:27.553] Build status: SCHEDULED
[12:47:27.553] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:39:0) after 1m43s
[12:49:10.709] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:39:0)
[12:49:10.723] Build status: SCHEDULED
[12:49:10.723] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:40:0) after 8m57s
[12:58:07.801] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:40:0)
[12:58:07.815] Build status: SCHEDULED
[12:58:07.815] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:41:0) after 6m33s
[13:04:40.833] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:41:0)
[13:04:40.847] Build status: SCHEDULED
[13:04:40.847] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:42:0) after 5m11s
[13:09:52.020] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:42:0)
[13:09:52.039] Build status: SCHEDULED
[13:09:52.039] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:43:0) after 1m30s
[13:11:22.057] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:43:0)
[13:11:22.073] Build status: SCHEDULED
[13:11:22.073] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:44:0) after 5m32s
[13:16:54.144] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:44:0)
[13:16:54.160] Build status: SCHEDULED
[13:16:54.160] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:45:0) after 1m44s
[13:18:38.175] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:45:0)
[13:18:38.188] Build status: SCHEDULED
[13:18:38.189] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:46:0) after 8m17s
[13:26:55.318] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:46:0)
[13:26:55.335] Build status: SCHEDULED
[13:26:55.336] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:47:0) after 4m42s
[13:31:37.750] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:47:0)
[13:31:37.773] Build status: SCHEDULED
[13:31:37.773] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:48:0) after 9m53s
[13:41:30.786] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:48:0)
[13:41:30.843] Build status: SCHEDULED
[13:41:30.843] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:49:0) after 8m18s
[13:49:48.856] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:49:0)
[13:49:48.994] Build status: SCHEDULED
[13:49:48.994] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:50:0) after 4m27s
[13:54:16.058] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:50:0)
[13:54:16.120] Build status: SCHEDULED
[13:54:16.120] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:51:0) after 3m4s
[13:57:20.134] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:51:0)
[13:57:20.148] Build status: SCHEDULED
[13:57:20.148] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:52:0) after 5m56s
[14:03:16.163] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:52:0)
[14:03:16.193] Build status: SCHEDULED
[14:03:16.194] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:53:0) after 2m8s
[14:05:24.223] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:53:0)
[14:05:24.243] Build status: SCHEDULED
[14:05:24.243] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:54:0) after 2m13s
[14:07:37.258] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:54:0)
[14:07:37.272] Build status: SCHEDULED
[14:07:37.272] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:55:0) after 4m20s
[14:11:57.344] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:55:0)
[14:11:57.357] Build status: SCHEDULED
[14:11:57.357] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:56:0) after 5m22s
[14:17:19.371] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:56:0)
[14:17:19.386] Build status: SCHEDULED
[14:17:19.387] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:57:0) after 6m49s
[14:24:08.409] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:57:0)
[14:24:08.427] Build status: SCHEDULED
[14:24:08.427] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:58:0) after 8m23s
[14:32:31.442] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:58:0)
[14:32:31.476] Build status: SCHEDULED
[14:32:31.476] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:59:0) after 7m20s
[14:39:51.452] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:59:0)
[14:39:51.472] Build status: SCHEDULED
[14:39:51.472] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:60:0) after 4m43s
[14:44:34.499] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:60:0)
[14:44:34.576] Build status: SCHEDULED
[14:44:34.576] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:61:0) after 8m2s
[14:52:36.597] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:61:0)
[14:52:36.614] Build status: SCHEDULED
[14:52:36.614] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:62:0) after 9m7s
[15:01:43.630] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:62:0)
[15:01:43.646] Build status: SCHEDULED
[15:01:43.646] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:63:0) after 2m58s
[15:04:41.706] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:63:0)
[15:04:41.766] Build status: SCHEDULED
[15:04:41.766] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:64:0) after 4m17s
[15:08:58.782] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:64:0)
[15:08:58.798] Build status: SCHEDULED
[15:08:58.798] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:65:0) after 7m34s
[15:16:32.902] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:65:0)
[15:16:32.915] Build status: SCHEDULED
[15:16:32.916] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:66:0) after 5m32s
[15:22:05.055] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:66:0)
[15:22:05.076] Build status: SCHEDULED
[15:22:05.076] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:67:0) after 6m14s
[15:28:19.153] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:67:0)
[15:28:19.241] Build status: SCHEDULED
[15:28:19.241] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:68:0) after 8m10s
[15:36:29.256] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:68:0)
[15:36:29.273] Build status: SCHEDULED
[15:36:29.273] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:69:0) after 2m47s
[15:39:16.366] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:69:0)
[15:39:16.395] Build status: SCHEDULED
[15:39:16.395] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:70:0) after 6m20s
[15:45:36.442] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:70:0)
[15:45:36.456] Build status: SCHEDULED
[15:45:36.457] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:71:0) after 2m12s
[15:47:48.429] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:71:0)
[15:47:48.454] Build status: SCHEDULED
[15:47:48.454] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:72:0) after 4m34s
[15:52:22.477] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:72:0)
[15:52:22.836] Build status: SCHEDULED
[15:52:22.836] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:73:0) after 6m36s
[15:58:58.914] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:73:0)
[15:58:58.941] Build status: SCHEDULED
[15:58:58.941] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:74:0) after 4m48s
[16:03:46.941] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:74:0)
[16:03:47.044] Build status: SCHEDULED
[16:03:47.044] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:75:0) after 9m47s
[16:13:34.078] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:75:0)
[16:13:34.217] Build status: SCHEDULED
[16:13:34.217] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:76:0) after 2m47s
[16:16:21.257] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:76:0)
[16:16:21.356] Build status: SCHEDULED
[16:16:21.356] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:77:0) after 2m24s
[16:17:49.230] Received PubSub notification, asking Buildbucket for the build status
[16:17:49.249] Build status: STARTED
[16:18:45.420] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:77:0)
[16:18:45.435] Build status: STARTED
[16:18:45.435] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:79:0) after 4m50s
[16:23:35.457] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:79:0)
[16:23:35.470] Build status: STARTED
[16:23:35.470] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:80:0) after 9m1s
[16:32:36.488] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:80:0)
[16:32:36.517] Build status: STARTED
[16:32:36.517] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:81:0) after 6m18s
[16:38:54.530] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:81:0)
[16:38:54.548] Build status: STARTED
[16:38:54.548] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:82:0) after 2m39s
[16:41:33.582] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:82:0)
[16:41:33.633] Build status: STARTED
[16:41:33.633] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:83:0) after 5m31s
[16:47:04.647] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:83:0)
[16:47:04.683] Build status: STARTED
[16:47:04.683] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:84:0) after 7m45s
[16:54:49.669] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:84:0)
[16:54:49.692] Build status: STARTED
[16:54:49.693] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:85:0) after 7m53s
[17:02:42.705] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:85:0)
[17:02:42.723] Build status: STARTED
[17:02:42.723] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8861310771787951120:86:0) after 9m1s
[17:07:55.087] Received PubSub notification, asking Buildbucket for the build status
[17:07:55.157] Build:
{
"id": "8695880709940378369",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-12-10T09:28:50.605893100Z",
"startTime": "2025-12-10T16:17:48.715008849Z",
"endTime": "2025-12-10T17:07:54.602763957Z",
"updateTime": "2025-12-10T17:07:54.602763957Z",
"status": "INFRA_FAILURE",
"input": {
}
}
[17:07:55.157] Invocation finished in 7h39m5.698700186s with status FAILED