[12:55:42.537] New invocation is queued and will start shortly
[12:55:43.659] Starting the invocation (attempt 1)
[12:55:43.706] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/123911bae1b45f2b238d582cf2aa93cbc9394314"
[12:55:43.706] Popped gitiles commit info from properties and tags
[12:55:43.707] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[12:55:43.707] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[12:55:43.707] Buildbucket request:
{
"requestId": "8869179691813548432",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8869179691813548432",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "123911bae1b45f2b238d582cf2aa93cbc9394314"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@123911bae1b45f2b238d582cf2aa93cbc9394314",
"title": "123911bae1b45f2b238d582cf2aa93cbc9394314",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/123911bae1b45f2b238d582cf2aa93cbc9394314"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "123911bae1b45f2b238d582cf2aa93cbc9394314",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8869179691813548432"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[12:55:44.055] Scheduled build:
{
"id": "8703749629879673905",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-09-14T12:55:43.761755520Z",
"updateTime": "2025-09-14T12:55:43.761755520Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "123911bae1b45f2b238d582cf2aa93cbc9394314",
"ref": "refs/heads/main"
}
}
}
[12:55:44.055] Task URL: https://cr-buildbucket.appspot.com/build/8703749629879673905
[12:55:44.055] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:2:0) after 6m7s
[12:55:45.015] Received PubSub notification, asking Buildbucket for the build status
[12:55:45.046] Build status: SCHEDULED
[13:01:51.069] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:2:0)
[13:01:51.098] Build status: SCHEDULED
[13:01:51.098] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:4:0) after 3m55s
[13:05:46.114] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:4:0)
[13:05:46.133] Build status: SCHEDULED
[13:05:46.133] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:5:0) after 7m10s
[13:12:56.241] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:5:0)
[13:12:56.330] Build status: SCHEDULED
[13:12:56.330] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:6:0) after 3m35s
[13:16:31.344] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:6:0)
[13:16:31.399] Build status: SCHEDULED
[13:16:31.399] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:7:0) after 8m59s
[13:25:30.546] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:7:0)
[13:25:30.564] Build status: SCHEDULED
[13:25:30.564] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:8:0) after 6m26s
[13:31:56.571] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:8:0)
[13:31:56.605] Build status: SCHEDULED
[13:31:56.605] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:9:0) after 8m55s
[13:40:51.663] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:9:0)
[13:40:51.685] Build status: SCHEDULED
[13:40:51.685] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:10:0) after 2m41s
[13:43:33.216] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:10:0)
[13:43:33.242] Build status: SCHEDULED
[13:43:33.243] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:11:0) after 3m26s
[13:46:59.264] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:11:0)
[13:46:59.297] Build status: SCHEDULED
[13:46:59.297] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:12:0) after 7m40s
[13:54:39.345] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:12:0)
[13:54:39.360] Build status: SCHEDULED
[13:54:39.360] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:13:0) after 5m18s
[13:59:57.686] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:13:0)
[13:59:57.707] Build status: SCHEDULED
[13:59:57.707] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:14:0) after 8m36s
[14:08:33.696] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:14:0)
[14:08:33.730] Build status: SCHEDULED
[14:08:33.730] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:15:0) after 6m45s
[14:15:18.742] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:15:0)
[14:15:18.776] Build status: SCHEDULED
[14:15:18.776] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:16:0) after 4m14s
[14:19:33.121] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:16:0)
[14:19:33.137] Build status: SCHEDULED
[14:19:33.137] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:17:0) after 4m57s
[14:24:30.257] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:17:0)
[14:24:30.276] Build status: SCHEDULED
[14:24:30.276] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:18:0) after 5m36s
[14:30:06.292] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:18:0)
[14:30:06.375] Build status: SCHEDULED
[14:30:06.375] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:19:0) after 2m37s
[14:32:43.408] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:19:0)
[14:32:43.427] Build status: SCHEDULED
[14:32:43.427] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:20:0) after 1m30s
[14:34:13.582] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:20:0)
[14:34:13.598] Build status: SCHEDULED
[14:34:13.598] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:21:0) after 1m56s
[14:36:09.612] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:21:0)
[14:36:09.626] Build status: SCHEDULED
[14:36:09.626] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:22:0) after 4m59s
[14:41:08.642] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:22:0)
[14:41:08.678] Build status: SCHEDULED
[14:41:08.678] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:23:0) after 5m8s
[14:46:16.895] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:23:0)
[14:46:16.923] Build status: SCHEDULED
[14:46:16.923] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:24:0) after 5m10s
[14:51:27.055] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:24:0)
[14:51:27.071] Build status: SCHEDULED
[14:51:27.071] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:25:0) after 9m52s
[15:01:19.184] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:25:0)
[15:01:19.205] Build status: SCHEDULED
[15:01:19.205] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:26:0) after 2m2s
[15:03:21.254] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:26:0)
[15:03:21.279] Build status: SCHEDULED
[15:03:21.280] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:27:0) after 8m29s
[15:11:50.334] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:27:0)
[15:11:50.356] Build status: SCHEDULED
[15:11:50.356] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:28:0) after 3m13s
[15:15:03.455] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:28:0)
[15:15:03.487] Build status: SCHEDULED
[15:15:03.487] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:29:0) after 6m1s
[15:21:04.601] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:29:0)
[15:21:04.669] Build status: SCHEDULED
[15:21:04.669] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:30:0) after 8m34s
[15:29:38.779] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:30:0)
[15:29:38.796] Build status: SCHEDULED
[15:29:38.796] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:31:0) after 4m16s
[15:33:55.213] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:31:0)
[15:33:55.241] Build status: SCHEDULED
[15:33:55.241] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:32:0) after 9m44s
[15:43:39.395] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:32:0)
[15:43:39.413] Build status: SCHEDULED
[15:43:39.413] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:33:0) after 4m53s
[15:48:32.429] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:33:0)
[15:48:32.448] Build status: SCHEDULED
[15:48:32.448] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:34:0) after 4m12s
[15:52:44.462] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:34:0)
[15:52:44.477] Build status: SCHEDULED
[15:52:44.477] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:35:0) after 6m52s
[15:59:36.594] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:35:0)
[15:59:36.610] Build status: SCHEDULED
[15:59:36.610] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:36:0) after 7m21s
[16:06:57.625] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:36:0)
[16:06:57.640] Build status: SCHEDULED
[16:06:57.640] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:37:0) after 3m16s
[16:10:13.733] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:37:0)
[16:10:13.868] Build status: SCHEDULED
[16:10:13.868] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:38:0) after 9m2s
[16:19:15.883] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:38:0)
[16:19:15.901] Build status: SCHEDULED
[16:19:15.901] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:39:0) after 7m0s
[16:26:16.130] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:39:0)
[16:26:16.154] Build status: SCHEDULED
[16:26:16.154] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:40:0) after 5m35s
[16:31:51.262] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:40:0)
[16:31:51.337] Build status: SCHEDULED
[16:31:51.337] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:41:0) after 3m28s
[16:35:19.550] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:41:0)
[16:35:19.583] Build status: SCHEDULED
[16:35:19.583] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:42:0) after 3m16s
[16:38:35.601] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:42:0)
[16:38:35.619] Build status: SCHEDULED
[16:38:35.619] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:43:0) after 4m57s
[16:43:32.654] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:43:0)
[16:43:32.702] Build status: SCHEDULED
[16:43:32.702] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:44:0) after 8m52s
[16:52:24.751] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:44:0)
[16:52:24.765] Build status: SCHEDULED
[16:52:24.766] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:45:0) after 5m26s
[16:57:50.782] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:45:0)
[16:57:50.801] Build status: SCHEDULED
[16:57:50.801] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:46:0) after 1m48s
[16:59:38.789] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:46:0)
[16:59:38.804] Build status: SCHEDULED
[16:59:38.804] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:47:0) after 2m53s
[17:02:31.818] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:47:0)
[17:02:31.835] Build status: SCHEDULED
[17:02:31.835] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:48:0) after 8m5s
[17:10:36.895] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:48:0)
[17:10:36.916] Build status: SCHEDULED
[17:10:36.916] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:49:0) after 1m12s
[17:11:49.038] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:49:0)
[17:11:49.054] Build status: SCHEDULED
[17:11:49.054] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:50:0) after 9m58s
[17:21:47.105] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:50:0)
[17:21:47.125] Build status: SCHEDULED
[17:21:47.125] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:51:0) after 8m14s
[17:30:01.148] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:51:0)
[17:30:01.170] Build status: SCHEDULED
[17:30:01.170] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:52:0) after 2m24s
[17:32:25.189] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:52:0)
[17:32:25.268] Build status: SCHEDULED
[17:32:25.268] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:53:0) after 8m53s
[17:41:18.287] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:53:0)
[17:41:18.341] Build status: SCHEDULED
[17:41:18.341] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:54:0) after 6m6s
[17:47:24.358] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:54:0)
[17:47:24.418] Build status: SCHEDULED
[17:47:24.418] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:55:0) after 3m1s
[17:50:25.481] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:55:0)
[17:50:25.497] Build status: SCHEDULED
[17:50:25.497] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:56:0) after 3m17s
[17:53:42.533] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:56:0)
[17:53:42.558] Build status: SCHEDULED
[17:53:42.558] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:57:0) after 2m50s
[17:56:32.588] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:57:0)
[17:56:32.607] Build status: SCHEDULED
[17:56:32.607] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:58:0) after 6m44s
[18:03:16.625] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:58:0)
[18:03:16.643] Build status: SCHEDULED
[18:03:16.643] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:59:0) after 6m30s
[18:09:46.666] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:59:0)
[18:09:46.681] Build status: SCHEDULED
[18:09:46.681] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:60:0) after 7m34s
[18:17:20.695] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:60:0)
[18:17:21.686] Build status: SCHEDULED
[18:17:21.686] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:61:0) after 9m31s
[18:26:52.705] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:61:0)
[18:26:52.879] Build status: SCHEDULED
[18:26:52.880] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:62:0) after 1m3s
[18:27:55.912] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:62:0)
[18:27:55.947] Build status: SCHEDULED
[18:27:55.947] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:63:0) after 7m45s
[18:35:41.268] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:63:0)
[18:35:41.283] Build status: SCHEDULED
[18:35:41.283] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:64:0) after 2m39s
[18:38:20.715] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:64:0)
[18:38:20.735] Build status: SCHEDULED
[18:38:20.735] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:65:0) after 3m52s
[18:42:12.751] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:65:0)
[18:42:12.769] Build status: SCHEDULED
[18:42:12.769] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:66:0) after 7m43s
[18:49:55.886] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:66:0)
[18:49:55.902] Build status: SCHEDULED
[18:49:55.902] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:67:0) after 1m57s
[18:51:53.013] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:67:0)
[18:51:53.030] Build status: SCHEDULED
[18:51:53.030] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:68:0) after 6m5s
[18:57:58.177] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:68:0)
[18:57:58.194] Build status: SCHEDULED
[18:57:58.194] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:69:0) after 6m8s
[19:04:06.312] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:69:0)
[19:04:06.330] Build status: SCHEDULED
[19:04:06.330] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:70:0) after 3m33s
[19:07:39.353] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:70:0)
[19:07:39.377] Build status: SCHEDULED
[19:07:39.377] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:71:0) after 1m37s
[19:09:16.395] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:71:0)
[19:09:16.415] Build status: SCHEDULED
[19:09:16.415] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:72:0) after 4m12s
[19:13:28.473] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:72:0)
[19:13:28.496] Build status: SCHEDULED
[19:13:28.496] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:73:0) after 2m32s
[19:16:00.512] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:73:0)
[19:16:00.528] Build status: SCHEDULED
[19:16:00.528] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:74:0) after 8m26s
[19:24:26.551] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:74:0)
[19:24:26.569] Build status: SCHEDULED
[19:24:26.569] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:75:0) after 3m9s
[19:27:35.615] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:75:0)
[19:27:35.636] Build status: SCHEDULED
[19:27:35.636] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:76:0) after 5m38s
[19:33:13.653] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:76:0)
[19:33:13.673] Build status: SCHEDULED
[19:33:13.673] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:77:0) after 4m59s
[19:38:12.792] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:77:0)
[19:38:12.814] Build status: SCHEDULED
[19:38:12.814] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:78:0) after 5m4s
[19:43:16.829] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:78:0)
[19:43:16.852] Build status: SCHEDULED
[19:43:16.852] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:79:0) after 3m10s
[19:46:26.916] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:79:0)
[19:46:26.934] Build status: SCHEDULED
[19:46:26.934] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:80:0) after 6m22s
[19:52:48.954] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:80:0)
[19:52:48.977] Build status: SCHEDULED
[19:52:48.977] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:81:0) after 6m15s
[19:59:04.005] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:81:0)
[19:59:04.022] Build status: SCHEDULED
[19:59:04.022] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:82:0) after 7m20s
[20:06:24.124] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:82:0)
[20:06:24.187] Build status: SCHEDULED
[20:06:24.187] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:83:0) after 2m7s
[20:08:31.203] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:83:0)
[20:08:31.254] Build status: SCHEDULED
[20:08:31.254] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:84:0) after 6m24s
[20:14:55.313] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:84:0)
[20:14:56.028] Build status: SCHEDULED
[20:14:56.028] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:85:0) after 8m19s
[20:23:15.046] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:85:0)
[20:23:15.062] Build status: SCHEDULED
[20:23:15.062] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:86:0) after 6m23s
[20:29:38.088] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:86:0)
[20:29:38.116] Build status: SCHEDULED
[20:29:38.116] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:87:0) after 3m50s
[20:33:28.140] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:87:0)
[20:33:28.171] Build status: SCHEDULED
[20:33:28.172] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:88:0) after 7m42s
[20:41:10.167] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:88:0)
[20:41:10.184] Build status: SCHEDULED
[20:41:10.184] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:89:0) after 9m54s
[20:51:04.991] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:89:0)
[20:51:05.004] Build status: SCHEDULED
[20:51:05.004] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:90:0) after 2m4s
[20:53:09.021] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:90:0)
[20:53:09.039] Build status: SCHEDULED
[20:53:09.039] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869179691813548432:91:0) after 7m22s
[20:55:50.665] Received PubSub notification, asking Buildbucket for the build status
[20:55:50.683] Build:
{
"id": "8703749629879673905",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-09-14T12:55:43.761755520Z",
"endTime": "2025-09-14T20:55:50.400686816Z",
"updateTime": "2025-09-14T20:55:50.400686816Z",
"status": "INFRA_FAILURE",
"statusDetails": {
"resourceExhaustion": {
},
"timeout": {
}
},
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "123911bae1b45f2b238d582cf2aa93cbc9394314",
"ref": "refs/heads/main"
}
}
}
[20:55:50.683] Invocation finished in 8h0m8.150387401s with status FAILED