[09:51:31.082] New invocation is queued and will start shortly
[09:51:31.082] Triggered by user:rerunner@pigweed-service-accounts.iam.gserviceaccount.com
[09:51:32.302] Starting the invocation (attempt 1)
[09:51:32.332] Ignoring gitiles_ref tag without the buildset tag
[09:51:32.332] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[09:51:32.332] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[09:51:32.332] Buildbucket request:
{
"requestId": "8868828892248821568",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8868828892248821568",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan",
"triggers": [
{
"buildbucket": {
"tags": [
"parent_buildername:rerunner",
"user_agent:rerunner"
]
},
"id": "7a204b57-564c-4447-b8f3-38fbc3d392d6",
"title": "rerunner/0",
"url": "https://cr-buildbucket.appspot.com/build/8703399046672428785"
}
]
}
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8868828892248821568"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "rerunner"
},
{
"key": "user_agent",
"value": "rerunner"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[09:51:32.660] Scheduled build:
{
"id": "8703398830217614401",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-09-18T09:51:32.389721085Z",
"updateTime": "2025-09-18T09:51:32.389721085Z",
"status": "SCHEDULED",
"input": {
}
}
[09:51:32.660] Task URL: https://cr-buildbucket.appspot.com/build/8703398830217614401
[09:51:32.660] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:2:0) after 1m1s
[09:51:33.617] Received PubSub notification, asking Buildbucket for the build status
[09:51:33.668] Build status: SCHEDULED
[09:52:33.676] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:2:0)
[09:52:33.731] Build status: SCHEDULED
[09:52:33.731] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:4:0) after 6m17s
[09:58:50.791] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:4:0)
[09:58:50.814] Build status: SCHEDULED
[09:58:50.814] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:5:0) after 7m8s
[10:05:59.427] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:5:0)
[10:05:59.448] Build status: SCHEDULED
[10:05:59.448] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:6:0) after 3m42s
[10:09:41.495] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:6:0)
[10:09:41.594] Build status: SCHEDULED
[10:09:41.594] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:7:0) after 9m40s
[10:19:21.602] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:7:0)
[10:19:21.629] Build status: SCHEDULED
[10:19:21.629] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:8:0) after 5m22s
[10:24:43.610] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:8:0)
[10:24:43.628] Build status: SCHEDULED
[10:24:43.628] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:9:0) after 1m33s
[10:26:16.824] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:9:0)
[10:26:16.861] Build status: SCHEDULED
[10:26:16.861] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:10:0) after 3m40s
[10:29:56.873] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:10:0)
[10:29:56.895] Build status: SCHEDULED
[10:29:56.895] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:11:0) after 4m53s
[10:34:49.909] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:11:0)
[10:34:49.937] Build status: SCHEDULED
[10:34:49.937] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:12:0) after 1m25s
[10:36:14.953] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:12:0)
[10:36:14.969] Build status: SCHEDULED
[10:36:14.969] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:13:0) after 7m7s
[10:43:21.984] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:13:0)
[10:43:22.010] Build status: SCHEDULED
[10:43:22.010] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:14:0) after 7m44s
[10:51:06.167] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:14:0)
[10:51:06.198] Build status: SCHEDULED
[10:51:06.198] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:15:0) after 7m23s
[10:58:29.214] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:15:0)
[10:58:29.273] Build status: SCHEDULED
[10:58:29.273] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:16:0) after 5m6s
[11:03:35.295] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:16:0)
[11:03:35.790] Build status: SCHEDULED
[11:03:35.790] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:17:0) after 4m38s
[11:08:13.979] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:17:0)
[11:08:14.005] Build status: SCHEDULED
[11:08:14.006] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:18:0) after 6m16s
[11:14:30.179] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:18:0)
[11:14:30.201] Build status: SCHEDULED
[11:14:30.202] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:19:0) after 8m0s
[11:22:30.607] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:19:0)
[11:22:30.975] Build status: SCHEDULED
[11:22:30.975] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:20:0) after 4m3s
[11:26:33.991] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:20:0)
[11:26:34.024] Build status: SCHEDULED
[11:26:34.024] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:21:0) after 2m13s
[11:28:47.013] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:21:0)
[11:28:47.030] Build status: SCHEDULED
[11:28:47.030] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:22:0) after 3m19s
[11:32:06.058] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:22:0)
[11:32:06.076] Build status: SCHEDULED
[11:32:06.076] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:23:0) after 2m18s
[11:34:24.178] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:23:0)
[11:34:24.201] Build status: SCHEDULED
[11:34:24.201] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:24:0) after 9m50s
[11:44:14.221] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:24:0)
[11:44:14.241] Build status: SCHEDULED
[11:44:14.241] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:25:0) after 6m41s
[11:50:55.299] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:25:0)
[11:50:55.340] Build status: SCHEDULED
[11:50:55.340] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:26:0) after 5m49s
[11:56:44.408] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:26:0)
[11:56:44.424] Build status: SCHEDULED
[11:56:44.424] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:27:0) after 2m20s
[11:59:04.442] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:27:0)
[11:59:04.470] Build status: SCHEDULED
[11:59:04.470] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:28:0) after 7m36s
[12:06:40.488] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:28:0)
[12:06:40.509] Build status: SCHEDULED
[12:06:40.509] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:29:0) after 1m47s
[12:08:27.492] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:29:0)
[12:08:27.517] Build status: SCHEDULED
[12:08:27.517] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:30:0) after 4m38s
[12:13:05.552] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:30:0)
[12:13:05.577] Build status: SCHEDULED
[12:13:05.578] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:31:0) after 6m21s
[12:19:26.591] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:31:0)
[12:19:26.608] Build status: SCHEDULED
[12:19:26.608] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:32:0) after 6m24s
[12:25:50.647] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:32:0)
[12:25:50.664] Build status: SCHEDULED
[12:25:50.664] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:33:0) after 8m12s
[12:34:02.654] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:33:0)
[12:34:02.676] Build status: SCHEDULED
[12:34:02.676] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:34:0) after 6m49s
[12:40:51.697] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:34:0)
[12:40:51.714] Build status: SCHEDULED
[12:40:51.714] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:35:0) after 2m21s
[12:43:12.730] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:35:0)
[12:43:12.766] Build status: SCHEDULED
[12:43:12.766] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:36:0) after 5m30s
[12:48:42.824] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:36:0)
[12:48:42.844] Build status: SCHEDULED
[12:48:42.844] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:37:0) after 3m13s
[12:51:55.864] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:37:0)
[12:51:55.887] Build status: SCHEDULED
[12:51:55.887] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:38:0) after 1m13s
[12:53:09.008] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:38:0)
[12:53:09.027] Build status: SCHEDULED
[12:53:09.027] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:39:0) after 1m44s
[12:54:53.042] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:39:0)
[12:54:53.058] Build status: SCHEDULED
[12:54:53.058] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:40:0) after 1m49s
[12:56:42.124] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:40:0)
[12:56:42.215] Build status: SCHEDULED
[12:56:42.215] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:41:0) after 1m36s
[12:58:18.232] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:41:0)
[12:58:18.259] Build status: SCHEDULED
[12:58:18.259] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:42:0) after 6m35s
[13:04:53.274] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:42:0)
[13:04:53.301] Build status: SCHEDULED
[13:04:53.301] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:43:0) after 8m9s
[13:13:02.369] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:43:0)
[13:13:02.400] Build status: SCHEDULED
[13:13:02.400] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:44:0) after 7m28s
[13:20:30.669] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:44:0)
[13:20:30.685] Build status: SCHEDULED
[13:20:30.685] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:45:0) after 3m25s
[13:23:55.702] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:45:0)
[13:23:55.846] Build status: SCHEDULED
[13:23:55.846] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:46:0) after 4m53s
[13:28:48.864] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:46:0)
[13:28:48.880] Build status: SCHEDULED
[13:28:48.880] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:47:0) after 1m1s
[13:29:49.968] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:47:0)
[13:29:50.039] Build status: SCHEDULED
[13:29:50.039] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:48:0) after 7m50s
[13:37:40.056] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:48:0)
[13:37:40.075] Build status: SCHEDULED
[13:37:40.075] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:49:0) after 4m55s
[13:42:35.176] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:49:0)
[13:42:35.195] Build status: SCHEDULED
[13:42:35.195] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:50:0) after 5m32s
[13:48:07.212] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:50:0)
[13:48:07.228] Build status: SCHEDULED
[13:48:07.228] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:51:0) after 3m23s
[13:51:30.338] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:51:0)
[13:51:30.358] Build status: SCHEDULED
[13:51:30.358] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:52:0) after 8m29s
[13:59:59.375] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:52:0)
[13:59:59.396] Build status: SCHEDULED
[13:59:59.396] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:53:0) after 7m12s
[14:07:11.373] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:53:0)
[14:07:11.397] Build status: SCHEDULED
[14:07:11.397] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:54:0) after 1m50s
[14:09:01.424] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:54:0)
[14:09:01.444] Build status: SCHEDULED
[14:09:01.444] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:55:0) after 4m32s
[14:13:33.462] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:55:0)
[14:13:33.479] Build status: SCHEDULED
[14:13:33.479] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:56:0) after 4m5s
[14:17:38.685] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:56:0)
[14:17:38.711] Build status: SCHEDULED
[14:17:38.711] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:57:0) after 2m38s
[14:20:16.736] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:57:0)
[14:20:16.755] Build status: SCHEDULED
[14:20:16.755] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:58:0) after 6m13s
[14:26:29.800] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:58:0)
[14:26:29.823] Build status: SCHEDULED
[14:26:29.823] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:59:0) after 1m2s
[14:27:31.860] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:59:0)
[14:27:31.880] Build status: SCHEDULED
[14:27:31.880] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:60:0) after 7m1s
[14:34:32.899] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:60:0)
[14:34:32.916] Build status: SCHEDULED
[14:34:32.916] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:61:0) after 9m36s
[14:44:08.933] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:61:0)
[14:44:08.953] Build status: SCHEDULED
[14:44:08.954] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:62:0) after 9m49s
[14:53:58.218] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:62:0)
[14:53:58.244] Build status: SCHEDULED
[14:53:58.244] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:63:0) after 5m30s
[14:59:28.312] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:63:0)
[14:59:28.331] Build status: SCHEDULED
[14:59:28.331] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:64:0) after 9m10s
[15:08:38.354] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:64:0)
[15:08:38.388] Build status: SCHEDULED
[15:08:38.388] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:65:0) after 3m48s
[15:12:26.403] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:65:0)
[15:12:26.421] Build status: SCHEDULED
[15:12:26.421] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:66:0) after 4m35s
[15:17:01.417] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:66:0)
[15:17:01.459] Build status: SCHEDULED
[15:17:01.459] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:67:0) after 4m13s
[15:21:14.483] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:67:0)
[15:21:14.579] Build status: SCHEDULED
[15:21:14.579] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:68:0) after 4m5s
[15:25:19.612] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:68:0)
[15:25:19.632] Build status: SCHEDULED
[15:25:19.632] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:69:0) after 8m57s
[15:34:16.696] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:69:0)
[15:34:16.715] Build status: SCHEDULED
[15:34:16.715] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:70:0) after 4m5s
[15:38:21.732] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:70:0)
[15:38:21.748] Build status: SCHEDULED
[15:38:21.748] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:71:0) after 3m33s
[15:41:54.762] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:71:0)
[15:41:54.783] Build status: SCHEDULED
[15:41:54.783] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:72:0) after 1m47s
[15:43:41.787] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:72:0)
[15:43:41.815] Build status: SCHEDULED
[15:43:41.815] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:73:0) after 5m4s
[15:48:45.862] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:73:0)
[15:48:45.923] Build status: SCHEDULED
[15:48:45.923] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:74:0) after 1m51s
[15:50:36.938] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:74:0)
[15:50:36.953] Build status: SCHEDULED
[15:50:36.954] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:75:0) after 1m0s
[15:51:37.019] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:75:0)
[15:51:37.047] Build status: SCHEDULED
[15:51:37.047] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:76:0) after 6m11s
[15:57:48.043] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:76:0)
[15:57:48.063] Build status: SCHEDULED
[15:57:48.064] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:77:0) after 7m36s
[16:05:24.129] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:77:0)
[16:05:24.148] Build status: SCHEDULED
[16:05:24.148] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:78:0) after 5m2s
[16:10:26.165] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:78:0)
[16:10:26.181] Build status: SCHEDULED
[16:10:26.181] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:79:0) after 5m43s
[16:16:09.197] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:79:0)
[16:16:09.217] Build status: SCHEDULED
[16:16:09.217] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:80:0) after 5m53s
[16:22:02.244] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:80:0)
[16:22:02.265] Build status: SCHEDULED
[16:22:02.265] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:81:0) after 7m20s
[16:29:22.278] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:81:0)
[16:29:22.306] Build status: SCHEDULED
[16:29:22.306] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:82:0) after 7m52s
[16:37:14.394] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:82:0)
[16:37:14.416] Build status: SCHEDULED
[16:37:14.416] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:83:0) after 4m51s
[16:42:05.434] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:83:0)
[16:42:05.456] Build status: SCHEDULED
[16:42:05.457] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:84:0) after 8m44s
[16:50:49.652] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:84:0)
[16:50:49.674] Build status: SCHEDULED
[16:50:49.674] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:85:0) after 7m23s
[16:58:12.719] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:85:0)
[16:58:12.736] Build status: SCHEDULED
[16:58:12.736] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:86:0) after 4m18s
[17:02:30.794] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:86:0)
[17:02:30.810] Build status: SCHEDULED
[17:02:30.810] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:87:0) after 7m39s
[17:10:09.864] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:87:0)
[17:10:09.880] Build status: SCHEDULED
[17:10:09.880] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:88:0) after 3m57s
[17:14:06.898] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:88:0)
[17:14:06.917] Build status: SCHEDULED
[17:14:06.917] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:89:0) after 4m31s
[17:18:37.934] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:89:0)
[17:18:37.957] Build status: SCHEDULED
[17:18:37.957] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:90:0) after 5m48s
[17:24:26.003] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:90:0)
[17:24:26.017] Build status: SCHEDULED
[17:24:26.018] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:91:0) after 2m58s
[17:27:24.036] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:91:0)
[17:27:24.054] Build status: SCHEDULED
[17:27:24.054] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:92:0) after 4m16s
[17:31:40.189] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:92:0)
[17:31:40.277] Build status: SCHEDULED
[17:31:40.277] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:93:0) after 3m14s
[17:34:54.442] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:93:0)
[17:34:54.460] Build status: SCHEDULED
[17:34:54.461] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:94:0) after 4m6s
[17:39:00.478] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:94:0)
[17:39:00.493] Build status: SCHEDULED
[17:39:00.493] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:95:0) after 6m6s
[17:45:06.512] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:95:0)
[17:45:06.633] Build status: SCHEDULED
[17:45:06.633] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:96:0) after 5m48s
[17:50:54.750] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:96:0)
[17:50:54.772] Build status: SCHEDULED
[17:50:54.773] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8868828892248821568:97:0) after 3m26s
[17:51:35.628] Received PubSub notification, asking Buildbucket for the build status
[17:51:35.647] Build:
{
"id": "8703398830217614401",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-09-18T09:51:32.389721085Z",
"endTime": "2025-09-18T17:51:35.410729075Z",
"updateTime": "2025-09-18T17:51:35.410729075Z",
"status": "INFRA_FAILURE",
"statusDetails": {
"resourceExhaustion": {
},
"timeout": {
}
},
"input": {
}
}
[17:51:35.647] Invocation finished in 8h0m4.578864822s with status FAILED