[05:24:46.160] New invocation is queued and will start shortly
[05:24:47.226] Starting the invocation (attempt 1)
[05:24:47.252] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/632e84a744694c7d1f089e256236236a6b47a877"
[05:24:47.252] Popped gitiles commit info from properties and tags
[05:24:47.252] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[05:24:47.252] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[05:24:47.253] Buildbucket request:
{
"requestId": "8875368656198346272",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-asan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8875368656198346272",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "bbc8139178f18d6ff81978ef448f14e9652431df"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@bbc8139178f18d6ff81978ef448f14e9652431df",
"title": "bbc8139178f18d6ff81978ef448f14e9652431df",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/bbc8139178f18d6ff81978ef448f14e9652431df"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "ecc6dfb536fc02c4dcf2925b67ccab4e60d5bda7"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@ecc6dfb536fc02c4dcf2925b67ccab4e60d5bda7",
"title": "ecc6dfb536fc02c4dcf2925b67ccab4e60d5bda7",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/ecc6dfb536fc02c4dcf2925b67ccab4e60d5bda7"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "27ef8558f97a09b3f9bb628573b31efb53b91034"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@27ef8558f97a09b3f9bb628573b31efb53b91034",
"title": "27ef8558f97a09b3f9bb628573b31efb53b91034",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/27ef8558f97a09b3f9bb628573b31efb53b91034"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "632e84a744694c7d1f089e256236236a6b47a877"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@632e84a744694c7d1f089e256236236a6b47a877",
"title": "632e84a744694c7d1f089e256236236a6b47a877",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/632e84a744694c7d1f089e256236236a6b47a877"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "632e84a744694c7d1f089e256236236a6b47a877",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8875368656198346272"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[05:24:48.012] Scheduled build:
{
"id": "8709938594324887969",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-07-08T05:24:47.324063743Z",
"updateTime": "2025-07-08T05:24:47.324063743Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "632e84a744694c7d1f089e256236236a6b47a877",
"ref": "refs/heads/main"
}
}
}
[05:24:48.012] Task URL: https://cr-buildbucket.appspot.com/build/8709938594324887969
[05:24:48.012] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:2:0) after 5m54s
[05:24:48.746] Received PubSub notification, asking Buildbucket for the build status
[05:24:48.766] Build status: SCHEDULED
[05:30:42.026] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:2:0)
[05:30:42.044] Build status: SCHEDULED
[05:30:42.044] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:4:0) after 8m32s
[05:39:14.069] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:4:0)
[05:39:14.088] Build status: SCHEDULED
[05:39:14.088] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:5:0) after 4m13s
[05:43:27.159] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:5:0)
[05:43:27.187] Build status: SCHEDULED
[05:43:27.187] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:6:0) after 7m23s
[05:50:50.476] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:6:0)
[05:50:50.503] Build status: SCHEDULED
[05:50:50.503] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:7:0) after 1m56s
[05:52:46.517] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:7:0)
[05:52:46.542] Build status: SCHEDULED
[05:52:46.542] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:8:0) after 4m24s
[05:57:10.556] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:8:0)
[05:57:10.581] Build status: SCHEDULED
[05:57:10.581] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:9:0) after 4m40s
[06:01:50.666] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:9:0)
[06:01:50.691] Build status: SCHEDULED
[06:01:50.691] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:10:0) after 4m35s
[06:06:25.709] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:10:0)
[06:06:25.803] Build status: SCHEDULED
[06:06:25.803] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:11:0) after 3m43s
[06:10:08.876] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:11:0)
[06:10:08.945] Build status: SCHEDULED
[06:10:08.945] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:12:0) after 8m54s
[06:19:03.019] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:12:0)
[06:19:03.038] Build status: SCHEDULED
[06:19:03.038] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:13:0) after 6m11s
[06:25:14.031] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:13:0)
[06:25:14.056] Build status: SCHEDULED
[06:25:14.056] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:14:0) after 3m13s
[06:28:27.079] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:14:0)
[06:28:27.097] Build status: SCHEDULED
[06:28:27.097] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:15:0) after 1m29s
[06:29:56.124] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:15:0)
[06:29:56.143] Build status: SCHEDULED
[06:29:56.143] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:16:0) after 6m5s
[06:36:01.268] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:16:0)
[06:36:01.292] Build status: SCHEDULED
[06:36:01.292] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:17:0) after 9m23s
[06:45:24.410] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:17:0)
[06:45:25.298] Build status: SCHEDULED
[06:45:25.298] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:18:0) after 4m2s
[06:49:27.308] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:18:0)
[06:49:27.330] Build status: SCHEDULED
[06:49:27.330] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:19:0) after 7m5s
[06:56:32.358] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:19:0)
[06:56:32.396] Build status: SCHEDULED
[06:56:32.396] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:20:0) after 2m45s
[06:59:17.422] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:20:0)
[06:59:17.440] Build status: SCHEDULED
[06:59:17.440] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:21:0) after 2m55s
[07:02:12.464] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:21:0)
[07:02:12.481] Build status: SCHEDULED
[07:02:12.481] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:22:0) after 4m36s
[07:06:48.504] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:22:0)
[07:06:48.522] Build status: SCHEDULED
[07:06:48.522] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:23:0) after 5m1s
[07:11:49.570] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:23:0)
[07:11:49.589] Build status: SCHEDULED
[07:11:49.589] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:24:0) after 7m36s
[07:19:25.614] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:24:0)
[07:19:25.641] Build status: SCHEDULED
[07:19:25.641] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:25:0) after 6m6s
[07:25:31.705] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:25:0)
[07:25:31.731] Build status: SCHEDULED
[07:25:31.731] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:26:0) after 5m16s
[07:30:47.741] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:26:0)
[07:30:47.764] Build status: SCHEDULED
[07:30:47.764] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:27:0) after 4m5s
[07:34:52.797] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:27:0)
[07:34:52.825] Build status: SCHEDULED
[07:34:52.825] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:28:0) after 6m3s
[07:40:56.760] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:28:0)
[07:40:56.798] Build status: SCHEDULED
[07:40:56.798] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:29:0) after 4m35s
[07:45:31.824] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:29:0)
[07:45:31.842] Build status: SCHEDULED
[07:45:31.842] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:30:0) after 8m38s
[07:54:09.959] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:30:0)
[07:54:10.063] Build status: SCHEDULED
[07:54:10.063] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:31:0) after 5m26s
[07:59:36.144] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:31:0)
[07:59:36.163] Build status: SCHEDULED
[07:59:36.163] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:32:0) after 2m1s
[08:01:37.230] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:32:0)
[08:01:37.272] Build status: SCHEDULED
[08:01:37.272] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:33:0) after 8m46s
[08:10:23.310] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:33:0)
[08:10:23.338] Build status: SCHEDULED
[08:10:23.338] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:34:0) after 8m34s
[08:18:57.419] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:34:0)
[08:18:57.480] Build status: SCHEDULED
[08:18:57.480] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:35:0) after 7m26s
[08:26:23.504] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:35:0)
[08:26:23.529] Build status: SCHEDULED
[08:26:23.529] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:36:0) after 1m49s
[08:28:12.552] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:36:0)
[08:28:12.582] Build status: SCHEDULED
[08:28:12.582] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:37:0) after 3m43s
[08:31:55.861] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:37:0)
[08:31:55.885] Build status: SCHEDULED
[08:31:55.885] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:38:0) after 5m2s
[08:36:57.957] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:38:0)
[08:36:57.982] Build status: SCHEDULED
[08:36:57.982] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:39:0) after 2m11s
[08:39:09.009] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:39:0)
[08:39:09.031] Build status: SCHEDULED
[08:39:09.031] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:40:0) after 6m30s
[08:45:39.384] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:40:0)
[08:45:39.416] Build status: SCHEDULED
[08:45:39.416] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:41:0) after 5m58s
[08:51:37.539] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:41:0)
[08:51:37.558] Build status: SCHEDULED
[08:51:37.558] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:42:0) after 1m22s
[08:52:59.531] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:42:0)
[08:52:59.556] Build status: SCHEDULED
[08:52:59.556] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:43:0) after 4m3s
[08:57:02.677] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:43:0)
[08:57:02.802] Build status: SCHEDULED
[08:57:02.802] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:44:0) after 6m30s
[09:03:32.872] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:44:0)
[09:03:32.899] Build status: SCHEDULED
[09:03:32.899] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:45:0) after 6m56s
[09:10:29.139] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:45:0)
[09:10:29.217] Build status: SCHEDULED
[09:10:29.217] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:46:0) after 5m27s
[09:15:56.242] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:46:0)
[09:15:56.270] Build status: SCHEDULED
[09:15:56.271] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:47:0) after 4m4s
[09:20:00.304] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:47:0)
[09:20:00.375] Build status: SCHEDULED
[09:20:00.375] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:48:0) after 1m33s
[09:21:33.401] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:48:0)
[09:21:33.421] Build status: SCHEDULED
[09:21:33.421] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:49:0) after 6m36s
[09:28:09.473] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:49:0)
[09:28:09.490] Build status: SCHEDULED
[09:28:09.490] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:50:0) after 3m35s
[09:31:44.513] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:50:0)
[09:31:44.583] Build status: SCHEDULED
[09:31:44.583] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:51:0) after 1m1s
[09:32:45.576] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:51:0)
[09:32:45.605] Build status: SCHEDULED
[09:32:45.605] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:52:0) after 1m59s
[09:34:44.676] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:52:0)
[09:34:44.744] Build status: SCHEDULED
[09:34:44.744] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:53:0) after 1m29s
[09:36:13.797] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:53:0)
[09:36:13.821] Build status: SCHEDULED
[09:36:13.821] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:54:0) after 7m16s
[09:43:29.890] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:54:0)
[09:43:29.918] Build status: SCHEDULED
[09:43:29.918] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:55:0) after 4m22s
[09:47:52.028] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:55:0)
[09:47:52.051] Build status: SCHEDULED
[09:47:52.051] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:56:0) after 4m20s
[09:52:12.113] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:56:0)
[09:52:12.187] Build status: SCHEDULED
[09:52:12.187] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:57:0) after 2m54s
[09:55:06.214] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:57:0)
[09:55:06.230] Build status: SCHEDULED
[09:55:06.230] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:58:0) after 3m48s
[09:58:54.273] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:58:0)
[09:58:54.291] Build status: SCHEDULED
[09:58:54.291] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:59:0) after 5m11s
[10:04:05.319] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:59:0)
[10:04:05.339] Build status: SCHEDULED
[10:04:05.339] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:60:0) after 3m38s
[10:07:43.513] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:60:0)
[10:07:43.537] Build status: SCHEDULED
[10:07:43.537] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:61:0) after 4m33s
[10:12:16.568] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:61:0)
[10:12:16.596] Build status: SCHEDULED
[10:12:16.596] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:62:0) after 4m44s
[10:17:00.619] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:62:0)
[10:17:01.852] Build status: SCHEDULED
[10:17:01.852] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:63:0) after 1m27s
[10:18:28.929] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:63:0)
[10:18:28.954] Build status: SCHEDULED
[10:18:28.954] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:64:0) after 5m45s
[10:24:14.021] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:64:0)
[10:24:14.046] Build status: SCHEDULED
[10:24:14.046] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:65:0) after 4m51s
[10:29:05.141] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:65:0)
[10:29:05.229] Build status: SCHEDULED
[10:29:05.229] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:66:0) after 2m11s
[10:31:16.255] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:66:0)
[10:31:16.275] Build status: SCHEDULED
[10:31:16.275] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:67:0) after 2m47s
[10:34:03.301] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:67:0)
[10:34:03.318] Build status: SCHEDULED
[10:34:03.318] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:68:0) after 9m6s
[10:43:09.379] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:68:0)
[10:43:09.397] Build status: SCHEDULED
[10:43:09.397] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:69:0) after 6m10s
[10:49:19.944] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:69:0)
[10:49:20.013] Build status: SCHEDULED
[10:49:20.013] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:70:0) after 4m29s
[10:53:49.262] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:70:0)
[10:53:49.294] Build status: SCHEDULED
[10:53:49.294] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:71:0) after 6m22s
[11:00:11.448] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:71:0)
[11:00:11.465] Build status: SCHEDULED
[11:00:11.465] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:72:0) after 5m39s
[11:05:50.753] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:72:0)
[11:05:50.777] Build status: SCHEDULED
[11:05:50.777] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:73:0) after 3m8s
[11:08:58.973] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:73:0)
[11:08:59.063] Build status: SCHEDULED
[11:08:59.063] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:74:0) after 7m35s
[11:16:34.095] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:74:0)
[11:16:34.116] Build status: SCHEDULED
[11:16:34.116] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:75:0) after 6m35s
[11:23:09.141] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:75:0)
[11:23:09.216] Build status: SCHEDULED
[11:23:09.216] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:76:0) after 3m40s
[11:26:49.999] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:76:0)
[11:26:50.013] Build status: SCHEDULED
[11:26:50.013] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:77:0) after 9m11s
[11:36:01.599] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:77:0)
[11:36:01.723] Build status: SCHEDULED
[11:36:01.723] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:78:0) after 4m29s
[11:40:30.969] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:78:0)
[11:40:30.984] Build status: SCHEDULED
[11:40:30.985] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:79:0) after 3m46s
[11:44:17.009] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:79:0)
[11:44:17.059] Build status: SCHEDULED
[11:44:17.059] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:80:0) after 2m53s
[11:47:10.042] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:80:0)
[11:47:10.064] Build status: SCHEDULED
[11:47:10.064] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:81:0) after 3m12s
[11:50:22.115] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:81:0)
[11:50:22.137] Build status: SCHEDULED
[11:50:22.137] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:82:0) after 2m8s
[11:52:30.164] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:82:0)
[11:52:30.197] Build status: SCHEDULED
[11:52:30.197] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:83:0) after 9m7s
[12:01:37.186] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:83:0)
[12:01:37.215] Build status: SCHEDULED
[12:01:37.215] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:84:0) after 1m9s
[12:02:46.234] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:84:0)
[12:02:46.301] Build status: SCHEDULED
[12:02:46.301] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:85:0) after 6m50s
[12:09:36.386] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:85:0)
[12:09:36.442] Build status: SCHEDULED
[12:09:36.442] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:86:0) after 6m46s
[12:16:22.503] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:86:0)
[12:16:22.525] Build status: SCHEDULED
[12:16:22.525] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:87:0) after 5m38s
[12:22:00.566] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:87:0)
[12:22:00.609] Build status: SCHEDULED
[12:22:00.609] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:88:0) after 3m3s
[12:25:03.630] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:88:0)
[12:25:03.666] Build status: SCHEDULED
[12:25:03.666] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:89:0) after 4m15s
[12:29:18.726] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:89:0)
[12:29:18.761] Build status: SCHEDULED
[12:29:18.761] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:90:0) after 7m23s
[12:36:41.786] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:90:0)
[12:36:41.808] Build status: SCHEDULED
[12:36:41.808] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:91:0) after 5m38s
[12:42:19.836] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:91:0)
[12:42:19.871] Build status: SCHEDULED
[12:42:19.871] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:92:0) after 9m28s
[12:51:47.893] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:92:0)
[12:51:47.919] Build status: SCHEDULED
[12:51:47.919] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:93:0) after 3m35s
[12:55:22.925] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:93:0)
[12:55:22.943] Build status: SCHEDULED
[12:55:22.944] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:94:0) after 5m13s
[13:00:35.959] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:94:0)
[13:00:35.980] Build status: SCHEDULED
[13:00:35.980] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:95:0) after 5m22s
[13:05:58.007] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:95:0)
[13:05:58.030] Build status: SCHEDULED
[13:05:58.030] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:96:0) after 3m14s
[13:09:12.154] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:96:0)
[13:09:12.174] Build status: SCHEDULED
[13:09:12.174] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:97:0) after 6m41s
[13:15:53.186] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:97:0)
[13:15:53.289] Build status: SCHEDULED
[13:15:53.289] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:98:0) after 2m17s
[13:18:10.344] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:98:0)
[13:18:10.370] Build status: SCHEDULED
[13:18:10.370] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:99:0) after 6m18s
[13:24:28.399] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:99:0)
[13:24:28.433] Build status: SCHEDULED
[13:24:28.433] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8875368656198346272:100:0) after 5m40s
[13:24:49.935] Received PubSub notification, asking Buildbucket for the build status
[13:24:50.013] Build:
{
"id": "8709938594324887969",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-07-08T05:24:47.324063743Z",
"endTime": "2025-07-08T13:24:49.586856238Z",
"updateTime": "2025-07-08T13:24:49.586856238Z",
"status": "INFRA_FAILURE",
"statusDetails": {
"resourceExhaustion": {
},
"timeout": {
}
},
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "632e84a744694c7d1f089e256236236a6b47a877",
"ref": "refs/heads/main"
}
}
}
[13:24:50.013] Invocation finished in 8h0m3.859738864s with status FAILED