[22:29:02.748] New invocation is queued and will start shortly
[22:29:03.837] Starting the invocation (attempt 1)
[22:29:03.860] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/c91fc7c107f021fb967af806fcedc5abc335c2b0"
[22:29:03.860] Popped gitiles commit info from properties and tags
[22:29:03.861] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[22:29:03.861] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[22:29:03.861] Buildbucket request:
{
"requestId": "8875123020367320768",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-ubsan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8875123020367320768",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "ad79a9071d7573626ea414c51811ed60e4a1c1d8"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@ad79a9071d7573626ea414c51811ed60e4a1c1d8",
"title": "ad79a9071d7573626ea414c51811ed60e4a1c1d8",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/ad79a9071d7573626ea414c51811ed60e4a1c1d8"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "664d0fc587419c800c7e2be561056e67716ed22c"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@664d0fc587419c800c7e2be561056e67716ed22c",
"title": "664d0fc587419c800c7e2be561056e67716ed22c",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/664d0fc587419c800c7e2be561056e67716ed22c"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "31964ac812fe8b306ca23e4b4db669e9e91b55c4"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@31964ac812fe8b306ca23e4b4db669e9e91b55c4",
"title": "31964ac812fe8b306ca23e4b4db669e9e91b55c4",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/31964ac812fe8b306ca23e4b4db669e9e91b55c4"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "60a4cc54044d92647890d5318f5a475d2f26369c"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@60a4cc54044d92647890d5318f5a475d2f26369c",
"title": "60a4cc54044d92647890d5318f5a475d2f26369c",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/60a4cc54044d92647890d5318f5a475d2f26369c"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "04aeff8fd3ca62306e3422e43e0bf45a43323d32"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@04aeff8fd3ca62306e3422e43e0bf45a43323d32",
"title": "04aeff8fd3ca62306e3422e43e0bf45a43323d32",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/04aeff8fd3ca62306e3422e43e0bf45a43323d32"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "cb5e7a324c13fd7bcc71646ad0c03386b52ea019"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@cb5e7a324c13fd7bcc71646ad0c03386b52ea019",
"title": "cb5e7a324c13fd7bcc71646ad0c03386b52ea019",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/cb5e7a324c13fd7bcc71646ad0c03386b52ea019"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "c91fc7c107f021fb967af806fcedc5abc335c2b0"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@c91fc7c107f021fb967af806fcedc5abc335c2b0",
"title": "c91fc7c107f021fb967af806fcedc5abc335c2b0",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/c91fc7c107f021fb967af806fcedc5abc335c2b0"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "c91fc7c107f021fb967af806fcedc5abc335c2b0",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8875123020367320768"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[22:29:04.211] Scheduled build:
{
"id": "8709692958469524561",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-ubsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-07-10T22:29:03.931983290Z",
"updateTime": "2025-07-10T22:29:03.931983290Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "c91fc7c107f021fb967af806fcedc5abc335c2b0",
"ref": "refs/heads/main"
}
}
}
[22:29:04.211] Task URL: https://cr-buildbucket.appspot.com/build/8709692958469524561
[22:29:04.211] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:2:0) after 9m33s
[22:29:05.204] Received PubSub notification, asking Buildbucket for the build status
[22:29:05.230] Build status: SCHEDULED
[22:38:37.249] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:2:0)
[22:38:37.275] Build status: SCHEDULED
[22:38:37.275] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:4:0) after 4m17s
[22:42:54.293] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:4:0)
[22:42:54.330] Build status: SCHEDULED
[22:42:54.330] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:5:0) after 3m40s
[22:46:34.393] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:5:0)
[22:46:34.424] Build status: SCHEDULED
[22:46:34.424] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:6:0) after 3m26s
[22:50:00.510] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:6:0)
[22:50:00.547] Build status: SCHEDULED
[22:50:00.547] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:7:0) after 1m22s
[22:51:22.645] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:7:0)
[22:51:22.677] Build status: SCHEDULED
[22:51:22.677] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:8:0) after 2m52s
[22:54:14.711] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:8:0)
[22:54:14.752] Build status: SCHEDULED
[22:54:14.752] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:9:0) after 4m28s
[22:58:42.775] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:9:0)
[22:58:42.795] Build status: SCHEDULED
[22:58:42.795] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:10:0) after 3m42s
[23:02:24.835] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:10:0)
[23:02:24.856] Build status: SCHEDULED
[23:02:24.856] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:11:0) after 8m36s
[23:11:00.999] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:11:0)
[23:11:01.021] Build status: SCHEDULED
[23:11:01.021] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:12:0) after 5m2s
[23:16:03.042] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:12:0)
[23:16:03.104] Build status: SCHEDULED
[23:16:03.104] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:13:0) after 7m43s
[23:23:46.118] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:13:0)
[23:23:46.140] Build status: SCHEDULED
[23:23:46.141] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:14:0) after 7m2s
[23:30:48.163] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:14:0)
[23:30:48.218] Build status: SCHEDULED
[23:30:48.218] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:15:0) after 2m16s
[23:33:04.368] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:15:0)
[23:33:04.391] Build status: SCHEDULED
[23:33:04.391] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:16:0) after 7m33s
[23:40:37.417] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:16:0)
[23:40:37.446] Build status: SCHEDULED
[23:40:37.446] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:17:0) after 6m1s
[23:46:38.466] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:17:0)
[23:46:38.487] Build status: SCHEDULED
[23:46:38.487] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:18:0) after 9m43s
[23:56:21.507] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:18:0)
[23:56:21.528] Build status: SCHEDULED
[23:56:21.528] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:19:0) after 5m25s
[00:01:46.510] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:19:0)
[00:01:46.561] Build status: SCHEDULED
[00:01:46.561] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:20:0) after 1m50s
[00:03:36.770] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:20:0)
[00:03:36.882] Build status: SCHEDULED
[00:03:36.882] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:21:0) after 7m17s
[00:10:53.858] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:21:0)
[00:10:53.948] Build status: SCHEDULED
[00:10:53.948] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:22:0) after 5m24s
[00:16:17.968] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:22:0)
[00:16:17.985] Build status: SCHEDULED
[00:16:17.985] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:23:0) after 5m11s
[00:21:29.045] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:23:0)
[00:21:29.059] Build status: SCHEDULED
[00:21:29.059] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:24:0) after 2m36s
[00:24:05.139] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:24:0)
[00:24:05.157] Build status: SCHEDULED
[00:24:05.157] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:25:0) after 4m43s
[00:28:48.155] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:25:0)
[00:28:48.177] Build status: SCHEDULED
[00:28:48.177] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:26:0) after 2m46s
[00:31:34.228] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:26:0)
[00:31:34.415] Build status: SCHEDULED
[00:31:34.415] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:27:0) after 7m21s
[00:38:55.447] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:27:0)
[00:38:55.508] Build status: SCHEDULED
[00:38:55.508] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:28:0) after 7m44s
[00:46:39.532] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:28:0)
[00:46:39.555] Build status: SCHEDULED
[00:46:39.555] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:29:0) after 8m18s
[00:54:57.661] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:29:0)
[00:54:57.681] Build status: SCHEDULED
[00:54:57.681] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:30:0) after 7m6s
[01:02:03.701] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:30:0)
[01:02:03.728] Build status: SCHEDULED
[01:02:03.728] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:31:0) after 8m44s
[01:10:47.709] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:31:0)
[01:10:47.728] Build status: SCHEDULED
[01:10:47.728] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:32:0) after 8m48s
[01:19:35.749] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:32:0)
[01:19:35.771] Build status: SCHEDULED
[01:19:35.771] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:33:0) after 8m7s
[01:27:42.845] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:33:0)
[01:27:42.880] Build status: SCHEDULED
[01:27:42.880] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:34:0) after 3m3s
[01:30:45.902] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:34:0)
[01:30:45.919] Build status: SCHEDULED
[01:30:45.919] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:35:0) after 9m35s
[01:40:20.982] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:35:0)
[01:40:21.026] Build status: SCHEDULED
[01:40:21.027] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:36:0) after 1m16s
[01:41:37.045] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:36:0)
[01:41:37.063] Build status: SCHEDULED
[01:41:37.063] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:37:0) after 2m28s
[01:44:05.213] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:37:0)
[01:44:05.468] Build status: SCHEDULED
[01:44:05.468] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:38:0) after 9m52s
[01:53:57.589] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:38:0)
[01:53:57.607] Build status: SCHEDULED
[01:53:57.607] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:39:0) after 1m20s
[01:55:17.628] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:39:0)
[01:55:17.649] Build status: SCHEDULED
[01:55:17.649] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:40:0) after 4m17s
[01:59:34.670] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:40:0)
[01:59:34.699] Build status: SCHEDULED
[01:59:34.699] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:41:0) after 3m0s
[02:02:34.721] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:41:0)
[02:02:34.810] Build status: SCHEDULED
[02:02:34.810] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:42:0) after 4m17s
[02:06:51.874] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:42:0)
[02:06:51.915] Build status: SCHEDULED
[02:06:51.915] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:43:0) after 6m35s
[02:13:27.038] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:43:0)
[02:13:27.087] Build status: SCHEDULED
[02:13:27.088] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:44:0) after 1m7s
[02:14:34.135] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:44:0)
[02:14:34.159] Build status: SCHEDULED
[02:14:34.159] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:45:0) after 1m20s
[02:15:54.175] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:45:0)
[02:15:54.198] Build status: SCHEDULED
[02:15:54.198] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:46:0) after 9m19s
[02:25:13.356] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:46:0)
[02:25:13.375] Build status: SCHEDULED
[02:25:13.375] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:47:0) after 5m49s
[02:31:02.397] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:47:0)
[02:31:02.590] Build status: SCHEDULED
[02:31:02.590] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:48:0) after 8m49s
[02:39:51.655] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:48:0)
[02:39:51.670] Build status: SCHEDULED
[02:39:51.671] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:49:0) after 2m24s
[02:42:15.761] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:49:0)
[02:42:15.780] Build status: SCHEDULED
[02:42:15.780] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:50:0) after 1m51s
[02:44:06.786] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:50:0)
[02:44:06.821] Build status: SCHEDULED
[02:44:06.821] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:51:0) after 8m37s
[02:52:43.832] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:51:0)
[02:52:43.868] Build status: SCHEDULED
[02:52:43.868] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:52:0) after 5m22s
[02:58:05.890] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:52:0)
[02:58:05.911] Build status: SCHEDULED
[02:58:05.911] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:53:0) after 4m26s
[03:02:32.014] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:53:0)
[03:02:32.029] Build status: SCHEDULED
[03:02:32.029] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:54:0) after 8m11s
[03:10:43.053] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:54:0)
[03:10:43.072] Build status: SCHEDULED
[03:10:43.072] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:55:0) after 7m30s
[03:18:13.126] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:55:0)
[03:18:13.141] Build status: SCHEDULED
[03:18:13.141] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:56:0) after 7m9s
[03:25:22.207] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:56:0)
[03:25:22.235] Build status: SCHEDULED
[03:25:22.235] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:57:0) after 6m54s
[03:32:16.262] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:57:0)
[03:32:16.307] Build status: SCHEDULED
[03:32:16.307] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:58:0) after 8m45s
[03:41:01.334] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:58:0)
[03:41:01.457] Build status: SCHEDULED
[03:41:01.457] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:59:0) after 3m53s
[03:44:54.478] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:59:0)
[03:44:54.498] Build status: SCHEDULED
[03:44:54.498] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:60:0) after 6m23s
[03:51:17.542] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:60:0)
[03:51:17.622] Build status: SCHEDULED
[03:51:17.622] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:61:0) after 7m3s
[03:58:20.634] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:61:0)
[03:58:20.658] Build status: SCHEDULED
[03:58:20.658] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:62:0) after 8m11s
[04:06:31.681] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:62:0)
[04:06:31.703] Build status: SCHEDULED
[04:06:31.703] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:63:0) after 5m26s
[04:11:57.722] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:63:0)
[04:11:57.741] Build status: SCHEDULED
[04:11:57.741] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:64:0) after 6m16s
[04:18:13.873] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:64:0)
[04:18:13.927] Build status: SCHEDULED
[04:18:13.927] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:65:0) after 5m6s
[04:23:19.942] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:65:0)
[04:23:19.959] Build status: SCHEDULED
[04:23:19.959] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:66:0) after 3m26s
[04:26:45.983] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:66:0)
[04:26:45.998] Build status: SCHEDULED
[04:26:45.998] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:67:0) after 2m38s
[04:29:24.022] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:67:0)
[04:29:24.057] Build status: SCHEDULED
[04:29:24.057] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:68:0) after 9m25s
[04:38:49.097] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:68:0)
[04:38:49.118] Build status: SCHEDULED
[04:38:49.118] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:69:0) after 8m32s
[04:47:21.160] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:69:0)
[04:47:21.241] Build status: SCHEDULED
[04:47:21.241] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:70:0) after 5m36s
[04:52:57.285] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:70:0)
[04:52:57.300] Build status: SCHEDULED
[04:52:57.300] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:71:0) after 4m59s
[04:57:56.324] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:71:0)
[04:57:56.340] Build status: SCHEDULED
[04:57:56.340] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:72:0) after 3m59s
[05:01:55.366] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:72:0)
[05:01:55.405] Build status: SCHEDULED
[05:01:55.405] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:73:0) after 3m39s
[05:05:34.371] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:73:0)
[05:05:34.442] Build status: SCHEDULED
[05:05:34.442] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:74:0) after 3m7s
[05:08:41.463] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:74:0)
[05:08:41.493] Build status: SCHEDULED
[05:08:41.493] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:75:0) after 6m43s
[05:15:24.549] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:75:0)
[05:15:24.568] Build status: SCHEDULED
[05:15:24.568] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:76:0) after 3m28s
[05:18:52.600] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:76:0)
[05:18:52.619] Build status: SCHEDULED
[05:18:52.619] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:77:0) after 2m9s
[05:21:01.649] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:77:0)
[05:21:01.680] Build status: SCHEDULED
[05:21:01.680] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:78:0) after 3m57s
[05:24:58.700] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:78:0)
[05:24:58.756] Build status: SCHEDULED
[05:24:58.756] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:79:0) after 8m43s
[05:33:41.825] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:79:0)
[05:33:42.033] Build status: SCHEDULED
[05:33:42.033] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:80:0) after 5m26s
[05:39:08.099] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:80:0)
[05:39:08.115] Build status: SCHEDULED
[05:39:08.115] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:81:0) after 6m31s
[05:45:39.136] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:81:0)
[05:45:39.195] Build status: SCHEDULED
[05:45:39.195] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:82:0) after 8m53s
[05:54:32.222] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:82:0)
[05:54:32.244] Build status: SCHEDULED
[05:54:32.244] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:83:0) after 9m55s
[06:04:27.239] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:83:0)
[06:04:27.264] Build status: SCHEDULED
[06:04:27.264] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:84:0) after 5m26s
[06:09:53.299] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:84:0)
[06:09:53.315] Build status: SCHEDULED
[06:09:53.315] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:85:0) after 4m23s
[06:14:16.388] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:85:0)
[06:14:16.457] Build status: SCHEDULED
[06:14:16.457] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:86:0) after 5m51s
[06:20:07.474] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:86:0)
[06:20:07.504] Build status: SCHEDULED
[06:20:07.504] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:87:0) after 7m19s
[06:27:26.522] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:87:0)
[06:27:26.549] Build status: SCHEDULED
[06:27:26.549] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8875123020367320768:88:0) after 3m27s
[06:29:11.124] Received PubSub notification, asking Buildbucket for the build status
[06:29:11.141] Build:
{
"id": "8709692958469524561",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-ubsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-07-10T22:29:03.931983290Z",
"endTime": "2025-07-11T06:29:10.847853086Z",
"updateTime": "2025-07-11T06:29:10.847853086Z",
"status": "INFRA_FAILURE",
"statusDetails": {
"resourceExhaustion": {
},
"timeout": {
}
},
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "c91fc7c107f021fb967af806fcedc5abc335c2b0",
"ref": "refs/heads/main"
}
}
}
[06:29:11.141] Invocation finished in 8h0m8.404814682s with status FAILED