[22:23:16.072] New invocation is queued and will start shortly
[22:23:17.330] Starting the invocation (attempt 1)
[22:23:17.359] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/b66729b90fcb9df2ee4818f6d4fff59385cdbc80"
[22:23:17.359] Popped gitiles commit info from properties and tags
[22:23:17.359] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[22:23:17.359] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[22:23:17.359] Buildbucket request:
{
"requestId": "8876572935348770864",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-tsan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8876572935348770864",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "434cf8e53f10a7cd36ec92ac7a8eb736f6600b48"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@434cf8e53f10a7cd36ec92ac7a8eb736f6600b48",
"title": "434cf8e53f10a7cd36ec92ac7a8eb736f6600b48",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/434cf8e53f10a7cd36ec92ac7a8eb736f6600b48"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "6ad38d7720ef2e75c02dd5365a0c5c13b919686f"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@6ad38d7720ef2e75c02dd5365a0c5c13b919686f",
"title": "6ad38d7720ef2e75c02dd5365a0c5c13b919686f",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/6ad38d7720ef2e75c02dd5365a0c5c13b919686f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "caf4417cf0b6e60983131444bd21f14104df4125"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@caf4417cf0b6e60983131444bd21f14104df4125",
"title": "caf4417cf0b6e60983131444bd21f14104df4125",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/caf4417cf0b6e60983131444bd21f14104df4125"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "9b084b679269663e45421bad4da08fdc687df2d9"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@9b084b679269663e45421bad4da08fdc687df2d9",
"title": "9b084b679269663e45421bad4da08fdc687df2d9",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/9b084b679269663e45421bad4da08fdc687df2d9"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "b66729b90fcb9df2ee4818f6d4fff59385cdbc80"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@b66729b90fcb9df2ee4818f6d4fff59385cdbc80",
"title": "b66729b90fcb9df2ee4818f6d4fff59385cdbc80",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/b66729b90fcb9df2ee4818f6d4fff59385cdbc80"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "b66729b90fcb9df2ee4818f6d4fff59385cdbc80",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8876572935348770864"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[22:23:17.758] Scheduled build:
{
"id": "8711142873268902945",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-06-24T22:23:17.426661530Z",
"updateTime": "2025-06-24T22:23:17.426661530Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "b66729b90fcb9df2ee4818f6d4fff59385cdbc80",
"ref": "refs/heads/main"
}
}
}
[22:23:17.758] Task URL: https://cr-buildbucket.appspot.com/build/8711142873268902945
[22:23:17.758] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:2:0) after 3m0s
[22:23:18.710] Received PubSub notification, asking Buildbucket for the build status
[22:23:18.735] Build status: SCHEDULED
[22:26:17.751] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:2:0)
[22:26:17.775] Build status: SCHEDULED
[22:26:17.776] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:4:0) after 1m28s
[22:27:45.811] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:4:0)
[22:27:45.837] Build status: SCHEDULED
[22:27:45.837] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:5:0) after 9m16s
[22:37:01.880] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:5:0)
[22:37:01.909] Build status: SCHEDULED
[22:37:01.909] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:6:0) after 5m39s
[22:42:40.999] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:6:0)
[22:42:41.060] Build status: SCHEDULED
[22:42:41.060] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:7:0) after 4m40s
[22:47:21.078] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:7:0)
[22:47:21.108] Build status: SCHEDULED
[22:47:21.108] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:8:0) after 6m31s
[22:53:52.265] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:8:0)
[22:53:52.299] Build status: SCHEDULED
[22:53:52.299] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:9:0) after 6m48s
[23:00:40.345] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:9:0)
[23:00:40.362] Build status: SCHEDULED
[23:00:40.362] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:10:0) after 9m28s
[23:10:08.458] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:10:0)
[23:10:08.508] Build status: SCHEDULED
[23:10:08.508] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:11:0) after 6m20s
[23:16:28.637] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:11:0)
[23:16:28.688] Build status: SCHEDULED
[23:16:28.689] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:12:0) after 3m38s
[23:20:06.711] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:12:0)
[23:20:06.733] Build status: SCHEDULED
[23:20:06.734] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:13:0) after 6m17s
[23:26:23.811] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:13:0)
[23:26:23.826] Build status: SCHEDULED
[23:26:23.826] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:14:0) after 5m14s
[23:31:37.844] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:14:0)
[23:31:37.867] Build status: SCHEDULED
[23:31:37.867] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:15:0) after 6m58s
[23:38:35.892] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:15:0)
[23:38:35.915] Build status: SCHEDULED
[23:38:35.915] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:16:0) after 6m24s
[23:44:59.987] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:16:0)
[23:45:00.029] Build status: SCHEDULED
[23:45:00.029] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:17:0) after 9m47s
[23:54:47.093] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:17:0)
[23:54:47.110] Build status: SCHEDULED
[23:54:47.110] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:18:0) after 7m5s
[00:01:52.132] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:18:0)
[00:01:52.193] Build status: SCHEDULED
[00:01:52.193] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:19:0) after 8m23s
[00:10:15.242] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:19:0)
[00:10:15.261] Build status: SCHEDULED
[00:10:15.261] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:20:0) after 6m40s
[00:16:55.286] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:20:0)
[00:16:55.314] Build status: SCHEDULED
[00:16:55.314] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:21:0) after 3m24s
[00:20:19.406] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:21:0)
[00:20:19.422] Build status: SCHEDULED
[00:20:19.422] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:22:0) after 3m33s
[00:23:52.523] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:22:0)
[00:23:52.588] Build status: SCHEDULED
[00:23:52.588] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:23:0) after 3m28s
[00:27:20.637] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:23:0)
[00:27:20.656] Build status: SCHEDULED
[00:27:20.656] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:24:0) after 3m45s
[00:31:05.779] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:24:0)
[00:31:05.794] Build status: SCHEDULED
[00:31:05.794] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:25:0) after 3m7s
[00:34:12.898] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:25:0)
[00:34:13.048] Build status: SCHEDULED
[00:34:13.048] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:26:0) after 7m43s
[00:41:56.073] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:26:0)
[00:41:56.100] Build status: SCHEDULED
[00:41:56.100] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:27:0) after 2m8s
[00:44:04.120] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:27:0)
[00:44:04.161] Build status: SCHEDULED
[00:44:04.161] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:28:0) after 6m30s
[00:50:34.242] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:28:0)
[00:50:34.259] Build status: SCHEDULED
[00:50:34.259] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:29:0) after 1m44s
[00:52:18.279] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:29:0)
[00:52:18.315] Build status: SCHEDULED
[00:52:18.315] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:30:0) after 6m7s
[00:58:25.398] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:30:0)
[00:58:25.419] Build status: SCHEDULED
[00:58:25.419] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:31:0) after 5m19s
[01:03:44.478] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:31:0)
[01:03:44.495] Build status: SCHEDULED
[01:03:44.495] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:32:0) after 2m56s
[01:06:40.517] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:32:0)
[01:06:40.547] Build status: SCHEDULED
[01:06:40.547] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:33:0) after 9m24s
[01:16:04.690] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:33:0)
[01:16:04.778] Build status: SCHEDULED
[01:16:04.778] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:34:0) after 3m12s
[01:19:16.790] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:34:0)
[01:19:16.812] Build status: SCHEDULED
[01:19:16.812] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:35:0) after 7m15s
[01:26:31.861] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:35:0)
[01:26:31.922] Build status: SCHEDULED
[01:26:31.922] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:36:0) after 3m1s
[01:29:32.948] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:36:0)
[01:29:33.026] Build status: SCHEDULED
[01:29:33.026] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:37:0) after 8m56s
[01:38:29.043] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:37:0)
[01:38:29.060] Build status: SCHEDULED
[01:38:29.060] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:38:0) after 3m18s
[01:41:47.135] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:38:0)
[01:41:47.159] Build status: SCHEDULED
[01:41:47.159] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:39:0) after 3m8s
[01:44:55.194] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:39:0)
[01:44:55.218] Build status: SCHEDULED
[01:44:55.218] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:40:0) after 1m57s
[01:46:52.233] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:40:0)
[01:46:52.253] Build status: SCHEDULED
[01:46:52.254] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:41:0) after 2m21s
[01:49:13.285] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:41:0)
[01:49:13.394] Build status: SCHEDULED
[01:49:13.394] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:42:0) after 5m20s
[01:54:33.421] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:42:0)
[01:54:33.446] Build status: SCHEDULED
[01:54:33.446] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:43:0) after 7m26s
[02:01:59.470] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:43:0)
[02:01:59.520] Build status: SCHEDULED
[02:01:59.520] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:44:0) after 6m52s
[02:08:51.600] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:44:0)
[02:08:51.635] Build status: SCHEDULED
[02:08:51.635] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:45:0) after 9m57s
[02:18:48.713] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:45:0)
[02:18:48.729] Build status: SCHEDULED
[02:18:48.730] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:46:0) after 2m30s
[02:21:18.716] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:46:0)
[02:21:18.732] Build status: SCHEDULED
[02:21:18.732] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:47:0) after 8m41s
[02:29:59.751] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:47:0)
[02:29:59.767] Build status: SCHEDULED
[02:29:59.767] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:48:0) after 3m2s
[02:33:01.834] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:48:0)
[02:33:01.854] Build status: SCHEDULED
[02:33:01.854] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:49:0) after 6m2s
[02:39:03.878] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:49:0)
[02:39:03.903] Build status: SCHEDULED
[02:39:03.903] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:50:0) after 5m30s
[02:44:33.925] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:50:0)
[02:44:33.954] Build status: SCHEDULED
[02:44:33.954] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:51:0) after 1m58s
[02:46:32.024] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:51:0)
[02:46:32.055] Build status: SCHEDULED
[02:46:32.055] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:52:0) after 1m17s
[02:47:49.115] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:52:0)
[02:47:49.133] Build status: SCHEDULED
[02:47:49.133] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:53:0) after 3m46s
[02:51:35.158] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:53:0)
[02:51:35.188] Build status: SCHEDULED
[02:51:35.188] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:54:0) after 2m29s
[02:54:04.215] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:54:0)
[02:54:04.239] Build status: SCHEDULED
[02:54:04.239] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:55:0) after 3m55s
[02:57:59.292] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:55:0)
[02:57:59.363] Build status: SCHEDULED
[02:57:59.363] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:56:0) after 8m58s
[03:06:57.382] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:56:0)
[03:06:57.414] Build status: SCHEDULED
[03:06:57.414] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:57:0) after 5m51s
[03:12:48.541] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:57:0)
[03:12:48.623] Build status: SCHEDULED
[03:12:48.623] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:58:0) after 5m5s
[03:17:53.649] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:58:0)
[03:17:53.670] Build status: SCHEDULED
[03:17:53.670] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:59:0) after 9m50s
[03:27:43.741] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:59:0)
[03:27:43.758] Build status: SCHEDULED
[03:27:43.758] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:60:0) after 9m53s
[03:37:36.957] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:60:0)
[03:37:36.979] Build status: SCHEDULED
[03:37:36.979] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:61:0) after 4m20s
[03:41:56.998] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:61:0)
[03:41:57.024] Build status: SCHEDULED
[03:41:57.024] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:62:0) after 8m56s
[03:50:53.097] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:62:0)
[03:50:53.111] Build status: SCHEDULED
[03:50:53.111] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:63:0) after 5m19s
[03:56:12.134] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:63:0)
[03:56:12.155] Build status: SCHEDULED
[03:56:12.155] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:64:0) after 3m15s
[03:59:27.559] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:64:0)
[03:59:27.596] Build status: SCHEDULED
[03:59:27.596] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:65:0) after 5m30s
[04:04:57.656] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:65:0)
[04:04:57.673] Build status: SCHEDULED
[04:04:57.674] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:66:0) after 5m37s
[04:10:34.795] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:66:0)
[04:10:34.810] Build status: SCHEDULED
[04:10:34.810] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:67:0) after 7m16s
[04:17:50.834] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:67:0)
[04:17:50.860] Build status: SCHEDULED
[04:17:50.860] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:68:0) after 4m33s
[04:22:24.057] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:68:0)
[04:22:24.080] Build status: SCHEDULED
[04:22:24.080] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:69:0) after 8m21s
[04:30:45.142] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:69:0)
[04:30:45.235] Build status: SCHEDULED
[04:30:45.235] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:70:0) after 1m46s
[04:32:31.512] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:70:0)
[04:32:31.534] Build status: SCHEDULED
[04:32:31.534] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:71:0) after 6m12s
[04:38:43.585] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:71:0)
[04:38:43.604] Build status: SCHEDULED
[04:38:43.604] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:72:0) after 9m59s
[04:48:42.631] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:72:0)
[04:48:42.648] Build status: SCHEDULED
[04:48:42.648] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:73:0) after 6m8s
[04:54:50.709] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:73:0)
[04:54:50.726] Build status: SCHEDULED
[04:54:50.726] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:74:0) after 7m47s
[05:02:37.799] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:74:0)
[05:02:37.874] Build status: SCHEDULED
[05:02:37.874] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:75:0) after 7m37s
[05:10:15.122] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:75:0)
[05:10:15.141] Build status: SCHEDULED
[05:10:15.141] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:76:0) after 1m15s
[05:11:30.891] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:76:0)
[05:11:30.929] Build status: SCHEDULED
[05:11:30.929] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:77:0) after 8m20s
[05:19:50.970] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:77:0)
[05:19:50.989] Build status: SCHEDULED
[05:19:50.989] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:78:0) after 8m58s
[05:28:49.009] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:78:0)
[05:28:49.029] Build status: SCHEDULED
[05:28:49.029] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:79:0) after 2m51s
[05:31:40.190] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:79:0)
[05:31:40.206] Build status: SCHEDULED
[05:31:40.206] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:80:0) after 6m14s
[05:37:54.231] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:80:0)
[05:37:54.266] Build status: SCHEDULED
[05:37:54.266] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:81:0) after 2m22s
[05:40:16.631] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:81:0)
[05:40:16.659] Build status: SCHEDULED
[05:40:16.659] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:82:0) after 4m2s
[05:44:19.071] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:82:0)
[05:44:19.090] Build status: SCHEDULED
[05:44:19.090] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:83:0) after 1m59s
[05:46:18.137] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:83:0)
[05:46:18.190] Build status: SCHEDULED
[05:46:18.190] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:84:0) after 2m7s
[05:48:25.376] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:84:0)
[05:48:25.395] Build status: SCHEDULED
[05:48:25.395] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:85:0) after 2m53s
[05:51:18.758] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:85:0)
[05:51:18.774] Build status: SCHEDULED
[05:51:18.774] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:86:0) after 2m23s
[05:53:41.801] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:86:0)
[05:53:41.819] Build status: SCHEDULED
[05:53:41.819] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:87:0) after 1m29s
[05:55:11.049] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:87:0)
[05:55:11.081] Build status: SCHEDULED
[05:55:11.081] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:88:0) after 8m8s
[06:03:19.120] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:88:0)
[06:03:19.141] Build status: SCHEDULED
[06:03:19.141] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:89:0) after 4m10s
[06:07:29.170] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:89:0)
[06:07:29.192] Build status: SCHEDULED
[06:07:29.192] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:90:0) after 9m5s
[06:16:34.221] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:90:0)
[06:16:34.252] Build status: SCHEDULED
[06:16:34.252] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8876572935348770864:91:0) after 7m49s
[06:23:20.383] Received PubSub notification, asking Buildbucket for the build status
[06:23:20.406] Build:
{
"id": "8711142873268902945",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-06-24T22:23:17.426661530Z",
"endTime": "2025-06-25T06:23:19.263285632Z",
"updateTime": "2025-06-25T06:23:19.263285632Z",
"status": "INFRA_FAILURE",
"statusDetails": {
"resourceExhaustion": {
},
"timeout": {
}
},
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "b66729b90fcb9df2ee4818f6d4fff59385cdbc80",
"ref": "refs/heads/main"
}
}
}
[06:23:20.407] Invocation finished in 8h0m4.347413967s with status FAILED