[18:31:53.094] New invocation is queued and will start shortly
[18:31:54.178] Starting the invocation (attempt 1)
[18:31:54.205] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/c8d7e4b3213334c06c63b8b53aa9b3ecee2ff206"
[18:31:54.205] Popped gitiles commit info from properties and tags
[18:31:54.205] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[18:31:54.205] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[18:31:54.205] Buildbucket request:
{
"requestId": "8875319135169376544",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-tsan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8875319135169376544",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "86a6be4ca0dc91c01a042183d2fed185de04619c"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@86a6be4ca0dc91c01a042183d2fed185de04619c",
"title": "86a6be4ca0dc91c01a042183d2fed185de04619c",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/86a6be4ca0dc91c01a042183d2fed185de04619c"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "10a055aad301f46cbe5fbe8f79e56d6a4ff71f4d"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@10a055aad301f46cbe5fbe8f79e56d6a4ff71f4d",
"title": "10a055aad301f46cbe5fbe8f79e56d6a4ff71f4d",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/10a055aad301f46cbe5fbe8f79e56d6a4ff71f4d"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "57d2afc82e321e6486ac98d279bf566b0a9f9952"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@57d2afc82e321e6486ac98d279bf566b0a9f9952",
"title": "57d2afc82e321e6486ac98d279bf566b0a9f9952",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/57d2afc82e321e6486ac98d279bf566b0a9f9952"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "3b7865b8bb9f5caf71fc79b17159793fe8d15624"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@3b7865b8bb9f5caf71fc79b17159793fe8d15624",
"title": "3b7865b8bb9f5caf71fc79b17159793fe8d15624",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/3b7865b8bb9f5caf71fc79b17159793fe8d15624"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "60e10377d598a9d65b46b9e41ad70d24a510c100"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@60e10377d598a9d65b46b9e41ad70d24a510c100",
"title": "60e10377d598a9d65b46b9e41ad70d24a510c100",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/60e10377d598a9d65b46b9e41ad70d24a510c100"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "c2055fc0124ccfa9d6aaaf5841d30aebc7f8fb3f"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@c2055fc0124ccfa9d6aaaf5841d30aebc7f8fb3f",
"title": "c2055fc0124ccfa9d6aaaf5841d30aebc7f8fb3f",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/c2055fc0124ccfa9d6aaaf5841d30aebc7f8fb3f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "c8d7e4b3213334c06c63b8b53aa9b3ecee2ff206"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@c8d7e4b3213334c06c63b8b53aa9b3ecee2ff206",
"title": "c8d7e4b3213334c06c63b8b53aa9b3ecee2ff206",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/c8d7e4b3213334c06c63b8b53aa9b3ecee2ff206"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "c8d7e4b3213334c06c63b8b53aa9b3ecee2ff206",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8875319135169376544"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[18:31:54.677] Scheduled build:
{
"id": "8709889073267292785",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-07-08T18:31:54.285914386Z",
"updateTime": "2025-07-08T18:31:54.285914386Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "c8d7e4b3213334c06c63b8b53aa9b3ecee2ff206",
"ref": "refs/heads/main"
}
},
"canary": true
}
[18:31:54.677] Task URL: https://cr-buildbucket.appspot.com/build/8709889073267292785
[18:31:54.677] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:2:0) after 6m58s
[18:31:55.661] Received PubSub notification, asking Buildbucket for the build status
[18:31:55.694] Build status: SCHEDULED
[18:38:52.932] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:2:0)
[18:38:52.965] Build status: SCHEDULED
[18:38:52.965] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:4:0) after 5m26s
[18:44:19.100] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:4:0)
[18:44:19.136] Build status: SCHEDULED
[18:44:19.136] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:5:0) after 3m17s
[18:47:36.244] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:5:0)
[18:47:36.268] Build status: SCHEDULED
[18:47:36.268] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:6:0) after 2m13s
[18:49:49.280] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:6:0)
[18:49:49.305] Build status: SCHEDULED
[18:49:49.305] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:7:0) after 6m55s
[18:56:44.512] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:7:0)
[18:56:44.528] Build status: SCHEDULED
[18:56:44.528] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:8:0) after 8m41s
[19:05:25.800] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:8:0)
[19:05:25.822] Build status: SCHEDULED
[19:05:25.822] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:9:0) after 3m27s
[19:08:52.921] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:9:0)
[19:08:52.942] Build status: SCHEDULED
[19:08:52.942] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:10:0) after 6m31s
[19:15:24.048] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:10:0)
[19:15:24.115] Build status: SCHEDULED
[19:15:24.115] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:11:0) after 2m38s
[19:18:02.282] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:11:0)
[19:18:02.347] Build status: SCHEDULED
[19:18:02.347] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:12:0) after 8m31s
[19:26:33.370] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:12:0)
[19:26:33.395] Build status: SCHEDULED
[19:26:33.395] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:13:0) after 6m49s
[19:33:22.521] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:13:0)
[19:33:22.547] Build status: SCHEDULED
[19:33:22.547] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:14:0) after 8m45s
[19:42:07.520] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:14:0)
[19:42:07.576] Build status: SCHEDULED
[19:42:07.576] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:15:0) after 2m36s
[19:44:43.557] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:15:0)
[19:44:43.632] Build status: SCHEDULED
[19:44:43.632] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:16:0) after 8m2s
[19:52:45.758] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:16:0)
[19:52:45.779] Build status: SCHEDULED
[19:52:45.779] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:17:0) after 5m41s
[19:58:26.796] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:17:0)
[19:58:26.817] Build status: SCHEDULED
[19:58:26.817] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:18:0) after 2m36s
[20:01:02.810] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:18:0)
[20:01:02.880] Build status: SCHEDULED
[20:01:02.880] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:19:0) after 1m51s
[20:02:53.899] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:19:0)
[20:02:53.923] Build status: SCHEDULED
[20:02:53.923] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:20:0) after 5m8s
[20:08:01.968] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:20:0)
[20:08:02.005] Build status: SCHEDULED
[20:08:02.005] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:21:0) after 8m28s
[20:16:30.087] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:21:0)
[20:16:30.122] Build status: SCHEDULED
[20:16:30.122] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:22:0) after 2m1s
[20:18:31.401] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:22:0)
[20:18:31.430] Build status: SCHEDULED
[20:18:31.430] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:23:0) after 6m14s
[20:24:45.502] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:23:0)
[20:24:45.526] Build status: SCHEDULED
[20:24:45.526] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:24:0) after 4m22s
[20:29:07.631] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:24:0)
[20:29:07.652] Build status: SCHEDULED
[20:29:07.652] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:25:0) after 1m24s
[20:30:31.672] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:25:0)
[20:30:31.726] Build status: SCHEDULED
[20:30:31.726] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:26:0) after 5m39s
[20:36:10.833] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:26:0)
[20:36:11.016] Build status: SCHEDULED
[20:36:11.017] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:27:0) after 2m16s
[20:38:27.050] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:27:0)
[20:38:27.069] Build status: SCHEDULED
[20:38:27.069] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:28:0) after 4m33s
[20:43:00.092] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:28:0)
[20:43:00.125] Build status: SCHEDULED
[20:43:00.125] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:29:0) after 3m23s
[20:46:23.149] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:29:0)
[20:46:23.171] Build status: SCHEDULED
[20:46:23.171] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:30:0) after 2m7s
[20:48:30.317] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:30:0)
[20:48:30.358] Build status: SCHEDULED
[20:48:30.358] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:31:0) after 3m58s
[20:52:28.430] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:31:0)
[20:52:28.464] Build status: SCHEDULED
[20:52:28.464] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:32:0) after 2m11s
[20:54:39.538] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:32:0)
[20:54:39.634] Build status: SCHEDULED
[20:54:39.634] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:33:0) after 7m32s
[21:02:11.715] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:33:0)
[21:02:11.734] Build status: SCHEDULED
[21:02:11.734] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:34:0) after 8m35s
[21:10:46.893] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:34:0)
[21:10:46.909] Build status: SCHEDULED
[21:10:46.909] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:35:0) after 1m7s
[21:11:53.918] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:35:0)
[21:11:53.971] Build status: SCHEDULED
[21:11:53.971] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:36:0) after 7m38s
[21:19:32.096] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:36:0)
[21:19:32.117] Build status: SCHEDULED
[21:19:32.117] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:37:0) after 3m15s
[21:22:47.195] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:37:0)
[21:22:47.220] Build status: SCHEDULED
[21:22:47.220] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:38:0) after 9m12s
[21:31:59.268] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:38:0)
[21:31:59.347] Build status: SCHEDULED
[21:31:59.347] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:39:0) after 6m4s
[21:38:03.416] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:39:0)
[21:38:03.439] Build status: SCHEDULED
[21:38:03.439] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:40:0) after 6m16s
[21:44:19.464] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:40:0)
[21:44:19.548] Build status: SCHEDULED
[21:44:19.548] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:41:0) after 2m55s
[21:47:14.657] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:41:0)
[21:47:14.697] Build status: SCHEDULED
[21:47:14.697] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:42:0) after 7m17s
[21:54:31.742] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:42:0)
[21:54:31.761] Build status: SCHEDULED
[21:54:31.761] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:43:0) after 7m57s
[22:02:28.819] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:43:0)
[22:02:28.845] Build status: SCHEDULED
[22:02:28.845] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:44:0) after 1m8s
[22:03:36.878] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:44:0)
[22:03:36.903] Build status: SCHEDULED
[22:03:36.903] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:45:0) after 9m9s
[22:12:45.974] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:45:0)
[22:12:46.128] Build status: SCHEDULED
[22:12:46.128] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:46:0) after 2m9s
[22:14:55.152] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:46:0)
[22:14:55.176] Build status: SCHEDULED
[22:14:55.176] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:47:0) after 1m38s
[22:16:33.147] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:47:0)
[22:16:33.192] Build status: SCHEDULED
[22:16:33.192] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:48:0) after 8m36s
[22:25:09.219] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:48:0)
[22:25:09.241] Build status: SCHEDULED
[22:25:09.241] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:49:0) after 8m32s
[22:33:41.269] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:49:0)
[22:33:41.301] Build status: SCHEDULED
[22:33:41.301] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:50:0) after 3m24s
[22:37:05.330] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:50:0)
[22:37:05.351] Build status: SCHEDULED
[22:37:05.351] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:51:0) after 7m11s
[22:44:16.471] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:51:0)
[22:44:16.498] Build status: SCHEDULED
[22:44:16.498] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:52:0) after 9m44s
[22:54:00.501] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:52:0)
[22:54:00.519] Build status: SCHEDULED
[22:54:00.519] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:53:0) after 9m36s
[23:03:36.590] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:53:0)
[23:03:36.614] Build status: SCHEDULED
[23:03:36.614] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:54:0) after 2m8s
[23:05:44.640] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:54:0)
[23:05:44.688] Build status: SCHEDULED
[23:05:44.688] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:55:0) after 9m19s
[23:15:03.737] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:55:0)
[23:15:03.755] Build status: SCHEDULED
[23:15:03.755] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:56:0) after 6m31s
[23:21:34.776] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:56:0)
[23:21:34.910] Build status: SCHEDULED
[23:21:34.910] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:57:0) after 7m57s
[23:29:32.119] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:57:0)
[23:29:32.191] Build status: SCHEDULED
[23:29:32.191] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:58:0) after 4m12s
[23:33:44.401] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:58:0)
[23:33:44.423] Build status: SCHEDULED
[23:33:44.423] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:59:0) after 8m50s
[23:42:34.550] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:59:0)
[23:42:34.574] Build status: SCHEDULED
[23:42:34.574] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:60:0) after 3m1s
[23:45:35.650] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:60:0)
[23:45:35.670] Build status: SCHEDULED
[23:45:35.670] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:61:0) after 4m35s
[23:50:10.694] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:61:0)
[23:50:10.727] Build status: SCHEDULED
[23:50:10.727] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:62:0) after 5m24s
[23:55:34.752] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:62:0)
[23:55:34.777] Build status: SCHEDULED
[23:55:34.777] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:63:0) after 2m21s
[23:57:55.863] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:63:0)
[23:57:55.880] Build status: SCHEDULED
[23:57:55.880] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:64:0) after 5m3s
[00:02:58.895] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:64:0)
[00:02:58.914] Build status: SCHEDULED
[00:02:58.914] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:65:0) after 1m14s
[00:04:12.929] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:65:0)
[00:04:12.946] Build status: SCHEDULED
[00:04:12.946] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:66:0) after 5m59s
[00:10:11.971] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:66:0)
[00:10:11.993] Build status: SCHEDULED
[00:10:11.993] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:67:0) after 5m4s
[00:15:16.150] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:67:0)
[00:15:16.174] Build status: SCHEDULED
[00:15:16.174] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:68:0) after 5m51s
[00:21:07.219] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:68:0)
[00:21:07.250] Build status: SCHEDULED
[00:21:07.250] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:69:0) after 7m54s
[00:29:01.257] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:69:0)
[00:29:01.274] Build status: SCHEDULED
[00:29:01.274] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:70:0) after 1m27s
[00:30:28.301] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:70:0)
[00:30:28.323] Build status: SCHEDULED
[00:30:28.323] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:71:0) after 7m1s
[00:37:29.371] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:71:0)
[00:37:29.393] Build status: SCHEDULED
[00:37:29.393] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:72:0) after 6m55s
[00:44:24.466] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:72:0)
[00:44:24.496] Build status: SCHEDULED
[00:44:24.496] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:73:0) after 8m20s
[00:52:44.523] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:73:0)
[00:52:44.580] Build status: SCHEDULED
[00:52:44.580] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:74:0) after 2m36s
[00:55:20.697] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:74:0)
[00:55:20.717] Build status: SCHEDULED
[00:55:20.717] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:75:0) after 2m10s
[00:57:30.746] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:75:0)
[00:57:30.773] Build status: SCHEDULED
[00:57:30.773] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:76:0) after 7m54s
[01:05:24.796] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:76:0)
[01:05:24.819] Build status: SCHEDULED
[01:05:24.819] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:77:0) after 9m25s
[01:14:49.825] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:77:0)
[01:14:49.856] Build status: SCHEDULED
[01:14:49.856] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:78:0) after 6m28s
[01:21:17.908] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:78:0)
[01:21:17.927] Build status: SCHEDULED
[01:21:17.928] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:79:0) after 4m33s
[01:25:50.948] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:79:0)
[01:25:50.965] Build status: SCHEDULED
[01:25:50.965] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:80:0) after 7m49s
[01:33:40.100] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:80:0)
[01:33:40.118] Build status: SCHEDULED
[01:33:40.118] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:81:0) after 9m33s
[01:43:13.146] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:81:0)
[01:43:13.167] Build status: SCHEDULED
[01:43:13.167] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:82:0) after 3m40s
[01:46:53.190] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:82:0)
[01:46:53.213] Build status: SCHEDULED
[01:46:53.213] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:83:0) after 6m17s
[01:53:10.340] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:83:0)
[01:53:10.363] Build status: SCHEDULED
[01:53:10.363] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:84:0) after 2m18s
[01:55:28.391] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:84:0)
[01:55:28.411] Build status: SCHEDULED
[01:55:28.411] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:85:0) after 1m59s
[01:57:27.455] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:85:0)
[01:57:27.480] Build status: SCHEDULED
[01:57:27.480] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:86:0) after 3m52s
[02:01:19.552] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:86:0)
[02:01:19.594] Build status: SCHEDULED
[02:01:19.594] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:87:0) after 7m24s
[02:08:43.596] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:87:0)
[02:08:43.724] Build status: SCHEDULED
[02:08:43.724] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:88:0) after 5m33s
[02:14:16.785] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:88:0)
[02:14:16.805] Build status: SCHEDULED
[02:14:16.805] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:89:0) after 4m27s
[02:14:56.613] Received PubSub notification, asking Buildbucket for the build status
[02:14:56.634] Build status: STARTED
[02:18:43.834] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:89:0)
[02:18:43.861] Build status: STARTED
[02:18:43.861] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8875319135169376544:91:0) after 7m49s
[02:22:10.323] Received PubSub notification, asking Buildbucket for the build status
[02:22:10.345] Build:
{
"id": "8709889073267292785",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-07-08T18:31:54.285914386Z",
"startTime": "2025-07-09T02:14:56.346971208Z",
"endTime": "2025-07-09T02:22:09.930841138Z",
"updateTime": "2025-07-09T02:22:09.930841138Z",
"status": "SUCCESS",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "c8d7e4b3213334c06c63b8b53aa9b3ecee2ff206",
"ref": "refs/heads/main"
}
},
"canary": true
}
[02:22:10.345] Invocation finished in 7h50m17.257042583s with status SUCCEEDED