[07:51:51.608] New invocation is queued and will start shortly
[07:51:52.850] Starting the invocation (attempt 1)
[07:51:52.880] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/5d40066c740a4410d8862c1c0061099651c711ca"
[07:51:52.880] Popped gitiles commit info from properties and tags
[07:51:52.880] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[07:51:52.880] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[07:51:52.880] Buildbucket request:
{
"requestId": "8872369702164596768",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-tsan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8872369702164596768",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "18c001d1d0f5127aecd3f760d8e61c1c46d2d01b"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@18c001d1d0f5127aecd3f760d8e61c1c46d2d01b",
"title": "18c001d1d0f5127aecd3f760d8e61c1c46d2d01b",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/18c001d1d0f5127aecd3f760d8e61c1c46d2d01b"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "5d40066c740a4410d8862c1c0061099651c711ca"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@5d40066c740a4410d8862c1c0061099651c711ca",
"title": "5d40066c740a4410d8862c1c0061099651c711ca",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/5d40066c740a4410d8862c1c0061099651c711ca"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "5d40066c740a4410d8862c1c0061099651c711ca",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8872369702164596768"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[07:51:53.334] Scheduled build:
{
"id": "8706939640047411841",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-08-10T07:51:52.995246749Z",
"updateTime": "2025-08-10T07:51:52.995246749Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "5d40066c740a4410d8862c1c0061099651c711ca",
"ref": "refs/heads/main"
}
},
"canary": true
}
[07:51:53.334] Task URL: https://cr-buildbucket.appspot.com/build/8706939640047411841
[07:51:53.334] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:2:0) after 8m10s
[07:51:54.294] Received PubSub notification, asking Buildbucket for the build status
[07:51:54.312] Build status: SCHEDULED
[08:00:03.332] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:2:0)
[08:00:03.374] Build status: SCHEDULED
[08:00:03.374] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:4:0) after 7m45s
[08:07:48.757] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:4:0)
[08:07:48.794] Build status: SCHEDULED
[08:07:48.794] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:5:0) after 1m14s
[08:09:02.811] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:5:0)
[08:09:02.835] Build status: SCHEDULED
[08:09:02.835] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:6:0) after 8m45s
[08:17:47.904] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:6:0)
[08:17:47.962] Build status: SCHEDULED
[08:17:47.962] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:7:0) after 8m18s
[08:26:06.023] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:7:0)
[08:26:06.076] Build status: SCHEDULED
[08:26:06.076] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:8:0) after 8m51s
[08:34:57.190] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:8:0)
[08:34:57.215] Build status: SCHEDULED
[08:34:57.215] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:9:0) after 6m52s
[08:41:49.229] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:9:0)
[08:41:49.248] Build status: SCHEDULED
[08:41:49.248] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:10:0) after 6m20s
[08:48:09.305] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:10:0)
[08:48:09.326] Build status: SCHEDULED
[08:48:09.326] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:11:0) after 7m6s
[08:55:15.350] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:11:0)
[08:55:15.456] Build status: SCHEDULED
[08:55:15.456] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:12:0) after 9m49s
[09:05:04.475] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:12:0)
[09:05:04.525] Build status: SCHEDULED
[09:05:04.525] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:13:0) after 7m48s
[09:12:52.543] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:13:0)
[09:12:52.565] Build status: SCHEDULED
[09:12:52.565] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:14:0) after 3m38s
[09:16:30.695] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:14:0)
[09:16:30.715] Build status: SCHEDULED
[09:16:30.715] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:15:0) after 1m55s
[09:18:26.166] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:15:0)
[09:18:26.184] Build status: SCHEDULED
[09:18:26.184] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:16:0) after 8m43s
[09:27:09.278] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:16:0)
[09:27:09.300] Build status: SCHEDULED
[09:27:09.300] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:17:0) after 5m52s
[09:33:01.620] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:17:0)
[09:33:01.655] Build status: SCHEDULED
[09:33:01.655] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:18:0) after 2m19s
[09:35:20.775] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:18:0)
[09:35:20.846] Build status: SCHEDULED
[09:35:20.846] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:19:0) after 2m39s
[09:37:59.865] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:19:0)
[09:37:59.884] Build status: SCHEDULED
[09:37:59.884] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:20:0) after 1m38s
[09:39:37.900] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:20:0)
[09:39:37.918] Build status: SCHEDULED
[09:39:37.918] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:21:0) after 7m17s
[09:46:55.027] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:21:0)
[09:46:55.059] Build status: SCHEDULED
[09:46:55.059] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:22:0) after 7m1s
[09:53:56.077] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:22:0)
[09:53:56.101] Build status: SCHEDULED
[09:53:56.101] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:23:0) after 7m9s
[10:01:05.270] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:23:0)
[10:01:05.291] Build status: SCHEDULED
[10:01:05.291] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:24:0) after 3m11s
[10:04:16.320] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:24:0)
[10:04:16.339] Build status: SCHEDULED
[10:04:16.339] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:25:0) after 6m33s
[10:10:49.569] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:25:0)
[10:10:49.586] Build status: SCHEDULED
[10:10:49.586] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:26:0) after 9m21s
[10:20:10.771] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:26:0)
[10:20:10.827] Build status: SCHEDULED
[10:20:10.827] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:27:0) after 1m54s
[10:22:04.889] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:27:0)
[10:22:04.937] Build status: SCHEDULED
[10:22:04.937] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:28:0) after 3m11s
[10:25:16.047] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:28:0)
[10:25:16.070] Build status: SCHEDULED
[10:25:16.070] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:29:0) after 1m34s
[10:26:50.086] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:29:0)
[10:26:50.102] Build status: SCHEDULED
[10:26:50.102] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:30:0) after 7m18s
[10:34:08.169] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:30:0)
[10:34:08.193] Build status: SCHEDULED
[10:34:08.193] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:31:0) after 6m21s
[10:40:29.207] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:31:0)
[10:40:29.228] Build status: SCHEDULED
[10:40:29.228] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:32:0) after 4m45s
[10:45:14.304] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:32:0)
[10:45:14.321] Build status: SCHEDULED
[10:45:14.321] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:33:0) after 6m37s
[10:51:51.389] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:33:0)
[10:51:51.421] Build status: SCHEDULED
[10:51:51.421] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:34:0) after 1m8s
[10:52:59.438] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:34:0)
[10:52:59.476] Build status: SCHEDULED
[10:52:59.476] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:35:0) after 3m20s
[10:56:19.493] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:35:0)
[10:56:19.523] Build status: SCHEDULED
[10:56:19.523] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:36:0) after 7m21s
[11:03:40.761] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:36:0)
[11:03:40.787] Build status: SCHEDULED
[11:03:40.787] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:37:0) after 8m44s
[11:12:24.781] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:37:0)
[11:12:24.807] Build status: SCHEDULED
[11:12:24.807] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:38:0) after 6m59s
[11:19:23.831] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:38:0)
[11:19:23.860] Build status: SCHEDULED
[11:19:23.860] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:39:0) after 7m47s
[11:27:10.865] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:39:0)
[11:27:10.893] Build status: SCHEDULED
[11:27:10.893] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:40:0) after 8m25s
[11:35:36.017] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:40:0)
[11:35:36.039] Build status: SCHEDULED
[11:35:36.039] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:41:0) after 4m50s
[11:40:26.052] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:41:0)
[11:40:26.069] Build status: SCHEDULED
[11:40:26.069] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:42:0) after 6m32s
[11:46:58.083] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:42:0)
[11:46:58.101] Build status: SCHEDULED
[11:46:58.101] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:43:0) after 1m16s
[11:48:14.227] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:43:0)
[11:48:14.248] Build status: SCHEDULED
[11:48:14.248] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:44:0) after 8m8s
[11:56:22.348] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:44:0)
[11:56:22.366] Build status: SCHEDULED
[11:56:22.366] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:45:0) after 9m57s
[12:06:19.558] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:45:0)
[12:06:19.576] Build status: SCHEDULED
[12:06:19.576] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:46:0) after 1m55s
[12:08:14.936] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:46:0)
[12:08:15.019] Build status: SCHEDULED
[12:08:15.019] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:47:0) after 3m53s
[12:12:08.038] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:47:0)
[12:12:08.085] Build status: SCHEDULED
[12:12:08.085] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:48:0) after 5m3s
[12:17:11.203] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:48:0)
[12:17:11.363] Build status: SCHEDULED
[12:17:11.363] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:49:0) after 3m16s
[12:20:27.377] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:49:0)
[12:20:27.392] Build status: SCHEDULED
[12:20:27.392] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:50:0) after 6m55s
[12:27:22.436] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:50:0)
[12:27:22.453] Build status: SCHEDULED
[12:27:22.453] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:51:0) after 8m47s
[12:36:09.467] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:51:0)
[12:36:09.483] Build status: SCHEDULED
[12:36:09.483] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:52:0) after 4m2s
[12:40:11.511] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:52:0)
[12:40:11.581] Build status: SCHEDULED
[12:40:11.582] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:53:0) after 3m9s
[12:43:20.598] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:53:0)
[12:43:20.625] Build status: SCHEDULED
[12:43:20.625] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:54:0) after 1m3s
[12:44:23.666] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:54:0)
[12:44:23.688] Build status: SCHEDULED
[12:44:23.688] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:55:0) after 5m35s
[12:49:59.229] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:55:0)
[12:49:59.247] Build status: SCHEDULED
[12:49:59.247] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:56:0) after 2m24s
[12:52:23.265] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:56:0)
[12:52:23.329] Build status: SCHEDULED
[12:52:23.329] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:57:0) after 4m57s
[12:57:20.655] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:57:0)
[12:57:20.675] Build status: SCHEDULED
[12:57:20.676] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:58:0) after 6m1s
[13:03:21.698] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:58:0)
[13:03:21.722] Build status: SCHEDULED
[13:03:21.722] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:59:0) after 8m44s
[13:12:05.739] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:59:0)
[13:12:05.767] Build status: SCHEDULED
[13:12:05.767] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:60:0) after 1m52s
[13:13:57.890] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:60:0)
[13:13:57.911] Build status: SCHEDULED
[13:13:57.911] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:61:0) after 2m33s
[13:16:30.941] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:61:0)
[13:16:30.960] Build status: SCHEDULED
[13:16:30.960] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:62:0) after 1m43s
[13:18:13.985] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:62:0)
[13:18:14.008] Build status: SCHEDULED
[13:18:14.008] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:63:0) after 7m35s
[13:25:49.032] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:63:0)
[13:25:49.055] Build status: SCHEDULED
[13:25:49.056] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:64:0) after 9m54s
[13:35:43.069] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:64:0)
[13:35:43.085] Build status: SCHEDULED
[13:35:43.085] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:65:0) after 7m3s
[13:42:46.102] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:65:0)
[13:42:46.123] Build status: SCHEDULED
[13:42:46.123] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:66:0) after 9m39s
[13:52:25.575] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:66:0)
[13:52:25.764] Build status: SCHEDULED
[13:52:25.764] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:67:0) after 6m39s
[13:59:06.627] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:67:0)
[13:59:06.643] Build status: SCHEDULED
[13:59:06.643] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:68:0) after 8m2s
[14:07:08.657] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:68:0)
[14:07:08.675] Build status: SCHEDULED
[14:07:08.676] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:69:0) after 9m9s
[14:16:17.844] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:69:0)
[14:16:17.867] Build status: SCHEDULED
[14:16:17.867] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:70:0) after 4m11s
[14:20:28.921] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:70:0)
[14:20:28.947] Build status: SCHEDULED
[14:20:28.947] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:71:0) after 3m55s
[14:24:24.292] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:71:0)
[14:24:24.316] Build status: SCHEDULED
[14:24:24.316] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:72:0) after 7m50s
[14:32:14.446] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:72:0)
[14:32:14.463] Build status: SCHEDULED
[14:32:14.463] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:73:0) after 3m22s
[14:35:36.881] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:73:0)
[14:35:36.895] Build status: SCHEDULED
[14:35:36.895] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:74:0) after 1m3s
[14:36:39.913] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:74:0)
[14:36:39.952] Build status: SCHEDULED
[14:36:39.952] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:75:0) after 7m40s
[14:44:19.931] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:75:0)
[14:44:19.955] Build status: SCHEDULED
[14:44:19.955] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:76:0) after 3m52s
[14:48:12.412] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:76:0)
[14:48:12.429] Build status: SCHEDULED
[14:48:12.429] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:77:0) after 1m31s
[14:49:43.572] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:77:0)
[14:49:43.603] Build status: SCHEDULED
[14:49:43.603] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:78:0) after 4m35s
[14:54:18.805] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:78:0)
[14:54:18.834] Build status: SCHEDULED
[14:54:18.834] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:79:0) after 2m5s
[14:56:23.838] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:79:0)
[14:56:23.862] Build status: SCHEDULED
[14:56:23.862] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:80:0) after 1m36s
[14:57:59.880] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:80:0)
[14:57:59.895] Build status: SCHEDULED
[14:57:59.895] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:81:0) after 2m13s
[15:00:12.913] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:81:0)
[15:00:12.941] Build status: SCHEDULED
[15:00:12.941] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:82:0) after 5m2s
[15:05:14.968] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:82:0)
[15:05:14.986] Build status: SCHEDULED
[15:05:14.986] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:83:0) after 1m8s
[15:06:23.101] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:83:0)
[15:06:23.124] Build status: SCHEDULED
[15:06:23.124] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:84:0) after 6m46s
[15:13:09.403] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:84:0)
[15:13:09.434] Build status: SCHEDULED
[15:13:09.434] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:85:0) after 8m25s
[15:21:34.458] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:85:0)
[15:21:34.505] Build status: SCHEDULED
[15:21:34.505] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:86:0) after 2m40s
[15:24:14.523] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:86:0)
[15:24:14.536] Build status: SCHEDULED
[15:24:14.536] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:87:0) after 9m2s
[15:33:17.151] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:87:0)
[15:33:17.174] Build status: SCHEDULED
[15:33:17.174] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:88:0) after 7m55s
[15:41:12.233] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:88:0)
[15:41:12.252] Build status: SCHEDULED
[15:41:12.252] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:89:0) after 5m53s
[15:47:05.268] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:89:0)
[15:47:05.301] Build status: SCHEDULED
[15:47:05.301] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:90:0) after 4m40s
[15:51:45.367] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:90:0)
[15:51:45.383] Build status: SCHEDULED
[15:51:45.383] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8872369702164596768:91:0) after 9m57s
[15:51:52.786] Received PubSub notification, asking Buildbucket for the build status
[15:51:52.811] Build:
{
"id": "8706939640047411841",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-08-10T07:51:52.995246749Z",
"endTime": "2025-08-10T15:51:52.108396913Z",
"updateTime": "2025-08-10T15:51:52.108396913Z",
"status": "INFRA_FAILURE",
"statusDetails": {
"resourceExhaustion": {
},
"timeout": {
}
},
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "5d40066c740a4410d8862c1c0061099651c711ca",
"ref": "refs/heads/main"
}
},
"canary": true
}
[15:51:52.811] Invocation finished in 8h0m1.21900092s with status FAILED