[03:28:51.043] New invocation is queued and will start shortly
[03:28:52.163] Starting the invocation (attempt 1)
[03:28:52.193] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/5142754646435683c25c9b96a1e79abb6917a073"
[03:28:52.193] Popped gitiles commit info from properties and tags
[03:28:52.193] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[03:28:52.193] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[03:28:52.193] Buildbucket request:
{
"requestId": "8837868805075810288",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8837868805075810288",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "5142754646435683c25c9b96a1e79abb6917a073"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@5142754646435683c25c9b96a1e79abb6917a073",
"title": "5142754646435683c25c9b96a1e79abb6917a073",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/5142754646435683c25c9b96a1e79abb6917a073"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "5142754646435683c25c9b96a1e79abb6917a073",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8837868805075810288"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[03:28:52.600] Scheduled build:
{
"id": "8672438743139739345",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-26T03:28:52.269672031Z",
"updateTime": "2026-08-26T03:28:52.269672031Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "5142754646435683c25c9b96a1e79abb6917a073",
"ref": "refs/heads/main"
}
}
}
[03:28:52.600] Task URL: https://cr-buildbucket.appspot.com/build/8672438743139739345
[03:28:52.600] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:2:0) after 9m55s
[03:28:53.534] Received PubSub notification, asking Buildbucket for the build status
[03:28:53.556] Build status: SCHEDULED
[03:38:47.687] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:2:0)
[03:38:47.705] Build status: SCHEDULED
[03:38:47.705] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:4:0) after 3m43s
[03:42:30.879] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:4:0)
[03:42:30.897] Build status: SCHEDULED
[03:42:30.897] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:5:0) after 9m33s
[03:52:03.914] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:5:0)
[03:52:04.021] Build status: SCHEDULED
[03:52:04.021] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:6:0) after 5m1s
[03:57:05.087] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:6:0)
[03:57:05.119] Build status: SCHEDULED
[03:57:05.119] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:7:0) after 2m56s
[04:00:01.187] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:7:0)
[04:00:01.208] Build status: SCHEDULED
[04:00:01.208] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:8:0) after 6m40s
[04:06:41.310] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:8:0)
[04:06:41.343] Build status: SCHEDULED
[04:06:41.343] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:9:0) after 4m40s
[04:11:21.470] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:9:0)
[04:11:21.498] Build status: SCHEDULED
[04:11:21.498] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:10:0) after 7m16s
[04:18:37.563] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:10:0)
[04:18:37.583] Build status: SCHEDULED
[04:18:37.583] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:11:0) after 6m51s
[04:25:28.842] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:11:0)
[04:25:28.867] Build status: SCHEDULED
[04:25:28.867] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:12:0) after 2m16s
[04:27:44.892] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:12:0)
[04:27:44.910] Build status: SCHEDULED
[04:27:44.910] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:13:0) after 7m16s
[04:35:00.929] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:13:0)
[04:35:01.193] Build status: SCHEDULED
[04:35:01.193] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:14:0) after 1m33s
[04:36:34.210] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:14:0)
[04:36:34.228] Build status: SCHEDULED
[04:36:34.228] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:15:0) after 2m53s
[04:39:27.243] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:15:0)
[04:39:27.262] Build status: SCHEDULED
[04:39:27.262] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:16:0) after 1m12s
[04:40:39.320] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:16:0)
[04:40:39.384] Build status: SCHEDULED
[04:40:39.384] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:17:0) after 4m12s
[04:44:51.526] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:17:0)
[04:44:51.545] Build status: SCHEDULED
[04:44:51.545] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:18:0) after 1m8s
[04:45:59.614] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:18:0)
[04:45:59.633] Build status: SCHEDULED
[04:45:59.633] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:19:0) after 8m35s
[04:54:34.700] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:19:0)
[04:54:34.717] Build status: SCHEDULED
[04:54:34.717] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:20:0) after 3m26s
[04:58:00.814] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:20:0)
[04:58:00.891] Build status: SCHEDULED
[04:58:00.892] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:21:0) after 7m40s
[05:05:40.919] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:21:0)
[05:05:40.942] Build status: SCHEDULED
[05:05:40.942] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:22:0) after 3m19s
[05:09:00.180] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:22:0)
[05:09:00.198] Build status: SCHEDULED
[05:09:00.198] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:23:0) after 1m3s
[05:10:03.165] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:23:0)
[05:10:03.343] Build status: SCHEDULED
[05:10:03.343] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:24:0) after 9m58s
[05:20:01.347] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:24:0)
[05:20:01.368] Build status: SCHEDULED
[05:20:01.368] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:25:0) after 3m27s
[05:23:28.391] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:25:0)
[05:23:28.412] Build status: SCHEDULED
[05:23:28.412] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:26:0) after 4m11s
[05:27:39.512] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:26:0)
[05:27:39.529] Build status: SCHEDULED
[05:27:39.529] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:27:0) after 4m59s
[05:32:38.543] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:27:0)
[05:32:38.561] Build status: SCHEDULED
[05:32:38.561] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:28:0) after 9m37s
[05:42:15.632] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:28:0)
[05:42:15.653] Build status: SCHEDULED
[05:42:15.653] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:29:0) after 2m25s
[05:44:40.669] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:29:0)
[05:44:40.726] Build status: SCHEDULED
[05:44:40.726] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:30:0) after 3m42s
[05:48:22.784] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:30:0)
[05:48:22.807] Build status: SCHEDULED
[05:48:22.807] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:31:0) after 3m43s
[05:52:05.945] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:31:0)
[05:52:05.965] Build status: SCHEDULED
[05:52:05.965] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:32:0) after 2m3s
[05:54:08.942] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:32:0)
[05:54:09.010] Build status: SCHEDULED
[05:54:09.010] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:33:0) after 6m25s
[06:00:34.152] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:33:0)
[06:00:34.171] Build status: SCHEDULED
[06:00:34.171] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:34:0) after 7m5s
[06:07:39.319] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:34:0)
[06:07:39.351] Build status: SCHEDULED
[06:07:39.351] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8837868805075810288:35:0) after 6m9s
[06:09:48.639] Received PubSub notification, asking Buildbucket for the build status
[06:09:48.660] Build:
{
"id": "8672438743139739345",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-26T03:28:52.269672031Z",
"endTime": "2026-08-26T06:09:48.310770615Z",
"updateTime": "2026-08-26T06:09:48.310770615Z",
"status": "INFRA_FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "5142754646435683c25c9b96a1e79abb6917a073",
"ref": "refs/heads/main"
}
}
}
[06:09:48.660] Invocation finished in 2h40m57.621439805s with status FAILED