[00:26:18.512] New invocation is queued and will start shortly
[00:26:21.068] Starting the invocation (attempt 1)
[00:26:21.099] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/3e5311128ba6a7ede1525ce07de0b86453c1108b"
[00:26:21.099] Popped gitiles commit info from properties and tags
[00:26:21.099] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[00:26:21.099] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[00:26:21.099] Buildbucket request:
{
"requestId": "8843316107623406864",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8843316107623406864",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "3e5311128ba6a7ede1525ce07de0b86453c1108b"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@3e5311128ba6a7ede1525ce07de0b86453c1108b",
"title": "3e5311128ba6a7ede1525ce07de0b86453c1108b",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/3e5311128ba6a7ede1525ce07de0b86453c1108b"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "3e5311128ba6a7ede1525ce07de0b86453c1108b",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8843316107623406864"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[00:26:21.567] Scheduled build:
{
"id": "8677886044187304193",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-06-27T00:26:21.166385920Z",
"updateTime": "2026-06-27T00:26:21.166385920Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "3e5311128ba6a7ede1525ce07de0b86453c1108b",
"ref": "refs/heads/main"
}
}
}
[00:26:21.567] Task URL: https://cr-buildbucket.appspot.com/build/8677886044187304193
[00:26:21.567] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:2:0) after 5m46s
[00:26:21.907] Received PubSub notification, asking Buildbucket for the build status
[00:26:21.931] Build status: SCHEDULED
[00:32:07.633] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:2:0)
[00:32:07.701] Build status: SCHEDULED
[00:32:07.701] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:4:0) after 4m21s
[00:36:28.807] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:4:0)
[00:36:28.828] Build status: SCHEDULED
[00:36:28.828] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:5:0) after 7m43s
[00:44:11.820] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:5:0)
[00:44:11.840] Build status: SCHEDULED
[00:44:11.840] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:6:0) after 6m20s
[00:50:31.929] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:6:0)
[00:50:31.955] Build status: SCHEDULED
[00:50:31.955] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:7:0) after 5m37s
[00:56:09.037] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:7:0)
[00:56:09.056] Build status: SCHEDULED
[00:56:09.056] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:8:0) after 1m40s
[00:57:49.070] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:8:0)
[00:57:49.103] Build status: SCHEDULED
[00:57:49.103] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:9:0) after 5m1s
[01:02:50.125] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:9:0)
[01:02:50.162] Build status: SCHEDULED
[01:02:50.162] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:10:0) after 7m52s
[01:10:42.225] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:10:0)
[01:10:42.245] Build status: SCHEDULED
[01:10:42.245] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:11:0) after 4m40s
[01:15:22.272] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:11:0)
[01:15:22.295] Build status: SCHEDULED
[01:15:22.295] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:12:0) after 4m55s
[01:20:17.310] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:12:0)
[01:20:17.334] Build status: SCHEDULED
[01:20:17.334] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:13:0) after 2m37s
[01:22:54.358] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:13:0)
[01:22:54.394] Build status: SCHEDULED
[01:22:54.432] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:14:0) after 4m28s
[01:27:22.495] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:14:0)
[01:27:22.583] Build status: SCHEDULED
[01:27:22.583] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:15:0) after 7m28s
[01:34:50.654] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:15:0)
[01:34:50.678] Build status: SCHEDULED
[01:34:50.678] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:16:0) after 1m29s
[01:36:19.688] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:16:0)
[01:36:19.707] Build status: SCHEDULED
[01:36:19.707] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:17:0) after 3m54s
[01:40:13.705] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:17:0)
[01:40:13.728] Build status: SCHEDULED
[01:40:13.728] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:18:0) after 2m19s
[01:42:32.742] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:18:0)
[01:42:32.767] Build status: SCHEDULED
[01:42:32.767] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:19:0) after 5m33s
[01:48:05.831] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:19:0)
[01:48:05.845] Build status: SCHEDULED
[01:48:05.845] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:20:0) after 6m57s
[01:55:02.860] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:20:0)
[01:55:02.953] Build status: SCHEDULED
[01:55:02.953] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:21:0) after 9m48s
[02:04:51.016] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:21:0)
[02:04:51.037] Build status: SCHEDULED
[02:04:51.037] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:22:0) after 2m0s
[02:06:51.203] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:22:0)
[02:06:51.221] Build status: SCHEDULED
[02:06:51.221] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:23:0) after 3m50s
[02:10:41.235] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:23:0)
[02:10:41.257] Build status: SCHEDULED
[02:10:41.257] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:24:0) after 1m25s
[02:12:06.272] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:24:0)
[02:12:06.292] Build status: SCHEDULED
[02:12:06.292] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:25:0) after 9m47s
[02:21:53.315] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:25:0)
[02:21:53.343] Build status: SCHEDULED
[02:21:53.343] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:26:0) after 9m15s
[02:31:08.460] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:26:0)
[02:31:08.494] Build status: SCHEDULED
[02:31:08.494] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:27:0) after 2m41s
[02:33:49.620] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:27:0)
[02:33:49.682] Build status: SCHEDULED
[02:33:49.682] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:28:0) after 4m50s
[02:38:39.749] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:28:0)
[02:38:39.766] Build status: SCHEDULED
[02:38:39.766] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:29:0) after 7m34s
[02:46:13.833] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:29:0)
[02:46:13.894] Build status: SCHEDULED
[02:46:13.895] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:30:0) after 6m17s
[02:52:30.875] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:30:0)
[02:52:30.898] Build status: SCHEDULED
[02:52:30.898] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:31:0) after 8m49s
[03:01:19.914] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:31:0)
[03:01:19.931] Build status: SCHEDULED
[03:01:19.931] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:32:0) after 6m59s
[03:08:18.949] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:32:0)
[03:08:18.994] Build status: SCHEDULED
[03:08:18.994] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:33:0) after 3m30s
[03:11:49.035] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:33:0)
[03:11:49.067] Build status: SCHEDULED
[03:11:49.067] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:34:0) after 6m16s
[03:13:59.970] Received PubSub notification, asking Buildbucket for the build status
[03:14:00.099] Build status: STARTED
[03:18:05.085] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:34:0)
[03:18:05.116] Build status: STARTED
[03:18:05.116] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843316107623406864:36:0) after 8m0s
[03:21:00.632] Received PubSub notification, asking Buildbucket for the build status
[03:21:00.652] Build:
{
"id": "8677886044187304193",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-06-27T00:26:21.166385920Z",
"startTime": "2026-06-27T03:13:59.715433565Z",
"endTime": "2026-06-27T03:20:59.697801224Z",
"updateTime": "2026-06-27T03:20:59.697801224Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "3e5311128ba6a7ede1525ce07de0b86453c1108b",
"ref": "refs/heads/main"
}
}
}
[03:21:00.652] Invocation finished in 2h54m42.146964533s with status FAILED