[01:04:06.392] New invocation is queued and will start shortly
[01:04:08.461] Starting the invocation (attempt 1)
[01:04:08.490] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/876420bfa4be769498abed63fba117677c0b9ab0"
[01:04:08.490] Popped gitiles commit info from properties and tags
[01:04:08.490] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[01:04:08.490] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[01:04:08.491] Buildbucket request:
{
"requestId": "8844582087107746160",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8844582087107746160",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "c98af4092824047f575552e8f81bc1956ffe5e0c"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@c98af4092824047f575552e8f81bc1956ffe5e0c",
"title": "c98af4092824047f575552e8f81bc1956ffe5e0c",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/c98af4092824047f575552e8f81bc1956ffe5e0c"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "2862ae57eab31f3dc3d9b0c7e9963c5b9436fbf0"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@2862ae57eab31f3dc3d9b0c7e9963c5b9436fbf0",
"title": "2862ae57eab31f3dc3d9b0c7e9963c5b9436fbf0",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/2862ae57eab31f3dc3d9b0c7e9963c5b9436fbf0"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "1ccbe9441a3630fbb2dd6b9aa9bbf1f7d88be531"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@1ccbe9441a3630fbb2dd6b9aa9bbf1f7d88be531",
"title": "1ccbe9441a3630fbb2dd6b9aa9bbf1f7d88be531",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/1ccbe9441a3630fbb2dd6b9aa9bbf1f7d88be531"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "876420bfa4be769498abed63fba117677c0b9ab0"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@876420bfa4be769498abed63fba117677c0b9ab0",
"title": "876420bfa4be769498abed63fba117677c0b9ab0",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/876420bfa4be769498abed63fba117677c0b9ab0"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "876420bfa4be769498abed63fba117677c0b9ab0",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8844582087107746160"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[01:04:08.863] Scheduled build:
{
"id": "8679152024184425601",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
},
"createdBy": "project:pigweed",
"createTime": "2026-06-13T01:04:08.557514587Z",
"updateTime": "2026-06-13T01:04:08.557514587Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "876420bfa4be769498abed63fba117677c0b9ab0",
"ref": "refs/heads/main"
}
}
}
[01:04:08.863] Task URL: https://cr-buildbucket.appspot.com/build/8679152024184425601
[01:04:08.863] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:2:0) after 7m15s
[01:04:08.992] Received PubSub notification, asking Buildbucket for the build status
[01:04:09.020] Build status: SCHEDULED
[01:11:23.928] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:2:0)
[01:11:23.955] Build status: SCHEDULED
[01:11:23.955] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:4:0) after 5m6s
[01:16:29.968] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:4:0)
[01:16:30.051] Build status: SCHEDULED
[01:16:30.051] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:5:0) after 3m9s
[01:19:39.026] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:5:0)
[01:19:39.092] Build status: SCHEDULED
[01:19:39.092] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:6:0) after 4m47s
[01:24:26.146] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:6:0)
[01:24:26.167] Build status: SCHEDULED
[01:24:26.167] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:7:0) after 4m46s
[01:29:12.136] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:7:0)
[01:29:12.155] Build status: SCHEDULED
[01:29:12.155] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:8:0) after 1m43s
[01:30:55.134] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:8:0)
[01:30:55.150] Build status: SCHEDULED
[01:30:55.150] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:9:0) after 8m13s
[01:39:08.205] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:9:0)
[01:39:08.233] Build status: SCHEDULED
[01:39:08.233] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:10:0) after 7m20s
[01:46:28.248] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:10:0)
[01:46:28.267] Build status: SCHEDULED
[01:46:28.267] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:11:0) after 6m12s
[01:52:40.253] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:11:0)
[01:52:40.270] Build status: SCHEDULED
[01:52:40.270] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:12:0) after 5m42s
[01:58:22.285] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:12:0)
[01:58:22.349] Build status: SCHEDULED
[01:58:22.349] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:13:0) after 4m55s
[02:03:17.461] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:13:0)
[02:03:17.479] Build status: SCHEDULED
[02:03:17.479] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:14:0) after 3m17s
[02:06:34.545] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:14:0)
[02:06:34.591] Build status: SCHEDULED
[02:06:34.591] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:15:0) after 5m21s
[02:11:55.608] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:15:0)
[02:11:55.629] Build status: SCHEDULED
[02:11:55.629] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:16:0) after 6m12s
[02:18:07.714] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:16:0)
[02:18:07.731] Build status: SCHEDULED
[02:18:07.731] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:17:0) after 6m28s
[02:24:36.140] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:17:0)
[02:24:36.182] Build status: SCHEDULED
[02:24:36.182] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:18:0) after 1m18s
[02:25:54.387] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:18:0)
[02:25:54.407] Build status: SCHEDULED
[02:25:54.407] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:19:0) after 5m53s
[02:31:47.911] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:19:0)
[02:31:47.939] Build status: SCHEDULED
[02:31:47.939] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:20:0) after 8m52s
[02:40:39.960] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:20:0)
[02:40:39.979] Build status: SCHEDULED
[02:40:39.979] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:21:0) after 3m10s
[02:43:49.999] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:21:0)
[02:43:50.036] Build status: SCHEDULED
[02:43:50.036] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:22:0) after 7m42s
[02:51:32.156] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:22:0)
[02:51:32.252] Build status: SCHEDULED
[02:51:32.252] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:23:0) after 3m24s
[02:54:56.266] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:23:0)
[02:54:56.293] Build status: SCHEDULED
[02:54:56.293] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:24:0) after 3m30s
[02:58:26.311] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:24:0)
[02:58:26.363] Build status: SCHEDULED
[02:58:26.363] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:25:0) after 6m43s
[03:05:09.430] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:25:0)
[03:05:09.454] Build status: SCHEDULED
[03:05:09.454] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:26:0) after 5m1s
[03:10:10.488] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:26:0)
[03:10:10.505] Build status: SCHEDULED
[03:10:10.505] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:27:0) after 9m3s
[03:19:13.575] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:27:0)
[03:19:13.597] Build status: SCHEDULED
[03:19:13.597] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:28:0) after 2m0s
[03:21:13.631] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:28:0)
[03:21:13.664] Build status: SCHEDULED
[03:21:13.664] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:29:0) after 6m22s
[03:27:35.713] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:29:0)
[03:27:35.752] Build status: SCHEDULED
[03:27:35.752] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:30:0) after 3m40s
[03:31:15.947] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:30:0)
[03:31:15.968] Build status: SCHEDULED
[03:31:15.968] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:31:0) after 9m3s
[03:38:53.161] Received PubSub notification, asking Buildbucket for the build status
[03:38:53.182] Build status: STARTED
[03:40:18.985] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:31:0)
[03:40:19.040] Build status: STARTED
[03:40:19.040] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8844582087107746160:33:0) after 8m25s
[03:43:08.536] Received PubSub notification, asking Buildbucket for the build status
[03:43:08.561] Build:
{
"id": "8679152024184425601",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
},
"createdBy": "project:pigweed",
"createTime": "2026-06-13T01:04:08.557514587Z",
"startTime": "2026-06-13T03:38:52.891240896Z",
"endTime": "2026-06-13T03:43:05.883690962Z",
"updateTime": "2026-06-13T03:43:05.883690962Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "876420bfa4be769498abed63fba117677c0b9ab0",
"ref": "refs/heads/main"
}
}
}
[03:43:08.561] Invocation finished in 2h39m2.175247835s with status FAILED