[01:41:40.196] New invocation is queued and will start shortly
[01:41:41.315] Starting the invocation (attempt 1)
[01:41:41.356] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/1c407aad06cf78a06fb885b999f939a887eac8a3"
[01:41:41.356] Popped gitiles commit info from properties and tags
[01:41:41.356] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[01:41:41.357] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[01:41:41.357] Buildbucket request:
{
"requestId": "8841137039098541360",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8841137039098541360",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "b2d61c14f9359ecc1731c27d9eff52423cfb2f1a"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@b2d61c14f9359ecc1731c27d9eff52423cfb2f1a",
"title": "b2d61c14f9359ecc1731c27d9eff52423cfb2f1a",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/b2d61c14f9359ecc1731c27d9eff52423cfb2f1a"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "f759727f02169dc49d22e99a5a90b317203389aa"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@f759727f02169dc49d22e99a5a90b317203389aa",
"title": "f759727f02169dc49d22e99a5a90b317203389aa",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/f759727f02169dc49d22e99a5a90b317203389aa"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "98875eb208fd4e2be525570b0023e105c4bf82c3"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@98875eb208fd4e2be525570b0023e105c4bf82c3",
"title": "98875eb208fd4e2be525570b0023e105c4bf82c3",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/98875eb208fd4e2be525570b0023e105c4bf82c3"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "1c407aad06cf78a06fb885b999f939a887eac8a3"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@1c407aad06cf78a06fb885b999f939a887eac8a3",
"title": "1c407aad06cf78a06fb885b999f939a887eac8a3",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/1c407aad06cf78a06fb885b999f939a887eac8a3"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "1c407aad06cf78a06fb885b999f939a887eac8a3",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8841137039098541360"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[01:41:41.991] Scheduled build:
{
"id": "8675706976996525297",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-07-21T01:41:41.579102822Z",
"updateTime": "2026-07-21T01:41:41.579102822Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "1c407aad06cf78a06fb885b999f939a887eac8a3",
"ref": "refs/heads/main"
}
}
}
[01:41:41.991] Task URL: https://cr-buildbucket.appspot.com/build/8675706976996525297
[01:41:41.991] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:2:0) after 8m39s
[01:41:43.063] Received PubSub notification, asking Buildbucket for the build status
[01:41:43.144] Build status: SCHEDULED
[01:50:21.053] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:2:0)
[01:50:21.074] Build status: SCHEDULED
[01:50:21.074] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:4:0) after 1m26s
[01:51:47.090] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:4:0)
[01:51:47.117] Build status: SCHEDULED
[01:51:47.117] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:5:0) after 8m10s
[01:59:57.218] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:5:0)
[01:59:57.239] Build status: SCHEDULED
[01:59:57.239] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:6:0) after 1m20s
[02:01:17.257] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:6:0)
[02:01:17.298] Build status: SCHEDULED
[02:01:17.298] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:7:0) after 8m43s
[02:10:00.325] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:7:0)
[02:10:00.353] Build status: SCHEDULED
[02:10:00.353] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:8:0) after 3m10s
[02:13:10.437] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:8:0)
[02:13:10.498] Build status: SCHEDULED
[02:13:10.498] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:9:0) after 4m43s
[02:17:53.513] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:9:0)
[02:17:53.535] Build status: SCHEDULED
[02:17:53.535] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:10:0) after 9m16s
[02:27:09.599] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:10:0)
[02:27:09.713] Build status: SCHEDULED
[02:27:09.713] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:11:0) after 9m44s
[02:36:53.725] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:11:0)
[02:36:53.743] Build status: SCHEDULED
[02:36:53.743] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:12:0) after 5m23s
[02:42:16.729] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:12:0)
[02:42:16.749] Build status: SCHEDULED
[02:42:16.749] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:13:0) after 2m3s
[02:44:19.765] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:13:0)
[02:44:19.784] Build status: SCHEDULED
[02:44:19.784] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:14:0) after 3m59s
[02:48:18.798] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:14:0)
[02:48:18.836] Build status: SCHEDULED
[02:48:18.836] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:15:0) after 4m52s
[02:53:10.855] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:15:0)
[02:53:10.974] Build status: SCHEDULED
[02:53:10.974] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:16:0) after 2m23s
[02:55:33.950] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:16:0)
[02:55:33.969] Build status: SCHEDULED
[02:55:33.969] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:17:0) after 7m17s
[03:02:50.988] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:17:0)
[03:02:51.042] Build status: SCHEDULED
[03:02:51.042] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:18:0) after 4m39s
[03:07:30.159] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:18:0)
[03:07:30.179] Build status: SCHEDULED
[03:07:30.179] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:19:0) after 9m32s
[03:17:02.282] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:19:0)
[03:17:02.302] Build status: SCHEDULED
[03:17:02.302] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:20:0) after 8m25s
[03:25:27.320] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:20:0)
[03:25:27.345] Build status: SCHEDULED
[03:25:27.345] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:21:0) after 3m18s
[03:28:45.318] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:21:0)
[03:28:45.441] Build status: SCHEDULED
[03:28:45.445] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:22:0) after 9m53s
[03:38:38.471] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:22:0)
[03:38:38.493] Build status: SCHEDULED
[03:38:38.493] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:23:0) after 2m28s
[03:41:06.546] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:23:0)
[03:41:06.619] Build status: SCHEDULED
[03:41:06.619] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:24:0) after 5m53s
[03:46:59.657] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:24:0)
[03:46:59.692] Build status: SCHEDULED
[03:46:59.692] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:25:0) after 6m48s
[03:53:47.892] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:25:0)
[03:53:47.919] Build status: SCHEDULED
[03:53:47.919] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:26:0) after 1m44s
[03:55:32.252] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:26:0)
[03:55:32.288] Build status: SCHEDULED
[03:55:32.288] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:27:0) after 4m20s
[03:59:52.361] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:27:0)
[03:59:52.376] Build status: SCHEDULED
[03:59:52.376] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:28:0) after 1m59s
[04:01:51.529] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:28:0)
[04:01:51.597] Build status: SCHEDULED
[04:01:51.597] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:29:0) after 9m40s
[04:11:31.737] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:29:0)
[04:11:31.760] Build status: SCHEDULED
[04:11:31.760] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:30:0) after 1m13s
[04:12:44.760] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:30:0)
[04:12:44.984] Build status: SCHEDULED
[04:12:44.984] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:31:0) after 2m35s
[04:15:20.005] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:31:0)
[04:15:20.020] Build status: SCHEDULED
[04:15:20.020] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:32:0) after 9m57s
[04:25:17.040] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:32:0)
[04:25:17.061] Build status: SCHEDULED
[04:25:17.061] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:33:0) after 9m50s
[04:35:07.132] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:33:0)
[04:35:07.153] Build status: SCHEDULED
[04:35:07.153] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:34:0) after 1m16s
[04:36:23.171] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:34:0)
[04:36:23.197] Build status: SCHEDULED
[04:36:23.197] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:35:0) after 9m38s
[04:46:01.294] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:35:0)
[04:46:01.442] Build status: STARTED
[04:46:01.442] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:36:0) after 5m31s
[04:46:41.468] Received PubSub notification, asking Buildbucket for the build status
[04:46:41.493] Build status: STARTED
[04:51:32.464] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:36:0)
[04:51:32.486] Build status: STARTED
[04:51:32.486] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:38:0) after 1m58s
[04:53:30.505] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:38:0)
[04:53:30.529] Build status: STARTED
[04:53:30.529] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:39:0) after 5m26s
[04:58:56.578] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:39:0)
[04:58:56.595] Build status: STARTED
[04:58:56.595] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8841137039098541360:40:0) after 8m29s
[05:03:49.186] Received PubSub notification, asking Buildbucket for the build status
[05:03:49.209] Build:
{
"id": "8675706976996525297",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-07-21T01:41:41.579102822Z",
"startTime": "2026-07-21T04:45:59.834310034Z",
"endTime": "2026-07-21T05:03:48.820204395Z",
"updateTime": "2026-07-21T05:03:48.820204395Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "1c407aad06cf78a06fb885b999f939a887eac8a3",
"ref": "refs/heads/main"
}
}
}
[05:03:49.209] Invocation finished in 3h22m9.017933437s with status FAILED