[00:33:10.837] New invocation is queued and will start shortly
[00:33:11.895] Starting the invocation (attempt 1)
[00:33:11.919] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/f2b6e0fcf21c73b3089a03f0699084909d34608d"
[00:33:11.920] Popped gitiles commit info from properties and tags
[00:33:11.920] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[00:33:11.920] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[00:33:11.920] Buildbucket request:
{
"requestId": "8844674629762651712",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8844674629762651712",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "f2b6e0fcf21c73b3089a03f0699084909d34608d"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@f2b6e0fcf21c73b3089a03f0699084909d34608d",
"title": "f2b6e0fcf21c73b3089a03f0699084909d34608d",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/f2b6e0fcf21c73b3089a03f0699084909d34608d"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "f2b6e0fcf21c73b3089a03f0699084909d34608d",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8844674629762651712"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[00:33:12.481] Scheduled build:
{
"id": "8679244567877612241",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-06-12T00:33:12.014147640Z",
"updateTime": "2026-06-12T00:33:12.014147640Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "f2b6e0fcf21c73b3089a03f0699084909d34608d",
"ref": "refs/heads/main"
}
}
}
[00:33:12.481] Task URL: https://cr-buildbucket.appspot.com/build/8679244567877612241
[00:33:12.481] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:2:0) after 2m15s
[00:33:13.429] Received PubSub notification, asking Buildbucket for the build status
[00:33:13.458] Build status: SCHEDULED
[00:35:27.494] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:2:0)
[00:35:27.514] Build status: SCHEDULED
[00:35:27.514] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:4:0) after 3m15s
[00:38:42.657] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:4:0)
[00:38:42.675] Build status: SCHEDULED
[00:38:42.675] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:5:0) after 7m31s
[00:46:13.690] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:5:0)
[00:46:13.716] Build status: SCHEDULED
[00:46:13.716] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:6:0) after 8m7s
[00:54:20.782] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:6:0)
[00:54:20.862] Build status: SCHEDULED
[00:54:20.862] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:7:0) after 9m38s
[01:03:58.893] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:7:0)
[01:03:58.913] Build status: SCHEDULED
[01:03:58.913] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:8:0) after 5m48s
[01:09:46.927] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:8:0)
[01:09:46.974] Build status: SCHEDULED
[01:09:46.974] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:9:0) after 2m4s
[01:11:51.065] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:9:0)
[01:11:51.082] Build status: SCHEDULED
[01:11:51.082] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:10:0) after 7m50s
[01:19:41.097] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:10:0)
[01:19:41.145] Build status: SCHEDULED
[01:19:41.145] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:11:0) after 9m37s
[01:29:18.289] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:11:0)
[01:29:18.317] Build status: SCHEDULED
[01:29:18.317] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:12:0) after 3m4s
[01:32:22.321] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:12:0)
[01:32:22.340] Build status: SCHEDULED
[01:32:22.340] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:13:0) after 6m23s
[01:38:45.418] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:13:0)
[01:38:45.458] Build status: SCHEDULED
[01:38:45.458] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:14:0) after 8m11s
[01:46:56.523] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:14:0)
[01:46:56.542] Build status: SCHEDULED
[01:46:56.543] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:15:0) after 5m23s
[01:52:19.594] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:15:0)
[01:52:19.611] Build status: SCHEDULED
[01:52:19.611] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:16:0) after 2m26s
[01:54:45.639] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:16:0)
[01:54:45.658] Build status: SCHEDULED
[01:54:45.658] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:17:0) after 8m58s
[02:03:43.680] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:17:0)
[02:03:43.789] Build status: SCHEDULED
[02:03:43.790] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:18:0) after 3m35s
[02:07:18.986] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:18:0)
[02:07:19.019] Build status: SCHEDULED
[02:07:19.019] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:19:0) after 6m34s
[02:13:53.016] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:19:0)
[02:13:53.068] Build status: SCHEDULED
[02:13:53.068] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:20:0) after 2m43s
[02:16:36.132] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:20:0)
[02:16:36.190] Build status: SCHEDULED
[02:16:36.190] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:21:0) after 7m53s
[02:24:29.206] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:21:0)
[02:24:29.228] Build status: SCHEDULED
[02:24:29.228] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:22:0) after 3m16s
[02:27:45.266] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:22:0)
[02:27:45.285] Build status: SCHEDULED
[02:27:45.285] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:23:0) after 1m10s
[02:28:55.320] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:23:0)
[02:28:55.340] Build status: SCHEDULED
[02:28:55.340] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:24:0) after 2m47s
[02:31:42.352] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:24:0)
[02:31:42.394] Build status: SCHEDULED
[02:31:42.394] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:25:0) after 7m5s
[02:38:47.522] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:25:0)
[02:38:47.641] Build status: SCHEDULED
[02:38:47.641] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:26:0) after 9m45s
[02:48:32.660] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:26:0)
[02:48:32.678] Build status: SCHEDULED
[02:48:32.678] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:27:0) after 3m37s
[02:52:09.691] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:27:0)
[02:52:09.874] Build status: SCHEDULED
[02:52:09.874] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:28:0) after 7m29s
[02:59:38.890] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:28:0)
[02:59:38.907] Build status: SCHEDULED
[02:59:38.907] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:29:0) after 8m38s
[03:08:16.967] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:29:0)
[03:08:16.984] Build status: SCHEDULED
[03:08:16.984] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:30:0) after 6m56s
[03:15:13.003] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:30:0)
[03:15:13.172] Build status: SCHEDULED
[03:15:13.172] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:31:0) after 9m28s
[03:24:41.231] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:31:0)
[03:24:41.252] Build status: SCHEDULED
[03:24:41.252] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:32:0) after 8m12s
[03:32:53.268] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:32:0)
[03:32:53.296] Build status: SCHEDULED
[03:32:53.296] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:33:0) after 9m51s
[03:40:48.722] Received PubSub notification, asking Buildbucket for the build status
[03:40:48.745] Build status: STARTED
[03:42:44.713] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:33:0)
[03:42:44.752] Build status: STARTED
[03:42:44.752] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8844674629762651712:35:0) after 6m54s
[03:44:27.018] Received PubSub notification, asking Buildbucket for the build status
[03:44:27.039] Build:
{
"id": "8679244567877612241",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-06-12T00:33:12.014147640Z",
"startTime": "2026-06-12T03:40:45.479790580Z",
"endTime": "2026-06-12T03:44:12.998340159Z",
"updateTime": "2026-06-12T03:44:12.998340159Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "f2b6e0fcf21c73b3089a03f0699084909d34608d",
"ref": "refs/heads/main"
}
}
}
[03:44:27.039] Invocation finished in 3h11m16.205648433s with status FAILED