[00:19:33.239] New invocation is queued and will start shortly
[00:19:34.638] Starting the invocation (attempt 1)
[00:19:34.673] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/e9768050f21d2414979374e37f6b288cb6140215"
[00:19:34.673] Popped gitiles commit info from properties and tags
[00:19:34.673] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[00:19:34.673] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[00:19:34.674] Buildbucket request:
{
"requestId": "8840508026621741920",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-asan-fuzz"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8840508026621741920",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "a19bcbb7e3d8b15738466f053dd509ec43508e38"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@a19bcbb7e3d8b15738466f053dd509ec43508e38",
"title": "a19bcbb7e3d8b15738466f053dd509ec43508e38",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/a19bcbb7e3d8b15738466f053dd509ec43508e38"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "e7ae54b1d8fdfe6453658bf027355bce6c5b9cc3"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@e7ae54b1d8fdfe6453658bf027355bce6c5b9cc3",
"title": "e7ae54b1d8fdfe6453658bf027355bce6c5b9cc3",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/e7ae54b1d8fdfe6453658bf027355bce6c5b9cc3"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "7579337933d5bcd3669730c7bac538bc4f731cb7"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@7579337933d5bcd3669730c7bac538bc4f731cb7",
"title": "7579337933d5bcd3669730c7bac538bc4f731cb7",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/7579337933d5bcd3669730c7bac538bc4f731cb7"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "e9768050f21d2414979374e37f6b288cb6140215"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@e9768050f21d2414979374e37f6b288cb6140215",
"title": "e9768050f21d2414979374e37f6b288cb6140215",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/e9768050f21d2414979374e37f6b288cb6140215"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "e9768050f21d2414979374e37f6b288cb6140215",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8840508026621741920"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[00:19:35.602] Scheduled build:
{
"id": "8675077964316734033",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-asan-fuzz"
},
"createdBy": "project:pigweed",
"createTime": "2026-07-28T00:19:34.817555994Z",
"updateTime": "2026-07-28T00:19:34.817555994Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "e9768050f21d2414979374e37f6b288cb6140215",
"ref": "refs/heads/main"
}
}
}
[00:19:35.602] Task URL: https://cr-buildbucket.appspot.com/build/8675077964316734033
[00:19:35.603] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:2:0) after 5m3s
[00:19:36.450] Received PubSub notification, asking Buildbucket for the build status
[00:19:36.500] Build status: SCHEDULED
[00:24:38.649] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:2:0)
[00:24:38.751] Build status: SCHEDULED
[00:24:38.751] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:4:0) after 6m11s
[00:30:49.749] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:4:0)
[00:30:49.767] Build status: SCHEDULED
[00:30:49.767] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:5:0) after 6m2s
[00:36:51.760] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:5:0)
[00:36:51.782] Build status: SCHEDULED
[00:36:51.782] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:6:0) after 7m8s
[00:43:59.800] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:6:0)
[00:43:59.855] Build status: SCHEDULED
[00:43:59.855] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:7:0) after 4m9s
[00:48:08.822] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:7:0)
[00:48:08.844] Build status: SCHEDULED
[00:48:08.844] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:8:0) after 8m23s
[00:56:31.858] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:8:0)
[00:56:31.880] Build status: SCHEDULED
[00:56:31.880] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:9:0) after 1m16s
[00:57:47.910] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:9:0)
[00:57:47.926] Build status: SCHEDULED
[00:57:47.926] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:10:0) after 4m3s
[01:01:51.037] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:10:0)
[01:01:51.056] Build status: SCHEDULED
[01:01:51.056] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:11:0) after 2m47s
[01:04:38.070] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:11:0)
[01:04:38.089] Build status: SCHEDULED
[01:04:38.089] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:12:0) after 1m4s
[01:05:42.092] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:12:0)
[01:05:42.179] Build status: SCHEDULED
[01:05:42.179] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:13:0) after 4m30s
[01:10:12.221] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:13:0)
[01:10:12.243] Build status: SCHEDULED
[01:10:12.243] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:14:0) after 7m35s
[01:17:47.366] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:14:0)
[01:17:47.385] Build status: SCHEDULED
[01:17:47.385] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:15:0) after 8m23s
[01:26:10.432] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:15:0)
[01:26:10.451] Build status: SCHEDULED
[01:26:10.451] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:16:0) after 2m43s
[01:28:53.570] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:16:0)
[01:28:53.589] Build status: SCHEDULED
[01:28:53.589] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:17:0) after 5m29s
[01:34:22.602] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:17:0)
[01:34:22.728] Build status: SCHEDULED
[01:34:22.728] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:18:0) after 4m23s
[01:38:45.743] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:18:0)
[01:38:45.764] Build status: SCHEDULED
[01:38:45.764] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:19:0) after 6m18s
[01:45:03.797] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:19:0)
[01:45:03.980] Build status: SCHEDULED
[01:45:03.981] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:20:0) after 1m16s
[01:46:20.015] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:20:0)
[01:46:20.038] Build status: SCHEDULED
[01:46:20.038] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:21:0) after 4m22s
[01:50:42.051] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:21:0)
[01:50:42.069] Build status: SCHEDULED
[01:50:42.069] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:22:0) after 9m58s
[02:00:40.085] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:22:0)
[02:00:40.107] Build status: SCHEDULED
[02:00:40.107] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:23:0) after 6m7s
[02:06:47.123] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:23:0)
[02:06:47.145] Build status: SCHEDULED
[02:06:47.145] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:24:0) after 6m38s
[02:13:25.168] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:24:0)
[02:13:25.257] Build status: SCHEDULED
[02:13:25.257] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:25:0) after 7m34s
[02:20:59.280] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:25:0)
[02:20:59.303] Build status: SCHEDULED
[02:20:59.304] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:26:0) after 3m57s
[02:24:56.397] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:26:0)
[02:24:56.430] Build status: SCHEDULED
[02:24:56.430] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:27:0) after 2m20s
[02:27:16.541] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:27:0)
[02:27:16.559] Build status: SCHEDULED
[02:27:16.559] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:28:0) after 2m45s
[02:30:01.523] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:28:0)
[02:30:01.542] Build status: SCHEDULED
[02:30:01.542] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:29:0) after 8m10s
[02:38:11.616] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:29:0)
[02:38:11.639] Build status: SCHEDULED
[02:38:11.639] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:30:0) after 6m55s
[02:45:06.802] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:30:0)
[02:45:06.819] Build status: SCHEDULED
[02:45:06.819] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:31:0) after 9m48s
[02:54:54.842] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:31:0)
[02:54:54.862] Build status: SCHEDULED
[02:54:54.862] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:32:0) after 4m0s
[02:58:54.881] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:32:0)
[02:58:54.964] Build status: SCHEDULED
[02:58:54.964] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:33:0) after 8m22s
[03:07:16.997] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:33:0)
[03:07:17.015] Build status: SCHEDULED
[03:07:17.015] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:34:0) after 3m8s
[03:10:24.983] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:34:0)
[03:10:25.015] Build status: SCHEDULED
[03:10:25.015] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:35:0) after 5m3s
[03:15:28.154] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:35:0)
[03:15:28.177] Build status: SCHEDULED
[03:15:28.178] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:36:0) after 3m8s
[03:18:36.209] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:36:0)
[03:18:36.232] Build status: SCHEDULED
[03:18:36.232] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:37:0) after 5m53s
[03:24:29.245] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:37:0)
[03:24:29.537] Build status: SCHEDULED
[03:24:29.537] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:38:0) after 7m5s
[03:31:34.616] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:38:0)
[03:31:34.702] Build status: SCHEDULED
[03:31:34.702] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:39:0) after 5m54s
[03:33:49.944] Received PubSub notification, asking Buildbucket for the build status
[03:33:49.962] Build status: STARTED
[03:37:28.820] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:39:0)
[03:37:28.882] Build status: STARTED
[03:37:28.882] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-asan-fuzz:8840508026621741920:41:0) after 8m6s
[03:41:55.327] Received PubSub notification, asking Buildbucket for the build status
[03:41:55.391] Build:
{
"id": "8675077964316734033",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-asan-fuzz"
},
"createdBy": "project:pigweed",
"createTime": "2026-07-28T00:19:34.817555994Z",
"startTime": "2026-07-28T03:33:49.773624751Z",
"endTime": "2026-07-28T03:41:54.270427776Z",
"updateTime": "2026-07-28T03:41:54.270427776Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "e9768050f21d2414979374e37f6b288cb6140215",
"ref": "refs/heads/main"
}
}
}
[03:41:55.391] Invocation finished in 3h22m22.156239707s with status FAILED