[21:10:21.001] New invocation is queued and will start shortly
[21:10:22.544] Starting the invocation (attempt 1)
[21:10:22.567] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96"
[21:10:22.567] Popped gitiles commit info from properties and tags
[21:10:22.567] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[21:10:22.567] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[21:10:22.567] Buildbucket request:
{
"requestId": "8881469759122004400",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-san-asan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8881469759122004400",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "aac8b0e10b7c5b83ba1e98a059836364d733bff8"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@aac8b0e10b7c5b83ba1e98a059836364d733bff8",
"title": "aac8b0e10b7c5b83ba1e98a059836364d733bff8",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/aac8b0e10b7c5b83ba1e98a059836364d733bff8"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "7a7eaa8afb31013d8faaed44b523abcaa7eeb0e6"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@7a7eaa8afb31013d8faaed44b523abcaa7eeb0e6",
"title": "7a7eaa8afb31013d8faaed44b523abcaa7eeb0e6",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/7a7eaa8afb31013d8faaed44b523abcaa7eeb0e6"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "050cd228203dacc5609a63e5a951a6157d1ac494"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@050cd228203dacc5609a63e5a951a6157d1ac494",
"title": "050cd228203dacc5609a63e5a951a6157d1ac494",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/050cd228203dacc5609a63e5a951a6157d1ac494"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96",
"title": "ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8881469759122004400"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[21:10:22.969] Scheduled build:
{
"id": "8716039696768037937",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-san-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-05-01T21:10:22.623893383Z",
"updateTime": "2025-05-01T21:10:22.623893383Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96",
"ref": "refs/heads/main"
}
}
}
[21:10:22.969] Task URL: https://cr-buildbucket.appspot.com/build/8716039696768037937
[21:10:22.969] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:2:0) after 8m17s
[21:10:23.071] Received PubSub notification, asking Buildbucket for the build status
[21:10:23.096] Build status: SCHEDULED
[21:18:40.278] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:2:0)
[21:18:40.303] Build status: SCHEDULED
[21:18:40.303] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:4:0) after 6m4s
[21:24:44.303] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:4:0)
[21:24:44.326] Build status: SCHEDULED
[21:24:44.326] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:5:0) after 6m43s
[21:31:27.399] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:5:0)
[21:31:27.427] Build status: SCHEDULED
[21:31:27.427] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:6:0) after 9m39s
[21:41:06.507] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:6:0)
[21:41:06.572] Build status: SCHEDULED
[21:41:06.572] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:7:0) after 6m54s
[21:48:00.617] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:7:0)
[21:48:00.657] Build status: SCHEDULED
[21:48:00.657] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:8:0) after 1m35s
[21:49:35.736] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:8:0)
[21:49:35.755] Build status: SCHEDULED
[21:49:35.755] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:9:0) after 4m48s
[21:54:23.783] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:9:0)
[21:54:23.801] Build status: SCHEDULED
[21:54:23.801] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:10:0) after 3m10s
[21:57:33.856] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:10:0)
[21:57:33.880] Build status: SCHEDULED
[21:57:33.880] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:11:0) after 1m4s
[21:58:37.954] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:11:0)
[21:58:37.983] Build status: SCHEDULED
[21:58:37.983] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:12:0) after 1m21s
[21:59:59.003] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:12:0)
[21:59:59.103] Build status: SCHEDULED
[21:59:59.103] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:13:0) after 6m47s
[22:06:46.197] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:13:0)
[22:06:46.213] Build status: SCHEDULED
[22:06:46.213] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:14:0) after 8m47s
[22:15:33.271] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:14:0)
[22:15:33.289] Build status: SCHEDULED
[22:15:33.289] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:15:0) after 9m15s
[22:24:48.322] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:15:0)
[22:24:48.342] Build status: SCHEDULED
[22:24:48.342] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:16:0) after 6m18s
[22:31:06.370] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:16:0)
[22:31:06.389] Build status: SCHEDULED
[22:31:06.390] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:17:0) after 7m24s
[22:38:30.412] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:17:0)
[22:38:30.433] Build status: SCHEDULED
[22:38:30.433] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:18:0) after 9m29s
[22:47:59.462] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:18:0)
[22:47:59.482] Build status: SCHEDULED
[22:47:59.482] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:19:0) after 1m46s
[22:49:45.546] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:19:0)
[22:49:45.562] Build status: SCHEDULED
[22:49:45.562] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:20:0) after 9m27s
[22:59:12.582] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:20:0)
[22:59:12.648] Build status: SCHEDULED
[22:59:12.648] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:21:0) after 6m1s
[23:05:13.691] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:21:0)
[23:05:13.707] Build status: SCHEDULED
[23:05:13.707] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:22:0) after 8m29s
[23:13:42.850] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:22:0)
[23:13:42.864] Build status: SCHEDULED
[23:13:42.864] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:23:0) after 4m46s
[23:18:28.953] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:23:0)
[23:18:28.980] Build status: SCHEDULED
[23:18:28.980] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:24:0) after 7m19s
[23:25:48.002] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:24:0)
[23:25:48.017] Build status: SCHEDULED
[23:25:48.017] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:25:0) after 5m26s
[23:31:14.084] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:25:0)
[23:31:14.100] Build status: SCHEDULED
[23:31:14.100] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:26:0) after 9m34s
[23:40:48.123] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:26:0)
[23:40:48.142] Build status: SCHEDULED
[23:40:48.142] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:27:0) after 2m1s
[23:42:49.146] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:27:0)
[23:42:49.165] Build status: SCHEDULED
[23:42:49.165] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:28:0) after 4m21s
[23:47:10.197] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:28:0)
[23:47:10.216] Build status: SCHEDULED
[23:47:10.216] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:29:0) after 6m40s
[23:53:50.290] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:29:0)
[23:53:50.306] Build status: SCHEDULED
[23:53:50.306] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:30:0) after 2m40s
[23:56:30.315] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:30:0)
[23:56:30.396] Build status: SCHEDULED
[23:56:30.396] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:31:0) after 2m5s
[23:58:35.504] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:31:0)
[23:58:35.525] Build status: SCHEDULED
[23:58:35.525] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:32:0) after 9m47s
[00:08:22.612] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:32:0)
[00:08:22.630] Build status: SCHEDULED
[00:08:22.630] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:33:0) after 2m43s
[00:11:05.651] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:33:0)
[00:11:05.667] Build status: SCHEDULED
[00:11:05.667] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:34:0) after 1m34s
[00:12:39.647] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:34:0)
[00:12:39.699] Build status: SCHEDULED
[00:12:39.699] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:35:0) after 9m41s
[00:22:20.716] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:35:0)
[00:22:20.735] Build status: SCHEDULED
[00:22:20.735] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-asan:8881469759122004400:36:0) after 6m53s
[00:25:51.516] Received PubSub notification, asking Buildbucket for the build status
[00:25:51.538] Build:
{
"id": "8716039696768037937",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-san-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-05-01T21:10:22.623893383Z",
"endTime": "2025-05-02T00:25:51.004780218Z",
"updateTime": "2025-05-02T00:25:51.004780218Z",
"status": "INFRA_FAILURE",
"statusDetails": {
"resourceExhaustion": {
}
},
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "ce8bfdf9aa30cc6d2c6122f96c9f61d7f1c58b96",
"ref": "refs/heads/main"
}
}
}
[00:25:51.538] Invocation finished in 3h15m30.543296121s with status FAILED