[23:34:49.104] New invocation is queued and will start shortly
[23:34:50.185] Starting the invocation (attempt 1)
[23:34:50.215] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/c22e3b35dfa22d263d25e2113db2914e32ec587b"
[23:34:50.215] Popped gitiles commit info from properties and tags
[23:34:50.215] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[23:34:50.216] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[23:34:50.216] Buildbucket request:
{
"requestId": "8860532770739265440",
"builder": {
"project": "pigweed",
"bucket": "pigweed.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8860532770739265440",
"job": "pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "59a42dd81261e69b2c6afa501059b7f0c8eee202"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@59a42dd81261e69b2c6afa501059b7f0c8eee202",
"title": "59a42dd81261e69b2c6afa501059b7f0c8eee202",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/59a42dd81261e69b2c6afa501059b7f0c8eee202"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "9607ad1ad29693a60c1b0554177626a3ed9201e6"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@9607ad1ad29693a60c1b0554177626a3ed9201e6",
"title": "9607ad1ad29693a60c1b0554177626a3ed9201e6",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/9607ad1ad29693a60c1b0554177626a3ed9201e6"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "695d944ab3c661e740b7ec24d3e8999f84466160"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@695d944ab3c661e740b7ec24d3e8999f84466160",
"title": "695d944ab3c661e740b7ec24d3e8999f84466160",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/695d944ab3c661e740b7ec24d3e8999f84466160"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "c22e3b35dfa22d263d25e2113db2914e32ec587b"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@c22e3b35dfa22d263d25e2113db2914e32ec587b",
"title": "c22e3b35dfa22d263d25e2113db2914e32ec587b",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/c22e3b35dfa22d263d25e2113db2914e32ec587b"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "c22e3b35dfa22d263d25e2113db2914e32ec587b",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8860532770739265440"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[23:34:50.556] Scheduled build:
{
"id": "8695102708831014129",
"builder": {
"project": "pigweed",
"bucket": "pigweed.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-12-18T23:34:50.281421169Z",
"updateTime": "2025-12-18T23:34:50.281421169Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "c22e3b35dfa22d263d25e2113db2914e32ec587b",
"ref": "refs/heads/main"
}
}
}
[23:34:50.556] Task URL: https://cr-buildbucket.appspot.com/build/8695102708831014129
[23:34:50.556] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:2:0) after 1m21s
[23:34:55.629] Received PubSub notification, asking Buildbucket for the build status
[23:34:55.659] Build status: SCHEDULED
[23:36:11.616] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:2:0)
[23:36:11.659] Build status: SCHEDULED
[23:36:11.659] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:4:0) after 1m31s
[23:37:42.671] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:4:0)
[23:37:42.686] Build status: SCHEDULED
[23:37:42.686] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:5:0) after 1m37s
[23:39:19.824] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:5:0)
[23:39:19.846] Build status: SCHEDULED
[23:39:19.846] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:6:0) after 4m6s
[23:43:25.910] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:6:0)
[23:43:25.924] Build status: SCHEDULED
[23:43:25.924] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:7:0) after 2m29s
[23:45:54.986] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:7:0)
[23:45:55.011] Build status: SCHEDULED
[23:45:55.011] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:8:0) after 5m30s
[23:51:25.026] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:8:0)
[23:51:25.098] Build status: SCHEDULED
[23:51:25.098] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:9:0) after 7m0s
[23:58:25.187] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:9:0)
[23:58:25.249] Build status: SCHEDULED
[23:58:25.249] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:10:0) after 1m38s
[00:00:03.283] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:10:0)
[00:00:03.309] Build status: SCHEDULED
[00:00:03.309] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:11:0) after 8m40s
[00:08:43.372] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:11:0)
[00:08:43.386] Build status: SCHEDULED
[00:08:43.386] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:12:0) after 3m39s
[00:12:22.448] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:12:0)
[00:12:22.508] Build status: SCHEDULED
[00:12:22.508] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:13:0) after 9m23s
[00:21:45.528] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:13:0)
[00:21:45.570] Build status: SCHEDULED
[00:21:45.570] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:14:0) after 4m16s
[00:26:01.585] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:14:0)
[00:26:01.605] Build status: SCHEDULED
[00:26:01.605] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:15:0) after 1m33s
[00:27:34.754] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:15:0)
[00:27:34.780] Build status: SCHEDULED
[00:27:34.780] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:16:0) after 3m53s
[00:31:27.755] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:16:0)
[00:31:27.775] Build status: SCHEDULED
[00:31:27.775] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:17:0) after 4m24s
[00:35:51.779] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:17:0)
[00:35:51.793] Build status: SCHEDULED
[00:35:51.793] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:18:0) after 7m8s
[00:42:59.822] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:18:0)
[00:42:59.873] Build status: SCHEDULED
[00:42:59.873] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:19:0) after 7m18s
[00:50:17.888] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:19:0)
[00:50:17.903] Build status: SCHEDULED
[00:50:17.903] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:20:0) after 6m45s
[00:57:02.991] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:20:0)
[00:57:03.008] Build status: SCHEDULED
[00:57:03.008] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:21:0) after 9m57s
[01:07:00.045] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:21:0)
[01:07:00.058] Build status: SCHEDULED
[01:07:00.058] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:22:0) after 2m16s
[01:09:16.070] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:22:0)
[01:09:16.087] Build status: SCHEDULED
[01:09:16.087] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:23:0) after 8m34s
[01:17:50.139] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:23:0)
[01:17:50.158] Build status: SCHEDULED
[01:17:50.158] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:24:0) after 6m35s
[01:24:25.265] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:24:0)
[01:24:25.286] Build status: SCHEDULED
[01:24:25.286] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:25:0) after 4m22s
[01:28:47.289] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:25:0)
[01:28:47.302] Build status: SCHEDULED
[01:28:47.302] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:26:0) after 5m54s
[01:34:41.364] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:26:0)
[01:34:41.382] Build status: SCHEDULED
[01:34:41.382] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:27:0) after 3m36s
[01:37:16.445] Received PubSub notification, asking Buildbucket for the build status
[01:37:16.460] Build status: STARTED
[01:38:17.447] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:27:0)
[01:38:17.484] Build status: STARTED
[01:38:17.484] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:29:0) after 9m59s
[01:48:16.501] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:29:0)
[01:48:16.521] Build status: STARTED
[01:48:16.522] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:30:0) after 4m58s
[01:53:14.542] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:30:0)
[01:53:14.562] Build status: STARTED
[01:53:14.562] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:31:0) after 6m15s
[01:59:29.547] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:31:0)
[01:59:29.595] Build status: STARTED
[01:59:29.595] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:32:0) after 5m16s
[02:04:45.611] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:32:0)
[02:04:45.650] Build status: STARTED
[02:04:45.650] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:33:0) after 6m57s
[02:11:42.663] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:33:0)
[02:11:42.736] Build status: STARTED
[02:11:42.736] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:34:0) after 5m49s
[02:17:31.752] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:34:0)
[02:17:31.788] Build status: STARTED
[02:17:31.788] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:35:0) after 6m50s
[02:24:21.876] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:35:0)
[02:24:21.904] Build status: STARTED
[02:24:21.905] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.ci-pigweed-mac-x86-bazel-sanitizers-asan:8860532770739265440:36:0) after 7m50s
[02:24:41.119] Received PubSub notification, asking Buildbucket for the build status
[02:24:41.132] Build:
{
"id": "8695102708831014129",
"builder": {
"project": "pigweed",
"bucket": "pigweed.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-12-18T23:34:50.281421169Z",
"startTime": "2025-12-19T01:37:16.299330774Z",
"endTime": "2025-12-19T02:24:40.774803804Z",
"updateTime": "2025-12-19T02:24:40.774803804Z",
"status": "INFRA_FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "c22e3b35dfa22d263d25e2113db2914e32ec587b",
"ref": "refs/heads/main"
}
}
}
[02:24:41.132] Invocation finished in 2h49m52.053087279s with status FAILED