[01:19:14.291] New invocation is queued and will start shortly
[01:19:15.479] Starting the invocation (attempt 1)
[01:19:15.509] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/90c692ee80eb6466c10eac5df6cdb0685b67a430"
[01:19:15.509] Popped gitiles commit info from properties and tags
[01:19:15.509] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[01:19:15.509] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[01:19:15.509] Buildbucket request:
{
"requestId": "8868317540301637776",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-asan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8868317540301637776",
"job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "036f3b3596ee2b1bfb79daf5cf576e90cbd8d264"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@036f3b3596ee2b1bfb79daf5cf576e90cbd8d264",
"title": "036f3b3596ee2b1bfb79daf5cf576e90cbd8d264",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/036f3b3596ee2b1bfb79daf5cf576e90cbd8d264"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "7aae54cae811bee703e8418779278e531087b562"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@7aae54cae811bee703e8418779278e531087b562",
"title": "7aae54cae811bee703e8418779278e531087b562",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/7aae54cae811bee703e8418779278e531087b562"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "d35044a3c1063013c7683ff35c8723beedbdfe1a"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@d35044a3c1063013c7683ff35c8723beedbdfe1a",
"title": "d35044a3c1063013c7683ff35c8723beedbdfe1a",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/d35044a3c1063013c7683ff35c8723beedbdfe1a"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "666bd39c0ecdfbd7ab5568c6b62d6894b587ea6b"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@666bd39c0ecdfbd7ab5568c6b62d6894b587ea6b",
"title": "666bd39c0ecdfbd7ab5568c6b62d6894b587ea6b",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/666bd39c0ecdfbd7ab5568c6b62d6894b587ea6b"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "afda10d63370766a8fbc8fcff9254f1ae5be555b"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@afda10d63370766a8fbc8fcff9254f1ae5be555b",
"title": "afda10d63370766a8fbc8fcff9254f1ae5be555b",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/afda10d63370766a8fbc8fcff9254f1ae5be555b"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "90c692ee80eb6466c10eac5df6cdb0685b67a430"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@90c692ee80eb6466c10eac5df6cdb0685b67a430",
"title": "90c692ee80eb6466c10eac5df6cdb0685b67a430",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/90c692ee80eb6466c10eac5df6cdb0685b67a430"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "90c692ee80eb6466c10eac5df6cdb0685b67a430",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8868317540301637776"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[01:19:15.830] Scheduled build:
{
"id": "8702887478321785297",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-09-24T01:19:15.558741006Z",
"updateTime": "2025-09-24T01:19:15.558741006Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "90c692ee80eb6466c10eac5df6cdb0685b67a430",
"ref": "refs/heads/main"
}
}
}
[01:19:15.830] Task URL: https://cr-buildbucket.appspot.com/build/8702887478321785297
[01:19:15.830] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:2:0) after 4m23s
[01:19:16.801] Received PubSub notification, asking Buildbucket for the build status
[01:19:16.818] Build status: SCHEDULED
[01:23:38.844] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:2:0)
[01:23:38.863] Build status: SCHEDULED
[01:23:38.863] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:4:0) after 2m0s
[01:25:39.020] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:4:0)
[01:25:39.102] Build status: SCHEDULED
[01:25:39.102] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:5:0) after 6m42s
[01:32:21.120] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:5:0)
[01:32:21.181] Build status: SCHEDULED
[01:32:21.181] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:6:0) after 8m36s
[01:40:57.233] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:6:0)
[01:40:57.256] Build status: SCHEDULED
[01:40:57.256] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:7:0) after 8m2s
[01:48:59.319] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:7:0)
[01:48:59.347] Build status: SCHEDULED
[01:48:59.347] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:8:0) after 1m35s
[01:50:34.339] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:8:0)
[01:50:34.426] Build status: SCHEDULED
[01:50:34.426] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:9:0) after 6m41s
[01:57:15.440] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:9:0)
[01:57:15.457] Build status: SCHEDULED
[01:57:15.457] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:10:0) after 7m22s
[02:04:37.537] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:10:0)
[02:04:37.561] Build status: SCHEDULED
[02:04:37.561] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:11:0) after 4m27s
[02:09:04.589] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:11:0)
[02:09:04.738] Build status: SCHEDULED
[02:09:04.738] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:12:0) after 3m54s
[02:12:58.711] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:12:0)
[02:12:58.730] Build status: SCHEDULED
[02:12:58.730] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:13:0) after 2m23s
[02:15:21.713] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:13:0)
[02:15:21.736] Build status: SCHEDULED
[02:15:21.736] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:14:0) after 2m59s
[02:18:20.753] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:14:0)
[02:18:20.768] Build status: SCHEDULED
[02:18:20.769] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:15:0) after 2m10s
[02:20:30.785] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:15:0)
[02:20:30.804] Build status: SCHEDULED
[02:20:30.804] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:16:0) after 5m49s
[02:26:19.818] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:16:0)
[02:26:19.835] Build status: SCHEDULED
[02:26:19.835] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:17:0) after 3m53s
[02:30:12.852] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:17:0)
[02:30:12.869] Build status: SCHEDULED
[02:30:12.869] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:18:0) after 7m50s
[02:38:02.883] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:18:0)
[02:38:02.918] Build status: SCHEDULED
[02:38:02.918] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:19:0) after 7m37s
[02:45:39.935] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:19:0)
[02:45:39.964] Build status: SCHEDULED
[02:45:39.964] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:20:0) after 8m36s
[02:54:15.981] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:20:0)
[02:54:16.000] Build status: SCHEDULED
[02:54:16.000] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:21:0) after 2m8s
[02:56:24.037] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:21:0)
[02:56:24.061] Build status: SCHEDULED
[02:56:24.061] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:22:0) after 5m27s
[03:01:51.075] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:22:0)
[03:01:51.097] Build status: SCHEDULED
[03:01:51.097] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:23:0) after 9m0s
[03:10:51.163] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:23:0)
[03:10:51.225] Build status: SCHEDULED
[03:10:51.225] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:24:0) after 4m34s
[03:15:25.237] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:24:0)
[03:15:25.261] Build status: SCHEDULED
[03:15:25.261] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:25:0) after 9m52s
[03:25:17.283] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:25:0)
[03:25:17.307] Build status: SCHEDULED
[03:25:17.307] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:26:0) after 6m47s
[03:32:04.315] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:26:0)
[03:32:04.374] Build status: SCHEDULED
[03:32:04.374] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:27:0) after 8m31s
[03:40:35.425] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:27:0)
[03:40:35.490] Build status: SCHEDULED
[03:40:35.490] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:28:0) after 3m36s
[03:44:11.532] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:28:0)
[03:44:11.552] Build status: SCHEDULED
[03:44:11.552] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:29:0) after 5m34s
[03:49:45.665] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:29:0)
[03:49:45.685] Build status: SCHEDULED
[03:49:45.685] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:30:0) after 2m52s
[03:52:37.657] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:30:0)
[03:52:37.672] Build status: SCHEDULED
[03:52:37.672] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:31:0) after 4m36s
[03:57:13.696] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:31:0)
[03:57:13.715] Build status: SCHEDULED
[03:57:13.715] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:32:0) after 8m59s
[04:06:12.780] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:32:0)
[04:06:12.799] Build status: SCHEDULED
[04:06:12.799] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:33:0) after 1m39s
[04:07:51.875] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:33:0)
[04:07:51.894] Build status: SCHEDULED
[04:07:51.894] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:34:0) after 8m53s
[04:16:44.957] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:34:0)
[04:16:44.997] Build status: SCHEDULED
[04:16:44.997] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:35:0) after 7m2s
[04:23:46.978] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:35:0)
[04:23:47.001] Build status: SCHEDULED
[04:23:47.001] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:36:0) after 4m31s
[04:28:18.096] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:36:0)
[04:28:18.112] Build status: SCHEDULED
[04:28:18.112] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:37:0) after 5m58s
[04:34:16.129] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:37:0)
[04:34:16.286] Build status: SCHEDULED
[04:34:16.286] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:38:0) after 9m24s
[04:43:40.308] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:38:0)
[04:43:40.327] Build status: SCHEDULED
[04:43:40.328] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:39:0) after 7m43s
[04:51:23.369] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:39:0)
[04:51:23.391] Build status: SCHEDULED
[04:51:23.391] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:40:0) after 6m0s
[04:57:23.414] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:40:0)
[04:57:23.473] Build status: SCHEDULED
[04:57:23.473] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-asan:8868317540301637776:41:0) after 8m10s
[04:59:42.451] Received PubSub notification, asking Buildbucket for the build status
[04:59:42.469] Build status: STARTED
[05:02:22.405] Received PubSub notification, asking Buildbucket for the build status
[05:02:22.418] Build:
{
"id": "8702887478321785297",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-09-24T01:19:15.558741006Z",
"startTime": "2025-09-24T04:59:41.783094789Z",
"endTime": "2025-09-24T05:02:22.165800312Z",
"updateTime": "2025-09-24T05:02:22.165800312Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "90c692ee80eb6466c10eac5df6cdb0685b67a430",
"ref": "refs/heads/main"
}
}
}
[05:02:22.418] Invocation finished in 3h43m8.131078818s with status FAILED