[23:49:14.346] New invocation is queued and will start shortly
[23:49:15.396] Starting the invocation (attempt 1)
[23:49:15.419] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/243a329868f2c0bae9235255f25029e5f1c12013"
[23:49:15.420] Popped gitiles commit info from properties and tags
[23:49:15.420] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[23:49:15.420] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[23:49:15.420] Buildbucket request:
{
"requestId": "8877201705272231264",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-ubsan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8877201705272231264",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "e0ff413c3c4297b3efc86a326bd7abf5f97e3c07"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@e0ff413c3c4297b3efc86a326bd7abf5f97e3c07",
"title": "e0ff413c3c4297b3efc86a326bd7abf5f97e3c07",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/e0ff413c3c4297b3efc86a326bd7abf5f97e3c07"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "d214608ac080eb9211f2fe6b0729d03580e78727"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@d214608ac080eb9211f2fe6b0729d03580e78727",
"title": "d214608ac080eb9211f2fe6b0729d03580e78727",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/d214608ac080eb9211f2fe6b0729d03580e78727"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "243a329868f2c0bae9235255f25029e5f1c12013"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@243a329868f2c0bae9235255f25029e5f1c12013",
"title": "243a329868f2c0bae9235255f25029e5f1c12013",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/243a329868f2c0bae9235255f25029e5f1c12013"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "243a329868f2c0bae9235255f25029e5f1c12013",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8877201705272231264"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[23:49:15.807] Scheduled build:
{
"id": "8711771643419014033",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-ubsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-06-17T23:49:15.483340038Z",
"updateTime": "2025-06-17T23:49:15.483340038Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "243a329868f2c0bae9235255f25029e5f1c12013",
"ref": "refs/heads/main"
}
},
"canary": true
}
[23:49:15.808] Task URL: https://cr-buildbucket.appspot.com/build/8711771643419014033
[23:49:15.808] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:2:0) after 2m35s
[23:49:16.782] Received PubSub notification, asking Buildbucket for the build status
[23:49:16.816] Build status: SCHEDULED
[23:51:50.818] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:2:0)
[23:51:50.844] Build status: SCHEDULED
[23:51:50.845] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:4:0) after 1m19s
[23:53:09.979] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:4:0)
[23:53:10.009] Build status: SCHEDULED
[23:53:10.010] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:5:0) after 5m19s
[23:58:29.028] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:5:0)
[23:58:29.056] Build status: SCHEDULED
[23:58:29.056] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:6:0) after 2m14s
[00:00:43.070] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:6:0)
[00:00:43.093] Build status: SCHEDULED
[00:00:43.093] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:7:0) after 7m13s
[00:07:56.145] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:7:0)
[00:07:56.187] Build status: SCHEDULED
[00:07:56.187] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:8:0) after 7m36s
[00:15:32.188] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:8:0)
[00:15:32.217] Build status: SCHEDULED
[00:15:32.217] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:9:0) after 6m57s
[00:22:29.241] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:9:0)
[00:22:29.259] Build status: SCHEDULED
[00:22:29.259] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:10:0) after 4m22s
[00:26:51.276] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:10:0)
[00:26:51.303] Build status: SCHEDULED
[00:26:51.303] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:11:0) after 2m54s
[00:29:45.367] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:11:0)
[00:29:45.393] Build status: SCHEDULED
[00:29:45.393] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:12:0) after 3m22s
[00:33:07.575] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:12:0)
[00:33:07.607] Build status: SCHEDULED
[00:33:07.607] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:13:0) after 1m18s
[00:34:25.630] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:13:0)
[00:34:25.652] Build status: SCHEDULED
[00:34:25.652] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:14:0) after 5m41s
[00:40:06.672] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:14:0)
[00:40:06.693] Build status: SCHEDULED
[00:40:06.693] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:15:0) after 7m29s
[00:47:35.763] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:15:0)
[00:47:35.780] Build status: SCHEDULED
[00:47:35.780] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:16:0) after 3m56s
[00:51:31.822] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:16:0)
[00:51:31.855] Build status: SCHEDULED
[00:51:31.855] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:17:0) after 7m38s
[00:59:09.871] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:17:0)
[00:59:09.888] Build status: SCHEDULED
[00:59:09.888] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:18:0) after 9m49s
[01:08:58.929] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:18:0)
[01:08:59.241] Build status: SCHEDULED
[01:08:59.241] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:19:0) after 1m23s
[01:10:22.230] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:19:0)
[01:10:22.245] Build status: SCHEDULED
[01:10:22.245] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:20:0) after 4m2s
[01:14:24.233] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:20:0)
[01:14:24.249] Build status: SCHEDULED
[01:14:24.249] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:21:0) after 2m54s
[01:17:18.264] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:21:0)
[01:17:18.282] Build status: SCHEDULED
[01:17:18.282] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:22:0) after 8m44s
[01:26:02.312] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:22:0)
[01:26:02.340] Build status: SCHEDULED
[01:26:02.340] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:23:0) after 2m8s
[01:28:10.311] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:23:0)
[01:28:10.346] Build status: SCHEDULED
[01:28:10.346] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:24:0) after 1m2s
[01:29:12.368] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:24:0)
[01:29:12.387] Build status: SCHEDULED
[01:29:12.387] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:25:0) after 6m24s
[01:35:36.429] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:25:0)
[01:35:36.457] Build status: SCHEDULED
[01:35:36.457] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:26:0) after 9m58s
[01:45:34.508] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:26:0)
[01:45:34.576] Build status: SCHEDULED
[01:45:34.576] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:27:0) after 5m44s
[01:51:18.644] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:27:0)
[01:51:18.687] Build status: SCHEDULED
[01:51:18.687] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:28:0) after 1m14s
[01:52:32.710] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:28:0)
[01:52:32.729] Build status: SCHEDULED
[01:52:32.729] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:29:0) after 2m24s
[01:54:56.771] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:29:0)
[01:54:56.795] Build status: SCHEDULED
[01:54:56.795] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:30:0) after 9m26s
[02:04:22.852] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:30:0)
[02:04:22.871] Build status: SCHEDULED
[02:04:22.871] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:31:0) after 7m41s
[02:12:03.886] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:31:0)
[02:12:03.905] Build status: SCHEDULED
[02:12:03.905] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:32:0) after 2m58s
[02:15:01.899] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:32:0)
[02:15:01.916] Build status: SCHEDULED
[02:15:01.916] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:33:0) after 3m26s
[02:18:27.943] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:33:0)
[02:18:27.965] Build status: SCHEDULED
[02:18:27.965] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:34:0) after 8m43s
[02:27:11.057] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:34:0)
[02:27:11.130] Build status: SCHEDULED
[02:27:11.130] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:35:0) after 6m19s
[02:33:30.131] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:35:0)
[02:33:30.157] Build status: SCHEDULED
[02:33:30.157] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:36:0) after 9m14s
[02:42:44.280] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:36:0)
[02:42:44.299] Build status: SCHEDULED
[02:42:44.299] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:37:0) after 9m42s
[02:52:26.323] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:37:0)
[02:52:26.340] Build status: SCHEDULED
[02:52:26.340] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:38:0) after 7m35s
[03:00:01.361] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:38:0)
[03:00:01.380] Build status: SCHEDULED
[03:00:01.380] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:39:0) after 4m28s
[03:04:29.399] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:39:0)
[03:04:29.423] Build status: SCHEDULED
[03:04:29.423] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:40:0) after 8m52s
[03:13:21.430] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:40:0)
[03:13:21.446] Build status: SCHEDULED
[03:13:21.446] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:41:0) after 1m34s
[03:14:55.467] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:41:0)
[03:14:55.485] Build status: SCHEDULED
[03:14:55.485] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:42:0) after 6m6s
[03:21:01.503] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:42:0)
[03:21:01.521] Build status: SCHEDULED
[03:21:01.521] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:43:0) after 8m52s
[03:29:54.038] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:43:0)
[03:29:54.063] Build status: SCHEDULED
[03:29:54.063] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:44:0) after 7m51s
[03:37:45.256] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:44:0)
[03:37:45.279] Build status: SCHEDULED
[03:37:45.280] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:45:0) after 8m19s
[03:46:04.384] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:45:0)
[03:46:04.401] Build status: SCHEDULED
[03:46:04.401] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:46:0) after 4m47s
[03:50:51.491] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:46:0)
[03:50:51.513] Build status: SCHEDULED
[03:50:51.514] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:47:0) after 3m38s
[03:54:29.833] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:47:0)
[03:54:29.854] Build status: SCHEDULED
[03:54:29.854] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:48:0) after 6m24s
[04:00:53.974] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:48:0)
[04:00:54.049] Build status: SCHEDULED
[04:00:54.049] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:49:0) after 2m54s
[04:03:48.114] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:49:0)
[04:03:48.129] Build status: SCHEDULED
[04:03:48.129] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:50:0) after 4m33s
[04:08:21.251] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:50:0)
[04:08:21.305] Build status: SCHEDULED
[04:08:21.305] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:51:0) after 8m27s
[04:16:49.085] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:51:0)
[04:16:49.100] Build status: SCHEDULED
[04:16:49.100] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:52:0) after 3m17s
[04:20:06.096] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:52:0)
[04:20:06.114] Build status: SCHEDULED
[04:20:06.114] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:53:0) after 9m20s
[04:29:26.137] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:53:0)
[04:29:26.164] Build status: SCHEDULED
[04:29:26.164] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:54:0) after 2m7s
[04:31:33.238] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:54:0)
[04:31:33.314] Build status: SCHEDULED
[04:31:33.314] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:55:0) after 3m59s
[04:35:32.346] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:55:0)
[04:35:32.377] Build status: SCHEDULED
[04:35:32.377] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:56:0) after 2m45s
[04:38:17.404] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:56:0)
[04:38:17.424] Build status: SCHEDULED
[04:38:17.424] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:57:0) after 8m25s
[04:46:42.449] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:57:0)
[04:46:42.479] Build status: SCHEDULED
[04:46:42.479] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:58:0) after 6m47s
[04:53:29.465] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:58:0)
[04:53:29.499] Build status: SCHEDULED
[04:53:29.499] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:59:0) after 3m26s
[04:56:55.528] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:59:0)
[04:56:55.545] Build status: SCHEDULED
[04:56:55.545] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:60:0) after 7m32s
[05:04:27.541] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:60:0)
[05:04:27.559] Build status: SCHEDULED
[05:04:27.559] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:61:0) after 1m5s
[05:05:32.587] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:61:0)
[05:05:32.665] Build status: SCHEDULED
[05:05:32.665] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:62:0) after 8m34s
[05:14:06.686] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:62:0)
[05:14:06.704] Build status: SCHEDULED
[05:14:06.704] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:63:0) after 5m57s
[05:20:03.786] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:63:0)
[05:20:03.818] Build status: SCHEDULED
[05:20:03.819] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:64:0) after 5m10s
[05:25:13.944] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:64:0)
[05:25:13.964] Build status: SCHEDULED
[05:25:13.964] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:65:0) after 8m52s
[05:34:06.015] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:65:0)
[05:34:06.033] Build status: SCHEDULED
[05:34:06.033] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:66:0) after 9m1s
[05:43:07.057] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:66:0)
[05:43:07.079] Build status: SCHEDULED
[05:43:07.079] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:67:0) after 9m46s
[05:52:53.100] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:67:0)
[05:52:53.135] Build status: SCHEDULED
[05:52:53.135] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:68:0) after 5m0s
[05:57:53.153] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:68:0)
[05:57:53.172] Build status: SCHEDULED
[05:57:53.172] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:69:0) after 5m24s
[06:03:17.216] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:69:0)
[06:03:17.236] Build status: SCHEDULED
[06:03:17.236] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:70:0) after 2m33s
[06:05:50.264] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:70:0)
[06:05:50.292] Build status: SCHEDULED
[06:05:50.292] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:71:0) after 7m21s
[06:13:11.388] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:71:0)
[06:13:11.415] Build status: SCHEDULED
[06:13:11.415] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:72:0) after 9m18s
[06:22:29.487] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:72:0)
[06:22:29.507] Build status: SCHEDULED
[06:22:29.507] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:73:0) after 5m31s
[06:28:00.590] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:73:0)
[06:28:00.611] Build status: SCHEDULED
[06:28:00.612] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:74:0) after 5m56s
[06:33:56.668] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:74:0)
[06:33:56.685] Build status: SCHEDULED
[06:33:56.685] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:75:0) after 9m33s
[06:43:29.894] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:75:0)
[06:43:29.914] Build status: SCHEDULED
[06:43:29.914] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:76:0) after 6m8s
[06:49:38.006] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:76:0)
[06:49:38.038] Build status: SCHEDULED
[06:49:38.038] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:77:0) after 8m2s
[06:57:40.061] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:77:0)
[06:57:40.077] Build status: SCHEDULED
[06:57:40.077] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:78:0) after 9m8s
[07:06:48.062] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:78:0)
[07:06:48.138] Build status: SCHEDULED
[07:06:48.138] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:79:0) after 5m34s
[07:12:22.211] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:79:0)
[07:12:22.228] Build status: SCHEDULED
[07:12:22.228] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:80:0) after 9m57s
[07:22:19.247] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:80:0)
[07:22:19.270] Build status: SCHEDULED
[07:22:19.270] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:81:0) after 7m10s
[07:29:29.343] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:81:0)
[07:29:29.403] Build status: SCHEDULED
[07:29:29.403] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:82:0) after 8m45s
[07:38:14.405] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:82:0)
[07:38:14.418] Build status: SCHEDULED
[07:38:14.418] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:83:0) after 7m6s
[07:45:20.814] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:83:0)
[07:45:20.831] Build status: SCHEDULED
[07:45:20.831] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:84:0) after 2m17s
[07:47:37.853] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:84:0)
[07:47:37.922] Build status: SCHEDULED
[07:47:37.922] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8877201705272231264:85:0) after 7m44s
[07:49:15.387] Received PubSub notification, asking Buildbucket for the build status
[07:49:15.407] Build:
{
"id": "8711771643419014033",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-san-ubsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-06-17T23:49:15.483340038Z",
"endTime": "2025-06-18T07:49:15.179029288Z",
"updateTime": "2025-06-18T07:49:15.179029288Z",
"status": "INFRA_FAILURE",
"statusDetails": {
"resourceExhaustion": {
},
"timeout": {
}
},
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "243a329868f2c0bae9235255f25029e5f1c12013",
"ref": "refs/heads/main"
}
},
"canary": true
}
[07:49:15.407] Invocation finished in 8h0m1.075278374s with status FAILED