[22:34:18.914] New invocation is queued and will start shortly
[22:34:20.129] Starting the invocation (attempt 1)
[22:34:20.156] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/dda2b3579e6284742f46ffdfbe2e88e1e1ded646"
[22:34:20.156] Popped gitiles commit info from properties and tags
[22:34:20.156] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[22:34:20.156] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[22:34:20.157] Buildbucket request:
{
"requestId": "8863254486240989392",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8863254486240989392",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "98784853e4109173b58f9c61474828fab46b9134"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@98784853e4109173b58f9c61474828fab46b9134",
"title": "98784853e4109173b58f9c61474828fab46b9134",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/98784853e4109173b58f9c61474828fab46b9134"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "867b674b2d35419a484873d53acfb51ab858dea5"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@867b674b2d35419a484873d53acfb51ab858dea5",
"title": "867b674b2d35419a484873d53acfb51ab858dea5",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/867b674b2d35419a484873d53acfb51ab858dea5"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "94ba316553c829e1f0d6fdb11275cf770ea386da"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@94ba316553c829e1f0d6fdb11275cf770ea386da",
"title": "94ba316553c829e1f0d6fdb11275cf770ea386da",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/94ba316553c829e1f0d6fdb11275cf770ea386da"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "bd8b6564f74043eecd3ea797ac6d1a1f3b94f271"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@bd8b6564f74043eecd3ea797ac6d1a1f3b94f271",
"title": "bd8b6564f74043eecd3ea797ac6d1a1f3b94f271",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/bd8b6564f74043eecd3ea797ac6d1a1f3b94f271"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "9ed30c782d6d46608c82da875a658790c80d193f"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@9ed30c782d6d46608c82da875a658790c80d193f",
"title": "9ed30c782d6d46608c82da875a658790c80d193f",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/9ed30c782d6d46608c82da875a658790c80d193f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "dda2b3579e6284742f46ffdfbe2e88e1e1ded646"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@dda2b3579e6284742f46ffdfbe2e88e1e1ded646",
"title": "dda2b3579e6284742f46ffdfbe2e88e1e1ded646",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/dda2b3579e6284742f46ffdfbe2e88e1e1ded646"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "dda2b3579e6284742f46ffdfbe2e88e1e1ded646",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8863254486240989392"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[22:34:21.028] Scheduled build:
{
"id": "8697824424139629841",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-11-18T22:34:20.295097820Z",
"updateTime": "2025-11-18T22:34:20.295097820Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "dda2b3579e6284742f46ffdfbe2e88e1e1ded646",
"ref": "refs/heads/main"
}
}
}
[22:34:21.028] Task URL: https://cr-buildbucket.appspot.com/build/8697824424139629841
[22:34:21.028] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:2:0) after 7m37s
[22:34:23.630] Received PubSub notification, asking Buildbucket for the build status
[22:34:23.663] Build status: SCHEDULED
[22:41:58.088] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:2:0)
[22:41:58.150] Build status: SCHEDULED
[22:41:58.150] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:4:0) after 4m16s
[22:46:14.162] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:4:0)
[22:46:14.188] Build status: SCHEDULED
[22:46:14.188] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:5:0) after 2m58s
[22:49:12.224] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:5:0)
[22:49:12.241] Build status: SCHEDULED
[22:49:12.241] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:6:0) after 8m10s
[22:57:22.254] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:6:0)
[22:57:22.269] Build status: SCHEDULED
[22:57:22.269] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:7:0) after 8m12s
[23:05:34.269] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:7:0)
[23:05:34.283] Build status: SCHEDULED
[23:05:34.283] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:8:0) after 2m13s
[23:07:47.295] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:8:0)
[23:07:47.317] Build status: SCHEDULED
[23:07:47.317] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:9:0) after 3m11s
[23:10:58.476] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:9:0)
[23:10:58.522] Build status: SCHEDULED
[23:10:58.522] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:10:0) after 9m2s
[23:20:00.536] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:10:0)
[23:20:00.553] Build status: SCHEDULED
[23:20:00.553] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:11:0) after 4m16s
[23:24:16.568] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:11:0)
[23:24:16.585] Build status: SCHEDULED
[23:24:16.585] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:12:0) after 5m54s
[23:30:10.571] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:12:0)
[23:30:10.600] Build status: SCHEDULED
[23:30:10.601] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:13:0) after 4m9s
[23:34:19.613] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:13:0)
[23:34:19.651] Build status: SCHEDULED
[23:34:19.651] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:14:0) after 5m10s
[23:39:29.727] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:14:0)
[23:39:29.754] Build status: SCHEDULED
[23:39:29.754] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:15:0) after 9m25s
[23:48:54.869] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:15:0)
[23:48:54.883] Build status: SCHEDULED
[23:48:54.883] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:16:0) after 8m4s
[23:56:58.855] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:16:0)
[23:56:58.875] Build status: SCHEDULED
[23:56:58.876] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:17:0) after 4m21s
[00:01:19.953] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:17:0)
[00:01:19.979] Build status: SCHEDULED
[00:01:19.979] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:18:0) after 6m58s
[00:08:17.954] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:18:0)
[00:08:17.971] Build status: SCHEDULED
[00:08:17.971] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:19:0) after 1m23s
[00:09:40.985] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:19:0)
[00:09:41.041] Build status: SCHEDULED
[00:09:41.041] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:20:0) after 3m9s
[00:12:50.055] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:20:0)
[00:12:50.102] Build status: SCHEDULED
[00:12:50.102] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:21:0) after 7m28s
[00:20:18.153] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:21:0)
[00:20:18.207] Build status: SCHEDULED
[00:20:18.207] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:22:0) after 6m14s
[00:26:32.221] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:22:0)
[00:26:32.236] Build status: SCHEDULED
[00:26:32.237] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:23:0) after 8m10s
[00:34:42.252] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:23:0)
[00:34:42.278] Build status: SCHEDULED
[00:34:42.278] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:24:0) after 8m26s
[00:43:08.291] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:24:0)
[00:43:08.344] Build status: SCHEDULED
[00:43:08.344] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:25:0) after 2m29s
[00:45:37.413] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:25:0)
[00:45:37.432] Build status: SCHEDULED
[00:45:37.432] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:26:0) after 5m23s
[00:51:00.442] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:26:0)
[00:51:00.459] Build status: SCHEDULED
[00:51:00.459] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:27:0) after 3m59s
[00:54:59.509] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:27:0)
[00:54:59.527] Build status: SCHEDULED
[00:54:59.527] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:28:0) after 3m19s
[00:58:18.554] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:28:0)
[00:58:18.573] Build status: SCHEDULED
[00:58:18.573] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:29:0) after 4m42s
[01:03:00.585] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:29:0)
[01:03:00.600] Build status: SCHEDULED
[01:03:00.600] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:30:0) after 1m44s
[01:04:44.618] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:30:0)
[01:04:44.635] Build status: SCHEDULED
[01:04:44.635] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:31:0) after 1m57s
[01:06:41.910] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:31:0)
[01:06:41.935] Build status: SCHEDULED
[01:06:41.935] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:32:0) after 8m45s
[01:15:26.988] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:32:0)
[01:15:27.003] Build status: SCHEDULED
[01:15:27.004] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:33:0) after 3m50s
[01:19:17.020] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:33:0)
[01:19:17.037] Build status: SCHEDULED
[01:19:17.037] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:34:0) after 7m21s
[01:26:38.052] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:34:0)
[01:26:38.076] Build status: SCHEDULED
[01:26:38.076] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:35:0) after 8m11s
[01:34:49.072] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:35:0)
[01:34:49.090] Build status: SCHEDULED
[01:34:49.090] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:36:0) after 2m33s
[01:37:22.102] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:36:0)
[01:37:22.129] Build status: SCHEDULED
[01:37:22.129] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:37:0) after 6m4s
[01:43:26.140] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:37:0)
[01:43:26.157] Build status: SCHEDULED
[01:43:26.157] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:38:0) after 1m57s
[01:45:23.173] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:38:0)
[01:45:23.187] Build status: SCHEDULED
[01:45:23.187] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:39:0) after 9m21s
[01:54:44.202] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:39:0)
[01:54:44.219] Build status: SCHEDULED
[01:54:44.219] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:40:0) after 1m53s
[01:56:37.239] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:40:0)
[01:56:37.255] Build status: SCHEDULED
[01:56:37.255] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:41:0) after 4m10s
[02:00:47.270] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:41:0)
[02:00:47.287] Build status: SCHEDULED
[02:00:47.287] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:42:0) after 4m13s
[02:05:00.303] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:42:0)
[02:05:00.317] Build status: SCHEDULED
[02:05:00.317] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:43:0) after 1m51s
[02:06:51.358] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:43:0)
[02:06:51.412] Build status: SCHEDULED
[02:06:51.412] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:44:0) after 3m4s
[02:09:55.428] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:44:0)
[02:09:55.445] Build status: SCHEDULED
[02:09:55.445] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:45:0) after 4m48s
[02:14:43.463] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:45:0)
[02:14:43.514] Build status: SCHEDULED
[02:14:43.514] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:46:0) after 8m3s
[02:22:46.714] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:46:0)
[02:22:46.738] Build status: SCHEDULED
[02:22:46.739] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:47:0) after 8m55s
[02:31:41.919] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:47:0)
[02:31:41.934] Build status: SCHEDULED
[02:31:41.934] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:48:0) after 2m26s
[02:34:07.955] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:48:0)
[02:34:07.973] Build status: SCHEDULED
[02:34:07.973] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:49:0) after 8m55s
[02:43:03.012] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:49:0)
[02:43:03.045] Build status: SCHEDULED
[02:43:03.045] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:50:0) after 3m13s
[02:46:16.026] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:50:0)
[02:46:16.048] Build status: SCHEDULED
[02:46:16.048] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:51:0) after 7m2s
[02:53:18.106] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:51:0)
[02:53:18.245] Build status: SCHEDULED
[02:53:18.245] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:52:0) after 2m19s
[02:55:37.415] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:52:0)
[02:55:37.438] Build status: SCHEDULED
[02:55:37.438] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:53:0) after 7m43s
[03:03:20.416] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:53:0)
[03:03:20.442] Build status: SCHEDULED
[03:03:20.442] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:54:0) after 5m44s
[03:09:04.461] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:54:0)
[03:09:04.491] Build status: SCHEDULED
[03:09:04.492] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:55:0) after 3m2s
[03:12:06.562] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:55:0)
[03:12:06.580] Build status: SCHEDULED
[03:12:06.580] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:56:0) after 5m25s
[03:17:31.599] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:56:0)
[03:17:31.622] Build status: SCHEDULED
[03:17:31.622] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:57:0) after 8m23s
[03:25:54.668] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:57:0)
[03:25:54.697] Build status: SCHEDULED
[03:25:54.697] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:58:0) after 3m0s
[03:28:54.709] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:58:0)
[03:28:54.723] Build status: SCHEDULED
[03:28:54.723] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:59:0) after 5m27s
[03:34:21.882] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:59:0)
[03:34:21.917] Build status: SCHEDULED
[03:34:21.917] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:60:0) after 9m30s
[03:43:51.984] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:60:0)
[03:43:51.999] Build status: SCHEDULED
[03:43:51.999] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:61:0) after 6m6s
[03:49:58.089] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:61:0)
[03:49:58.106] Build status: SCHEDULED
[03:49:58.106] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:62:0) after 2m51s
[03:52:49.156] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:62:0)
[03:52:49.171] Build status: SCHEDULED
[03:52:49.171] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:63:0) after 4m44s
[03:57:33.184] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:63:0)
[03:57:33.202] Build status: SCHEDULED
[03:57:33.202] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:64:0) after 5m1s
[04:02:34.632] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:64:0)
[04:02:34.653] Build status: SCHEDULED
[04:02:34.653] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:65:0) after 3m17s
[04:05:51.731] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:65:0)
[04:05:51.752] Build status: SCHEDULED
[04:05:51.752] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:66:0) after 4m32s
[04:10:23.768] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:66:0)
[04:10:23.782] Build status: SCHEDULED
[04:10:23.782] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:67:0) after 9m13s
[04:19:36.802] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:67:0)
[04:19:36.816] Build status: SCHEDULED
[04:19:36.816] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:68:0) after 6m37s
[04:26:13.921] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:68:0)
[04:26:13.949] Build status: SCHEDULED
[04:26:13.949] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:69:0) after 9m10s
[04:35:24.063] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:69:0)
[04:35:24.087] Build status: SCHEDULED
[04:35:24.087] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:70:0) after 1m8s
[04:36:32.190] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:70:0)
[04:36:32.207] Build status: SCHEDULED
[04:36:32.207] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:71:0) after 4m33s
[04:41:05.221] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:71:0)
[04:41:05.238] Build status: SCHEDULED
[04:41:05.238] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:72:0) after 7m9s
[04:48:14.270] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:72:0)
[04:48:14.285] Build status: SCHEDULED
[04:48:14.285] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:73:0) after 5m31s
[04:53:45.293] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:73:0)
[04:53:45.310] Build status: SCHEDULED
[04:53:45.310] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:74:0) after 3m21s
[04:57:06.308] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:74:0)
[04:57:06.323] Build status: SCHEDULED
[04:57:06.323] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:75:0) after 5m55s
[05:03:01.366] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:75:0)
[05:03:01.384] Build status: SCHEDULED
[05:03:01.384] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:76:0) after 9m56s
[05:12:57.361] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:76:0)
[05:12:57.379] Build status: SCHEDULED
[05:12:57.379] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:77:0) after 9m39s
[05:22:36.458] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:77:0)
[05:22:36.486] Build status: SCHEDULED
[05:22:36.486] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:78:0) after 1m9s
[05:23:45.547] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:78:0)
[05:23:45.566] Build status: SCHEDULED
[05:23:45.566] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:79:0) after 5m49s
[05:29:34.667] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:79:0)
[05:29:34.686] Build status: SCHEDULED
[05:29:34.686] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:80:0) after 3m43s
[05:33:18.352] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:80:0)
[05:33:18.365] Build status: SCHEDULED
[05:33:18.366] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:81:0) after 1m8s
[05:34:26.717] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:81:0)
[05:34:26.737] Build status: SCHEDULED
[05:34:26.737] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:82:0) after 2m15s
[05:36:41.881] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:82:0)
[05:36:41.898] Build status: SCHEDULED
[05:36:41.898] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:83:0) after 3m53s
[05:40:34.915] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:83:0)
[05:40:34.930] Build status: SCHEDULED
[05:40:34.930] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:84:0) after 3m50s
[05:43:50.672] Received PubSub notification, asking Buildbucket for the build status
[05:43:50.745] Build status: STARTED
[05:44:24.961] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:84:0)
[05:44:24.978] Build status: STARTED
[05:44:24.978] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:86:0) after 6m54s
[05:51:19.031] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:86:0)
[05:51:19.049] Build status: STARTED
[05:51:19.049] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:87:0) after 8m0s
[05:59:19.073] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:87:0)
[05:59:19.087] Build status: STARTED
[05:59:19.087] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:88:0) after 2m42s
[06:02:01.230] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:88:0)
[06:02:01.251] Build status: STARTED
[06:02:01.251] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:89:0) after 4m15s
[06:06:16.275] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:89:0)
[06:06:16.313] Build status: STARTED
[06:06:16.313] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:90:0) after 7m47s
[06:14:03.423] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:90:0)
[06:14:03.561] Build status: STARTED
[06:14:03.561] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:91:0) after 8m49s
[06:22:52.582] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:91:0)
[06:22:52.596] Build status: STARTED
[06:22:52.596] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:92:0) after 4m32s
[06:27:24.599] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:92:0)
[06:27:24.620] Build status: STARTED
[06:27:24.620] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8863254486240989392:93:0) after 7m5s
[06:32:38.973] Received PubSub notification, asking Buildbucket for the build status
[06:32:39.042] Build:
{
"id": "8697824424139629841",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-asan"
},
"createdBy": "project:pigweed",
"createTime": "2025-11-18T22:34:20.295097820Z",
"startTime": "2025-11-19T05:43:44.473381662Z",
"endTime": "2025-11-19T06:32:38.434715556Z",
"updateTime": "2025-11-19T06:32:38.434715556Z",
"status": "INFRA_FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "dda2b3579e6284742f46ffdfbe2e88e1e1ded646",
"ref": "refs/heads/main"
}
}
}
[06:32:39.043] Invocation finished in 7h58m20.135405096s with status FAILED