[02:19:48.563] New invocation is queued and will start shortly
[02:19:49.796] Starting the invocation (attempt 1)
[02:19:49.820] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/d901ac7bb0b57f48f7923a67bd38845c031b59a3"
[02:19:49.820] Popped gitiles commit info from properties and tags
[02:19:49.820] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[02:19:49.820] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[02:19:49.820] Buildbucket request:
{
"requestId": "8865233432635201664",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-tsan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8865233432635201664",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "f40abea2c6ac67c65bfe5650922b9128c8bf6c6f"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@f40abea2c6ac67c65bfe5650922b9128c8bf6c6f",
"title": "f40abea2c6ac67c65bfe5650922b9128c8bf6c6f",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/f40abea2c6ac67c65bfe5650922b9128c8bf6c6f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "cc91dbff7944fb31453b3f0f5a308e1ef78d6d1b"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@cc91dbff7944fb31453b3f0f5a308e1ef78d6d1b",
"title": "cc91dbff7944fb31453b3f0f5a308e1ef78d6d1b",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/cc91dbff7944fb31453b3f0f5a308e1ef78d6d1b"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "9f6d1750fa3f1981eecc173d2f8660cb12c0702d"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@9f6d1750fa3f1981eecc173d2f8660cb12c0702d",
"title": "9f6d1750fa3f1981eecc173d2f8660cb12c0702d",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/9f6d1750fa3f1981eecc173d2f8660cb12c0702d"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "fb8325f2253fcf172fa6eecbe536f77c2cec6c23"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@fb8325f2253fcf172fa6eecbe536f77c2cec6c23",
"title": "fb8325f2253fcf172fa6eecbe536f77c2cec6c23",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/fb8325f2253fcf172fa6eecbe536f77c2cec6c23"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "d901ac7bb0b57f48f7923a67bd38845c031b59a3"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@d901ac7bb0b57f48f7923a67bd38845c031b59a3",
"title": "d901ac7bb0b57f48f7923a67bd38845c031b59a3",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/d901ac7bb0b57f48f7923a67bd38845c031b59a3"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "d901ac7bb0b57f48f7923a67bd38845c031b59a3",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8865233432635201664"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[02:19:50.234] Scheduled build:
{
"id": "8699803370622034513",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-10-28T02:19:49.861443516Z",
"updateTime": "2025-10-28T02:19:49.861443516Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "d901ac7bb0b57f48f7923a67bd38845c031b59a3",
"ref": "refs/heads/main"
}
}
}
[02:19:50.234] Task URL: https://cr-buildbucket.appspot.com/build/8699803370622034513
[02:19:50.234] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:2:0) after 5m45s
[02:19:51.143] Received PubSub notification, asking Buildbucket for the build status
[02:19:51.172] Build status: SCHEDULED
[02:25:35.248] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:2:0)
[02:25:35.266] Build status: SCHEDULED
[02:25:35.266] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:4:0) after 2m47s
[02:28:22.283] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:4:0)
[02:28:22.306] Build status: SCHEDULED
[02:28:22.306] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:5:0) after 2m11s
[02:30:33.323] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:5:0)
[02:30:33.347] Build status: SCHEDULED
[02:30:33.347] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:6:0) after 6m26s
[02:36:59.380] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:6:0)
[02:36:59.410] Build status: SCHEDULED
[02:36:59.410] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:7:0) after 1m31s
[02:38:30.422] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:7:0)
[02:38:30.475] Build status: SCHEDULED
[02:38:30.475] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:8:0) after 9m19s
[02:47:49.569] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:8:0)
[02:47:49.592] Build status: SCHEDULED
[02:47:49.592] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:9:0) after 8m45s
[02:56:34.655] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:9:0)
[02:56:34.671] Build status: SCHEDULED
[02:56:34.671] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:10:0) after 1m40s
[02:58:14.685] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:10:0)
[02:58:14.729] Build status: SCHEDULED
[02:58:14.729] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:11:0) after 7m18s
[03:05:32.819] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:11:0)
[03:05:32.839] Build status: SCHEDULED
[03:05:32.839] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:12:0) after 2m44s
[03:08:16.851] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:12:0)
[03:08:16.874] Build status: SCHEDULED
[03:08:16.874] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:13:0) after 1m44s
[03:10:01.203] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:13:0)
[03:10:01.270] Build status: SCHEDULED
[03:10:01.270] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:14:0) after 6m56s
[03:16:57.286] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:14:0)
[03:16:57.316] Build status: SCHEDULED
[03:16:57.316] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:15:0) after 9m9s
[03:26:06.328] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:15:0)
[03:26:06.367] Build status: SCHEDULED
[03:26:06.367] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:16:0) after 8m7s
[03:34:13.445] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:16:0)
[03:34:13.461] Build status: SCHEDULED
[03:34:13.461] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:17:0) after 9m44s
[03:43:57.583] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:17:0)
[03:43:57.654] Build status: SCHEDULED
[03:43:57.654] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:18:0) after 4m57s
[03:48:54.670] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:18:0)
[03:48:54.689] Build status: SCHEDULED
[03:48:54.689] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:19:0) after 9m6s
[03:58:00.715] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:19:0)
[03:58:00.747] Build status: SCHEDULED
[03:58:00.747] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:20:0) after 6m7s
[04:04:07.789] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:20:0)
[04:04:07.808] Build status: SCHEDULED
[04:04:07.808] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:21:0) after 8m47s
[04:12:54.874] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:21:0)
[04:12:54.950] Build status: SCHEDULED
[04:12:54.950] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:22:0) after 8m11s
[04:21:05.968] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:22:0)
[04:21:05.993] Build status: SCHEDULED
[04:21:05.993] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:23:0) after 4m45s
[04:25:51.004] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:23:0)
[04:25:51.023] Build status: SCHEDULED
[04:25:51.023] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:24:0) after 4m59s
[04:30:50.077] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:24:0)
[04:30:50.095] Build status: SCHEDULED
[04:30:50.095] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:25:0) after 2m23s
[04:33:13.165] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:25:0)
[04:33:13.218] Build status: SCHEDULED
[04:33:13.218] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:26:0) after 6m16s
[04:39:29.257] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:26:0)
[04:39:29.385] Build status: SCHEDULED
[04:39:29.385] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:27:0) after 1m45s
[04:41:14.397] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:27:0)
[04:41:14.412] Build status: SCHEDULED
[04:41:14.412] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:28:0) after 1m29s
[04:42:43.460] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:28:0)
[04:42:43.474] Build status: SCHEDULED
[04:42:43.474] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:29:0) after 1m10s
[04:43:53.501] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:29:0)
[04:43:53.592] Build status: SCHEDULED
[04:43:53.592] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:30:0) after 6m2s
[04:49:55.614] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:30:0)
[04:49:55.631] Build status: SCHEDULED
[04:49:55.631] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:31:0) after 9m59s
[04:59:54.745] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:31:0)
[04:59:54.800] Build status: SCHEDULED
[04:59:54.800] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:32:0) after 6m45s
[05:06:39.816] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:32:0)
[05:06:39.834] Build status: SCHEDULED
[05:06:39.834] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:33:0) after 3m34s
[05:10:13.818] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:33:0)
[05:10:13.835] Build status: SCHEDULED
[05:10:13.835] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:34:0) after 9m46s
[05:19:59.856] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:34:0)
[05:19:59.883] Build status: SCHEDULED
[05:19:59.883] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:35:0) after 2m22s
[05:22:21.905] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:35:0)
[05:22:22.000] Build status: SCHEDULED
[05:22:22.000] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:36:0) after 3m10s
[05:25:32.023] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:36:0)
[05:25:32.051] Build status: SCHEDULED
[05:25:32.052] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:37:0) after 7m15s
[05:32:47.085] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:37:0)
[05:32:47.113] Build status: SCHEDULED
[05:32:47.113] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:38:0) after 7m55s
[05:40:42.133] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:38:0)
[05:40:42.158] Build status: SCHEDULED
[05:40:42.158] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:39:0) after 5m27s
[05:46:09.203] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:39:0)
[05:46:09.256] Build status: SCHEDULED
[05:46:09.256] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:40:0) after 9m10s
[05:55:19.318] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:40:0)
[05:55:19.336] Build status: SCHEDULED
[05:55:19.336] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:41:0) after 7m23s
[06:02:42.367] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:41:0)
[06:02:42.439] Build status: SCHEDULED
[06:02:42.439] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:42:0) after 6m58s
[06:09:40.513] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:42:0)
[06:09:40.534] Build status: SCHEDULED
[06:09:40.534] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:43:0) after 2m13s
[06:11:53.552] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:43:0)
[06:11:53.582] Build status: SCHEDULED
[06:11:53.582] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:44:0) after 9m6s
[06:20:59.599] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:44:0)
[06:20:59.614] Build status: SCHEDULED
[06:20:59.614] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:45:0) after 1m52s
[06:22:51.644] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:45:0)
[06:22:51.667] Build status: SCHEDULED
[06:22:51.667] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:46:0) after 3m44s
[06:26:35.683] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:46:0)
[06:26:35.782] Build status: SCHEDULED
[06:26:35.782] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:47:0) after 2m0s
[06:28:35.852] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:47:0)
[06:28:35.869] Build status: SCHEDULED
[06:28:35.869] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:48:0) after 4m57s
[06:33:32.964] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:48:0)
[06:33:32.982] Build status: SCHEDULED
[06:33:32.982] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:49:0) after 8m21s
[06:41:54.001] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:49:0)
[06:41:54.082] Build status: SCHEDULED
[06:41:54.082] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:50:0) after 6m33s
[06:48:27.144] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:50:0)
[06:48:27.161] Build status: SCHEDULED
[06:48:27.161] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:51:0) after 4m45s
[06:53:12.173] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:51:0)
[06:53:12.191] Build status: SCHEDULED
[06:53:12.191] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:52:0) after 9m9s
[07:02:21.345] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:52:0)
[07:02:21.360] Build status: SCHEDULED
[07:02:21.360] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:53:0) after 8m2s
[07:10:23.372] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:53:0)
[07:10:23.390] Build status: SCHEDULED
[07:10:23.390] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:54:0) after 9m48s
[07:20:11.405] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:54:0)
[07:20:11.433] Build status: SCHEDULED
[07:20:11.433] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:55:0) after 2m29s
[07:22:40.451] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:55:0)
[07:22:40.473] Build status: SCHEDULED
[07:22:40.473] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:56:0) after 4m58s
[07:27:38.491] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:56:0)
[07:27:38.504] Build status: SCHEDULED
[07:27:38.504] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:57:0) after 9m45s
[07:37:23.521] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:57:0)
[07:37:23.539] Build status: SCHEDULED
[07:37:23.539] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:58:0) after 7m58s
[07:45:21.554] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:58:0)
[07:45:21.569] Build status: SCHEDULED
[07:45:21.569] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:59:0) after 1m55s
[07:47:16.585] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:59:0)
[07:47:16.599] Build status: SCHEDULED
[07:47:16.599] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:60:0) after 7m42s
[07:54:58.708] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:60:0)
[07:54:58.725] Build status: SCHEDULED
[07:54:58.725] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:61:0) after 4m1s
[07:58:59.741] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:61:0)
[07:58:59.765] Build status: SCHEDULED
[07:58:59.765] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:62:0) after 3m46s
[08:02:45.851] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:62:0)
[08:02:45.885] Build status: SCHEDULED
[08:02:45.885] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:63:0) after 5m4s
[08:07:49.873] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:63:0)
[08:07:49.900] Build status: SCHEDULED
[08:07:49.900] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:64:0) after 5m48s
[08:13:37.965] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:64:0)
[08:13:37.981] Build status: SCHEDULED
[08:13:37.981] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:65:0) after 7m51s
[08:21:29.064] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:65:0)
[08:21:29.085] Build status: SCHEDULED
[08:21:29.085] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:66:0) after 3m46s
[08:25:15.129] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:66:0)
[08:25:15.148] Build status: SCHEDULED
[08:25:15.148] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:67:0) after 6m9s
[08:31:24.162] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:67:0)
[08:31:24.185] Build status: SCHEDULED
[08:31:24.185] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:68:0) after 9m57s
[08:41:21.249] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:68:0)
[08:41:21.317] Build status: SCHEDULED
[08:41:21.317] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:69:0) after 7m56s
[08:49:17.415] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:69:0)
[08:49:17.436] Build status: SCHEDULED
[08:49:17.436] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:70:0) after 2m11s
[08:51:28.453] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:70:0)
[08:51:28.476] Build status: SCHEDULED
[08:51:28.476] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:71:0) after 8m50s
[09:00:18.571] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:71:0)
[09:00:18.587] Build status: SCHEDULED
[09:00:18.587] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:72:0) after 8m58s
[09:09:16.613] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:72:0)
[09:09:16.644] Build status: SCHEDULED
[09:09:16.644] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:73:0) after 5m30s
[09:14:46.658] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:73:0)
[09:14:46.693] Build status: SCHEDULED
[09:14:46.693] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:74:0) after 5m59s
[09:20:45.712] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:74:0)
[09:20:45.732] Build status: SCHEDULED
[09:20:45.732] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:75:0) after 6m41s
[09:27:26.746] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:75:0)
[09:27:26.775] Build status: SCHEDULED
[09:27:26.775] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:76:0) after 8m16s
[09:35:42.808] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:76:0)
[09:35:42.910] Build status: SCHEDULED
[09:35:42.910] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:77:0) after 6m10s
[09:41:52.932] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:77:0)
[09:41:52.950] Build status: SCHEDULED
[09:41:52.950] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:78:0) after 7m10s
[09:49:02.977] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:78:0)
[09:49:02.992] Build status: SCHEDULED
[09:49:02.992] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:79:0) after 7m32s
[09:56:35.006] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:79:0)
[09:56:35.066] Build status: SCHEDULED
[09:56:35.066] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:80:0) after 5m28s
[10:02:03.166] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:80:0)
[10:02:03.189] Build status: SCHEDULED
[10:02:03.189] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:81:0) after 8m25s
[10:10:28.221] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:81:0)
[10:10:28.437] Build status: SCHEDULED
[10:10:28.437] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:82:0) after 2m37s
[10:13:05.455] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:82:0)
[10:13:05.475] Build status: SCHEDULED
[10:13:05.475] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:83:0) after 2m13s
[10:15:18.491] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:83:0)
[10:15:18.509] Build status: SCHEDULED
[10:15:18.509] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:84:0) after 4m13s
[10:19:31.604] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:84:0)
[10:19:31.648] Build status: SCHEDULED
[10:19:31.648] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:85:0) after 9m44s
[10:26:28.779] Received PubSub notification, asking Buildbucket for the build status
[10:26:28.880] Build status: STARTED
[10:29:15.669] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:85:0)
[10:29:15.795] Build status: STARTED
[10:29:15.795] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:87:0) after 3m10s
[10:32:25.849] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:87:0)
[10:32:25.929] Build status: STARTED
[10:32:25.929] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865233432635201664:88:0) after 8m48s
[10:40:12.928] Received PubSub notification, asking Buildbucket for the build status
[10:40:12.958] Build:
{
"id": "8699803370622034513",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-10-28T02:19:49.861443516Z",
"startTime": "2025-10-28T10:26:28.610695753Z",
"endTime": "2025-10-28T10:40:12.604516951Z",
"updateTime": "2025-10-28T10:40:12.604516951Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "d901ac7bb0b57f48f7923a67bd38845c031b59a3",
"ref": "refs/heads/main"
}
}
}
[10:40:12.958] Invocation finished in 8h20m24.401409534s with status FAILED