[04:49:37.405] New invocation is queued and will start shortly
[04:49:38.826] Starting the invocation (attempt 1)
[04:49:38.857] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/263c47e26d9258cf87f45a96267e6f6b5d70c454"
[04:49:38.857] Popped gitiles commit info from properties and tags
[04:49:38.857] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[04:49:38.857] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[04:49:38.857] Buildbucket request:
{
"requestId": "8866039379857201840",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-tsan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8866039379857201840",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "263c47e26d9258cf87f45a96267e6f6b5d70c454"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@263c47e26d9258cf87f45a96267e6f6b5d70c454",
"title": "263c47e26d9258cf87f45a96267e6f6b5d70c454",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/263c47e26d9258cf87f45a96267e6f6b5d70c454"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "263c47e26d9258cf87f45a96267e6f6b5d70c454",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8866039379857201840"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[04:49:39.319] Scheduled build:
{
"id": "8700609317588626769",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-10-19T04:49:38.938055937Z",
"updateTime": "2025-10-19T04:49:38.938055937Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "263c47e26d9258cf87f45a96267e6f6b5d70c454",
"ref": "refs/heads/main"
}
}
}
[04:49:39.319] Task URL: https://cr-buildbucket.appspot.com/build/8700609317588626769
[04:49:39.319] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:2:0) after 6m58s
[04:49:40.248] Received PubSub notification, asking Buildbucket for the build status
[04:49:40.262] Build status: SCHEDULED
[04:56:37.333] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:2:0)
[04:56:37.361] Build status: SCHEDULED
[04:56:37.361] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:4:0) after 4m41s
[05:01:18.467] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:4:0)
[05:01:18.482] Build status: SCHEDULED
[05:01:18.482] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:5:0) after 8m57s
[05:10:15.497] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:5:0)
[05:10:15.529] Build status: SCHEDULED
[05:10:15.529] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:6:0) after 6m40s
[05:16:55.928] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:6:0)
[05:16:55.951] Build status: SCHEDULED
[05:16:55.951] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:7:0) after 7m59s
[05:24:55.284] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:7:0)
[05:24:55.311] Build status: SCHEDULED
[05:24:55.311] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:8:0) after 7m5s
[05:32:00.388] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:8:0)
[05:32:00.408] Build status: SCHEDULED
[05:32:00.408] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:9:0) after 1m46s
[05:33:46.421] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:9:0)
[05:33:46.709] Build status: SCHEDULED
[05:33:46.709] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:10:0) after 2m33s
[05:36:19.734] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:10:0)
[05:36:19.776] Build status: SCHEDULED
[05:36:19.776] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:11:0) after 6m12s
[05:42:31.764] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:11:0)
[05:42:31.783] Build status: SCHEDULED
[05:42:31.783] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:12:0) after 5m4s
[05:47:35.811] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:12:0)
[05:47:35.839] Build status: SCHEDULED
[05:47:35.839] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:13:0) after 5m37s
[05:53:12.972] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:13:0)
[05:53:12.999] Build status: SCHEDULED
[05:53:12.999] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:14:0) after 3m44s
[05:56:57.204] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:14:0)
[05:56:57.222] Build status: SCHEDULED
[05:56:57.222] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:15:0) after 5m34s
[06:02:31.235] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:15:0)
[06:02:31.265] Build status: SCHEDULED
[06:02:31.265] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:16:0) after 6m51s
[06:09:22.371] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:16:0)
[06:09:22.387] Build status: SCHEDULED
[06:09:22.387] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:17:0) after 2m52s
[06:12:14.504] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:17:0)
[06:12:14.522] Build status: SCHEDULED
[06:12:14.522] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:18:0) after 8m21s
[06:20:35.672] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:18:0)
[06:20:35.703] Build status: SCHEDULED
[06:20:35.703] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:19:0) after 4m58s
[06:25:33.716] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:19:0)
[06:25:33.793] Build status: SCHEDULED
[06:25:33.793] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:20:0) after 5m27s
[06:31:00.815] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:20:0)
[06:31:00.843] Build status: SCHEDULED
[06:31:00.843] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:21:0) after 8m58s
[06:39:58.855] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:21:0)
[06:39:58.886] Build status: SCHEDULED
[06:39:58.886] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:22:0) after 3m44s
[06:43:43.081] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:22:0)
[06:43:43.102] Build status: SCHEDULED
[06:43:43.102] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:23:0) after 2m12s
[06:45:55.142] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:23:0)
[06:45:55.198] Build status: SCHEDULED
[06:45:55.198] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:24:0) after 1m51s
[06:47:46.211] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:24:0)
[06:47:46.365] Build status: SCHEDULED
[06:47:46.365] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:25:0) after 3m57s
[06:51:43.470] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:25:0)
[06:51:43.543] Build status: SCHEDULED
[06:51:43.543] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:26:0) after 8m53s
[07:00:36.561] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:26:0)
[07:00:36.610] Build status: SCHEDULED
[07:00:36.610] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:27:0) after 5m3s
[07:05:39.725] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:27:0)
[07:05:39.752] Build status: SCHEDULED
[07:05:39.752] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:28:0) after 5m49s
[07:11:28.852] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:28:0)
[07:11:28.868] Build status: SCHEDULED
[07:11:28.868] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:29:0) after 8m30s
[07:19:58.841] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:29:0)
[07:19:58.859] Build status: SCHEDULED
[07:19:58.859] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:30:0) after 4m56s
[07:24:55.058] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:30:0)
[07:24:55.080] Build status: SCHEDULED
[07:24:55.080] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:31:0) after 9m0s
[07:33:55.395] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:31:0)
[07:33:55.430] Build status: SCHEDULED
[07:33:55.430] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:32:0) after 7m38s
[07:41:33.465] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:32:0)
[07:41:33.487] Build status: SCHEDULED
[07:41:33.487] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:33:0) after 4m45s
[07:46:18.459] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:33:0)
[07:46:18.498] Build status: SCHEDULED
[07:46:18.498] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:34:0) after 6m34s
[07:52:52.675] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:34:0)
[07:52:52.700] Build status: SCHEDULED
[07:52:52.701] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:35:0) after 8m18s
[08:01:10.763] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:35:0)
[08:01:10.792] Build status: SCHEDULED
[08:01:10.792] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:36:0) after 7m55s
[08:09:05.810] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:36:0)
[08:09:05.822] Build status: SCHEDULED
[08:09:05.823] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:37:0) after 1m7s
[08:10:12.840] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:37:0)
[08:10:12.925] Build status: SCHEDULED
[08:10:12.925] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:38:0) after 3m32s
[08:13:45.070] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:38:0)
[08:13:45.088] Build status: SCHEDULED
[08:13:45.088] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:39:0) after 8m26s
[08:22:11.087] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:39:0)
[08:22:11.120] Build status: SCHEDULED
[08:22:11.120] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:40:0) after 1m7s
[08:23:18.262] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:40:0)
[08:23:18.280] Build status: SCHEDULED
[08:23:18.280] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:41:0) after 8m5s
[08:31:24.008] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:41:0)
[08:31:24.025] Build status: SCHEDULED
[08:31:24.025] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:42:0) after 4m8s
[08:35:32.213] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:42:0)
[08:35:32.241] Build status: SCHEDULED
[08:35:32.241] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:43:0) after 3m45s
[08:39:17.288] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:43:0)
[08:39:17.308] Build status: SCHEDULED
[08:39:17.308] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:44:0) after 8m49s
[08:48:06.362] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:44:0)
[08:48:06.629] Build status: SCHEDULED
[08:48:06.629] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:45:0) after 1m56s
[08:50:02.740] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:45:0)
[08:50:02.771] Build status: SCHEDULED
[08:50:02.771] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:46:0) after 1m13s
[08:51:16.034] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:46:0)
[08:51:16.050] Build status: SCHEDULED
[08:51:16.050] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:47:0) after 6m13s
[08:57:29.066] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:47:0)
[08:57:29.083] Build status: SCHEDULED
[08:57:29.083] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:48:0) after 9m20s
[09:06:49.100] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:48:0)
[09:06:49.118] Build status: SCHEDULED
[09:06:49.118] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:49:0) after 2m21s
[09:09:10.184] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:49:0)
[09:09:10.213] Build status: SCHEDULED
[09:09:10.213] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:50:0) after 5m13s
[09:14:23.230] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:50:0)
[09:14:23.247] Build status: SCHEDULED
[09:14:23.247] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:51:0) after 9m12s
[09:23:35.301] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:51:0)
[09:23:35.369] Build status: SCHEDULED
[09:23:35.369] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:52:0) after 6m32s
[09:30:07.449] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:52:0)
[09:30:07.466] Build status: SCHEDULED
[09:30:07.466] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:53:0) after 1m54s
[09:32:01.453] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:53:0)
[09:32:01.491] Build status: SCHEDULED
[09:32:01.491] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:54:0) after 9m7s
[09:41:08.608] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:54:0)
[09:41:08.629] Build status: SCHEDULED
[09:41:08.629] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:55:0) after 2m12s
[09:43:20.671] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:55:0)
[09:43:21.435] Build status: SCHEDULED
[09:43:21.435] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:56:0) after 4m21s
[09:47:42.451] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:56:0)
[09:47:42.470] Build status: SCHEDULED
[09:47:42.470] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:57:0) after 1m30s
[09:49:12.485] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:57:0)
[09:49:12.534] Build status: SCHEDULED
[09:49:12.534] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:58:0) after 2m59s
[09:52:11.573] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:58:0)
[09:52:11.589] Build status: SCHEDULED
[09:52:11.589] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:59:0) after 9m49s
[10:02:00.611] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:59:0)
[10:02:00.706] Build status: SCHEDULED
[10:02:00.706] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:60:0) after 6m44s
[10:08:44.757] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:60:0)
[10:08:44.777] Build status: SCHEDULED
[10:08:44.777] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:61:0) after 9m31s
[10:18:15.795] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:61:0)
[10:18:15.820] Build status: SCHEDULED
[10:18:15.820] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:62:0) after 5m25s
[10:23:40.838] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:62:0)
[10:23:40.850] Build status: SCHEDULED
[10:23:40.850] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:63:0) after 7m2s
[10:30:43.003] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:63:0)
[10:30:43.026] Build status: SCHEDULED
[10:30:43.026] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:64:0) after 1m6s
[10:31:49.047] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:64:0)
[10:31:49.068] Build status: SCHEDULED
[10:31:49.068] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:65:0) after 1m50s
[10:33:39.084] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:65:0)
[10:33:39.148] Build status: SCHEDULED
[10:33:39.148] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:66:0) after 9m25s
[10:43:04.164] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:66:0)
[10:43:04.177] Build status: SCHEDULED
[10:43:04.177] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:67:0) after 1m48s
[10:44:52.257] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:67:0)
[10:44:52.274] Build status: SCHEDULED
[10:44:52.274] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:68:0) after 7m28s
[10:52:20.298] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:68:0)
[10:52:20.374] Build status: SCHEDULED
[10:52:20.374] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:69:0) after 2m58s
[10:55:18.389] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:69:0)
[10:55:18.447] Build status: SCHEDULED
[10:55:18.447] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:70:0) after 2m52s
[10:58:10.555] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:70:0)
[10:58:10.573] Build status: SCHEDULED
[10:58:10.573] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:71:0) after 9m4s
[11:07:14.656] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:71:0)
[11:07:14.671] Build status: SCHEDULED
[11:07:14.671] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:72:0) after 4m1s
[11:11:15.676] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:72:0)
[11:11:15.695] Build status: SCHEDULED
[11:11:15.695] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:73:0) after 4m41s
[11:15:56.693] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:73:0)
[11:15:56.720] Build status: SCHEDULED
[11:15:56.720] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:74:0) after 4m25s
[11:20:21.734] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:74:0)
[11:20:21.764] Build status: SCHEDULED
[11:20:21.765] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:75:0) after 5m25s
[11:25:46.781] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:75:0)
[11:25:46.824] Build status: SCHEDULED
[11:25:46.824] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:76:0) after 1m20s
[11:27:06.838] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:76:0)
[11:27:06.857] Build status: SCHEDULED
[11:27:06.857] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:77:0) after 6m17s
[11:33:23.890] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:77:0)
[11:33:23.915] Build status: SCHEDULED
[11:33:23.916] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:78:0) after 7m57s
[11:41:21.094] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:78:0)
[11:41:21.185] Build status: SCHEDULED
[11:41:21.185] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:79:0) after 4m54s
[11:46:15.200] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:79:0)
[11:46:15.263] Build status: SCHEDULED
[11:46:15.263] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:80:0) after 1m52s
[11:48:07.377] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:80:0)
[11:48:07.445] Build status: SCHEDULED
[11:48:07.445] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:81:0) after 7m8s
[11:55:15.515] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:81:0)
[11:55:15.530] Build status: SCHEDULED
[11:55:15.530] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:82:0) after 1m2s
[11:56:17.663] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:82:0)
[11:56:17.716] Build status: SCHEDULED
[11:56:17.716] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:83:0) after 4m43s
[12:01:00.838] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:83:0)
[12:01:00.857] Build status: SCHEDULED
[12:01:00.857] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:84:0) after 5m9s
[12:06:09.929] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:84:0)
[12:06:09.952] Build status: SCHEDULED
[12:06:09.952] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:85:0) after 1m4s
[12:07:13.966] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:85:0)
[12:07:13.986] Build status: SCHEDULED
[12:07:13.986] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:86:0) after 1m5s
[12:08:19.079] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:86:0)
[12:08:19.097] Build status: SCHEDULED
[12:08:19.097] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:87:0) after 6m38s
[12:14:57.134] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:87:0)
[12:14:57.177] Build status: SCHEDULED
[12:14:57.177] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:88:0) after 6m59s
[12:21:56.248] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:88:0)
[12:21:56.264] Build status: SCHEDULED
[12:21:56.264] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:89:0) after 3m49s
[12:25:45.313] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:89:0)
[12:25:45.333] Build status: SCHEDULED
[12:25:45.333] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:90:0) after 7m22s
[12:31:17.614] Received PubSub notification, asking Buildbucket for the build status
[12:31:17.648] Build status: STARTED
[12:33:07.416] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:90:0)
[12:33:07.429] Build status: STARTED
[12:33:07.429] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:92:0) after 2m10s
[12:35:17.454] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:92:0)
[12:35:17.809] Build status: STARTED
[12:35:17.809] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:93:0) after 4m55s
[12:40:12.987] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:93:0)
[12:40:13.004] Build status: STARTED
[12:40:13.004] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8866039379857201840:94:0) after 7m5s
[12:40:19.870] Received PubSub notification, asking Buildbucket for the build status
[12:40:19.931] Build:
{
"id": "8700609317588626769",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-x86-bazel-sanitizers-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-10-19T04:49:38.938055937Z",
"startTime": "2025-10-19T12:30:42.461451060Z",
"endTime": "2025-10-19T12:40:19.532867595Z",
"updateTime": "2025-10-19T12:40:19.532867595Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "263c47e26d9258cf87f45a96267e6f6b5d70c454",
"ref": "refs/heads/main"
}
}
}
[12:40:19.931] Invocation finished in 7h50m42.540142222s with status FAILED