[05:16:43.344] New invocation is queued and will start shortly
[05:16:44.488] Starting the invocation (attempt 1)
[05:16:44.514] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/6fef9caa5d75d90877e8edc032227d0ed585995f"
[05:16:44.514] Popped gitiles commit info from properties and tags
[05:16:44.514] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[05:16:44.514] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[05:16:44.515] Buildbucket request:
{
"requestId": "8867668420323244944",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8867668420323244944",
"job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "bedd9e04833b94415a4ca0098f0fe9162a07ef0d"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@bedd9e04833b94415a4ca0098f0fe9162a07ef0d",
"title": "bedd9e04833b94415a4ca0098f0fe9162a07ef0d",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/bedd9e04833b94415a4ca0098f0fe9162a07ef0d"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "dc8c8025c045a5c868c87c7d62ec714cbf2ccadf"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@dc8c8025c045a5c868c87c7d62ec714cbf2ccadf",
"title": "dc8c8025c045a5c868c87c7d62ec714cbf2ccadf",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/dc8c8025c045a5c868c87c7d62ec714cbf2ccadf"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "6fef9caa5d75d90877e8edc032227d0ed585995f"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@6fef9caa5d75d90877e8edc032227d0ed585995f",
"title": "6fef9caa5d75d90877e8edc032227d0ed585995f",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/6fef9caa5d75d90877e8edc032227d0ed585995f"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "6fef9caa5d75d90877e8edc032227d0ed585995f",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8867668420323244944"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[05:16:44.845] Scheduled build:
{
"id": "8702238358375918065",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-10-01T05:16:44.579665448Z",
"updateTime": "2025-10-01T05:16:44.579665448Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "6fef9caa5d75d90877e8edc032227d0ed585995f",
"ref": "refs/heads/main"
}
}
}
[05:16:44.845] Task URL: https://cr-buildbucket.appspot.com/build/8702238358375918065
[05:16:44.845] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:2:0) after 8m7s
[05:16:45.837] Received PubSub notification, asking Buildbucket for the build status
[05:16:45.877] Build status: SCHEDULED
[05:24:51.871] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:2:0)
[05:24:51.899] Build status: SCHEDULED
[05:24:51.899] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:4:0) after 6m11s
[05:31:03.163] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:4:0)
[05:31:03.187] Build status: SCHEDULED
[05:31:03.187] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:5:0) after 6m1s
[05:37:04.255] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:5:0)
[05:37:04.317] Build status: SCHEDULED
[05:37:04.317] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:6:0) after 6m38s
[05:43:42.330] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:6:0)
[05:43:42.362] Build status: SCHEDULED
[05:43:42.362] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:7:0) after 8m0s
[05:51:42.376] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:7:0)
[05:51:42.398] Build status: SCHEDULED
[05:51:42.398] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:8:0) after 1m9s
[05:52:51.503] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:8:0)
[05:52:51.525] Build status: SCHEDULED
[05:52:51.525] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:9:0) after 4m55s
[05:57:46.542] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:9:0)
[05:57:46.565] Build status: SCHEDULED
[05:57:46.565] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:10:0) after 3m50s
[06:01:36.635] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:10:0)
[06:01:36.744] Build status: SCHEDULED
[06:01:36.745] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:11:0) after 3m0s
[06:04:36.725] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:11:0)
[06:04:36.743] Build status: SCHEDULED
[06:04:36.743] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:12:0) after 7m56s
[06:12:32.822] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:12:0)
[06:12:32.836] Build status: SCHEDULED
[06:12:32.836] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:13:0) after 1m1s
[06:13:33.849] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:13:0)
[06:13:33.879] Build status: SCHEDULED
[06:13:33.879] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:14:0) after 1m24s
[06:14:57.943] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:14:0)
[06:14:57.963] Build status: SCHEDULED
[06:14:57.963] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:15:0) after 9m50s
[06:24:48.041] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:15:0)
[06:24:48.055] Build status: SCHEDULED
[06:24:48.055] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:16:0) after 4m22s
[06:29:10.209] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:16:0)
[06:29:10.232] Build status: SCHEDULED
[06:29:10.232] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:17:0) after 1m15s
[06:30:25.310] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:17:0)
[06:30:25.336] Build status: SCHEDULED
[06:30:25.336] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:18:0) after 1m9s
[06:31:34.355] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:18:0)
[06:31:34.374] Build status: SCHEDULED
[06:31:34.374] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:19:0) after 5m51s
[06:37:25.430] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:19:0)
[06:37:25.497] Build status: SCHEDULED
[06:37:25.497] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:20:0) after 9m11s
[06:46:36.588] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:20:0)
[06:46:36.615] Build status: SCHEDULED
[06:46:36.615] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:21:0) after 4m10s
[06:50:46.626] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:21:0)
[06:50:46.651] Build status: SCHEDULED
[06:50:46.651] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:22:0) after 1m13s
[06:51:59.667] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:22:0)
[06:51:59.694] Build status: SCHEDULED
[06:51:59.694] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:23:0) after 6m45s
[06:58:44.735] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:23:0)
[06:58:44.749] Build status: SCHEDULED
[06:58:44.749] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:24:0) after 4m44s
[07:03:28.720] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:24:0)
[07:03:28.740] Build status: SCHEDULED
[07:03:28.740] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:25:0) after 6m28s
[07:09:56.756] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:25:0)
[07:09:56.769] Build status: SCHEDULED
[07:09:56.769] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:26:0) after 7m15s
[07:17:11.784] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:26:0)
[07:17:11.804] Build status: SCHEDULED
[07:17:11.804] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:27:0) after 3m5s
[07:20:16.770] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:27:0)
[07:20:16.788] Build status: SCHEDULED
[07:20:16.788] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:28:0) after 7m43s
[07:27:59.826] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:28:0)
[07:28:00.050] Build status: SCHEDULED
[07:28:00.050] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:29:0) after 6m8s
[07:34:08.067] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:29:0)
[07:34:08.085] Build status: SCHEDULED
[07:34:08.085] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:30:0) after 1m6s
[07:35:14.150] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:30:0)
[07:35:14.167] Build status: SCHEDULED
[07:35:14.167] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:31:0) after 8m27s
[07:43:41.182] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:31:0)
[07:43:41.215] Build status: SCHEDULED
[07:43:41.215] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:32:0) after 4m13s
[07:47:54.239] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:32:0)
[07:47:54.254] Build status: SCHEDULED
[07:47:54.254] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:33:0) after 8m20s
[07:56:14.283] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:33:0)
[07:56:14.298] Build status: SCHEDULED
[07:56:14.298] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:34:0) after 7m12s
[08:03:26.279] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:34:0)
[08:03:26.298] Build status: SCHEDULED
[08:03:26.298] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:35:0) after 5m13s
[08:08:39.342] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:35:0)
[08:08:39.357] Build status: SCHEDULED
[08:08:39.357] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:36:0) after 6m33s
[08:15:12.464] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:36:0)
[08:15:12.490] Build status: SCHEDULED
[08:15:12.490] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:37:0) after 2m11s
[08:17:23.557] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:37:0)
[08:17:23.573] Build status: SCHEDULED
[08:17:23.573] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867668420323244944:38:0) after 4m47s
[08:19:51.867] Received PubSub notification, asking Buildbucket for the build status
[08:19:51.889] Build status: STARTED
[08:20:11.785] Received PubSub notification, asking Buildbucket for the build status
[08:20:11.803] Build:
{
"id": "8702238358375918065",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-10-01T05:16:44.579665448Z",
"startTime": "2025-10-01T08:19:51.528604393Z",
"endTime": "2025-10-01T08:20:11.416505362Z",
"updateTime": "2025-10-01T08:20:11.416505362Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "6fef9caa5d75d90877e8edc032227d0ed585995f",
"ref": "refs/heads/main"
}
}
}
[08:20:11.803] Invocation finished in 3h3m28.465504088s with status FAILED