[20:10:48.255] New invocation is queued and will start shortly
[20:10:49.398] Starting the invocation (attempt 1)
[20:10:49.434] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/69d5623f785890709079ef626835bd9ddf57d810"
[20:10:49.434] Popped gitiles commit info from properties and tags
[20:10:49.434] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[20:10:49.434] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[20:10:49.434] Buildbucket request:
{
"requestId": "8835269052551092304",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-gtest"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8835269052551092304",
"job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "467b79f025e89f87d2e3c88acb18015e19a6a77c"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@467b79f025e89f87d2e3c88acb18015e19a6a77c",
"title": "467b79f025e89f87d2e3c88acb18015e19a6a77c",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/467b79f025e89f87d2e3c88acb18015e19a6a77c"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "27316536f2ef05d900934aacd7593699aa2707e1"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@27316536f2ef05d900934aacd7593699aa2707e1",
"title": "27316536f2ef05d900934aacd7593699aa2707e1",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/27316536f2ef05d900934aacd7593699aa2707e1"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "69d5623f785890709079ef626835bd9ddf57d810"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@69d5623f785890709079ef626835bd9ddf57d810",
"title": "69d5623f785890709079ef626835bd9ddf57d810",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/69d5623f785890709079ef626835bd9ddf57d810"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "69d5623f785890709079ef626835bd9ddf57d810",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8835269052551092304"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[20:10:49.951] Scheduled build:
{
"id": "8669838990551358945",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-gtest"
},
"createdBy": "project:pigweed",
"createTime": "2026-09-23T20:10:49.541342576Z",
"updateTime": "2026-09-23T20:10:49.541342576Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "69d5623f785890709079ef626835bd9ddf57d810",
"ref": "refs/heads/main"
}
},
"canary": true
}
[20:10:49.951] Task URL: https://cr-buildbucket.appspot.com/build/8669838990551358945
[20:10:49.951] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:2:0) after 8m39s
[20:10:50.893] Received PubSub notification, asking Buildbucket for the build status
[20:10:50.914] Build status: SCHEDULED
[20:19:28.967] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:2:0)
[20:19:28.992] Build status: SCHEDULED
[20:19:28.992] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:4:0) after 9m51s
[20:29:20.043] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:4:0)
[20:29:20.064] Build status: SCHEDULED
[20:29:20.064] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:5:0) after 7m42s
[20:37:02.105] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:5:0)
[20:37:02.131] Build status: SCHEDULED
[20:37:02.131] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:6:0) after 4m20s
[20:41:22.172] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:6:0)
[20:41:22.192] Build status: SCHEDULED
[20:41:22.192] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:7:0) after 9m24s
[20:50:46.209] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:7:0)
[20:50:46.227] Build status: SCHEDULED
[20:50:46.227] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:8:0) after 7m43s
[20:58:29.297] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:8:0)
[20:58:29.383] Build status: SCHEDULED
[20:58:29.383] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:9:0) after 7m46s
[21:06:15.468] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:9:0)
[21:06:15.487] Build status: SCHEDULED
[21:06:15.487] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:10:0) after 7m57s
[21:14:12.521] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:10:0)
[21:14:12.544] Build status: SCHEDULED
[21:14:12.544] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:11:0) after 8m28s
[21:22:40.590] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:11:0)
[21:22:40.612] Build status: SCHEDULED
[21:22:40.613] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:12:0) after 9m55s
[21:32:35.705] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:12:0)
[21:32:35.723] Build status: SCHEDULED
[21:32:35.723] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:13:0) after 1m45s
[21:34:20.978] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:13:0)
[21:34:21.004] Build status: SCHEDULED
[21:34:21.004] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:14:0) after 4m49s
[21:39:10.147] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:14:0)
[21:39:10.168] Build status: SCHEDULED
[21:39:10.168] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:15:0) after 8m15s
[21:47:25.196] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:15:0)
[21:47:25.231] Build status: SCHEDULED
[21:47:25.231] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:16:0) after 5m51s
[21:53:16.246] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:16:0)
[21:53:16.272] Build status: SCHEDULED
[21:53:16.272] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:17:0) after 8m6s
[22:01:22.394] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:17:0)
[22:01:22.417] Build status: SCHEDULED
[22:01:22.417] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:18:0) after 8m15s
[22:09:37.465] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:18:0)
[22:09:37.523] Build status: SCHEDULED
[22:09:37.523] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:19:0) after 8m20s
[22:17:57.542] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:19:0)
[22:17:57.677] Build status: SCHEDULED
[22:17:57.677] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:20:0) after 1m39s
[22:19:36.693] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:20:0)
[22:19:36.711] Build status: SCHEDULED
[22:19:36.711] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:21:0) after 7m40s
[22:27:16.819] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:21:0)
[22:27:16.838] Build status: SCHEDULED
[22:27:16.838] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:22:0) after 6m20s
[22:33:36.856] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:22:0)
[22:33:36.938] Build status: SCHEDULED
[22:33:36.938] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:23:0) after 9m30s
[22:43:06.953] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:23:0)
[22:43:06.975] Build status: SCHEDULED
[22:43:06.975] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:24:0) after 7m17s
[22:50:23.998] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:24:0)
[22:50:24.024] Build status: SCHEDULED
[22:50:24.024] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:25:0) after 1m13s
[22:51:37.075] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:25:0)
[22:51:37.310] Build status: SCHEDULED
[22:51:37.310] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:26:0) after 7m4s
[22:58:41.430] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:26:0)
[22:58:41.455] Build status: SCHEDULED
[22:58:41.455] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:27:0) after 6m18s
[23:04:59.515] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:27:0)
[23:04:59.537] Build status: SCHEDULED
[23:04:59.537] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:28:0) after 1m16s
[23:06:15.625] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:28:0)
[23:06:15.656] Build status: SCHEDULED
[23:06:15.656] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:29:0) after 3m38s
[23:09:53.747] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:29:0)
[23:09:53.772] Build status: SCHEDULED
[23:09:53.772] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8835269052551092304:30:0) after 5m42s
[23:12:41.667] Received PubSub notification, asking Buildbucket for the build status
[23:12:41.693] Build status: STARTED
[23:13:51.697] Received PubSub notification, asking Buildbucket for the build status
[23:13:51.713] Build:
{
"id": "8669838990551358945",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-gtest"
},
"createdBy": "project:pigweed",
"createTime": "2026-09-23T20:10:49.541342576Z",
"startTime": "2026-09-23T23:12:41.316883604Z",
"endTime": "2026-09-23T23:13:49.145937560Z",
"updateTime": "2026-09-23T23:13:49.145937560Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "69d5623f785890709079ef626835bd9ddf57d810",
"ref": "refs/heads/main"
}
},
"canary": true
}
[23:13:51.713] Invocation finished in 3h3m3.462189558s with status FAILED