buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-linux-bazel-sanitizers-tsan-gtest"
>

• 278f49430d9cbf06ebed4fb7c2e65b6236950663 (3 weeks ago)

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "278f49430d9cbf06ebed4fb7c2e65b6236950663"
}

• buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/278f49430d9cbf06ebed4fb7c2e65b6236950663
• gitiles_ref:refs/heads/main

[05:53:20.218] New invocation is queued and will start shortly
[05:53:21.359] Starting the invocation (attempt 1)
[05:53:21.383] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/278f49430d9cbf06ebed4fb7c2e65b6236950663"
[05:53:21.383] Popped gitiles commit info from properties and tags
[05:53:21.383] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[05:53:21.383] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[05:53:21.384] Buildbucket request:
{
  "requestId": "8843386129738183936",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-linux-bazel-sanitizers-tsan-gtest"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8843386129738183936",
            "job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "278f49430d9cbf06ebed4fb7c2e65b6236950663"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@278f49430d9cbf06ebed4fb7c2e65b6236950663",
                        "title": "278f49430d9cbf06ebed4fb7c2e65b6236950663",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/278f49430d9cbf06ebed4fb7c2e65b6236950663"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "278f49430d9cbf06ebed4fb7c2e65b6236950663",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8843386129738183936"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[05:53:21.798] Scheduled build:
{
  "id": "8677956067737806705",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-linux-bazel-sanitizers-tsan-gtest"
  },
  "createdBy": "project:pigweed",
  "createTime": "2026-06-26T05:53:21.504173519Z",
  "updateTime": "2026-06-26T05:53:21.504173519Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "278f49430d9cbf06ebed4fb7c2e65b6236950663",
      "ref": "refs/heads/main"
    }
  }
}
[05:53:21.798] Task URL: https://cr-buildbucket.appspot.com/build/8677956067737806705
[05:53:21.798] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:2:0) after 8m40s
[05:53:22.746] Received PubSub notification, asking Buildbucket for the build status
[05:53:22.771] Build status: SCHEDULED
[06:02:01.939] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:2:0)
[06:02:02.047] Build status: SCHEDULED
[06:02:02.047] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:4:0) after 9m6s
[06:11:08.036] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:4:0)
[06:11:08.056] Build status: SCHEDULED
[06:11:08.056] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:5:0) after 7m46s
[06:18:54.079] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:5:0)
[06:18:54.150] Build status: SCHEDULED
[06:18:54.150] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:6:0) after 2m59s
[06:21:53.168] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:6:0)
[06:21:53.190] Build status: SCHEDULED
[06:21:53.190] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:7:0) after 6m49s
[06:28:42.305] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:7:0)
[06:28:42.326] Build status: SCHEDULED
[06:28:42.326] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:8:0) after 9m5s
[06:37:47.341] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:8:0)
[06:37:47.410] Build status: SCHEDULED
[06:37:47.410] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:9:0) after 8m57s
[06:46:44.435] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:9:0)
[06:46:44.455] Build status: SCHEDULED
[06:46:44.455] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:10:0) after 4m3s
[06:50:47.451] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:10:0)
[06:50:47.471] Build status: SCHEDULED
[06:50:47.471] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:11:0) after 7m26s
[06:58:13.495] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:11:0)
[06:58:13.513] Build status: SCHEDULED
[06:58:13.513] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:12:0) after 8m9s
[07:06:22.528] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:12:0)
[07:06:22.552] Build status: SCHEDULED
[07:06:22.552] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:13:0) after 8m36s
[07:14:58.594] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:13:0)
[07:14:58.610] Build status: SCHEDULED
[07:14:58.610] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:14:0) after 2m6s
[07:17:04.725] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:14:0)
[07:17:04.743] Build status: SCHEDULED
[07:17:04.743] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:15:0) after 8m17s
[07:25:21.724] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:15:0)
[07:25:21.815] Build status: SCHEDULED
[07:25:21.815] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:16:0) after 2m44s
[07:28:05.829] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:16:0)
[07:28:05.849] Build status: SCHEDULED
[07:28:05.849] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:17:0) after 2m27s
[07:30:32.914] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:17:0)
[07:30:32.955] Build status: SCHEDULED
[07:30:32.955] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:18:0) after 4m52s
[07:35:24.971] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:18:0)
[07:35:24.999] Build status: SCHEDULED
[07:35:24.999] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:19:0) after 9m11s
[07:44:36.040] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:19:0)
[07:44:36.068] Build status: SCHEDULED
[07:44:36.068] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:20:0) after 7m23s
[07:51:59.072] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:20:0)
[07:51:59.091] Build status: SCHEDULED
[07:51:59.091] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:21:0) after 3m24s
[07:55:23.152] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:21:0)
[07:55:23.255] Build status: SCHEDULED
[07:55:23.255] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:22:0) after 2m45s
[07:58:08.328] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:22:0)
[07:58:08.379] Build status: SCHEDULED
[07:58:08.379] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:23:0) after 5m20s
[08:03:28.452] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:23:0)
[08:03:28.475] Build status: SCHEDULED
[08:03:28.475] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:24:0) after 9m1s
[08:12:29.478] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:24:0)
[08:12:29.517] Build status: SCHEDULED
[08:12:29.517] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:25:0) after 9m31s
[08:22:00.534] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:25:0)
[08:22:00.568] Build status: SCHEDULED
[08:22:00.568] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:26:0) after 4m10s
[08:26:10.582] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:26:0)
[08:26:10.600] Build status: SCHEDULED
[08:26:10.600] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:27:0) after 4m26s
[08:30:36.666] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:27:0)
[08:30:36.686] Build status: SCHEDULED
[08:30:36.686] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:28:0) after 1m46s
[08:32:22.727] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:28:0)
[08:32:22.752] Build status: SCHEDULED
[08:32:22.752] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:29:0) after 6m17s
[08:38:39.769] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:29:0)
[08:38:39.788] Build status: SCHEDULED
[08:38:39.788] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:30:0) after 5m54s
[08:44:33.858] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:30:0)
[08:44:33.886] Build status: SCHEDULED
[08:44:33.886] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:31:0) after 2m46s
[08:47:19.903] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:31:0)
[08:47:19.935] Build status: SCHEDULED
[08:47:19.935] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:32:0) after 3m37s
[08:50:56.951] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:32:0)
[08:50:56.968] Build status: SCHEDULED
[08:50:56.968] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:33:0) after 2m18s
[08:53:15.053] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:33:0)
[08:53:15.118] Build status: SCHEDULED
[08:53:15.118] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:34:0) after 5m21s
[08:58:36.186] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:34:0)
[08:58:36.209] Build status: SCHEDULED
[08:58:36.209] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:35:0) after 6m48s
[09:05:24.244] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:35:0)
[09:05:24.264] Build status: SCHEDULED
[09:05:24.264] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:36:0) after 6m25s
[09:11:49.281] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:36:0)
[09:11:49.302] Build status: SCHEDULED
[09:11:49.302] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:37:0) after 9m40s
[09:21:29.370] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:37:0)
[09:21:29.594] Build status: SCHEDULED
[09:21:29.594] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:38:0) after 8m45s
[09:30:14.659] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:38:0)
[09:30:14.719] Build status: SCHEDULED
[09:30:14.719] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:39:0) after 9m37s
[09:33:44.106] Received PubSub notification, asking Buildbucket for the build status
[09:33:44.196] Build status: STARTED
[09:39:51.734] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:39:0)
[09:39:51.750] Build status: STARTED
[09:39:51.750] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-gtest:8843386129738183936:41:0) after 7m35s
[09:40:17.758] Received PubSub notification, asking Buildbucket for the build status
[09:40:17.777] Build:
{
  "id": "8677956067737806705",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-linux-bazel-sanitizers-tsan-gtest"
  },
  "createdBy": "project:pigweed",
  "createTime": "2026-06-26T05:53:21.504173519Z",
  "startTime": "2026-06-26T09:32:09.033535181Z",
  "endTime": "2026-06-26T09:40:17.508978061Z",
  "updateTime": "2026-06-26T09:40:17.508978061Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "278f49430d9cbf06ebed4fb7c2e65b6236950663",
      "ref": "refs/heads/main"
    }
  }
}
[09:40:17.777] Invocation finished in 3h46m57.563565966s with status FAILED