buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-arm-bazel-sanitizers-tsan-build"
>

• 11987af0771a4fc67b1f9f4f9d207f89909193f5 (2 weeks ago)

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "11987af0771a4fc67b1f9f4f9d207f89909193f5"
}

• buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/11987af0771a4fc67b1f9f4f9d207f89909193f5
• gitiles_ref:refs/heads/main

[04:50:01.314] New invocation is queued and will start shortly
[04:50:02.372] Starting the invocation (attempt 1)
[04:50:02.399] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/11987af0771a4fc67b1f9f4f9d207f89909193f5"
[04:50:02.399] Popped gitiles commit info from properties and tags
[04:50:02.399] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[04:50:02.399] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[04:50:02.399] Buildbucket request:
{
  "requestId": "8843480710145098960",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8843480710145098960",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "11987af0771a4fc67b1f9f4f9d207f89909193f5"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@11987af0771a4fc67b1f9f4f9d207f89909193f5",
                        "title": "11987af0771a4fc67b1f9f4f9d207f89909193f5",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/11987af0771a4fc67b1f9f4f9d207f89909193f5"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "11987af0771a4fc67b1f9f4f9d207f89909193f5",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8843480710145098960"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[04:50:02.788] Scheduled build:
{
  "id": "8678050648263528913",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
  },
  "createdBy": "project:pigweed",
  "createTime": "2026-06-25T04:50:02.486377308Z",
  "updateTime": "2026-06-25T04:50:02.486377308Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "11987af0771a4fc67b1f9f4f9d207f89909193f5",
      "ref": "refs/heads/main"
    }
  }
}
[04:50:02.789] Task URL: https://cr-buildbucket.appspot.com/build/8678050648263528913
[04:50:02.789] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:2:0) after 9m6s
[04:50:03.741] Received PubSub notification, asking Buildbucket for the build status
[04:50:03.813] Build status: SCHEDULED
[04:59:08.823] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:2:0)
[04:59:08.848] Build status: SCHEDULED
[04:59:08.848] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:4:0) after 4m4s
[05:03:12.887] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:4:0)
[05:03:12.948] Build status: SCHEDULED
[05:03:12.948] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:5:0) after 9m11s
[05:12:24.056] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:5:0)
[05:12:24.077] Build status: SCHEDULED
[05:12:24.077] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:6:0) after 1m31s
[05:13:55.250] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:6:0)
[05:13:55.329] Build status: SCHEDULED
[05:13:55.329] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:7:0) after 3m35s
[05:17:30.348] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:7:0)
[05:17:30.382] Build status: SCHEDULED
[05:17:30.382] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:8:0) after 4m33s
[05:22:03.459] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:8:0)
[05:22:03.500] Build status: SCHEDULED
[05:22:03.500] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:9:0) after 2m13s
[05:24:16.601] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:9:0)
[05:24:16.635] Build status: SCHEDULED
[05:24:16.635] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:10:0) after 1m28s
[05:25:44.649] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:10:0)
[05:25:44.670] Build status: SCHEDULED
[05:25:44.670] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:11:0) after 3m30s
[05:29:14.683] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:11:0)
[05:29:14.703] Build status: SCHEDULED
[05:29:14.703] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:12:0) after 5m19s
[05:34:33.717] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:12:0)
[05:34:33.734] Build status: SCHEDULED
[05:34:33.735] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:13:0) after 7m32s
[05:42:05.984] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:13:0)
[05:42:06.016] Build status: SCHEDULED
[05:42:06.016] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:14:0) after 8m56s
[05:51:02.060] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:14:0)
[05:51:02.105] Build status: SCHEDULED
[05:51:02.105] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:15:0) after 7m57s
[05:58:59.121] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:15:0)
[05:58:59.145] Build status: SCHEDULED
[05:58:59.145] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:16:0) after 2m47s
[06:01:46.173] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:16:0)
[06:01:46.193] Build status: SCHEDULED
[06:01:46.193] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:17:0) after 2m36s
[06:04:22.282] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:17:0)
[06:04:22.307] Build status: SCHEDULED
[06:04:22.308] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:18:0) after 3m43s
[06:08:05.337] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:18:0)
[06:08:05.375] Build status: SCHEDULED
[06:08:05.375] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:19:0) after 3m52s
[06:11:57.392] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:19:0)
[06:11:57.466] Build status: SCHEDULED
[06:11:57.466] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:20:0) after 4m58s
[06:16:55.451] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:20:0)
[06:16:55.489] Build status: SCHEDULED
[06:16:55.489] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:21:0) after 9m16s
[06:26:11.506] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:21:0)
[06:26:11.524] Build status: SCHEDULED
[06:26:11.524] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:22:0) after 2m57s
[06:29:08.604] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:22:0)
[06:29:08.766] Build status: SCHEDULED
[06:29:08.766] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:23:0) after 9m49s
[06:38:57.935] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:23:0)
[06:38:57.977] Build status: SCHEDULED
[06:38:57.977] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:24:0) after 9m31s
[06:48:29.117] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:24:0)
[06:48:29.219] Build status: SCHEDULED
[06:48:29.219] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:25:0) after 1m49s
[06:50:18.191] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:25:0)
[06:50:18.210] Build status: SCHEDULED
[06:50:18.210] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:26:0) after 1m45s
[06:52:03.240] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:26:0)
[06:52:03.269] Build status: SCHEDULED
[06:52:03.270] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:27:0) after 6m21s
[06:58:24.284] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:27:0)
[06:58:24.425] Build status: SCHEDULED
[06:58:24.425] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:28:0) after 6m15s
[07:04:39.441] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:28:0)
[07:04:39.471] Build status: SCHEDULED
[07:04:39.471] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:29:0) after 8m47s
[07:13:26.568] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:29:0)
[07:13:26.605] Build status: SCHEDULED
[07:13:26.605] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:30:0) after 4m17s
[07:17:43.624] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:30:0)
[07:17:43.665] Build status: SCHEDULED
[07:17:43.665] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:31:0) after 8m28s
[07:26:11.679] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:31:0)
[07:26:11.704] Build status: SCHEDULED
[07:26:11.704] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:32:0) after 9m34s
[07:35:45.830] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:32:0)
[07:35:45.849] Build status: SCHEDULED
[07:35:45.849] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:33:0) after 4m54s
[07:40:39.863] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:33:0)
[07:40:39.882] Build status: SCHEDULED
[07:40:39.882] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:34:0) after 9m11s
[07:49:18.032] Received PubSub notification, asking Buildbucket for the build status
[07:49:18.050] Build status: STARTED
[07:49:50.987] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:34:0)
[07:49:51.009] Build status: STARTED
[07:49:51.009] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:36:0) after 2m49s
[07:52:40.019] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:36:0)
[07:52:40.056] Build status: STARTED
[07:52:40.056] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:37:0) after 3m42s
[07:56:22.083] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:37:0)
[07:56:22.101] Build status: STARTED
[07:56:22.101] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:38:0) after 1m5s
[07:57:27.166] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:38:0)
[07:57:27.188] Build status: STARTED
[07:57:27.188] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-build:8843480710145098960:39:0) after 8m33s
[07:59:16.456] Received PubSub notification, asking Buildbucket for the build status
[07:59:16.496] Build:
{
  "id": "8678050648263528913",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan-build"
  },
  "createdBy": "project:pigweed",
  "createTime": "2026-06-25T04:50:02.486377308Z",
  "startTime": "2026-06-25T07:49:17.866166169Z",
  "endTime": "2026-06-25T07:59:15.903651171Z",
  "updateTime": "2026-06-25T07:59:15.903651171Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "11987af0771a4fc67b1f9f4f9d207f89909193f5",
      "ref": "refs/heads/main"
    }
  }
}
[07:59:16.496] Invocation finished in 3h9m15.187438249s with status FAILED