buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
>

• b4b46ec292a1eac3adae35fdcb6d775ca36d19db (2 weeks ago)

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "b4b46ec292a1eac3adae35fdcb6d775ca36d19db"
}

• buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/b4b46ec292a1eac3adae35fdcb6d775ca36d19db
• gitiles_ref:refs/heads/main

[19:34:35.933] New invocation is queued and will start shortly
[19:34:40.795] Starting the invocation (attempt 1)
[19:34:40.822] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/b4b46ec292a1eac3adae35fdcb6d775ca36d19db"
[19:34:40.822] Popped gitiles commit info from properties and tags
[19:34:40.822] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[19:34:40.822] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[19:34:40.822] Buildbucket request:
{
  "requestId": "8843062669510772688",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8843062669510772688",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "b4b46ec292a1eac3adae35fdcb6d775ca36d19db"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@b4b46ec292a1eac3adae35fdcb6d775ca36d19db",
                        "title": "b4b46ec292a1eac3adae35fdcb6d775ca36d19db",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/b4b46ec292a1eac3adae35fdcb6d775ca36d19db"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "b4b46ec292a1eac3adae35fdcb6d775ca36d19db",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8843062669510772688"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[19:34:41.255] Scheduled build:
{
  "id": "8677632603647671121",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
  },
  "createdBy": "project:pigweed",
  "createTime": "2026-06-29T19:34:40.899213594Z",
  "updateTime": "2026-06-29T19:34:40.899213594Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "b4b46ec292a1eac3adae35fdcb6d775ca36d19db",
      "ref": "refs/heads/main"
    }
  }
}
[19:34:41.255] Task URL: https://cr-buildbucket.appspot.com/build/8677632603647671121
[19:34:41.255] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:2:0) after 5m46s
[19:34:42.193] Received PubSub notification, asking Buildbucket for the build status
[19:34:42.237] Build status: SCHEDULED
[19:40:27.366] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:2:0)
[19:40:27.387] Build status: SCHEDULED
[19:40:27.387] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:4:0) after 6m11s
[19:46:38.462] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:4:0)
[19:46:38.483] Build status: SCHEDULED
[19:46:38.483] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:5:0) after 2m4s
[19:48:42.550] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:5:0)
[19:48:42.580] Build status: SCHEDULED
[19:48:42.580] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:6:0) after 1m38s
[19:50:20.675] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:6:0)
[19:50:20.699] Build status: SCHEDULED
[19:50:20.699] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:7:0) after 7m6s
[19:57:26.832] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:7:0)
[19:57:26.852] Build status: SCHEDULED
[19:57:26.852] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:8:0) after 1m21s
[19:58:47.872] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:8:0)
[19:58:47.896] Build status: SCHEDULED
[19:58:47.896] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:9:0) after 1m27s
[20:00:14.934] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:9:0)
[20:00:14.956] Build status: SCHEDULED
[20:00:14.956] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:10:0) after 1m54s
[20:02:09.018] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:10:0)
[20:02:09.040] Build status: SCHEDULED
[20:02:09.040] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:11:0) after 6m21s
[20:08:30.297] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:11:0)
[20:08:30.315] Build status: SCHEDULED
[20:08:30.315] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:12:0) after 5m37s
[20:14:07.430] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:12:0)
[20:14:07.450] Build status: SCHEDULED
[20:14:07.450] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:13:0) after 9m43s
[20:23:50.488] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:13:0)
[20:23:50.509] Build status: SCHEDULED
[20:23:50.509] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:14:0) after 1m56s
[20:25:46.482] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:14:0)
[20:25:46.510] Build status: SCHEDULED
[20:25:46.510] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:15:0) after 6m39s
[20:32:25.621] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:15:0)
[20:32:25.660] Build status: SCHEDULED
[20:32:25.660] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:16:0) after 8m19s
[20:40:44.789] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:16:0)
[20:40:44.830] Build status: SCHEDULED
[20:40:44.830] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:17:0) after 7m21s
[20:48:05.810] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:17:0)
[20:48:05.828] Build status: SCHEDULED
[20:48:05.829] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:18:0) after 9m28s
[20:57:33.893] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:18:0)
[20:57:33.913] Build status: SCHEDULED
[20:57:33.913] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:19:0) after 4m41s
[21:02:14.935] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:19:0)
[21:02:14.953] Build status: SCHEDULED
[21:02:14.953] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:20:0) after 9m42s
[21:11:57.073] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:20:0)
[21:11:57.146] Build status: SCHEDULED
[21:11:57.146] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:21:0) after 5m49s
[21:17:46.174] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:21:0)
[21:17:46.208] Build status: SCHEDULED
[21:17:46.208] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:22:0) after 9m4s
[21:26:50.221] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:22:0)
[21:26:50.240] Build status: SCHEDULED
[21:26:50.240] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:23:0) after 5m14s
[21:32:04.290] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:23:0)
[21:32:04.305] Build status: SCHEDULED
[21:32:04.305] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:24:0) after 6m55s
[21:38:59.328] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:24:0)
[21:38:59.347] Build status: SCHEDULED
[21:38:59.347] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:25:0) after 2m21s
[21:41:20.375] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:25:0)
[21:41:20.399] Build status: SCHEDULED
[21:41:20.400] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:26:0) after 1m54s
[21:43:14.524] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:26:0)
[21:43:14.552] Build status: SCHEDULED
[21:43:14.552] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:27:0) after 1m32s
[21:44:46.580] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:27:0)
[21:44:46.597] Build status: SCHEDULED
[21:44:46.597] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:28:0) after 6m17s
[21:51:03.631] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:28:0)
[21:51:03.649] Build status: SCHEDULED
[21:51:03.649] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:29:0) after 2m48s
[21:53:51.730] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:29:0)
[21:53:51.749] Build status: SCHEDULED
[21:53:51.749] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:30:0) after 1m36s
[21:55:27.873] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:30:0)
[21:55:27.893] Build status: SCHEDULED
[21:55:27.893] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:31:0) after 4m1s
[21:59:28.866] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:31:0)
[21:59:28.936] Build status: SCHEDULED
[21:59:28.936] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:32:0) after 7m33s
[22:07:01.954] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:32:0)
[22:07:01.973] Build status: SCHEDULED
[22:07:01.973] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:33:0) after 6m34s
[22:12:15.510] Received PubSub notification, asking Buildbucket for the build status
[22:12:15.534] Build status: STARTED
[22:13:36.139] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:33:0)
[22:13:36.159] Build status: STARTED
[22:13:36.159] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:35:0) after 2m51s
[22:16:27.174] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:35:0)
[22:16:27.188] Build status: STARTED
[22:16:27.188] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:36:0) after 3m20s
[22:19:47.201] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:36:0)
[22:19:47.223] Build status: STARTED
[22:19:47.223] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843062669510772688:37:0) after 7m12s
[22:19:58.221] Received PubSub notification, asking Buildbucket for the build status
[22:19:58.243] Build:
{
  "id": "8677632603647671121",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
  },
  "createdBy": "project:pigweed",
  "createTime": "2026-06-29T19:34:40.899213594Z",
  "startTime": "2026-06-29T22:12:15.029403798Z",
  "endTime": "2026-06-29T22:19:57.410801076Z",
  "updateTime": "2026-06-29T22:19:57.410801076Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "b4b46ec292a1eac3adae35fdcb6d775ca36d19db",
      "ref": "refs/heads/main"
    }
  }
}
[22:19:58.243] Invocation finished in 2h45m22.319668035s with status FAILED