Status: FAILED
Triggered by: -
Duration: 3 hours
Actions:

Definition (rev b10570f0bf2a66ff95088887bb25ca04e788f46a)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "6ce4987872e7a726cb76ec077df8c8664306ac0e"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/6ce4987872e7a726cb76ec077df8c8664306ac0e
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[22:20:00.140] New invocation is queued and will start shortly
[22:20:01.213] Starting the invocation (attempt 1)
[22:20:01.234] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/6ce4987872e7a726cb76ec077df8c8664306ac0e"
[22:20:01.234] Popped gitiles commit info from properties and tags
[22:20:01.234] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[22:20:01.234] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[22:20:01.234] Buildbucket request:
{
  "requestId": "8843052263230215392",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8843052263230215392",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "6ce4987872e7a726cb76ec077df8c8664306ac0e"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@6ce4987872e7a726cb76ec077df8c8664306ac0e",
                        "title": "6ce4987872e7a726cb76ec077df8c8664306ac0e",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/6ce4987872e7a726cb76ec077df8c8664306ac0e"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "6ce4987872e7a726cb76ec077df8c8664306ac0e",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8843052263230215392"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[22:20:01.782] Scheduled build:
{
  "id": "8677622201337421297",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
  },
  "createdBy": "project:pigweed",
  "createTime": "2026-06-29T22:20:01.315001686Z",
  "updateTime": "2026-06-29T22:20:01.315001686Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "6ce4987872e7a726cb76ec077df8c8664306ac0e",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[22:20:01.782] Task URL: https://cr-buildbucket.appspot.com/build/8677622201337421297
[22:20:01.782] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:2:0) after 4m52s
[22:20:02.438] Received PubSub notification, asking Buildbucket for the build status
[22:20:02.467] Build status: SCHEDULED
[22:24:53.806] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:2:0)
[22:24:53.832] Build status: SCHEDULED
[22:24:53.833] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:4:0) after 9m27s
[22:34:20.891] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:4:0)
[22:34:20.943] Build status: SCHEDULED
[22:34:20.943] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:5:0) after 8m3s
[22:42:23.955] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:5:0)
[22:42:23.976] Build status: SCHEDULED
[22:42:23.976] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:6:0) after 8m54s
[22:51:17.952] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:6:0)
[22:51:18.009] Build status: SCHEDULED
[22:51:18.010] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:7:0) after 6m38s
[22:57:56.026] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:7:0)
[22:57:56.047] Build status: SCHEDULED
[22:57:56.047] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:8:0) after 1m34s
[22:59:30.106] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:8:0)
[22:59:30.156] Build status: SCHEDULED
[22:59:30.156] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:9:0) after 2m53s
[23:02:23.168] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:9:0)
[23:02:23.190] Build status: SCHEDULED
[23:02:23.190] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:10:0) after 4m57s
[23:07:20.204] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:10:0)
[23:07:20.224] Build status: SCHEDULED
[23:07:20.224] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:11:0) after 2m29s
[23:09:49.292] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:11:0)
[23:09:49.314] Build status: SCHEDULED
[23:09:49.314] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:12:0) after 9m36s
[23:19:25.330] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:12:0)
[23:19:25.361] Build status: SCHEDULED
[23:19:25.361] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:13:0) after 4m6s
[23:23:31.441] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:13:0)
[23:23:31.464] Build status: SCHEDULED
[23:23:31.464] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:14:0) after 9m13s
[23:32:44.482] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:14:0)
[23:32:44.503] Build status: SCHEDULED
[23:32:44.504] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:15:0) after 4m31s
[23:37:15.536] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:15:0)
[23:37:15.560] Build status: SCHEDULED
[23:37:15.560] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:16:0) after 2m53s
[23:40:08.569] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:16:0)
[23:40:08.589] Build status: SCHEDULED
[23:40:08.589] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:17:0) after 6m49s
[23:46:57.629] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:17:0)
[23:46:57.658] Build status: SCHEDULED
[23:46:57.658] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:18:0) after 3m53s
[23:50:50.729] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:18:0)
[23:50:50.749] Build status: SCHEDULED
[23:50:50.749] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:19:0) after 3m15s
[23:54:05.844] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:19:0)
[23:54:05.864] Build status: SCHEDULED
[23:54:05.864] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:20:0) after 6m7s
[00:00:12.861] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:20:0)
[00:00:12.884] Build status: SCHEDULED
[00:00:12.884] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:21:0) after 1m54s
[00:02:06.945] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:21:0)
[00:02:06.967] Build status: SCHEDULED
[00:02:06.967] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:22:0) after 1m26s
[00:03:32.991] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:22:0)
[00:03:33.035] Build status: SCHEDULED
[00:03:33.035] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:23:0) after 4m1s
[00:07:34.106] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:23:0)
[00:07:34.124] Build status: SCHEDULED
[00:07:34.124] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:24:0) after 3m41s
[00:11:15.203] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:24:0)
[00:11:15.229] Build status: SCHEDULED
[00:11:15.229] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:25:0) after 8m51s
[00:20:06.344] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:25:0)
[00:20:06.367] Build status: SCHEDULED
[00:20:06.367] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:26:0) after 3m13s
[00:23:19.429] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:26:0)
[00:23:19.511] Build status: SCHEDULED
[00:23:19.511] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:27:0) after 8m8s
[00:31:27.589] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:27:0)
[00:31:27.649] Build status: SCHEDULED
[00:31:27.649] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:28:0) after 1m44s
[00:33:11.662] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:28:0)
[00:33:11.682] Build status: SCHEDULED
[00:33:11.682] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:29:0) after 5m54s
[00:39:05.698] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:29:0)
[00:39:05.718] Build status: SCHEDULED
[00:39:05.718] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:30:0) after 6m3s
[00:45:08.782] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:30:0)
[00:45:08.807] Build status: SCHEDULED
[00:45:08.807] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:31:0) after 7m50s
[00:52:58.903] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:31:0)
[00:52:58.920] Build status: SCHEDULED
[00:52:58.920] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:32:0) after 3m3s
[00:56:01.944] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:32:0)
[00:56:01.981] Build status: SCHEDULED
[00:56:01.981] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:33:0) after 3m45s
[00:59:46.998] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:33:0)
[00:59:47.025] Build status: SCHEDULED
[00:59:47.025] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:34:0) after 7m55s
[01:07:42.082] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:34:0)
[01:07:42.099] Build status: SCHEDULED
[01:07:42.099] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:35:0) after 4m45s
[01:12:27.219] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:35:0)
[01:12:27.236] Build status: SCHEDULED
[01:12:27.236] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:36:0) after 8m31s
[01:20:58.285] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:36:0)
[01:20:58.311] Build status: SCHEDULED
[01:20:58.311] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:37:0) after 1m54s
[01:22:52.326] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:37:0)
[01:22:52.352] Build status: SCHEDULED
[01:22:52.352] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:38:0) after 2m27s
[01:25:19.460] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:38:0)
[01:25:19.484] Build status: SCHEDULED
[01:25:19.484] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:39:0) after 8m10s
[01:33:29.469] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:39:0)
[01:33:29.489] Build status: SCHEDULED
[01:33:29.490] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:40:0) after 1m57s
[01:35:26.577] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:40:0)
[01:35:26.593] Build status: SCHEDULED
[01:35:26.593] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:41:0) after 3m32s
[01:38:58.580] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:41:0)
[01:38:58.612] Build status: SCHEDULED
[01:38:58.613] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:42:0) after 3m18s
[01:42:16.610] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:42:0)
[01:42:16.627] Build status: SCHEDULED
[01:42:16.627] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:43:0) after 4m10s
[01:46:26.686] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:43:0)
[01:46:26.709] Build status: SCHEDULED
[01:46:26.709] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:44:0) after 5m36s
[01:52:02.704] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:44:0)
[01:52:02.721] Build status: SCHEDULED
[01:52:02.721] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:45:0) after 7m7s
[01:59:09.737] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:45:0)
[01:59:09.755] Build status: SCHEDULED
[01:59:09.755] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:46:0) after 2m31s
[02:01:40.807] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:46:0)
[02:01:40.828] Build status: SCHEDULED
[02:01:40.828] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan-gtest:8843052263230215392:47:0) after 7m37s
[02:03:56.104] Received PubSub notification, asking Buildbucket for the build status
[02:03:56.128] Build status: STARTED
[02:07:44.252] Received PubSub notification, asking Buildbucket for the build status
[02:07:44.299] Build:
{
  "id": "8677622201337421297",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-arm-bazel-sanitizers-tsan-gtest"
  },
  "createdBy": "project:pigweed",
  "createTime": "2026-06-29T22:20:01.315001686Z",
  "startTime": "2026-06-30T02:03:55.857405389Z",
  "endTime": "2026-06-30T02:07:43.879686882Z",
  "updateTime": "2026-06-30T02:07:43.879686882Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "6ce4987872e7a726cb76ec077df8c8664306ac0e",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[02:07:44.299] Invocation finished in 3h47m44.171395935s with status FAILED