1. All projects
  2. pigweed
  3. pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan
  4. 8867950428784367152
Status: FAILED
Triggered by: -
Duration: 6 hours
Actions:

Definition (rev ed331b29843b850a4bc37ad64e7e3a240af6f486) 

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-x86-bazel-sanitizers-asan"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "e8d1563a34ee20ed1b26f4493697c94b165e9572"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/e8d1563a34ee20ed1b26f4493697c94b165e9572
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[02:34:19.132] New invocation is queued and will start shortly
[02:34:20.291] Starting the invocation (attempt 1)
[02:34:20.326] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/e8d1563a34ee20ed1b26f4493697c94b165e9572"
[02:34:20.326] Popped gitiles commit info from properties and tags
[02:34:20.326] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[02:34:20.328] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[02:34:20.328] Buildbucket request:
{
  "requestId": "8867950428784367152",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-sanitizers-asan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8867950428784367152",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "e8d1563a34ee20ed1b26f4493697c94b165e9572"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@e8d1563a34ee20ed1b26f4493697c94b165e9572",
                        "title": "e8d1563a34ee20ed1b26f4493697c94b165e9572",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/e8d1563a34ee20ed1b26f4493697c94b165e9572"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "e8d1563a34ee20ed1b26f4493697c94b165e9572",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8867950428784367152"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[02:34:20.693] Scheduled build:
{
  "id": "8702520366783226929",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-sanitizers-asan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-09-28T02:34:20.404494014Z",
  "updateTime": "2025-09-28T02:34:20.404494014Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "e8d1563a34ee20ed1b26f4493697c94b165e9572",
      "ref": "refs/heads/main"
    }
  }
}
[02:34:20.693] Task URL: https://cr-buildbucket.appspot.com/build/8702520366783226929
[02:34:20.693] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:2:0) after 4m57s
[02:34:31.060] Received PubSub notification, asking Buildbucket for the build status
[02:34:31.076] Build status: SCHEDULED
[02:39:17.681] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:2:0)
[02:39:17.708] Build status: SCHEDULED
[02:39:17.708] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:4:0) after 6m7s
[02:45:24.750] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:4:0)
[02:45:24.779] Build status: SCHEDULED
[02:45:24.779] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:5:0) after 7m44s
[02:53:08.779] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:5:0)
[02:53:08.880] Build status: SCHEDULED
[02:53:08.880] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:6:0) after 5m2s
[02:58:10.942] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:6:0)
[02:58:10.966] Build status: SCHEDULED
[02:58:10.966] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:7:0) after 2m56s
[03:01:07.391] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:7:0)
[03:01:07.414] Build status: SCHEDULED
[03:01:07.414] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:8:0) after 6m0s
[03:07:07.426] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:8:0)
[03:07:07.519] Build status: SCHEDULED
[03:07:07.519] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:9:0) after 6m3s
[03:13:10.747] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:9:0)
[03:13:10.768] Build status: SCHEDULED
[03:13:10.768] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:10:0) after 1m0s
[03:14:10.781] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:10:0)
[03:14:10.794] Build status: SCHEDULED
[03:14:10.794] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:11:0) after 7m38s
[03:21:48.844] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:11:0)
[03:21:48.897] Build status: SCHEDULED
[03:21:48.897] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:12:0) after 4m29s
[03:26:17.930] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:12:0)
[03:26:17.945] Build status: SCHEDULED
[03:26:17.945] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:13:0) after 8m1s
[03:34:18.955] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:13:0)
[03:34:18.972] Build status: SCHEDULED
[03:34:18.972] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:14:0) after 5m58s
[03:40:17.511] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:14:0)
[03:40:17.527] Build status: SCHEDULED
[03:40:17.527] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:15:0) after 6m44s
[03:47:01.543] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:15:0)
[03:47:01.557] Build status: SCHEDULED
[03:47:01.557] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:16:0) after 4m0s
[03:51:01.644] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:16:0)
[03:51:01.676] Build status: SCHEDULED
[03:51:01.676] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:17:0) after 4m34s
[03:55:35.688] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:17:0)
[03:55:35.703] Build status: SCHEDULED
[03:55:35.703] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:18:0) after 3m24s
[03:58:59.718] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:18:0)
[03:58:59.736] Build status: SCHEDULED
[03:58:59.736] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:19:0) after 4m31s
[04:03:30.749] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:19:0)
[04:03:30.764] Build status: SCHEDULED
[04:03:30.764] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:20:0) after 2m35s
[04:06:05.884] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:20:0)
[04:06:06.024] Build status: SCHEDULED
[04:06:06.024] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:21:0) after 5m40s
[04:11:46.007] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:21:0)
[04:11:46.027] Build status: SCHEDULED
[04:11:46.027] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:22:0) after 1m0s
[04:12:46.927] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:22:0)
[04:12:46.944] Build status: SCHEDULED
[04:12:46.944] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:23:0) after 3m53s
[04:16:40.724] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:23:0)
[04:16:40.739] Build status: SCHEDULED
[04:16:40.739] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:24:0) after 9m46s
[04:26:26.795] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:24:0)
[04:26:26.814] Build status: SCHEDULED
[04:26:26.814] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:25:0) after 1m11s
[04:27:37.875] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:25:0)
[04:27:37.891] Build status: SCHEDULED
[04:27:37.891] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:26:0) after 7m12s
[04:34:50.072] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:26:0)
[04:34:50.090] Build status: SCHEDULED
[04:34:50.090] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:27:0) after 8m57s
[04:43:47.190] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:27:0)
[04:43:47.210] Build status: SCHEDULED
[04:43:47.210] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:28:0) after 7m46s
[04:51:33.223] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:28:0)
[04:51:33.274] Build status: SCHEDULED
[04:51:33.274] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:29:0) after 3m35s
[04:55:08.668] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:29:0)
[04:55:08.684] Build status: SCHEDULED
[04:55:08.684] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:30:0) after 7m6s
[05:02:14.727] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:30:0)
[05:02:14.752] Build status: SCHEDULED
[05:02:14.752] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:31:0) after 2m11s
[05:04:26.325] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:31:0)
[05:04:26.397] Build status: SCHEDULED
[05:04:26.397] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:32:0) after 1m39s
[05:06:05.399] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:32:0)
[05:06:05.992] Build status: SCHEDULED
[05:06:05.992] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:33:0) after 2m0s
[05:08:06.007] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:33:0)
[05:08:06.028] Build status: SCHEDULED
[05:08:06.029] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:34:0) after 3m57s
[05:12:03.209] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:34:0)
[05:12:03.226] Build status: SCHEDULED
[05:12:03.226] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:35:0) after 6m41s
[05:18:44.241] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:35:0)
[05:18:44.257] Build status: SCHEDULED
[05:18:44.257] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:36:0) after 9m55s
[05:28:39.285] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:36:0)
[05:28:39.343] Build status: SCHEDULED
[05:28:39.343] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:37:0) after 1m22s
[05:30:01.362] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:37:0)
[05:30:01.535] Build status: SCHEDULED
[05:30:01.535] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:38:0) after 9m15s
[05:39:16.551] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:38:0)
[05:39:16.578] Build status: SCHEDULED
[05:39:16.578] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:39:0) after 1m4s
[05:40:20.674] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:39:0)
[05:40:20.692] Build status: SCHEDULED
[05:40:20.692] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:40:0) after 4m37s
[05:44:57.762] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:40:0)
[05:44:57.874] Build status: SCHEDULED
[05:44:57.874] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:41:0) after 6m28s
[05:51:26.012] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:41:0)
[05:51:26.027] Build status: SCHEDULED
[05:51:26.027] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:42:0) after 8m24s
[05:59:50.138] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:42:0)
[05:59:50.154] Build status: SCHEDULED
[05:59:50.154] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:43:0) after 7m23s
[06:07:13.272] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:43:0)
[06:07:13.296] Build status: SCHEDULED
[06:07:13.297] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:44:0) after 2m14s
[06:09:27.317] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:44:0)
[06:09:27.337] Build status: SCHEDULED
[06:09:27.337] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:45:0) after 7m37s
[06:17:04.422] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:45:0)
[06:17:04.441] Build status: SCHEDULED
[06:17:04.441] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:46:0) after 8m14s
[06:25:18.458] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:46:0)
[06:25:18.513] Build status: SCHEDULED
[06:25:18.513] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:47:0) after 7m23s
[06:32:42.063] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:47:0)
[06:32:42.092] Build status: SCHEDULED
[06:32:42.092] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:48:0) after 7m23s
[06:40:05.109] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:48:0)
[06:40:05.132] Build status: SCHEDULED
[06:40:05.132] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:49:0) after 6m51s
[06:46:56.147] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:49:0)
[06:46:56.168] Build status: SCHEDULED
[06:46:56.168] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:50:0) after 1m46s
[06:48:42.318] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:50:0)
[06:48:42.332] Build status: SCHEDULED
[06:48:42.332] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:51:0) after 5m47s
[06:54:29.350] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:51:0)
[06:54:29.437] Build status: SCHEDULED
[06:54:29.437] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:52:0) after 8m13s
[07:02:42.804] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:52:0)
[07:02:42.931] Build status: SCHEDULED
[07:02:42.931] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:53:0) after 9m36s
[07:12:18.921] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:53:0)
[07:12:18.941] Build status: SCHEDULED
[07:12:18.941] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:54:0) after 1m57s
[07:14:16.136] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:54:0)
[07:14:16.156] Build status: SCHEDULED
[07:14:16.156] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:55:0) after 5m24s
[07:19:40.287] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:55:0)
[07:19:40.305] Build status: SCHEDULED
[07:19:40.305] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:56:0) after 3m49s
[07:23:29.545] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:56:0)
[07:23:29.562] Build status: SCHEDULED
[07:23:29.562] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:57:0) after 5m41s
[07:29:10.765] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:57:0)
[07:29:10.801] Build status: SCHEDULED
[07:29:10.801] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:58:0) after 9m37s
[07:38:47.798] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:58:0)
[07:38:47.901] Build status: SCHEDULED
[07:38:47.901] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:59:0) after 5m59s
[07:44:47.130] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:59:0)
[07:44:47.148] Build status: SCHEDULED
[07:44:47.148] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:60:0) after 2m34s
[07:47:21.175] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:60:0)
[07:47:21.193] Build status: SCHEDULED
[07:47:21.193] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:61:0) after 1m30s
[07:48:51.571] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:61:0)
[07:48:51.603] Build status: SCHEDULED
[07:48:51.603] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:62:0) after 8m9s
[07:57:00.784] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:62:0)
[07:57:00.805] Build status: SCHEDULED
[07:57:00.805] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:63:0) after 1m34s
[07:58:34.821] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:63:0)
[07:58:34.877] Build status: SCHEDULED
[07:58:34.878] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:64:0) after 5m53s
[08:04:27.951] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:64:0)
[08:04:27.974] Build status: SCHEDULED
[08:04:27.974] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:65:0) after 6m25s
[08:10:53.177] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:65:0)
[08:10:53.195] Build status: SCHEDULED
[08:10:53.195] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:66:0) after 8m57s
[08:19:50.300] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:66:0)
[08:19:50.437] Build status: SCHEDULED
[08:19:50.437] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:67:0) after 2m9s
[08:21:59.458] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:67:0)
[08:21:59.474] Build status: SCHEDULED
[08:21:59.474] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:68:0) after 7m23s
[08:29:22.489] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:68:0)
[08:29:22.503] Build status: SCHEDULED
[08:29:22.503] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:69:0) after 8m5s
[08:37:27.524] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:69:0)
[08:37:27.542] Build status: SCHEDULED
[08:37:27.542] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:70:0) after 8m29s
[08:45:56.563] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:70:0)
[08:45:56.597] Build status: SCHEDULED
[08:45:56.597] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:71:0) after 5m10s
[08:49:22.406] Received PubSub notification, asking Buildbucket for the build status
[08:49:22.442] Build status: STARTED
[08:51:06.703] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:71:0)
[08:51:06.720] Build status: STARTED
[08:51:06.720] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:73:0) after 2m8s
[08:53:14.733] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:73:0)
[08:53:14.751] Build status: STARTED
[08:53:14.751] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8867950428784367152:74:0) after 4m44s
[08:55:06.752] Received PubSub notification, asking Buildbucket for the build status
[08:55:06.848] Build:
{
  "id": "8702520366783226929",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-sanitizers-asan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-09-28T02:34:20.404494014Z",
  "startTime": "2025-09-28T08:49:22.237444638Z",
  "endTime": "2025-09-28T08:55:06.410634427Z",
  "updateTime": "2025-09-28T08:55:06.410634427Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "e8d1563a34ee20ed1b26f4493697c94b165e9572",
      "ref": "refs/heads/main"
    }
  }
}
[08:55:06.848] Invocation finished in 6h20m47.736797083s with status FAILED