LUCI Scheduler :: pigweed :: pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan

Status: FAILED

Triggered by: -

Duration: 8 hours

Actions:

Definition (rev ed331b29843b850a4bc37ad64e7e3a240af6f486)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-x86-bazel-sanitizers-asan"
>

Incoming triggers

c4c8c7be6a2edd18076865fd4e9b66830c1dee48 (1 week ago)
e8d3556b3dd3fdd500c1ab8ad64573faca86a8c6 (1 week ago)
e1a57a6a46203effb449a4c1d910ba0e290f5d72 (1 week ago)
ab421e291f28316fdb926c5c06ddb4da7be3ff89 (1 week ago)

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "ab421e291f28316fdb926c5c06ddb4da7be3ff89"
}

Incoming tags

buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/ab421e291f28316fdb926c5c06ddb4da7be3ff89
gitiles_ref:refs/heads/main

Debug log (UTC)

[04:26:16.078] New invocation is queued and will start shortly
[04:26:17.124] Starting the invocation (attempt 1)
[04:26:17.146] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/ab421e291f28316fdb926c5c06ddb4da7be3ff89"
[04:26:17.146] Popped gitiles commit info from properties and tags
[04:26:17.146] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[04:26:17.147] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[04:26:17.147] Buildbucket request:
{
  "requestId": "8862507567557491056",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-sanitizers-asan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8862507567557491056",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "c4c8c7be6a2edd18076865fd4e9b66830c1dee48"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@c4c8c7be6a2edd18076865fd4e9b66830c1dee48",
                        "title": "c4c8c7be6a2edd18076865fd4e9b66830c1dee48",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/c4c8c7be6a2edd18076865fd4e9b66830c1dee48"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "e8d3556b3dd3fdd500c1ab8ad64573faca86a8c6"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@e8d3556b3dd3fdd500c1ab8ad64573faca86a8c6",
                        "title": "e8d3556b3dd3fdd500c1ab8ad64573faca86a8c6",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/e8d3556b3dd3fdd500c1ab8ad64573faca86a8c6"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "e1a57a6a46203effb449a4c1d910ba0e290f5d72"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@e1a57a6a46203effb449a4c1d910ba0e290f5d72",
                        "title": "e1a57a6a46203effb449a4c1d910ba0e290f5d72",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/e1a57a6a46203effb449a4c1d910ba0e290f5d72"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "ab421e291f28316fdb926c5c06ddb4da7be3ff89"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@ab421e291f28316fdb926c5c06ddb4da7be3ff89",
                        "title": "ab421e291f28316fdb926c5c06ddb4da7be3ff89",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/ab421e291f28316fdb926c5c06ddb4da7be3ff89"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "ab421e291f28316fdb926c5c06ddb4da7be3ff89",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8862507567557491056"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[04:26:17.578] Scheduled build:
{
  "id": "8697077505729020801",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-sanitizers-asan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-11-27T04:26:17.199351388Z",
  "updateTime": "2025-11-27T04:26:17.199351388Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "ab421e291f28316fdb926c5c06ddb4da7be3ff89",
      "ref": "refs/heads/main"
    }
  }
}
[04:26:17.578] Task URL: https://cr-buildbucket.appspot.com/build/8697077505729020801
[04:26:17.578] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:2:0) after 3m23s
[04:26:18.607] Received PubSub notification, asking Buildbucket for the build status
[04:26:18.632] Build status: SCHEDULED
[04:29:40.634] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:2:0)
[04:29:40.692] Build status: SCHEDULED
[04:29:40.692] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:4:0) after 4m45s
[04:34:25.678] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:4:0)
[04:34:25.691] Build status: SCHEDULED
[04:34:25.691] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:5:0) after 6m31s
[04:40:56.710] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:5:0)
[04:40:56.733] Build status: SCHEDULED
[04:40:56.733] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:6:0) after 7m24s
[04:48:20.715] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:6:0)
[04:48:20.796] Build status: SCHEDULED
[04:48:20.796] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:7:0) after 3m8s
[04:51:28.809] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:7:0)
[04:51:28.843] Build status: SCHEDULED
[04:51:28.843] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:8:0) after 5m35s
[04:57:03.868] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:8:0)
[04:57:03.936] Build status: SCHEDULED
[04:57:03.936] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:9:0) after 2m25s
[04:59:29.036] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:9:0)
[04:59:29.070] Build status: SCHEDULED
[04:59:29.071] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:10:0) after 9m35s
[05:09:04.195] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:10:0)
[05:09:04.211] Build status: SCHEDULED
[05:09:04.211] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:11:0) after 9m0s
[05:18:04.274] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:11:0)
[05:18:04.564] Build status: SCHEDULED
[05:18:04.564] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:12:0) after 5m11s
[05:23:15.627] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:12:0)
[05:23:15.643] Build status: SCHEDULED
[05:23:15.643] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:13:0) after 4m28s
[05:27:43.611] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:13:0)
[05:27:43.628] Build status: SCHEDULED
[05:27:43.628] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:14:0) after 5m54s
[05:33:37.690] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:14:0)
[05:33:37.735] Build status: SCHEDULED
[05:33:37.735] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:15:0) after 7m54s
[05:41:31.780] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:15:0)
[05:41:31.799] Build status: SCHEDULED
[05:41:31.799] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:16:0) after 3m21s
[05:44:52.810] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:16:0)
[05:44:52.826] Build status: SCHEDULED
[05:44:52.826] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:17:0) after 1m8s
[05:46:00.841] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:17:0)
[05:46:00.856] Build status: SCHEDULED
[05:46:00.856] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:18:0) after 2m6s
[05:48:07.652] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:18:0)
[05:48:07.735] Build status: SCHEDULED
[05:48:07.735] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:19:0) after 8m49s
[05:56:56.751] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:19:0)
[05:56:56.855] Build status: SCHEDULED
[05:56:56.855] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:20:0) after 1m37s
[05:58:33.971] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:20:0)
[05:58:33.988] Build status: SCHEDULED
[05:58:33.988] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:21:0) after 6m40s
[06:05:14.016] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:21:0)
[06:05:14.039] Build status: SCHEDULED
[06:05:14.039] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:22:0) after 3m14s
[06:08:28.054] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:22:0)
[06:08:28.138] Build status: SCHEDULED
[06:08:28.138] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:23:0) after 9m18s
[06:17:46.154] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:23:0)
[06:17:46.172] Build status: SCHEDULED
[06:17:46.172] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:24:0) after 6m1s
[06:23:47.205] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:24:0)
[06:23:47.287] Build status: SCHEDULED
[06:23:47.287] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:25:0) after 6m26s
[06:30:13.304] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:25:0)
[06:30:13.366] Build status: SCHEDULED
[06:30:13.366] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:26:0) after 4m38s
[06:34:51.383] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:26:0)
[06:34:51.399] Build status: SCHEDULED
[06:34:51.399] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:27:0) after 1m44s
[06:36:35.410] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:27:0)
[06:36:35.435] Build status: SCHEDULED
[06:36:35.435] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:28:0) after 6m1s
[06:42:36.479] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:28:0)
[06:42:36.496] Build status: SCHEDULED
[06:42:36.497] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:29:0) after 9m57s
[06:52:33.514] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:29:0)
[06:52:33.533] Build status: SCHEDULED
[06:52:33.533] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:30:0) after 7m43s
[07:00:16.529] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:30:0)
[07:00:16.557] Build status: SCHEDULED
[07:00:16.557] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:31:0) after 5m9s
[07:05:25.572] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:31:0)
[07:05:25.589] Build status: SCHEDULED
[07:05:25.589] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:32:0) after 3m9s
[07:08:34.679] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:32:0)
[07:08:34.694] Build status: SCHEDULED
[07:08:34.694] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:33:0) after 2m50s
[07:11:24.687] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:33:0)
[07:11:24.789] Build status: SCHEDULED
[07:11:24.789] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:34:0) after 5m53s
[07:17:17.838] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:34:0)
[07:17:17.854] Build status: SCHEDULED
[07:17:17.854] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:35:0) after 8m7s
[07:25:24.919] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:35:0)
[07:25:24.943] Build status: SCHEDULED
[07:25:24.943] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:36:0) after 9m0s
[07:34:25.044] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:36:0)
[07:34:25.066] Build status: SCHEDULED
[07:34:25.066] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:37:0) after 4m49s
[07:39:14.087] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:37:0)
[07:39:14.115] Build status: SCHEDULED
[07:39:14.115] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:38:0) after 9m45s
[07:48:59.138] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:38:0)
[07:48:59.157] Build status: SCHEDULED
[07:48:59.157] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:39:0) after 6m20s
[07:55:19.233] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:39:0)
[07:55:19.250] Build status: SCHEDULED
[07:55:19.250] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:40:0) after 9m16s
[08:04:35.310] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:40:0)
[08:04:35.348] Build status: SCHEDULED
[08:04:35.348] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:41:0) after 4m16s
[08:08:51.362] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:41:0)
[08:08:51.374] Build status: SCHEDULED
[08:08:51.375] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:42:0) after 2m43s
[08:11:34.515] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:42:0)
[08:11:34.540] Build status: SCHEDULED
[08:11:34.540] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:43:0) after 7m3s
[08:18:37.612] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:43:0)
[08:18:37.628] Build status: SCHEDULED
[08:18:37.628] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:44:0) after 7m0s
[08:25:37.751] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:44:0)
[08:25:37.774] Build status: SCHEDULED
[08:25:37.774] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:45:0) after 3m25s
[08:29:02.790] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:45:0)
[08:29:02.819] Build status: SCHEDULED
[08:29:02.819] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:46:0) after 7m57s
[08:36:59.838] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:46:0)
[08:36:59.853] Build status: SCHEDULED
[08:36:59.853] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:47:0) after 6m22s
[08:43:21.883] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:47:0)
[08:43:21.905] Build status: SCHEDULED
[08:43:21.905] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:48:0) after 2m43s
[08:46:04.925] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:48:0)
[08:46:04.939] Build status: SCHEDULED
[08:46:04.939] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:49:0) after 9m28s
[08:55:32.958] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:49:0)
[08:55:32.988] Build status: SCHEDULED
[08:55:32.988] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:50:0) after 5m29s
[09:01:02.004] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:50:0)
[09:01:02.036] Build status: SCHEDULED
[09:01:02.036] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:51:0) after 9m11s
[09:10:13.168] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:51:0)
[09:10:13.186] Build status: SCHEDULED
[09:10:13.186] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:52:0) after 8m23s
[09:18:36.241] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:52:0)
[09:18:36.272] Build status: SCHEDULED
[09:18:36.272] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:53:0) after 8m49s
[09:27:25.298] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:53:0)
[09:27:25.317] Build status: SCHEDULED
[09:27:25.317] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:54:0) after 9m15s
[09:36:40.344] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:54:0)
[09:36:40.365] Build status: SCHEDULED
[09:36:40.365] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:55:0) after 3m30s
[09:40:10.355] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:55:0)
[09:40:10.424] Build status: SCHEDULED
[09:40:10.424] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:56:0) after 7m18s
[09:47:28.526] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:56:0)
[09:47:28.581] Build status: SCHEDULED
[09:47:28.581] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:57:0) after 5m58s
[09:53:26.600] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:57:0)
[09:53:26.620] Build status: SCHEDULED
[09:53:26.620] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:58:0) after 3m16s
[09:56:42.644] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:58:0)
[09:56:42.670] Build status: SCHEDULED
[09:56:42.670] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:59:0) after 5m16s
[10:01:58.735] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:59:0)
[10:01:58.805] Build status: SCHEDULED
[10:01:58.805] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:60:0) after 9m59s
[10:11:57.833] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:60:0)
[10:11:57.857] Build status: SCHEDULED
[10:11:57.857] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:61:0) after 4m30s
[10:16:27.870] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:61:0)
[10:16:27.891] Build status: SCHEDULED
[10:16:27.891] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:62:0) after 9m40s
[10:26:07.907] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:62:0)
[10:26:07.987] Build status: SCHEDULED
[10:26:07.987] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:63:0) after 6m47s
[10:32:55.094] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:63:0)
[10:32:55.115] Build status: SCHEDULED
[10:32:55.115] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:64:0) after 4m16s
[10:37:11.136] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:64:0)
[10:37:11.163] Build status: SCHEDULED
[10:37:11.163] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:65:0) after 3m38s
[10:40:49.259] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:65:0)
[10:40:49.276] Build status: SCHEDULED
[10:40:49.276] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:66:0) after 1m21s
[10:42:10.313] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:66:0)
[10:42:10.337] Build status: SCHEDULED
[10:42:10.337] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:67:0) after 9m37s
[10:51:47.403] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:67:0)
[10:51:47.485] Build status: SCHEDULED
[10:51:47.485] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:68:0) after 1m1s
[10:52:48.525] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:68:0)
[10:52:48.562] Build status: SCHEDULED
[10:52:48.562] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:69:0) after 6m8s
[10:58:56.596] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:69:0)
[10:58:56.615] Build status: SCHEDULED
[10:58:56.615] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:70:0) after 7m54s
[11:06:50.664] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:70:0)
[11:06:50.679] Build status: SCHEDULED
[11:06:50.679] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:71:0) after 8m16s
[11:15:06.672] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:71:0)
[11:15:06.686] Build status: SCHEDULED
[11:15:06.686] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:72:0) after 6m8s
[11:21:14.702] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:72:0)
[11:21:14.735] Build status: SCHEDULED
[11:21:14.735] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:73:0) after 5m13s
[11:26:27.757] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:73:0)
[11:26:27.775] Build status: SCHEDULED
[11:26:27.775] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:74:0) after 8m25s
[11:34:52.898] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:74:0)
[11:34:52.923] Build status: SCHEDULED
[11:34:52.923] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:75:0) after 6m4s
[11:39:38.671] Received PubSub notification, asking Buildbucket for the build status
[11:39:38.688] Build status: STARTED
[11:40:57.029] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:75:0)
[11:40:57.045] Build status: STARTED
[11:40:57.045] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:77:0) after 7m2s
[11:47:59.103] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:77:0)
[11:47:59.127] Build status: STARTED
[11:47:59.127] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:78:0) after 5m53s
[11:53:52.181] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:78:0)
[11:53:52.216] Build status: STARTED
[11:53:52.216] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:79:0) after 2m41s
[11:56:33.236] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:79:0)
[11:56:33.267] Build status: STARTED
[11:56:33.267] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:80:0) after 6m30s
[12:03:03.364] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:80:0)
[12:03:03.431] Build status: STARTED
[12:03:03.431] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:81:0) after 4m24s
[12:07:27.547] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:81:0)
[12:07:27.561] Build status: STARTED
[12:07:27.561] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:82:0) after 9m2s
[12:16:29.590] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:82:0)
[12:16:29.607] Build status: STARTED
[12:16:29.607] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:83:0) after 6m29s
[12:22:58.629] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:83:0)
[12:22:58.651] Build status: STARTED
[12:22:58.651] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8862507567557491056:84:0) after 5m40s
[12:28:32.896] Received PubSub notification, asking Buildbucket for the build status
[12:28:32.911] Build:
{
  "id": "8697077505729020801",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-sanitizers-asan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-11-27T04:26:17.199351388Z",
  "startTime": "2025-11-27T11:39:38.127616560Z",
  "endTime": "2025-11-27T12:28:32.596848946Z",
  "updateTime": "2025-11-27T12:28:32.596848946Z",
  "status": "INFRA_FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "ab421e291f28316fdb926c5c06ddb4da7be3ff89",
      "ref": "refs/heads/main"
    }
  }
}
[12:28:32.911] Invocation finished in 8h2m16.839416871s with status FAILED