1. All projects
  2. pigweed
  3. pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan
  4. 8869417824498327456
Status: FAILED
Triggered by: -
Duration: 8 hours
Actions:

Definition (rev ed331b29843b850a4bc37ad64e7e3a240af6f486) 

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-x86-bazel-sanitizers-asan"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "7559f62b892b7945209dfbc3ba98d11a191c3e45"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/7559f62b892b7945209dfbc3ba98d11a191c3e45
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[21:50:41.519] New invocation is queued and will start shortly
[21:50:42.967] Starting the invocation (attempt 1)
[21:50:43.013] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/7559f62b892b7945209dfbc3ba98d11a191c3e45"
[21:50:43.013] Popped gitiles commit info from properties and tags
[21:50:43.014] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[21:50:43.014] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[21:50:43.014] Buildbucket request:
{
  "requestId": "8869417824498327456",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-sanitizers-asan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8869417824498327456",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "7559f62b892b7945209dfbc3ba98d11a191c3e45"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@7559f62b892b7945209dfbc3ba98d11a191c3e45",
                        "title": "7559f62b892b7945209dfbc3ba98d11a191c3e45",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/7559f62b892b7945209dfbc3ba98d11a191c3e45"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "7559f62b892b7945209dfbc3ba98d11a191c3e45",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8869417824498327456"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[21:50:43.400] Scheduled build:
{
  "id": "8703987762183264353",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-sanitizers-asan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-09-11T21:50:43.099471300Z",
  "updateTime": "2025-09-11T21:50:43.099471300Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "7559f62b892b7945209dfbc3ba98d11a191c3e45",
      "ref": "refs/heads/main"
    }
  }
}
[21:50:43.400] Task URL: https://cr-buildbucket.appspot.com/build/8703987762183264353
[21:50:43.400] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:2:0) after 2m42s
[21:50:44.362] Received PubSub notification, asking Buildbucket for the build status
[21:50:44.381] Build status: SCHEDULED
[21:53:25.416] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:2:0)
[21:53:25.433] Build status: SCHEDULED
[21:53:25.433] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:4:0) after 9m15s
[22:02:40.448] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:4:0)
[22:02:40.472] Build status: SCHEDULED
[22:02:40.472] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:5:0) after 1m27s
[22:04:07.477] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:5:0)
[22:04:07.492] Build status: SCHEDULED
[22:04:07.492] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:6:0) after 8m43s
[22:12:50.510] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:6:0)
[22:12:50.539] Build status: SCHEDULED
[22:12:50.539] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:7:0) after 9m57s
[22:22:47.661] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:7:0)
[22:22:47.696] Build status: SCHEDULED
[22:22:47.696] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:8:0) after 7m44s
[22:30:31.811] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:8:0)
[22:30:31.855] Build status: SCHEDULED
[22:30:31.855] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:9:0) after 7m16s
[22:37:47.867] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:9:0)
[22:37:47.888] Build status: SCHEDULED
[22:37:47.888] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:10:0) after 3m4s
[22:40:51.954] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:10:0)
[22:40:51.977] Build status: SCHEDULED
[22:40:51.977] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:11:0) after 9m38s
[22:50:29.992] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:11:0)
[22:50:30.013] Build status: SCHEDULED
[22:50:30.014] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:12:0) after 3m30s
[22:54:00.044] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:12:0)
[22:54:00.069] Build status: SCHEDULED
[22:54:00.069] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:13:0) after 4m46s
[22:58:46.085] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:13:0)
[22:58:46.105] Build status: SCHEDULED
[22:58:46.105] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:14:0) after 4m22s
[23:03:08.170] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:14:0)
[23:03:08.208] Build status: SCHEDULED
[23:03:08.208] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:15:0) after 2m6s
[23:05:14.262] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:15:0)
[23:05:14.293] Build status: SCHEDULED
[23:05:14.293] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:16:0) after 5m2s
[23:10:16.378] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:16:0)
[23:10:16.448] Build status: SCHEDULED
[23:10:16.448] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:17:0) after 6m58s
[23:17:14.563] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:17:0)
[23:17:14.580] Build status: SCHEDULED
[23:17:14.580] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:18:0) after 2m46s
[23:20:00.600] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:18:0)
[23:20:00.619] Build status: SCHEDULED
[23:20:00.619] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:19:0) after 6m37s
[23:26:37.684] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:19:0)
[23:26:37.748] Build status: SCHEDULED
[23:26:37.748] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:20:0) after 9m13s
[23:35:50.851] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:20:0)
[23:35:50.933] Build status: SCHEDULED
[23:35:50.933] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:21:0) after 6m23s
[23:42:14.018] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:21:0)
[23:42:14.036] Build status: SCHEDULED
[23:42:14.036] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:22:0) after 4m20s
[23:46:34.127] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:22:0)
[23:46:34.149] Build status: SCHEDULED
[23:46:34.149] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:23:0) after 4m1s
[23:50:35.197] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:23:0)
[23:50:35.214] Build status: SCHEDULED
[23:50:35.214] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:24:0) after 9m10s
[23:59:45.235] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:24:0)
[23:59:45.253] Build status: SCHEDULED
[23:59:45.253] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:25:0) after 4m58s
[00:04:43.298] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:25:0)
[00:04:43.316] Build status: SCHEDULED
[00:04:43.316] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:26:0) after 6m26s
[00:11:09.354] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:26:0)
[00:11:09.384] Build status: SCHEDULED
[00:11:09.384] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:27:0) after 7m47s
[00:18:56.375] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:27:0)
[00:18:56.391] Build status: SCHEDULED
[00:18:56.391] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:28:0) after 1m48s
[00:20:44.392] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:28:0)
[00:20:44.410] Build status: SCHEDULED
[00:20:44.410] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:29:0) after 7m44s
[00:28:28.423] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:29:0)
[00:28:28.454] Build status: SCHEDULED
[00:28:28.454] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:30:0) after 5m52s
[00:34:20.458] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:30:0)
[00:34:20.483] Build status: SCHEDULED
[00:34:20.483] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:31:0) after 4m19s
[00:38:39.498] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:31:0)
[00:38:39.517] Build status: SCHEDULED
[00:38:39.517] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:32:0) after 3m50s
[00:42:29.609] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:32:0)
[00:42:29.633] Build status: SCHEDULED
[00:42:29.633] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:33:0) after 7m24s
[00:49:53.760] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:33:0)
[00:49:53.832] Build status: SCHEDULED
[00:49:53.832] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:34:0) after 8m43s
[00:58:36.866] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:34:0)
[00:58:36.884] Build status: SCHEDULED
[00:58:36.884] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:35:0) after 6m51s
[01:05:27.905] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:35:0)
[01:05:27.930] Build status: SCHEDULED
[01:05:27.930] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:36:0) after 6m45s
[01:12:12.931] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:36:0)
[01:12:12.951] Build status: SCHEDULED
[01:12:12.951] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:37:0) after 8m31s
[01:20:44.129] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:37:0)
[01:20:44.151] Build status: SCHEDULED
[01:20:44.151] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:38:0) after 9m29s
[01:30:13.217] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:38:0)
[01:30:13.234] Build status: SCHEDULED
[01:30:13.234] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:39:0) after 9m16s
[01:39:29.319] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:39:0)
[01:39:29.350] Build status: SCHEDULED
[01:39:29.350] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:40:0) after 9m57s
[01:49:26.456] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:40:0)
[01:49:26.470] Build status: SCHEDULED
[01:49:26.470] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:41:0) after 3m36s
[01:53:02.636] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:41:0)
[01:53:02.652] Build status: SCHEDULED
[01:53:02.652] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:42:0) after 2m41s
[01:55:43.720] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:42:0)
[01:55:43.772] Build status: SCHEDULED
[01:55:43.772] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:43:0) after 2m37s
[01:58:20.774] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:43:0)
[01:58:20.792] Build status: SCHEDULED
[01:58:20.792] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:44:0) after 3m20s
[02:01:40.806] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:44:0)
[02:01:40.823] Build status: SCHEDULED
[02:01:40.823] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:45:0) after 4m15s
[02:05:55.819] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:45:0)
[02:05:55.835] Build status: SCHEDULED
[02:05:55.836] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:46:0) after 1m11s
[02:07:06.913] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:46:0)
[02:07:06.930] Build status: SCHEDULED
[02:07:06.930] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:47:0) after 4m47s
[02:11:53.899] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:47:0)
[02:11:53.917] Build status: SCHEDULED
[02:11:53.917] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:48:0) after 4m41s
[02:16:34.935] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:48:0)
[02:16:34.987] Build status: SCHEDULED
[02:16:34.987] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:49:0) after 5m43s
[02:22:18.003] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:49:0)
[02:22:18.019] Build status: SCHEDULED
[02:22:18.019] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:50:0) after 9m43s
[02:32:01.063] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:50:0)
[02:32:01.080] Build status: SCHEDULED
[02:32:01.080] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:51:0) after 7m12s
[02:39:13.095] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:51:0)
[02:39:13.112] Build status: SCHEDULED
[02:39:13.112] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:52:0) after 4m0s
[02:43:13.128] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:52:0)
[02:43:13.145] Build status: SCHEDULED
[02:43:13.145] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:53:0) after 1m50s
[02:45:03.189] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:53:0)
[02:45:03.222] Build status: SCHEDULED
[02:45:03.222] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:54:0) after 4m10s
[02:49:13.255] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:54:0)
[02:49:13.274] Build status: SCHEDULED
[02:49:13.274] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:55:0) after 4m35s
[02:53:48.271] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:55:0)
[02:53:48.287] Build status: SCHEDULED
[02:53:48.287] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:56:0) after 7m50s
[03:01:38.422] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:56:0)
[03:01:38.459] Build status: SCHEDULED
[03:01:38.459] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:57:0) after 1m20s
[03:02:58.475] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:57:0)
[03:02:58.500] Build status: SCHEDULED
[03:02:58.500] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:58:0) after 8m25s
[03:11:23.554] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:58:0)
[03:11:23.645] Build status: SCHEDULED
[03:11:23.646] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:59:0) after 2m27s
[03:13:50.625] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:59:0)
[03:13:50.643] Build status: SCHEDULED
[03:13:50.643] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:60:0) after 6m54s
[03:20:44.662] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:60:0)
[03:20:44.680] Build status: SCHEDULED
[03:20:44.680] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:61:0) after 3m4s
[03:23:48.750] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:61:0)
[03:23:48.776] Build status: SCHEDULED
[03:23:48.776] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:62:0) after 2m44s
[03:26:32.796] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:62:0)
[03:26:32.813] Build status: SCHEDULED
[03:26:32.813] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:63:0) after 4m31s
[03:31:03.826] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:63:0)
[03:31:03.847] Build status: SCHEDULED
[03:31:03.847] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:64:0) after 5m19s
[03:36:22.866] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:64:0)
[03:36:22.884] Build status: SCHEDULED
[03:36:22.884] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:65:0) after 4m26s
[03:40:48.916] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:65:0)
[03:40:48.948] Build status: SCHEDULED
[03:40:48.948] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:66:0) after 8m58s
[03:49:46.968] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:66:0)
[03:49:47.054] Build status: SCHEDULED
[03:49:47.054] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:67:0) after 2m20s
[03:52:07.150] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:67:0)
[03:52:07.169] Build status: SCHEDULED
[03:52:07.169] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:68:0) after 5m34s
[03:57:41.248] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:68:0)
[03:57:41.263] Build status: SCHEDULED
[03:57:41.263] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:69:0) after 4m48s
[04:02:29.282] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:69:0)
[04:02:29.316] Build status: SCHEDULED
[04:02:29.317] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:70:0) after 7m19s
[04:09:48.316] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:70:0)
[04:09:48.332] Build status: SCHEDULED
[04:09:48.332] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:71:0) after 4m46s
[04:14:34.347] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:71:0)
[04:14:34.370] Build status: SCHEDULED
[04:14:34.370] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:72:0) after 9m25s
[04:23:59.388] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:72:0)
[04:23:59.426] Build status: SCHEDULED
[04:23:59.426] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:73:0) after 1m56s
[04:25:55.460] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:73:0)
[04:25:55.483] Build status: SCHEDULED
[04:25:55.483] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:74:0) after 2m46s
[04:28:41.553] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:74:0)
[04:28:41.569] Build status: SCHEDULED
[04:28:41.569] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:75:0) after 4m33s
[04:33:14.635] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:75:0)
[04:33:14.650] Build status: SCHEDULED
[04:33:14.650] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:76:0) after 4m36s
[04:37:50.669] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:76:0)
[04:37:50.687] Build status: SCHEDULED
[04:37:50.687] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:77:0) after 5m36s
[04:43:26.673] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:77:0)
[04:43:26.801] Build status: SCHEDULED
[04:43:26.801] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:78:0) after 3m32s
[04:46:59.033] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:78:0)
[04:46:59.085] Build status: SCHEDULED
[04:46:59.085] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:79:0) after 2m32s
[04:49:31.497] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:79:0)
[04:49:31.554] Build status: SCHEDULED
[04:49:31.554] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:80:0) after 8m8s
[04:57:39.620] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:80:0)
[04:57:39.644] Build status: SCHEDULED
[04:57:39.644] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:81:0) after 6m6s
[05:03:45.675] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:81:0)
[05:03:45.697] Build status: SCHEDULED
[05:03:45.697] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:82:0) after 7m56s
[05:11:41.814] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:82:0)
[05:11:41.834] Build status: SCHEDULED
[05:11:41.834] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:83:0) after 5m33s
[05:17:14.848] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:83:0)
[05:17:14.868] Build status: SCHEDULED
[05:17:14.868] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:84:0) after 3m0s
[05:20:14.843] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:84:0)
[05:20:14.887] Build status: SCHEDULED
[05:20:14.887] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:85:0) after 2m17s
[05:22:31.925] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:85:0)
[05:22:31.943] Build status: SCHEDULED
[05:22:31.943] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:86:0) after 9m23s
[05:31:54.960] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:86:0)
[05:31:55.021] Build status: SCHEDULED
[05:31:55.021] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:87:0) after 1m39s
[05:33:34.039] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:87:0)
[05:33:34.088] Build status: SCHEDULED
[05:33:34.088] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:88:0) after 7m48s
[05:41:22.102] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:88:0)
[05:41:22.117] Build status: SCHEDULED
[05:41:22.117] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:89:0) after 5m16s
[05:46:38.131] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:89:0)
[05:46:38.168] Build status: SCHEDULED
[05:46:38.168] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-asan:8869417824498327456:90:0) after 5m36s
[05:50:47.909] Received PubSub notification, asking Buildbucket for the build status
[05:50:47.925] Build:
{
  "id": "8703987762183264353",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-sanitizers-asan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-09-11T21:50:43.099471300Z",
  "endTime": "2025-09-12T05:50:47.704624841Z",
  "updateTime": "2025-09-12T05:50:47.704624841Z",
  "status": "INFRA_FAILURE",
  "statusDetails": {
    "resourceExhaustion": {

    },
    "timeout": {

    }
  },
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "7559f62b892b7945209dfbc3ba98d11a191c3e45",
      "ref": "refs/heads/main"
    }
  }
}
[05:50:47.925] Invocation finished in 8h0m6.416707484s with status FAILED