1. All projects
  2. pigweed
  3. pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan
  4. 8886655598304642304
Status: FAILED
Triggered by: -
Duration: 8 hours
Actions:

Definition (rev d2b949be221fc8dd3acbfdb77921a85f7b1b8e9d) 

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-x86-bazel-san-ubsan"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "0449995468dd677674cbf3908eed420e63ce659e"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/0449995468dd677674cbf3908eed420e63ce659e
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[15:23:39.370] New invocation is queued and will start shortly
[15:23:40.819] Starting the invocation (attempt 1)
[15:23:40.849] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/0449995468dd677674cbf3908eed420e63ce659e"
[15:23:40.849] Popped gitiles commit info from properties and tags
[15:23:40.849] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[15:23:40.849] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[15:23:40.849] Buildbucket request:
{
  "requestId": "8886655598304642304",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-ubsan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8886655598304642304",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "0449995468dd677674cbf3908eed420e63ce659e"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@0449995468dd677674cbf3908eed420e63ce659e",
                        "title": "0449995468dd677674cbf3908eed420e63ce659e",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/0449995468dd677674cbf3908eed420e63ce659e"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "0449995468dd677674cbf3908eed420e63ce659e",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8886655598304642304"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[15:23:41.258] Scheduled build:
{
  "id": "8721225536024355073",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-ubsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-03-05T15:23:40.915523758Z",
  "updateTime": "2025-03-05T15:23:40.915523758Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "0449995468dd677674cbf3908eed420e63ce659e",
      "ref": "refs/heads/main"
    }
  }
}
[15:23:41.258] Task URL: https://cr-buildbucket.appspot.com/build/8721225536024355073
[15:23:41.258] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:2:0) after 3m47s
[15:23:42.346] Received PubSub notification, asking Buildbucket for the build status
[15:23:42.374] Build status: SCHEDULED
[15:27:28.281] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:2:0)
[15:27:28.313] Build status: SCHEDULED
[15:27:28.313] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:4:0) after 8m12s
[15:35:40.337] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:4:0)
[15:35:40.362] Build status: SCHEDULED
[15:35:40.362] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:5:0) after 9m2s
[15:44:42.427] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:5:0)
[15:44:42.447] Build status: SCHEDULED
[15:44:42.447] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:6:0) after 1m52s
[15:46:34.506] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:6:0)
[15:46:34.534] Build status: SCHEDULED
[15:46:34.534] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:7:0) after 1m1s
[15:47:35.553] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:7:0)
[15:47:35.577] Build status: SCHEDULED
[15:47:35.577] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:8:0) after 6m6s
[15:53:41.689] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:8:0)
[15:53:41.716] Build status: SCHEDULED
[15:53:41.716] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:9:0) after 7m48s
[16:01:29.786] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:9:0)
[16:01:29.816] Build status: SCHEDULED
[16:01:29.816] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:10:0) after 7m16s
[16:08:45.875] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:10:0)
[16:08:45.911] Build status: SCHEDULED
[16:08:45.911] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:11:0) after 7m57s
[16:16:42.936] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:11:0)
[16:16:42.969] Build status: SCHEDULED
[16:16:42.969] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:12:0) after 8m7s
[16:24:50.005] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:12:0)
[16:24:50.022] Build status: SCHEDULED
[16:24:50.022] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:13:0) after 2m8s
[16:26:58.040] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:13:0)
[16:26:58.057] Build status: SCHEDULED
[16:26:58.057] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:14:0) after 9m32s
[16:36:30.071] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:14:0)
[16:36:30.096] Build status: SCHEDULED
[16:36:30.096] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:15:0) after 4m41s
[16:41:11.200] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:15:0)
[16:41:11.217] Build status: SCHEDULED
[16:41:11.217] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:16:0) after 7m59s
[16:49:10.360] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:16:0)
[16:49:10.420] Build status: SCHEDULED
[16:49:10.420] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:17:0) after 6m19s
[16:55:29.490] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:17:0)
[16:55:29.514] Build status: SCHEDULED
[16:55:29.514] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:18:0) after 5m13s
[17:00:42.528] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:18:0)
[17:00:42.544] Build status: SCHEDULED
[17:00:42.544] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:19:0) after 7m49s
[17:08:31.703] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:19:0)
[17:08:31.719] Build status: SCHEDULED
[17:08:31.719] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:20:0) after 4m43s
[17:13:14.713] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:20:0)
[17:13:14.727] Build status: SCHEDULED
[17:13:14.727] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:21:0) after 1m31s
[17:14:45.801] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:21:0)
[17:14:45.817] Build status: SCHEDULED
[17:14:45.817] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:22:0) after 4m35s
[17:19:20.845] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:22:0)
[17:19:20.864] Build status: SCHEDULED
[17:19:20.864] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:23:0) after 2m4s
[17:21:24.987] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:23:0)
[17:21:25.011] Build status: SCHEDULED
[17:21:25.011] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:24:0) after 9m41s
[17:31:06.057] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:24:0)
[17:31:06.076] Build status: SCHEDULED
[17:31:06.076] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:25:0) after 9m30s
[17:40:36.136] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:25:0)
[17:40:36.157] Build status: SCHEDULED
[17:40:36.157] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:26:0) after 8m30s
[17:49:06.272] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:26:0)
[17:49:06.297] Build status: SCHEDULED
[17:49:06.297] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:27:0) after 3m17s
[17:52:23.329] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:27:0)
[17:52:23.354] Build status: SCHEDULED
[17:52:23.354] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:28:0) after 4m20s
[17:56:43.404] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:28:0)
[17:56:43.421] Build status: SCHEDULED
[17:56:43.421] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:29:0) after 1m37s
[17:58:20.449] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:29:0)
[17:58:20.476] Build status: SCHEDULED
[17:58:20.477] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:30:0) after 8m55s
[18:07:15.521] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:30:0)
[18:07:15.539] Build status: SCHEDULED
[18:07:15.539] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:31:0) after 1m4s
[18:08:19.557] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:31:0)
[18:08:19.599] Build status: SCHEDULED
[18:08:19.599] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:32:0) after 8m57s
[18:17:16.621] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:32:0)
[18:17:16.647] Build status: SCHEDULED
[18:17:16.647] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:33:0) after 1m34s
[18:18:50.736] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:33:0)
[18:18:50.753] Build status: SCHEDULED
[18:18:50.753] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:34:0) after 1m18s
[18:20:08.829] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:34:0)
[18:20:08.868] Build status: SCHEDULED
[18:20:08.868] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:35:0) after 9m23s
[18:29:31.916] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:35:0)
[18:29:31.934] Build status: SCHEDULED
[18:29:31.934] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:36:0) after 9m42s
[18:39:13.955] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:36:0)
[18:39:13.996] Build status: SCHEDULED
[18:39:13.996] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:37:0) after 7m13s
[18:46:27.055] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:37:0)
[18:46:27.073] Build status: SCHEDULED
[18:46:27.073] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:38:0) after 6m41s
[18:53:08.193] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:38:0)
[18:53:08.211] Build status: SCHEDULED
[18:53:08.211] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:39:0) after 2m20s
[18:55:28.372] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:39:0)
[18:55:28.386] Build status: SCHEDULED
[18:55:28.386] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:40:0) after 4m42s
[19:00:10.424] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:40:0)
[19:00:10.442] Build status: SCHEDULED
[19:00:10.442] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:41:0) after 7m19s
[19:07:29.553] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:41:0)
[19:07:29.574] Build status: SCHEDULED
[19:07:29.574] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:42:0) after 5m59s
[19:13:28.606] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:42:0)
[19:13:28.625] Build status: SCHEDULED
[19:13:28.625] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:43:0) after 7m35s
[19:21:03.654] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:43:0)
[19:21:03.694] Build status: SCHEDULED
[19:21:03.694] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:44:0) after 3m19s
[19:24:22.822] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:44:0)
[19:24:22.882] Build status: SCHEDULED
[19:24:22.882] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:45:0) after 2m22s
[19:26:44.987] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:45:0)
[19:26:45.023] Build status: SCHEDULED
[19:26:45.024] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:46:0) after 4m7s
[19:30:52.095] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:46:0)
[19:30:52.115] Build status: SCHEDULED
[19:30:52.115] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:47:0) after 3m50s
[19:34:42.136] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:47:0)
[19:34:42.153] Build status: SCHEDULED
[19:34:42.153] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:48:0) after 2m46s
[19:37:28.230] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:48:0)
[19:37:28.257] Build status: SCHEDULED
[19:37:28.257] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:49:0) after 2m57s
[19:40:25.228] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:49:0)
[19:40:25.249] Build status: SCHEDULED
[19:40:25.249] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:50:0) after 4m34s
[19:44:59.314] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:50:0)
[19:44:59.332] Build status: SCHEDULED
[19:44:59.332] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:51:0) after 8m25s
[19:53:24.371] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:51:0)
[19:53:24.390] Build status: SCHEDULED
[19:53:24.390] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:52:0) after 9m51s
[20:03:15.448] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:52:0)
[20:03:15.463] Build status: SCHEDULED
[20:03:15.463] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:53:0) after 3m53s
[20:07:08.481] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:53:0)
[20:07:08.500] Build status: SCHEDULED
[20:07:08.500] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:54:0) after 5m33s
[20:12:41.715] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:54:0)
[20:12:41.757] Build status: SCHEDULED
[20:12:41.757] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:55:0) after 9m52s
[20:22:33.836] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:55:0)
[20:22:33.855] Build status: SCHEDULED
[20:22:33.855] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:56:0) after 3m53s
[20:26:26.930] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:56:0)
[20:26:26.947] Build status: SCHEDULED
[20:26:26.947] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:57:0) after 8m47s
[20:35:14.073] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:57:0)
[20:35:14.096] Build status: SCHEDULED
[20:35:14.096] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:58:0) after 4m52s
[20:40:06.163] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:58:0)
[20:40:06.344] Build status: SCHEDULED
[20:40:06.344] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:59:0) after 7m33s
[20:47:39.456] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:59:0)
[20:47:39.473] Build status: SCHEDULED
[20:47:39.473] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:60:0) after 2m42s
[20:50:21.544] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:60:0)
[20:50:21.563] Build status: SCHEDULED
[20:50:21.563] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:61:0) after 1m37s
[20:51:58.634] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:61:0)
[20:51:58.648] Build status: SCHEDULED
[20:51:58.648] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:62:0) after 8m44s
[21:00:42.748] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:62:0)
[21:00:42.764] Build status: SCHEDULED
[21:00:42.764] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:63:0) after 3m37s
[21:04:19.742] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:63:0)
[21:04:19.760] Build status: SCHEDULED
[21:04:19.760] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:64:0) after 1m56s
[21:06:15.776] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:64:0)
[21:06:15.801] Build status: SCHEDULED
[21:06:15.801] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:65:0) after 4m46s
[21:11:01.825] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:65:0)
[21:11:01.851] Build status: SCHEDULED
[21:11:01.851] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:66:0) after 7m44s
[21:18:45.879] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:66:0)
[21:18:45.902] Build status: SCHEDULED
[21:18:45.902] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:67:0) after 5m10s
[21:23:55.934] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:67:0)
[21:23:56.051] Build status: SCHEDULED
[21:23:56.051] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:68:0) after 4m4s
[21:28:00.081] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:68:0)
[21:28:00.102] Build status: SCHEDULED
[21:28:00.102] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:69:0) after 2m25s
[21:30:25.123] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:69:0)
[21:30:25.139] Build status: SCHEDULED
[21:30:25.139] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:70:0) after 9m25s
[21:39:50.206] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:70:0)
[21:39:50.232] Build status: SCHEDULED
[21:39:50.232] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:71:0) after 5m41s
[21:45:31.259] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:71:0)
[21:45:31.280] Build status: SCHEDULED
[21:45:31.280] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:72:0) after 8m27s
[21:53:58.335] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:72:0)
[21:53:58.352] Build status: SCHEDULED
[21:53:58.353] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:73:0) after 1m20s
[21:55:18.413] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:73:0)
[21:55:18.429] Build status: SCHEDULED
[21:55:18.429] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:74:0) after 7m49s
[22:03:07.510] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:74:0)
[22:03:07.548] Build status: SCHEDULED
[22:03:07.548] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:75:0) after 8m34s
[22:11:41.634] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:75:0)
[22:11:41.655] Build status: SCHEDULED
[22:11:41.655] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:76:0) after 8m33s
[22:20:14.732] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:76:0)
[22:20:14.746] Build status: SCHEDULED
[22:20:14.747] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:77:0) after 3m42s
[22:23:56.803] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:77:0)
[22:23:56.824] Build status: SCHEDULED
[22:23:56.824] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:78:0) after 5m55s
[22:29:51.893] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:78:0)
[22:29:51.913] Build status: SCHEDULED
[22:29:51.913] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:79:0) after 3m59s
[22:33:51.028] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:79:0)
[22:33:51.042] Build status: SCHEDULED
[22:33:51.042] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:80:0) after 3m8s
[22:36:59.070] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:80:0)
[22:36:59.088] Build status: SCHEDULED
[22:36:59.088] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:81:0) after 3m31s
[22:40:30.305] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:81:0)
[22:40:30.329] Build status: SCHEDULED
[22:40:30.329] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:82:0) after 6m23s
[22:46:53.526] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:82:0)
[22:46:53.543] Build status: SCHEDULED
[22:46:53.543] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:83:0) after 1m56s
[22:48:49.578] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:83:0)
[22:48:49.644] Build status: SCHEDULED
[22:48:49.644] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:84:0) after 6m35s
[22:55:24.672] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:84:0)
[22:55:24.747] Build status: SCHEDULED
[22:55:24.747] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:85:0) after 4m53s
[23:00:17.771] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:85:0)
[23:00:17.786] Build status: SCHEDULED
[23:00:17.786] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:86:0) after 4m24s
[23:04:41.805] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:86:0)
[23:04:41.830] Build status: SCHEDULED
[23:04:41.830] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:87:0) after 7m48s
[23:12:29.881] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:87:0)
[23:12:29.955] Build status: SCHEDULED
[23:12:29.955] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:88:0) after 3m55s
[23:16:25.029] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:88:0)
[23:16:25.048] Build status: SCHEDULED
[23:16:25.048] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:89:0) after 6m55s
[23:23:20.072] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:89:0)
[23:23:20.090] Build status: SCHEDULED
[23:23:20.090] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-ubsan:8886655598304642304:90:0) after 5m59s
[23:23:45.137] Received PubSub notification, asking Buildbucket for the build status
[23:23:45.158] Build:
{
  "id": "8721225536024355073",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-ubsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-03-05T15:23:40.915523758Z",
  "endTime": "2025-03-05T23:23:44.879115633Z",
  "updateTime": "2025-03-05T23:23:44.879115633Z",
  "status": "INFRA_FAILURE",
  "statusDetails": {
    "resourceExhaustion": {

    },
    "timeout": {

    }
  },
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "0449995468dd677674cbf3908eed420e63ce659e",
      "ref": "refs/heads/main"
    }
  }
}
[23:23:45.158] Invocation finished in 8h0m5.800209425s with status FAILED