1. All projects
  2. pigweed
  3. pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan
  4. 8886654044110212224
Status: FAILED
Triggered by: -
Duration: 8 hours
Actions:

Definition (rev d2b949be221fc8dd3acbfdb77921a85f7b1b8e9d) 

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-x86-bazel-san-tsan"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "0449995468dd677674cbf3908eed420e63ce659e"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/0449995468dd677674cbf3908eed420e63ce659e
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[15:48:21.562] New invocation is queued and will start shortly
[15:48:22.605] Starting the invocation (attempt 1)
[15:48:22.629] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/0449995468dd677674cbf3908eed420e63ce659e"
[15:48:22.629] Popped gitiles commit info from properties and tags
[15:48:22.629] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[15:48:22.629] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[15:48:22.629] Buildbucket request:
{
  "requestId": "8886654044110212224",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-tsan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8886654044110212224",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "0449995468dd677674cbf3908eed420e63ce659e"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@0449995468dd677674cbf3908eed420e63ce659e",
                        "title": "0449995468dd677674cbf3908eed420e63ce659e",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/0449995468dd677674cbf3908eed420e63ce659e"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "0449995468dd677674cbf3908eed420e63ce659e",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8886654044110212224"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[15:48:23.047] Scheduled build:
{
  "id": "8721223982275592273",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-03-05T15:48:22.685109383Z",
  "updateTime": "2025-03-05T15:48:22.685109383Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "0449995468dd677674cbf3908eed420e63ce659e",
      "ref": "refs/heads/main"
    }
  }
}
[15:48:23.047] Task URL: https://cr-buildbucket.appspot.com/build/8721223982275592273
[15:48:23.047] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:2:0) after 1m39s
[15:48:24.017] Received PubSub notification, asking Buildbucket for the build status
[15:48:24.043] Build status: SCHEDULED
[15:50:02.092] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:2:0)
[15:50:02.198] Build status: SCHEDULED
[15:50:02.198] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:4:0) after 6m26s
[15:56:28.229] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:4:0)
[15:56:28.249] Build status: SCHEDULED
[15:56:28.249] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:5:0) after 4m36s
[16:01:04.385] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:5:0)
[16:01:04.412] Build status: SCHEDULED
[16:01:04.412] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:6:0) after 1m3s
[16:02:07.723] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:6:0)
[16:02:07.748] Build status: SCHEDULED
[16:02:07.748] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:7:0) after 9m24s
[16:11:31.949] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:7:0)
[16:11:31.974] Build status: SCHEDULED
[16:11:31.974] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:8:0) after 8m25s
[16:19:57.017] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:8:0)
[16:19:57.038] Build status: SCHEDULED
[16:19:57.038] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:9:0) after 4m16s
[16:24:13.103] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:9:0)
[16:24:13.124] Build status: SCHEDULED
[16:24:13.124] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:10:0) after 7m48s
[16:32:01.233] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:10:0)
[16:32:01.253] Build status: SCHEDULED
[16:32:01.253] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:11:0) after 4m16s
[16:36:17.283] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:11:0)
[16:36:17.347] Build status: SCHEDULED
[16:36:17.347] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:12:0) after 7m46s
[16:44:03.340] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:12:0)
[16:44:03.366] Build status: SCHEDULED
[16:44:03.366] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:13:0) after 9m45s
[16:53:48.444] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:13:0)
[16:53:48.457] Build status: SCHEDULED
[16:53:48.457] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:14:0) after 9m8s
[17:02:56.584] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:14:0)
[17:02:56.618] Build status: SCHEDULED
[17:02:56.618] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:15:0) after 3m18s
[17:06:14.639] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:15:0)
[17:06:14.655] Build status: SCHEDULED
[17:06:14.655] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:16:0) after 3m32s
[17:09:46.778] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:16:0)
[17:09:46.795] Build status: SCHEDULED
[17:09:46.795] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:17:0) after 2m22s
[17:12:08.941] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:17:0)
[17:12:08.991] Build status: SCHEDULED
[17:12:08.991] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:18:0) after 8m5s
[17:20:14.163] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:18:0)
[17:20:14.179] Build status: SCHEDULED
[17:20:14.179] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:19:0) after 4m35s
[17:24:49.196] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:19:0)
[17:24:49.215] Build status: SCHEDULED
[17:24:49.215] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:20:0) after 8m40s
[17:33:29.233] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:20:0)
[17:33:29.253] Build status: SCHEDULED
[17:33:29.253] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:21:0) after 9m42s
[17:43:11.325] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:21:0)
[17:43:11.346] Build status: SCHEDULED
[17:43:11.346] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:22:0) after 2m27s
[17:45:38.393] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:22:0)
[17:45:38.536] Build status: SCHEDULED
[17:45:38.536] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:23:0) after 9m42s
[17:55:20.653] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:23:0)
[17:55:20.695] Build status: SCHEDULED
[17:55:20.695] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:24:0) after 7m52s
[18:03:12.716] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:24:0)
[18:03:12.731] Build status: SCHEDULED
[18:03:12.731] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:25:0) after 5m24s
[18:08:36.768] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:25:0)
[18:08:36.789] Build status: SCHEDULED
[18:08:36.789] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:26:0) after 6m55s
[18:15:31.814] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:26:0)
[18:15:31.837] Build status: SCHEDULED
[18:15:31.838] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:27:0) after 9m53s
[18:25:24.858] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:27:0)
[18:25:24.887] Build status: SCHEDULED
[18:25:24.887] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:28:0) after 9m57s
[18:35:21.933] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:28:0)
[18:35:21.967] Build status: SCHEDULED
[18:35:21.967] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:29:0) after 7m56s
[18:43:18.026] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:29:0)
[18:43:18.043] Build status: SCHEDULED
[18:43:18.043] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:30:0) after 4m52s
[18:48:10.077] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:30:0)
[18:48:10.126] Build status: SCHEDULED
[18:48:10.126] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:31:0) after 1m46s
[18:49:56.212] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:31:0)
[18:49:56.251] Build status: SCHEDULED
[18:49:56.251] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:32:0) after 9m8s
[18:59:04.381] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:32:0)
[18:59:04.401] Build status: SCHEDULED
[18:59:04.401] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:33:0) after 5m23s
[19:04:27.476] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:33:0)
[19:04:27.493] Build status: SCHEDULED
[19:04:27.493] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:34:0) after 2m59s
[19:07:26.531] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:34:0)
[19:07:26.551] Build status: SCHEDULED
[19:07:26.551] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:35:0) after 2m2s
[19:09:28.575] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:35:0)
[19:09:28.600] Build status: SCHEDULED
[19:09:28.600] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:36:0) after 2m42s
[19:12:10.644] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:36:0)
[19:12:10.665] Build status: SCHEDULED
[19:12:10.665] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:37:0) after 9m31s
[19:21:41.778] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:37:0)
[19:21:41.795] Build status: SCHEDULED
[19:21:41.795] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:38:0) after 2m51s
[19:24:32.927] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:38:0)
[19:24:32.946] Build status: SCHEDULED
[19:24:32.946] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:39:0) after 4m26s
[19:28:59.081] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:39:0)
[19:28:59.096] Build status: SCHEDULED
[19:28:59.096] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:40:0) after 3m8s
[19:32:07.194] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:40:0)
[19:32:07.211] Build status: SCHEDULED
[19:32:07.212] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:41:0) after 2m6s
[19:34:13.228] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:41:0)
[19:34:13.254] Build status: SCHEDULED
[19:34:13.254] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:42:0) after 9m5s
[19:43:18.271] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:42:0)
[19:43:18.298] Build status: SCHEDULED
[19:43:18.298] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:43:0) after 6m39s
[19:49:57.416] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:43:0)
[19:49:57.438] Build status: SCHEDULED
[19:49:57.438] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:44:0) after 6m37s
[19:56:34.483] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:44:0)
[19:56:34.499] Build status: SCHEDULED
[19:56:34.499] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:45:0) after 7m33s
[20:04:07.571] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:45:0)
[20:04:07.594] Build status: SCHEDULED
[20:04:07.595] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:46:0) after 3m42s
[20:07:49.602] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:46:0)
[20:07:49.632] Build status: SCHEDULED
[20:07:49.632] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:47:0) after 6m34s
[20:14:23.655] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:47:0)
[20:14:23.673] Build status: SCHEDULED
[20:14:23.673] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:48:0) after 9m47s
[20:24:10.703] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:48:0)
[20:24:10.748] Build status: SCHEDULED
[20:24:10.748] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:49:0) after 1m49s
[20:25:59.889] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:49:0)
[20:25:59.907] Build status: SCHEDULED
[20:25:59.907] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:50:0) after 7m37s
[20:33:36.929] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:50:0)
[20:33:37.065] Build status: SCHEDULED
[20:33:37.065] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:51:0) after 5m57s
[20:39:34.201] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:51:0)
[20:39:34.220] Build status: SCHEDULED
[20:39:34.220] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:52:0) after 2m26s
[20:42:00.249] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:52:0)
[20:42:00.329] Build status: SCHEDULED
[20:42:00.329] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:53:0) after 2m8s
[20:44:08.370] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:53:0)
[20:44:08.388] Build status: SCHEDULED
[20:44:08.388] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:54:0) after 7m25s
[20:51:33.396] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:54:0)
[20:51:33.425] Build status: SCHEDULED
[20:51:33.425] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:55:0) after 8m42s
[21:00:15.447] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:55:0)
[21:00:15.525] Build status: SCHEDULED
[21:00:15.525] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:56:0) after 7m32s
[21:07:47.642] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:56:0)
[21:07:47.660] Build status: SCHEDULED
[21:07:47.660] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:57:0) after 5m4s
[21:12:51.684] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:57:0)
[21:12:51.700] Build status: SCHEDULED
[21:12:51.700] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:58:0) after 7m11s
[21:20:02.774] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:58:0)
[21:20:02.792] Build status: SCHEDULED
[21:20:02.792] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:59:0) after 7m22s
[21:27:24.877] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:59:0)
[21:27:24.912] Build status: SCHEDULED
[21:27:24.912] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:60:0) after 1m47s
[21:29:11.906] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:60:0)
[21:29:11.933] Build status: SCHEDULED
[21:29:11.933] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:61:0) after 4m19s
[21:33:30.951] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:61:0)
[21:33:30.970] Build status: SCHEDULED
[21:33:30.970] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:62:0) after 1m24s
[21:34:54.989] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:62:0)
[21:34:55.006] Build status: SCHEDULED
[21:34:55.006] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:63:0) after 6m42s
[21:41:37.007] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:63:0)
[21:41:37.032] Build status: SCHEDULED
[21:41:37.032] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:64:0) after 3m49s
[21:45:26.118] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:64:0)
[21:45:26.144] Build status: SCHEDULED
[21:45:26.144] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:65:0) after 6m56s
[21:52:22.168] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:65:0)
[21:52:22.192] Build status: SCHEDULED
[21:52:22.192] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:66:0) after 1m15s
[21:53:37.226] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:66:0)
[21:53:37.251] Build status: SCHEDULED
[21:53:37.251] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:67:0) after 2m17s
[21:55:54.278] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:67:0)
[21:55:54.304] Build status: SCHEDULED
[21:55:54.304] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:68:0) after 3m52s
[21:59:46.375] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:68:0)
[21:59:46.392] Build status: SCHEDULED
[21:59:46.392] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:69:0) after 7m50s
[22:07:36.424] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:69:0)
[22:07:36.580] Build status: SCHEDULED
[22:07:36.580] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:70:0) after 6m18s
[22:13:54.675] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:70:0)
[22:13:54.755] Build status: SCHEDULED
[22:13:54.755] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:71:0) after 7m28s
[22:21:22.833] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:71:0)
[22:21:22.902] Build status: SCHEDULED
[22:21:22.902] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:72:0) after 2m7s
[22:23:29.969] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:72:0)
[22:23:29.987] Build status: SCHEDULED
[22:23:29.988] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:73:0) after 6m13s
[22:29:43.056] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:73:0)
[22:29:43.076] Build status: SCHEDULED
[22:29:43.076] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:74:0) after 4m10s
[22:33:53.231] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:74:0)
[22:33:53.349] Build status: SCHEDULED
[22:33:53.349] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:75:0) after 4m1s
[22:37:54.484] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:75:0)
[22:37:54.500] Build status: SCHEDULED
[22:37:54.500] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:76:0) after 3m7s
[22:41:01.723] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:76:0)
[22:41:01.741] Build status: SCHEDULED
[22:41:01.741] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:77:0) after 2m23s
[22:43:24.763] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:77:0)
[22:43:24.786] Build status: SCHEDULED
[22:43:24.786] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:78:0) after 2m32s
[22:45:56.819] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:78:0)
[22:45:56.839] Build status: SCHEDULED
[22:45:56.839] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:79:0) after 5m58s
[22:51:54.940] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:79:0)
[22:51:54.956] Build status: SCHEDULED
[22:51:54.956] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:80:0) after 1m33s
[22:53:28.014] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:80:0)
[22:53:28.033] Build status: SCHEDULED
[22:53:28.033] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:81:0) after 9m27s
[23:02:55.231] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:81:0)
[23:02:55.249] Build status: SCHEDULED
[23:02:55.249] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:82:0) after 5m59s
[23:08:54.364] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:82:0)
[23:08:54.387] Build status: SCHEDULED
[23:08:54.387] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:83:0) after 6m45s
[23:15:39.472] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:83:0)
[23:15:39.489] Build status: SCHEDULED
[23:15:39.489] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:84:0) after 5m5s
[23:20:44.520] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:84:0)
[23:20:44.575] Build status: SCHEDULED
[23:20:44.575] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:85:0) after 2m7s
[23:22:51.639] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:85:0)
[23:22:51.658] Build status: SCHEDULED
[23:22:51.658] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:86:0) after 2m34s
[23:25:25.677] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:86:0)
[23:25:25.696] Build status: SCHEDULED
[23:25:25.696] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:87:0) after 4m33s
[23:29:58.720] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:87:0)
[23:29:58.735] Build status: SCHEDULED
[23:29:58.735] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:88:0) after 6m52s
[23:36:50.752] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:88:0)
[23:36:50.767] Build status: SCHEDULED
[23:36:50.767] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:89:0) after 5m19s
[23:42:09.818] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:89:0)
[23:42:09.838] Build status: SCHEDULED
[23:42:09.838] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:90:0) after 4m31s
[23:46:40.874] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:90:0)
[23:46:40.891] Build status: SCHEDULED
[23:46:40.891] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8886654044110212224:91:0) after 2m38s
[23:48:24.712] Received PubSub notification, asking Buildbucket for the build status
[23:48:24.732] Build:
{
  "id": "8721223982275592273",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-03-05T15:48:22.685109383Z",
  "endTime": "2025-03-05T23:48:24.414222853Z",
  "updateTime": "2025-03-05T23:48:24.414222853Z",
  "status": "INFRA_FAILURE",
  "statusDetails": {
    "resourceExhaustion": {

    },
    "timeout": {

    }
  },
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "0449995468dd677674cbf3908eed420e63ce659e",
      "ref": "refs/heads/main"
    }
  }
}
[23:48:24.732] Invocation finished in 8h0m3.180039675s with status FAILED