1. All projects
  2. pigweed
  3. pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan
  4. 8886653121431674992
Status: FAILED
Triggered by: -
Duration: 8 hours
Actions:

Definition (rev d2b949be221fc8dd3acbfdb77921a85f7b1b8e9d) 

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-x86-bazel-san-asan"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "0449995468dd677674cbf3908eed420e63ce659e"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/0449995468dd677674cbf3908eed420e63ce659e
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[16:03:01.498] New invocation is queued and will start shortly
[16:03:02.988] Starting the invocation (attempt 1)
[16:03:03.016] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/0449995468dd677674cbf3908eed420e63ce659e"
[16:03:03.016] Popped gitiles commit info from properties and tags
[16:03:03.016] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[16:03:03.016] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[16:03:03.016] Buildbucket request:
{
  "requestId": "8886653121431674992",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-asan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8886653121431674992",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "0449995468dd677674cbf3908eed420e63ce659e"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@0449995468dd677674cbf3908eed420e63ce659e",
                        "title": "0449995468dd677674cbf3908eed420e63ce659e",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/0449995468dd677674cbf3908eed420e63ce659e"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "0449995468dd677674cbf3908eed420e63ce659e",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8886653121431674992"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[16:03:03.409] Scheduled build:
{
  "id": "8721223059115741329",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-asan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-03-05T16:03:03.079960248Z",
  "updateTime": "2025-03-05T16:03:03.079960248Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "0449995468dd677674cbf3908eed420e63ce659e",
      "ref": "refs/heads/main"
    }
  }
}
[16:03:03.409] Task URL: https://cr-buildbucket.appspot.com/build/8721223059115741329
[16:03:03.409] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:2:0) after 1m8s
[16:03:04.375] Received PubSub notification, asking Buildbucket for the build status
[16:03:04.416] Build status: SCHEDULED
[16:04:11.453] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:2:0)
[16:04:11.481] Build status: SCHEDULED
[16:04:11.481] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:4:0) after 9m5s
[16:13:16.593] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:4:0)
[16:13:16.658] Build status: SCHEDULED
[16:13:16.658] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:5:0) after 4m58s
[16:18:14.846] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:5:0)
[16:18:14.869] Build status: SCHEDULED
[16:18:14.869] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:6:0) after 7m40s
[16:25:54.988] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:6:0)
[16:25:55.018] Build status: SCHEDULED
[16:25:55.018] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:7:0) after 9m55s
[16:35:50.067] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:7:0)
[16:35:50.106] Build status: SCHEDULED
[16:35:50.106] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:8:0) after 4m25s
[16:40:15.120] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:8:0)
[16:40:15.167] Build status: SCHEDULED
[16:40:15.167] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:9:0) after 1m59s
[16:42:14.192] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:9:0)
[16:42:14.229] Build status: SCHEDULED
[16:42:14.229] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:10:0) after 3m51s
[16:46:05.483] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:10:0)
[16:46:05.501] Build status: SCHEDULED
[16:46:05.501] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:11:0) after 7m39s
[16:53:44.518] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:11:0)
[16:53:44.551] Build status: SCHEDULED
[16:53:44.551] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:12:0) after 8m41s
[17:02:25.627] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:12:0)
[17:02:25.650] Build status: SCHEDULED
[17:02:25.650] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:13:0) after 1m53s
[17:04:18.665] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:13:0)
[17:04:18.687] Build status: SCHEDULED
[17:04:18.687] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:14:0) after 4m27s
[17:08:45.830] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:14:0)
[17:08:45.858] Build status: SCHEDULED
[17:08:45.858] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:15:0) after 2m17s
[17:11:02.910] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:15:0)
[17:11:02.934] Build status: SCHEDULED
[17:11:02.934] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:16:0) after 5m33s
[17:16:35.973] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:16:0)
[17:16:35.999] Build status: SCHEDULED
[17:16:35.999] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:17:0) after 8m44s
[17:25:20.000] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:17:0)
[17:25:20.077] Build status: SCHEDULED
[17:25:20.077] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:18:0) after 5m14s
[17:30:34.114] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:18:0)
[17:30:34.149] Build status: SCHEDULED
[17:30:34.149] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:19:0) after 3m39s
[17:34:13.186] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:19:0)
[17:34:13.204] Build status: SCHEDULED
[17:34:13.204] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:20:0) after 1m51s
[17:36:04.264] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:20:0)
[17:36:04.279] Build status: SCHEDULED
[17:36:04.279] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:21:0) after 2m4s
[17:38:08.320] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:21:0)
[17:38:08.339] Build status: SCHEDULED
[17:38:08.339] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:22:0) after 3m3s
[17:41:11.358] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:22:0)
[17:41:11.375] Build status: SCHEDULED
[17:41:11.375] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:23:0) after 3m24s
[17:44:35.494] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:23:0)
[17:44:35.514] Build status: SCHEDULED
[17:44:35.514] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:24:0) after 9m19s
[17:53:54.583] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:24:0)
[17:53:54.599] Build status: SCHEDULED
[17:53:54.599] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:25:0) after 8m12s
[18:02:06.744] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:25:0)
[18:02:06.761] Build status: SCHEDULED
[18:02:06.761] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:26:0) after 4m17s
[18:06:23.773] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:26:0)
[18:06:23.792] Build status: SCHEDULED
[18:06:23.792] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:27:0) after 4m12s
[18:10:35.830] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:27:0)
[18:10:35.893] Build status: SCHEDULED
[18:10:35.893] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:28:0) after 2m9s
[18:12:44.911] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:28:0)
[18:12:44.928] Build status: SCHEDULED
[18:12:44.928] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:29:0) after 5m9s
[18:17:53.953] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:29:0)
[18:17:53.979] Build status: SCHEDULED
[18:17:53.979] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:30:0) after 1m15s
[18:19:09.046] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:30:0)
[18:19:09.062] Build status: SCHEDULED
[18:19:09.062] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:31:0) after 5m41s
[18:24:50.219] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:31:0)
[18:24:50.265] Build status: SCHEDULED
[18:24:50.265] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:32:0) after 1m39s
[18:26:29.308] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:32:0)
[18:26:29.339] Build status: SCHEDULED
[18:26:29.339] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:33:0) after 9m31s
[18:36:00.374] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:33:0)
[18:36:00.401] Build status: SCHEDULED
[18:36:00.401] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:34:0) after 5m54s
[18:41:54.506] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:34:0)
[18:41:54.528] Build status: SCHEDULED
[18:41:54.528] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:35:0) after 3m47s
[18:45:41.778] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:35:0)
[18:45:42.062] Build status: SCHEDULED
[18:45:42.062] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:36:0) after 3m50s
[18:49:32.121] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:36:0)
[18:49:32.143] Build status: SCHEDULED
[18:49:32.143] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:37:0) after 1m45s
[18:51:17.226] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:37:0)
[18:51:17.644] Build status: SCHEDULED
[18:51:17.644] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:38:0) after 4m43s
[18:56:00.671] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:38:0)
[18:56:00.837] Build status: SCHEDULED
[18:56:00.837] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:39:0) after 7m29s
[19:03:29.952] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:39:0)
[19:03:29.969] Build status: SCHEDULED
[19:03:29.969] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:40:0) after 9m28s
[19:12:58.035] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:40:0)
[19:12:58.053] Build status: SCHEDULED
[19:12:58.053] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:41:0) after 5m12s
[19:18:10.157] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:41:0)
[19:18:10.174] Build status: SCHEDULED
[19:18:10.174] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:42:0) after 6m3s
[19:24:13.234] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:42:0)
[19:24:13.259] Build status: SCHEDULED
[19:24:13.259] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:43:0) after 3m15s
[19:27:28.280] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:43:0)
[19:27:28.355] Build status: SCHEDULED
[19:27:28.355] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:44:0) after 5m57s
[19:33:25.380] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:44:0)
[19:33:25.413] Build status: SCHEDULED
[19:33:25.413] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:45:0) after 8m23s
[19:41:48.432] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:45:0)
[19:41:48.451] Build status: SCHEDULED
[19:41:48.451] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:46:0) after 1m45s
[19:43:33.477] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:46:0)
[19:43:33.495] Build status: SCHEDULED
[19:43:33.495] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:47:0) after 1m7s
[19:44:40.570] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:47:0)
[19:44:40.595] Build status: SCHEDULED
[19:44:40.595] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:48:0) after 7m55s
[19:52:35.633] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:48:0)
[19:52:35.650] Build status: SCHEDULED
[19:52:35.650] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:49:0) after 6m33s
[19:59:08.738] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:49:0)
[19:59:08.757] Build status: SCHEDULED
[19:59:08.757] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:50:0) after 5m0s
[20:04:08.745] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:50:0)
[20:04:08.767] Build status: SCHEDULED
[20:04:08.767] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:51:0) after 3m2s
[20:07:10.797] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:51:0)
[20:07:10.817] Build status: SCHEDULED
[20:07:10.817] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:52:0) after 2m19s
[20:09:29.913] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:52:0)
[20:09:29.930] Build status: SCHEDULED
[20:09:29.931] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:53:0) after 5m5s
[20:14:35.006] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:53:0)
[20:14:35.029] Build status: SCHEDULED
[20:14:35.029] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:54:0) after 1m15s
[20:15:50.049] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:54:0)
[20:15:50.074] Build status: SCHEDULED
[20:15:50.074] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:55:0) after 7m33s
[20:23:23.310] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:55:0)
[20:23:23.324] Build status: SCHEDULED
[20:23:23.324] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:56:0) after 7m37s
[20:31:00.353] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:56:0)
[20:31:00.371] Build status: SCHEDULED
[20:31:00.371] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:57:0) after 1m35s
[20:32:35.452] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:57:0)
[20:32:35.473] Build status: SCHEDULED
[20:32:35.473] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:58:0) after 2m35s
[20:35:10.506] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:58:0)
[20:35:10.524] Build status: SCHEDULED
[20:35:10.524] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:59:0) after 3m31s
[20:38:41.610] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:59:0)
[20:38:41.647] Build status: SCHEDULED
[20:38:41.647] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:60:0) after 9m12s
[20:47:53.673] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:60:0)
[20:47:53.690] Build status: SCHEDULED
[20:47:53.690] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:61:0) after 5m39s
[20:53:32.705] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:61:0)
[20:53:32.723] Build status: SCHEDULED
[20:53:32.723] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:62:0) after 5m45s
[20:59:17.792] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:62:0)
[20:59:17.809] Build status: SCHEDULED
[20:59:17.809] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:63:0) after 4m38s
[21:03:55.794] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:63:0)
[21:03:55.812] Build status: SCHEDULED
[21:03:55.812] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:64:0) after 1m32s
[21:05:27.834] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:64:0)
[21:05:27.853] Build status: SCHEDULED
[21:05:27.853] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:65:0) after 3m30s
[21:08:57.871] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:65:0)
[21:08:57.914] Build status: SCHEDULED
[21:08:57.914] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:66:0) after 6m47s
[21:15:44.933] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:66:0)
[21:15:44.955] Build status: SCHEDULED
[21:15:44.955] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:67:0) after 7m18s
[21:23:02.938] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:67:0)
[21:23:03.001] Build status: SCHEDULED
[21:23:03.001] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:68:0) after 3m44s
[21:26:47.091] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:68:0)
[21:26:47.106] Build status: SCHEDULED
[21:26:47.106] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:69:0) after 9m59s
[21:36:46.097] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:69:0)
[21:36:46.198] Build status: SCHEDULED
[21:36:46.198] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:70:0) after 8m10s
[21:44:56.247] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:70:0)
[21:44:56.274] Build status: SCHEDULED
[21:44:56.274] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:71:0) after 2m24s
[21:47:20.362] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:71:0)
[21:47:20.379] Build status: SCHEDULED
[21:47:20.379] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:72:0) after 5m16s
[21:52:36.358] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:72:0)
[21:52:36.376] Build status: SCHEDULED
[21:52:36.376] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:73:0) after 7m8s
[21:59:44.395] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:73:0)
[21:59:44.414] Build status: SCHEDULED
[21:59:44.414] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:74:0) after 3m47s
[22:03:31.437] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:74:0)
[22:03:31.453] Build status: SCHEDULED
[22:03:31.453] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:75:0) after 5m10s
[22:08:41.535] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:75:0)
[22:08:41.566] Build status: SCHEDULED
[22:08:41.566] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:76:0) after 1m44s
[22:10:25.638] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:76:0)
[22:10:25.661] Build status: SCHEDULED
[22:10:25.661] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:77:0) after 1m37s
[22:12:02.714] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:77:0)
[22:12:02.751] Build status: SCHEDULED
[22:12:02.751] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:78:0) after 1m1s
[22:13:03.847] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:78:0)
[22:13:03.865] Build status: SCHEDULED
[22:13:03.865] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:79:0) after 6m50s
[22:19:54.169] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:79:0)
[22:19:54.191] Build status: SCHEDULED
[22:19:54.191] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:80:0) after 7m48s
[22:27:42.323] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:80:0)
[22:27:42.356] Build status: SCHEDULED
[22:27:42.356] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:81:0) after 4m50s
[22:32:32.384] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:81:0)
[22:32:32.418] Build status: SCHEDULED
[22:32:32.418] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:82:0) after 6m49s
[22:39:21.531] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:82:0)
[22:39:21.619] Build status: SCHEDULED
[22:39:21.619] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:83:0) after 6m28s
[22:45:49.641] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:83:0)
[22:45:49.661] Build status: SCHEDULED
[22:45:49.661] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:84:0) after 8m25s
[22:54:14.718] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:84:0)
[22:54:14.734] Build status: SCHEDULED
[22:54:14.734] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:85:0) after 7m33s
[23:01:47.808] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:85:0)
[23:01:47.831] Build status: SCHEDULED
[23:01:47.831] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:86:0) after 8m53s
[23:10:40.868] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:86:0)
[23:10:40.886] Build status: SCHEDULED
[23:10:40.886] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:87:0) after 4m20s
[23:15:00.923] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:87:0)
[23:15:00.941] Build status: SCHEDULED
[23:15:00.941] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:88:0) after 9m22s
[23:24:22.968] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:88:0)
[23:24:23.145] Build status: SCHEDULED
[23:24:23.145] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:89:0) after 1m6s
[23:25:29.204] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:89:0)
[23:25:29.221] Build status: SCHEDULED
[23:25:29.221] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:90:0) after 4m45s
[23:30:14.246] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:90:0)
[23:30:14.266] Build status: SCHEDULED
[23:30:14.266] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:91:0) after 9m0s
[23:39:14.327] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:91:0)
[23:39:14.347] Build status: SCHEDULED
[23:39:14.347] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:92:0) after 8m17s
[23:47:31.404] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:92:0)
[23:47:31.419] Build status: SCHEDULED
[23:47:31.419] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:93:0) after 1m9s
[23:48:40.521] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:93:0)
[23:48:40.574] Build status: SCHEDULED
[23:48:40.574] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:94:0) after 7m52s
[23:56:32.638] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:94:0)
[23:56:32.656] Build status: SCHEDULED
[23:56:32.656] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:95:0) after 3m0s
[23:59:32.723] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:95:0)
[23:59:32.741] Build status: SCHEDULED
[23:59:32.741] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8886653121431674992:96:0) after 7m45s
[00:03:04.828] Received PubSub notification, asking Buildbucket for the build status
[00:03:04.845] Build:
{
  "id": "8721223059115741329",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-asan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-03-05T16:03:03.079960248Z",
  "endTime": "2025-03-06T00:03:04.584394830Z",
  "updateTime": "2025-03-06T00:03:04.584394830Z",
  "status": "INFRA_FAILURE",
  "statusDetails": {
    "resourceExhaustion": {

    },
    "timeout": {

    }
  },
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "0449995468dd677674cbf3908eed420e63ce659e",
      "ref": "refs/heads/main"
    }
  }
}
[00:03:04.845] Invocation finished in 8h0m3.358402798s with status FAILED