1. All projects
  2. pigweed
  3. pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan
  4. 8877691697122352272
Status: FAILED
Triggered by: -
Duration: 6 hours
Actions:

Definition (rev d2b949be221fc8dd3acbfdb77921a85f7b1b8e9d) 

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-x86-bazel-san-tsan"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "df47878ac5b09a469f3da3781d5b5cb09a016637"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/df47878ac5b09a469f3da3781d5b5cb09a016637
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[14:01:01.700] New invocation is queued and will start shortly
[14:01:03.219] Starting the invocation (attempt 1)
[14:01:03.246] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/df47878ac5b09a469f3da3781d5b5cb09a016637"
[14:01:03.246] Popped gitiles commit info from properties and tags
[14:01:03.246] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[14:01:03.246] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[14:01:03.247] Buildbucket request:
{
  "requestId": "8877691697122352272",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-tsan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8877691697122352272",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "df47878ac5b09a469f3da3781d5b5cb09a016637"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@df47878ac5b09a469f3da3781d5b5cb09a016637",
                        "title": "df47878ac5b09a469f3da3781d5b5cb09a016637",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/df47878ac5b09a469f3da3781d5b5cb09a016637"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "df47878ac5b09a469f3da3781d5b5cb09a016637",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8877691697122352272"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[14:01:03.862] Scheduled build:
{
  "id": "8712261634742965473",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-06-12T14:01:03.342598333Z",
  "updateTime": "2025-06-12T14:01:03.342598333Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "df47878ac5b09a469f3da3781d5b5cb09a016637",
      "ref": "refs/heads/main"
    }
  }
}
[14:01:03.862] Task URL: https://cr-buildbucket.appspot.com/build/8712261634742965473
[14:01:03.862] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:2:0) after 2m55s
[14:01:04.691] Received PubSub notification, asking Buildbucket for the build status
[14:01:04.774] Build status: SCHEDULED
[14:03:58.878] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:2:0)
[14:03:58.911] Build status: SCHEDULED
[14:03:58.911] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:4:0) after 5m54s
[14:09:52.942] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:4:0)
[14:09:52.982] Build status: SCHEDULED
[14:09:52.982] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:5:0) after 8m17s
[14:18:10.064] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:5:0)
[14:18:10.090] Build status: SCHEDULED
[14:18:10.090] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:6:0) after 1m28s
[14:19:38.138] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:6:0)
[14:19:38.173] Build status: SCHEDULED
[14:19:38.173] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:7:0) after 3m22s
[14:23:00.226] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:7:0)
[14:23:00.258] Build status: SCHEDULED
[14:23:00.258] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:8:0) after 2m36s
[14:25:36.229] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:8:0)
[14:25:36.253] Build status: SCHEDULED
[14:25:36.253] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:9:0) after 7m54s
[14:33:30.269] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:9:0)
[14:33:30.300] Build status: SCHEDULED
[14:33:30.300] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:10:0) after 8m7s
[14:41:37.325] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:10:0)
[14:41:37.360] Build status: SCHEDULED
[14:41:37.360] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:11:0) after 2m32s
[14:44:09.421] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:11:0)
[14:44:09.446] Build status: SCHEDULED
[14:44:09.446] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:12:0) after 3m20s
[14:47:29.513] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:12:0)
[14:47:29.527] Build status: SCHEDULED
[14:47:29.527] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:13:0) after 4m51s
[14:52:20.597] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:13:0)
[14:52:20.613] Build status: SCHEDULED
[14:52:20.613] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:14:0) after 9m42s
[15:02:02.628] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:14:0)
[15:02:02.646] Build status: SCHEDULED
[15:02:02.646] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:15:0) after 1m1s
[15:03:03.664] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:15:0)
[15:03:03.698] Build status: SCHEDULED
[15:03:03.698] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:16:0) after 7m48s
[15:10:51.772] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:16:0)
[15:10:51.799] Build status: SCHEDULED
[15:10:51.799] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:17:0) after 7m20s
[15:18:11.826] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:17:0)
[15:18:11.845] Build status: SCHEDULED
[15:18:11.845] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:18:0) after 1m26s
[15:19:37.861] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:18:0)
[15:19:37.884] Build status: SCHEDULED
[15:19:37.884] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:19:0) after 3m28s
[15:23:05.900] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:19:0)
[15:23:05.935] Build status: SCHEDULED
[15:23:05.935] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:20:0) after 3m51s
[15:26:57.045] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:20:0)
[15:26:57.065] Build status: SCHEDULED
[15:26:57.065] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:21:0) after 2m55s
[15:29:52.133] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:21:0)
[15:29:52.153] Build status: SCHEDULED
[15:29:52.153] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:22:0) after 4m37s
[15:34:29.187] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:22:0)
[15:34:29.276] Build status: SCHEDULED
[15:34:29.276] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:23:0) after 7m34s
[15:42:03.342] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:23:0)
[15:42:03.364] Build status: SCHEDULED
[15:42:03.364] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:24:0) after 5m40s
[15:47:43.381] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:24:0)
[15:47:43.399] Build status: SCHEDULED
[15:47:43.399] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:25:0) after 5m38s
[15:53:21.421] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:25:0)
[15:53:21.479] Build status: SCHEDULED
[15:53:21.479] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:26:0) after 1m36s
[15:54:57.495] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:26:0)
[15:54:57.516] Build status: SCHEDULED
[15:54:57.516] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:27:0) after 2m3s
[15:57:00.536] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:27:0)
[15:57:00.586] Build status: SCHEDULED
[15:57:00.587] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:28:0) after 2m27s
[15:59:27.651] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:28:0)
[15:59:27.678] Build status: SCHEDULED
[15:59:27.678] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:29:0) after 2m3s
[16:01:30.700] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:29:0)
[16:01:30.741] Build status: SCHEDULED
[16:01:30.741] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:30:0) after 8m26s
[16:09:56.764] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:30:0)
[16:09:56.781] Build status: SCHEDULED
[16:09:56.781] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:31:0) after 4m34s
[16:14:30.797] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:31:0)
[16:14:30.814] Build status: SCHEDULED
[16:14:30.814] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:32:0) after 3m24s
[16:17:54.878] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:32:0)
[16:17:55.920] Build status: SCHEDULED
[16:17:55.920] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:33:0) after 1m6s
[16:19:01.943] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:33:0)
[16:19:02.047] Build status: SCHEDULED
[16:19:02.047] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:34:0) after 2m6s
[16:21:08.047] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:34:0)
[16:21:08.126] Build status: SCHEDULED
[16:21:08.126] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:35:0) after 3m33s
[16:24:41.151] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:35:0)
[16:24:41.227] Build status: SCHEDULED
[16:24:41.227] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:36:0) after 7m30s
[16:32:11.249] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:36:0)
[16:32:11.410] Build status: SCHEDULED
[16:32:11.410] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:37:0) after 1m43s
[16:33:54.431] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:37:0)
[16:33:54.478] Build status: SCHEDULED
[16:33:54.478] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:38:0) after 8m23s
[16:42:17.555] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:38:0)
[16:42:17.574] Build status: SCHEDULED
[16:42:17.574] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:39:0) after 8m25s
[16:50:42.604] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:39:0)
[16:50:42.635] Build status: SCHEDULED
[16:50:42.635] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:40:0) after 7m31s
[16:58:13.616] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:40:0)
[16:58:13.658] Build status: SCHEDULED
[16:58:13.658] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:41:0) after 4m41s
[17:02:54.705] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:41:0)
[17:02:54.762] Build status: SCHEDULED
[17:02:54.762] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:42:0) after 8m42s
[17:11:36.801] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:42:0)
[17:11:36.818] Build status: SCHEDULED
[17:11:36.818] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:43:0) after 4m8s
[17:15:44.858] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:43:0)
[17:15:44.875] Build status: SCHEDULED
[17:15:44.875] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:44:0) after 6m39s
[17:22:23.949] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:44:0)
[17:22:24.011] Build status: SCHEDULED
[17:22:24.011] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:45:0) after 8m34s
[17:30:58.031] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:45:0)
[17:30:58.051] Build status: SCHEDULED
[17:30:58.051] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:46:0) after 7m42s
[17:38:40.089] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:46:0)
[17:38:40.106] Build status: SCHEDULED
[17:38:40.106] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:47:0) after 4m3s
[17:42:43.119] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:47:0)
[17:42:43.136] Build status: SCHEDULED
[17:42:43.136] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:48:0) after 2m11s
[17:44:54.157] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:48:0)
[17:44:54.201] Build status: SCHEDULED
[17:44:54.201] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:49:0) after 6m34s
[17:51:28.223] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:49:0)
[17:51:28.362] Build status: SCHEDULED
[17:51:28.362] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:50:0) after 5m42s
[17:57:10.363] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:50:0)
[17:57:10.377] Build status: SCHEDULED
[17:57:10.377] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:51:0) after 9m49s
[18:06:59.447] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:51:0)
[18:06:59.471] Build status: SCHEDULED
[18:06:59.471] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:52:0) after 1m27s
[18:08:26.487] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:52:0)
[18:08:31.515] Build status: SCHEDULED
[18:08:31.515] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:53:0) after 9m55s
[18:18:26.538] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:53:0)
[18:18:31.572] Build status: SCHEDULED
[18:18:31.572] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:54:0) after 7m23s
[18:25:54.548] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:54:0)
[18:25:59.575] Build status: SCHEDULED
[18:25:59.575] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:55:0) after 9m26s
[18:35:25.909] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:55:0)
[18:35:25.949] Build status: SCHEDULED
[18:35:25.949] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:56:0) after 4m17s
[18:39:43.021] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:56:0)
[18:39:48.044] Build status: SCHEDULED
[18:39:48.045] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:57:0) after 1m12s
[18:41:00.067] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:57:0)
[18:41:00.084] Build status: SCHEDULED
[18:41:00.084] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:58:0) after 1m11s
[18:42:11.311] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:58:0)
[18:42:11.331] Build status: SCHEDULED
[18:42:11.331] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:59:0) after 3m43s
[18:45:54.809] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:59:0)
[18:45:54.824] Build status: SCHEDULED
[18:45:54.824] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:60:0) after 4m33s
[18:50:28.253] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:60:0)
[18:50:28.275] Build status: SCHEDULED
[18:50:28.275] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:61:0) after 7m16s
[18:57:44.294] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:61:0)
[18:57:44.317] Build status: SCHEDULED
[18:57:44.317] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:62:0) after 6m56s
[19:04:40.658] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:62:0)
[19:04:40.679] Build status: SCHEDULED
[19:04:40.679] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:63:0) after 5m52s
[19:10:33.072] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:63:0)
[19:10:33.089] Build status: SCHEDULED
[19:10:33.089] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:64:0) after 9m21s
[19:19:54.189] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:64:0)
[19:19:54.267] Build status: SCHEDULED
[19:19:54.267] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:65:0) after 6m59s
[19:26:53.299] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:65:0)
[19:26:53.322] Build status: SCHEDULED
[19:26:53.322] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:66:0) after 4m14s
[19:31:07.442] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:66:0)
[19:31:07.462] Build status: SCHEDULED
[19:31:07.462] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:67:0) after 1m33s
[19:32:45.405] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:67:0)
[19:32:45.428] Build status: SCHEDULED
[19:32:45.428] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:68:0) after 4m24s
[19:38:15.925] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:68:0)
[19:38:17.577] Build status: SCHEDULED
[19:38:17.577] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:69:0) after 7m41s
[19:57:43.314] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:69:0)
[19:57:43.342] Build status: SCHEDULED
[19:57:43.342] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:70:0) after 8m49s
[20:06:34.185] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:70:0)
[20:06:34.322] Build status: SCHEDULED
[20:06:34.322] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:71:0) after 5m2s
[20:11:36.382] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:71:0)
[20:11:36.402] Build status: SCHEDULED
[20:11:36.402] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:72:0) after 4m51s
[20:16:27.424] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:72:0)
[20:16:27.443] Build status: SCHEDULED
[20:16:27.443] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:73:0) after 5m26s
[20:21:53.475] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:73:0)
[20:21:53.620] Build status: SCHEDULED
[20:21:53.620] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:74:0) after 7m23s
[20:29:16.771] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:74:0)
[20:29:16.822] Build status: SCHEDULED
[20:29:16.822] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:75:0) after 5m8s
[20:34:24.838] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:75:0)
[20:34:24.859] Build status: SCHEDULED
[20:34:24.859] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8877691697122352272:76:0) after 8m59s
[20:36:15.619] Received PubSub notification, asking Buildbucket for the build status
[20:36:15.644] Build:
{
  "id": "8712261634742965473",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-06-12T14:01:03.342598333Z",
  "endTime": "2025-06-12T20:36:09.188836786Z",
  "updateTime": "2025-06-12T20:36:09.188836786Z",
  "status": "INFRA_FAILURE",
  "statusDetails": {
    "resourceExhaustion": {

    }
  },
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "df47878ac5b09a469f3da3781d5b5cb09a016637",
      "ref": "refs/heads/main"
    }
  }
}
[20:36:15.644] Invocation finished in 6h35m13.955291477s with status FAILED