Status: FAILED
Triggered by: fuchsia-prod-builder@fuchsia-infra.iam.gserviceaccount.com
Duration: 6 hours
Actions:

Definition (rev 0663a629884a7698d2f3baaf5156926202c4609e)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.toolchain"
  builder: "toolchain-prod-pigweed-mac-x86"
>

Incoming triggers

  • clang-mac-x64/0 (8 months ago by fuchsia-prod-builder@fuchsia-infra.iam.gserviceaccount.com)

Incoming properties

{
  "$fuchsia/checkout": {
    "clang_toolchain": {
      "cas_digest": "946193f0d7773e9fe6b264b25c620853fed421b3679be4c8447b9ab556af723d/578",
      "cas_instance": "projects/chromium-swarm/instances/default_instance"
    },
    "gitiles_commit": {
      "host": "fuchsia.googlesource.com",
      "id": "f31bdbd7f577bc2069c6d0ccf438e0cb2e0a2afc",
      "project": "integration"
    }
  },
  "clang_toolchain": {
    "git_repository": "https://llvm.googlesource.com/llvm-project",
    "git_revision": "bf3f86623c4712bff146b5d168cad5f7e658ac56"
  }
}

Incoming tags

  • buildset:commit/gitiles/llvm.googlesource.com/llvm-project/+/bf3f86623c4712bff146b5d168cad5f7e658ac56
  • gitiles_ref:refs/heads/main
  • parent_buildername:clang-mac-x64
  • user_agent:recipe

Debug log (UTC)

[19:44:13.466] New invocation is queued and will start shortly
[19:44:13.466] Triggered by user:fuchsia-prod-builder@fuchsia-infra.iam.gserviceaccount.com
[19:44:14.689] Starting the invocation (attempt 1)
[19:44:14.729] Popped buildset tag "buildset:commit/gitiles/llvm.googlesource.com/llvm-project/+/bf3f86623c4712bff146b5d168cad5f7e658ac56"
[19:44:14.729] Reconstructed gitiles commit from tags
[19:44:14.729] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[19:44:14.729] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[19:44:14.729] Buildbucket request:
{
  "requestId": "8918801127831564640",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.toolchain",
    "builder": "toolchain-prod-pigweed-mac-x86"
  },
  "properties": {
      "$fuchsia/checkout": {
            "clang_toolchain": {
                  "cas_digest": "946193f0d7773e9fe6b264b25c620853fed421b3679be4c8447b9ab556af723d/578",
                  "cas_instance": "projects/chromium-swarm/instances/default_instance"
                },
            "gitiles_commit": {
                  "host": "fuchsia.googlesource.com",
                  "id": "f31bdbd7f577bc2069c6d0ccf438e0cb2e0a2afc",
                  "project": "integration"
                }
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8918801127831564640",
            "job": "pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86",
            "triggers": [
                  {
                        "buildbucket": {
                              "properties": {
                                    "$fuchsia/checkout": {
                                          "clang_toolchain": {
                                                "cas_digest": "946193f0d7773e9fe6b264b25c620853fed421b3679be4c8447b9ab556af723d/578",
                                                "cas_instance": "projects/chromium-swarm/instances/default_instance"
                                              },
                                          "gitiles_commit": {
                                                "host": "fuchsia.googlesource.com",
                                                "id": "f31bdbd7f577bc2069c6d0ccf438e0cb2e0a2afc",
                                                "project": "integration"
                                              }
                                        },
                                    "clang_toolchain": {
                                          "git_repository": "https://llvm.googlesource.com/llvm-project",
                                          "git_revision": "bf3f86623c4712bff146b5d168cad5f7e658ac56"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/llvm.googlesource.com/llvm-project/+/bf3f86623c4712bff146b5d168cad5f7e658ac56",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:clang-mac-x64",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "89f91dab-b394-4edc-8755-12ada933797e",
                        "title": "clang-mac-x64/0"
                      }
                ]
          },
      "clang_toolchain": {
            "git_repository": "https://llvm.googlesource.com/llvm-project",
            "git_revision": "bf3f86623c4712bff146b5d168cad5f7e658ac56"
          }
    },
  "gitilesCommit": {
    "host": "llvm.googlesource.com",
    "project": "llvm-project",
    "id": "bf3f86623c4712bff146b5d168cad5f7e658ac56",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8918801127831564640"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "clang-mac-x64"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[19:44:15.840] Scheduled build:
{
  "id": "8753371065626795617",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.toolchain",
    "builder": "toolchain-prod-pigweed-mac-x86"
  },
  "createdBy": "project:pigweed",
  "createTime": "2024-03-15T19:44:14.946306842Z",
  "updateTime": "2024-03-15T19:44:14.946306842Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "llvm.googlesource.com",
      "project": "llvm-project",
      "id": "bf3f86623c4712bff146b5d168cad5f7e658ac56",
      "ref": "refs/heads/main"
    }
  }
}
[19:44:15.840] Task URL: https://cr-buildbucket.appspot.com/build/8753371065626795617
[19:44:15.840] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:2:0) after 3m7s
[19:44:16.579] Received PubSub notification, asking Buildbucket for the build status
[19:44:16.622] Build status: SCHEDULED
[19:47:22.835] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:2:0)
[19:47:22.884] Build status: SCHEDULED
[19:47:22.884] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:4:0) after 5m44s
[19:53:06.912] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:4:0)
[19:53:07.048] Build status: SCHEDULED
[19:53:07.048] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:5:0) after 5m50s
[19:58:57.034] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:5:0)
[19:58:57.075] Build status: SCHEDULED
[19:58:57.075] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:6:0) after 8m53s
[20:07:50.099] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:6:0)
[20:07:50.149] Build status: SCHEDULED
[20:07:50.149] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:7:0) after 8m7s
[20:15:57.192] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:7:0)
[20:15:57.224] Build status: SCHEDULED
[20:15:57.224] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:8:0) after 6m56s
[20:22:53.242] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:8:0)
[20:22:53.269] Build status: SCHEDULED
[20:22:53.269] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:9:0) after 5m10s
[20:28:03.302] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:9:0)
[20:28:03.341] Build status: SCHEDULED
[20:28:03.341] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:10:0) after 6m17s
[20:34:20.364] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:10:0)
[20:34:20.396] Build status: SCHEDULED
[20:34:20.396] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:11:0) after 4m27s
[20:38:47.485] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:11:0)
[20:38:47.519] Build status: SCHEDULED
[20:38:47.519] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:12:0) after 4m56s
[20:43:43.658] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:12:0)
[20:43:43.712] Build status: SCHEDULED
[20:43:43.712] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:13:0) after 3m33s
[20:47:16.762] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:13:0)
[20:47:16.795] Build status: SCHEDULED
[20:47:16.795] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:14:0) after 5m43s
[20:52:59.810] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:14:0)
[20:52:59.828] Build status: SCHEDULED
[20:52:59.828] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:15:0) after 8m54s
[21:01:53.816] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:15:0)
[21:01:53.930] Build status: SCHEDULED
[21:01:53.930] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:16:0) after 9m38s
[21:11:31.975] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:16:0)
[21:11:32.013] Build status: SCHEDULED
[21:11:32.013] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:17:0) after 2m9s
[21:13:41.076] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:17:0)
[21:13:41.094] Build status: SCHEDULED
[21:13:41.094] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:18:0) after 8m35s
[21:22:16.115] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:18:0)
[21:22:16.157] Build status: SCHEDULED
[21:22:16.158] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:19:0) after 1m32s
[21:23:48.185] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:19:0)
[21:23:48.211] Build status: SCHEDULED
[21:23:48.211] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:20:0) after 6m26s
[21:30:14.347] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:20:0)
[21:30:14.382] Build status: SCHEDULED
[21:30:14.382] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:21:0) after 6m9s
[21:36:23.439] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:21:0)
[21:36:23.464] Build status: SCHEDULED
[21:36:23.464] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:22:0) after 8m32s
[21:44:55.522] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:22:0)
[21:44:55.609] Build status: SCHEDULED
[21:44:55.609] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:23:0) after 4m16s
[21:49:11.626] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:23:0)
[21:49:11.665] Build status: SCHEDULED
[21:49:11.665] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:24:0) after 1m0s
[21:50:11.789] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:24:0)
[21:50:11.828] Build status: SCHEDULED
[21:50:11.828] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:25:0) after 6m27s
[21:56:38.893] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:25:0)
[21:56:38.919] Build status: SCHEDULED
[21:56:38.919] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:26:0) after 9m10s
[22:05:49.259] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:26:0)
[22:05:49.309] Build status: SCHEDULED
[22:05:49.309] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:27:0) after 9m57s
[22:15:46.805] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:27:0)
[22:15:46.834] Build status: SCHEDULED
[22:15:46.834] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:28:0) after 3m31s
[22:19:18.051] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:28:0)
[22:19:18.080] Build status: SCHEDULED
[22:19:18.080] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:29:0) after 8m32s
[22:27:50.105] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:29:0)
[22:27:50.156] Build status: SCHEDULED
[22:27:50.156] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:30:0) after 7m39s
[22:35:29.177] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:30:0)
[22:35:29.215] Build status: SCHEDULED
[22:35:29.215] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:31:0) after 8m4s
[22:43:33.705] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:31:0)
[22:43:33.754] Build status: SCHEDULED
[22:43:33.754] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:32:0) after 5m29s
[22:49:02.869] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:32:0)
[22:49:02.890] Build status: SCHEDULED
[22:49:02.890] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:33:0) after 2m38s
[22:51:41.083] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:33:0)
[22:51:41.143] Build status: SCHEDULED
[22:51:41.143] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:34:0) after 2m52s
[22:54:33.205] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:34:0)
[22:54:33.243] Build status: SCHEDULED
[22:54:33.243] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:35:0) after 3m30s
[22:58:03.257] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:35:0)
[22:58:03.297] Build status: SCHEDULED
[22:58:03.297] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:36:0) after 1m22s
[22:59:25.334] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:36:0)
[22:59:25.381] Build status: SCHEDULED
[22:59:25.381] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:37:0) after 9m8s
[23:08:33.924] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:37:0)
[23:08:33.944] Build status: SCHEDULED
[23:08:33.944] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:38:0) after 4m13s
[23:12:46.990] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:38:0)
[23:12:47.020] Build status: SCHEDULED
[23:12:47.020] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:39:0) after 7m8s
[23:19:55.119] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:39:0)
[23:19:55.342] Build status: SCHEDULED
[23:19:55.342] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:40:0) after 6m15s
[23:26:10.379] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:40:0)
[23:26:10.401] Build status: SCHEDULED
[23:26:10.401] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:41:0) after 3m52s
[23:30:02.432] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:41:0)
[23:30:02.468] Build status: SCHEDULED
[23:30:02.468] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:42:0) after 5m21s
[23:35:23.488] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:42:0)
[23:35:23.509] Build status: SCHEDULED
[23:35:23.509] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:43:0) after 3m46s
[23:39:09.595] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:43:0)
[23:39:09.619] Build status: SCHEDULED
[23:39:09.619] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:44:0) after 1m19s
[23:40:28.945] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:44:0)
[23:40:28.985] Build status: SCHEDULED
[23:40:28.985] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:45:0) after 2m25s
[23:42:54.306] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:45:0)
[23:42:54.342] Build status: SCHEDULED
[23:42:54.342] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:46:0) after 3m49s
[23:46:43.835] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:46:0)
[23:46:43.869] Build status: SCHEDULED
[23:46:43.869] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:47:0) after 3m25s
[23:50:08.893] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:47:0)
[23:50:08.972] Build status: SCHEDULED
[23:50:08.972] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:48:0) after 1m13s
[23:51:22.096] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:48:0)
[23:51:22.130] Build status: SCHEDULED
[23:51:22.130] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:49:0) after 2m50s
[23:54:12.280] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:49:0)
[23:54:12.301] Build status: SCHEDULED
[23:54:12.301] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:50:0) after 1m37s
[23:55:49.341] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:50:0)
[23:55:49.369] Build status: SCHEDULED
[23:55:49.369] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:51:0) after 3m42s
[23:59:31.643] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:51:0)
[23:59:31.670] Build status: SCHEDULED
[23:59:31.670] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:52:0) after 4m46s
[00:04:17.696] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:52:0)
[00:04:17.776] Build status: SCHEDULED
[00:04:17.776] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:53:0) after 5m49s
[00:10:06.801] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:53:0)
[00:10:06.837] Build status: SCHEDULED
[00:10:06.837] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:54:0) after 6m38s
[00:16:45.200] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:54:0)
[00:16:45.279] Build status: SCHEDULED
[00:16:45.279] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:55:0) after 5m32s
[00:22:17.314] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:55:0)
[00:22:17.346] Build status: SCHEDULED
[00:22:17.346] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:56:0) after 8m53s
[00:31:10.631] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:56:0)
[00:31:10.718] Build status: SCHEDULED
[00:31:10.718] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:57:0) after 3m44s
[00:34:54.819] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:57:0)
[00:34:54.838] Build status: SCHEDULED
[00:34:54.838] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:58:0) after 5m31s
[00:40:25.918] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:58:0)
[00:40:25.994] Build status: SCHEDULED
[00:40:25.994] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:59:0) after 5m19s
[00:45:45.039] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:59:0)
[00:45:45.136] Build status: SCHEDULED
[00:45:45.136] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:60:0) after 9m21s
[00:55:06.174] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:60:0)
[00:55:06.217] Build status: SCHEDULED
[00:55:06.217] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:61:0) after 3m13s
[00:58:19.574] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:61:0)
[00:58:19.592] Build status: SCHEDULED
[00:58:19.592] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:62:0) after 4m55s
[01:03:14.679] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:62:0)
[01:03:14.696] Build status: SCHEDULED
[01:03:14.696] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:63:0) after 1m8s
[01:04:22.882] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:63:0)
[01:04:23.024] Build status: SCHEDULED
[01:04:23.024] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:64:0) after 5m15s
[01:09:38.075] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:64:0)
[01:09:38.098] Build status: SCHEDULED
[01:09:38.098] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:65:0) after 8m46s
[01:18:24.191] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:65:0)
[01:18:24.222] Build status: SCHEDULED
[01:18:24.222] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:66:0) after 9m19s
[01:27:43.242] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:66:0)
[01:27:43.310] Build status: SCHEDULED
[01:27:43.310] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:67:0) after 1m25s
[01:29:08.431] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:67:0)
[01:29:08.452] Build status: SCHEDULED
[01:29:08.453] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:68:0) after 9m43s
[01:38:51.471] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:68:0)
[01:38:51.488] Build status: SCHEDULED
[01:38:51.488] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-prod-pigweed-mac-x86:8918801127831564640:69:0) after 6m58s
[01:44:23.970] Received PubSub notification, asking Buildbucket for the build status
[01:44:24.026] Build:
{
  "id": "8753371065626795617",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.toolchain",
    "builder": "toolchain-prod-pigweed-mac-x86"
  },
  "createdBy": "project:pigweed",
  "createTime": "2024-03-15T19:44:14.946306842Z",
  "endTime": "2024-03-16T01:44:21.442506Z",
  "updateTime": "2024-03-16T01:44:21.442506Z",
  "status": "INFRA_FAILURE",
  "statusDetails": {
    "resourceExhaustion": {

    },
    "timeout": {

    }
  },
  "input": {
    "gitilesCommit": {
      "host": "llvm.googlesource.com",
      "project": "llvm-project",
      "id": "bf3f86623c4712bff146b5d168cad5f7e658ac56",
      "ref": "refs/heads/main"
    }
  }
}
[01:44:24.026] Invocation finished in 6h0m10.566944496s with status FAILED