Status: FAILED
Triggered by: fuchsia-ci-builder@fuchsia-infra.iam.gserviceaccount.com
Duration: 6 hours
Actions:

Definition (rev 0663a629884a7698d2f3baaf5156926202c4609e)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.toolchain"
  builder: "toolchain-ci-pigweed-mac-x86"
>

Incoming triggers

  • clang-mac-x64/0 (2 years ago by fuchsia-ci-builder@fuchsia-infra.iam.gserviceaccount.com)
  • clang-mac-x64/0 (2 years ago by fuchsia-ci-builder@fuchsia-infra.iam.gserviceaccount.com)
  • clang-mac-x64/0 (2 years ago by fuchsia-ci-builder@fuchsia-infra.iam.gserviceaccount.com)
  • clang-mac-x64/0 (2 years ago by fuchsia-ci-builder@fuchsia-infra.iam.gserviceaccount.com)

Incoming properties

{
  "$fuchsia/checkout": {
    "clang_toolchain": {
      "cas_digest": "35866314b5d22571a8f5d63a14223c1131c536fb592b35d63b40cc2f20b883ea/578",
      "cas_instance": "projects/chromium-swarm/instances/default_instance"
    },
    "gitiles_commit": {
      "host": "fuchsia.googlesource.com",
      "id": "603af2f762fe9a9235da0c42e4accdaada09ac0e",
      "project": "integration"
    }
  },
  "clang_toolchain": {
    "git_repository": "https://llvm.googlesource.com/llvm-project",
    "git_revision": "8603a7b21f301508d3a6af9f2238c7b92ce19617"
  }
}

Incoming tags

  • buildset:commit/gitiles/llvm.googlesource.com/llvm-project/+/8603a7b21f301508d3a6af9f2238c7b92ce19617
  • gitiles_ref:refs/heads/main
  • parent_buildername:clang-mac-x64
  • user_agent:recipe

Debug log (UTC)

[03:58:51.191] New invocation is queued and will start shortly
[03:58:51.191] Triggered by user:fuchsia-ci-builder@fuchsia-infra.iam.gserviceaccount.com
[03:58:52.404] Starting the invocation (attempt 1)
[03:58:52.433] Popped buildset tag "buildset:commit/gitiles/llvm.googlesource.com/llvm-project/+/8603a7b21f301508d3a6af9f2238c7b92ce19617"
[03:58:52.434] Reconstructed gitiles commit from tags
[03:58:52.434] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[03:58:52.435] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[03:58:52.435] Buildbucket request:
{
  "requestId": "8920944335683377328",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.toolchain",
    "builder": "toolchain-ci-pigweed-mac-x86"
  },
  "properties": {
      "$fuchsia/checkout": {
            "clang_toolchain": {
                  "cas_digest": "35866314b5d22571a8f5d63a14223c1131c536fb592b35d63b40cc2f20b883ea/578",
                  "cas_instance": "projects/chromium-swarm/instances/default_instance"
                },
            "gitiles_commit": {
                  "host": "fuchsia.googlesource.com",
                  "id": "603af2f762fe9a9235da0c42e4accdaada09ac0e",
                  "project": "integration"
                }
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8920944335683377328",
            "job": "pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86",
            "triggers": [
                  {
                        "buildbucket": {
                              "properties": {
                                    "$fuchsia/checkout": {
                                          "clang_toolchain": {
                                                "cas_digest": "fd5880a0459b765dd917494e309a2f78685fbd2827d3c2bcae0cc54a15588d21/578",
                                                "cas_instance": "projects/chromium-swarm/instances/default_instance"
                                              },
                                          "gitiles_commit": {
                                                "host": "fuchsia.googlesource.com",
                                                "id": "5459dc050b75e8176f423046b62a95280071288e",
                                                "project": "integration"
                                              }
                                        },
                                    "clang_toolchain": {
                                          "git_repository": "https://llvm.googlesource.com/llvm-project",
                                          "git_revision": "1219214a3bcc51022492928b8bb4ff4bdb75d0cb"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/llvm.googlesource.com/llvm-project/+/1219214a3bcc51022492928b8bb4ff4bdb75d0cb",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:clang-mac-x64",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "05b7fed4-8f04-439c-9660-732a61097e26",
                        "title": "clang-mac-x64/0"
                      },
                  {
                        "buildbucket": {
                              "properties": {
                                    "$fuchsia/checkout": {
                                          "clang_toolchain": {
                                                "cas_digest": "fa02b7cd5d21a8e2d3c07c4f763bdd46492c978e3a083d15fb4347d5f423a2ab/578",
                                                "cas_instance": "projects/chromium-swarm/instances/default_instance"
                                              },
                                          "gitiles_commit": {
                                                "host": "fuchsia.googlesource.com",
                                                "id": "5459dc050b75e8176f423046b62a95280071288e",
                                                "project": "integration"
                                              }
                                        },
                                    "clang_toolchain": {
                                          "git_repository": "https://llvm.googlesource.com/llvm-project",
                                          "git_revision": "563ef306017a47d387f1c36dd562b172c1ad0626"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/llvm.googlesource.com/llvm-project/+/563ef306017a47d387f1c36dd562b172c1ad0626",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:clang-mac-x64",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "cc1f1ca2-9b9c-4e6d-9597-f9f8fb80b6b2",
                        "title": "clang-mac-x64/0"
                      },
                  {
                        "buildbucket": {
                              "properties": {
                                    "$fuchsia/checkout": {
                                          "clang_toolchain": {
                                                "cas_digest": "b5e5c8f87d92421543a3adf6b64d0861287d698532d709152664bef9d36c6da1/578",
                                                "cas_instance": "projects/chromium-swarm/instances/default_instance"
                                              },
                                          "gitiles_commit": {
                                                "host": "fuchsia.googlesource.com",
                                                "id": "5459dc050b75e8176f423046b62a95280071288e",
                                                "project": "integration"
                                              }
                                        },
                                    "clang_toolchain": {
                                          "git_repository": "https://llvm.googlesource.com/llvm-project",
                                          "git_revision": "3ff805540173b83d73b673b39ac5760fc19bac15"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/llvm.googlesource.com/llvm-project/+/3ff805540173b83d73b673b39ac5760fc19bac15",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:clang-mac-x64",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "f7e56690-f5ff-4827-b4a9-69672170e313",
                        "title": "clang-mac-x64/0"
                      },
                  {
                        "buildbucket": {
                              "properties": {
                                    "$fuchsia/checkout": {
                                          "clang_toolchain": {
                                                "cas_digest": "35866314b5d22571a8f5d63a14223c1131c536fb592b35d63b40cc2f20b883ea/578",
                                                "cas_instance": "projects/chromium-swarm/instances/default_instance"
                                              },
                                          "gitiles_commit": {
                                                "host": "fuchsia.googlesource.com",
                                                "id": "603af2f762fe9a9235da0c42e4accdaada09ac0e",
                                                "project": "integration"
                                              }
                                        },
                                    "clang_toolchain": {
                                          "git_repository": "https://llvm.googlesource.com/llvm-project",
                                          "git_revision": "8603a7b21f301508d3a6af9f2238c7b92ce19617"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/llvm.googlesource.com/llvm-project/+/8603a7b21f301508d3a6af9f2238c7b92ce19617",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:clang-mac-x64",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "083ddf18-56cb-44d8-92fc-251ffa7c4a10",
                        "title": "clang-mac-x64/0"
                      }
                ]
          },
      "clang_toolchain": {
            "git_repository": "https://llvm.googlesource.com/llvm-project",
            "git_revision": "8603a7b21f301508d3a6af9f2238c7b92ce19617"
          }
    },
  "gitilesCommit": {
    "host": "llvm.googlesource.com",
    "project": "llvm-project",
    "id": "8603a7b21f301508d3a6af9f2238c7b92ce19617",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8920944335683377328"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "clang-mac-x64"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[03:58:53.183] Scheduled build:
{
  "id": "8755514273560653057",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.toolchain",
    "builder": "toolchain-ci-pigweed-mac-x86"
  },
  "createdBy": "project:pigweed",
  "createTime": "2024-02-21T03:58:52.584001169Z",
  "updateTime": "2024-02-21T03:58:52.584001169Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "llvm.googlesource.com",
      "project": "llvm-project",
      "id": "8603a7b21f301508d3a6af9f2238c7b92ce19617",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[03:58:53.183] Task URL: https://cr-buildbucket.appspot.com/build/8755514273560653057
[03:58:53.183] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:2:0) after 1m54s
[03:58:54.173] Received PubSub notification, asking Buildbucket for the build status
[03:58:54.228] Build status: SCHEDULED
[04:00:47.301] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:2:0)
[04:00:47.332] Build status: SCHEDULED
[04:00:47.332] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:4:0) after 3m57s
[04:04:44.471] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:4:0)
[04:04:44.550] Build status: SCHEDULED
[04:04:44.550] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:5:0) after 1m15s
[04:05:59.845] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:5:0)
[04:05:59.875] Build status: SCHEDULED
[04:05:59.875] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:6:0) after 8m4s
[04:14:04.000] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:6:0)
[04:14:04.051] Build status: SCHEDULED
[04:14:04.051] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:7:0) after 7m23s
[04:21:27.149] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:7:0)
[04:21:27.169] Build status: SCHEDULED
[04:21:27.169] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:8:0) after 8m33s
[04:30:00.469] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:8:0)
[04:30:00.528] Build status: SCHEDULED
[04:30:00.528] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:9:0) after 6m33s
[04:36:33.647] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:9:0)
[04:36:33.669] Build status: SCHEDULED
[04:36:33.669] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:10:0) after 8m58s
[04:45:31.862] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:10:0)
[04:45:31.893] Build status: SCHEDULED
[04:45:31.893] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:11:0) after 3m38s
[04:49:10.021] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:11:0)
[04:49:10.049] Build status: SCHEDULED
[04:49:10.049] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:12:0) after 4m30s
[04:53:40.067] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:12:0)
[04:53:40.094] Build status: SCHEDULED
[04:53:40.094] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:13:0) after 8m59s
[05:02:39.113] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:13:0)
[05:02:39.223] Build status: SCHEDULED
[05:02:39.223] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:14:0) after 9m15s
[05:11:54.248] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:14:0)
[05:11:54.265] Build status: SCHEDULED
[05:11:54.265] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:15:0) after 5m56s
[05:17:50.878] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:15:0)
[05:17:50.896] Build status: SCHEDULED
[05:17:50.896] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:16:0) after 3m28s
[05:21:19.019] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:16:0)
[05:21:19.040] Build status: SCHEDULED
[05:21:19.040] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:17:0) after 9m14s
[05:30:33.063] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:17:0)
[05:30:33.095] Build status: SCHEDULED
[05:30:33.096] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:18:0) after 9m16s
[05:39:49.219] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:18:0)
[05:39:49.246] Build status: SCHEDULED
[05:39:49.246] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:19:0) after 9m58s
[05:49:47.266] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:19:0)
[05:49:47.302] Build status: SCHEDULED
[05:49:47.302] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:20:0) after 8m57s
[05:58:44.643] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:20:0)
[05:58:44.722] Build status: SCHEDULED
[05:58:44.722] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:21:0) after 7m44s
[06:06:29.115] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:21:0)
[06:06:29.132] Build status: SCHEDULED
[06:06:29.132] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:22:0) after 8m27s
[06:14:56.164] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:22:0)
[06:14:56.200] Build status: SCHEDULED
[06:14:56.200] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:23:0) after 9m13s
[06:24:09.668] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:23:0)
[06:24:09.694] Build status: SCHEDULED
[06:24:09.694] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:24:0) after 4m53s
[06:29:02.715] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:24:0)
[06:29:02.744] Build status: SCHEDULED
[06:29:02.744] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:25:0) after 8m48s
[06:37:50.963] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:25:0)
[06:37:51.082] Build status: SCHEDULED
[06:37:51.082] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:26:0) after 4m0s
[06:41:51.113] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:26:0)
[06:41:51.136] Build status: SCHEDULED
[06:41:51.136] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:27:0) after 5m19s
[06:47:10.243] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:27:0)
[06:47:10.277] Build status: SCHEDULED
[06:47:10.279] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:28:0) after 3m29s
[06:50:39.316] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:28:0)
[06:50:39.351] Build status: SCHEDULED
[06:50:39.351] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:29:0) after 2m47s
[06:53:26.552] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:29:0)
[06:53:26.584] Build status: SCHEDULED
[06:53:26.584] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:30:0) after 5m24s
[06:58:51.396] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:30:0)
[06:58:51.426] Build status: SCHEDULED
[06:58:51.426] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:31:0) after 3m20s
[07:02:11.486] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:31:0)
[07:02:11.540] Build status: SCHEDULED
[07:02:11.540] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:32:0) after 6m37s
[07:08:48.713] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:32:0)
[07:08:48.731] Build status: SCHEDULED
[07:08:48.731] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:33:0) after 3m30s
[07:12:18.887] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:33:0)
[07:12:18.984] Build status: SCHEDULED
[07:12:18.984] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:34:0) after 7m20s
[07:19:39.006] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:34:0)
[07:19:39.037] Build status: SCHEDULED
[07:19:39.037] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:35:0) after 4m50s
[07:24:29.197] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:35:0)
[07:24:29.220] Build status: SCHEDULED
[07:24:29.221] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:36:0) after 9m47s
[07:34:16.301] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:36:0)
[07:34:16.319] Build status: SCHEDULED
[07:34:16.319] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:37:0) after 2m48s
[07:37:04.455] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:37:0)
[07:37:04.499] Build status: SCHEDULED
[07:37:04.499] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:38:0) after 7m51s
[07:44:55.517] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:38:0)
[07:44:55.591] Build status: SCHEDULED
[07:44:55.591] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:39:0) after 2m12s
[07:47:07.623] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:39:0)
[07:47:07.655] Build status: SCHEDULED
[07:47:07.655] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:40:0) after 1m22s
[07:48:29.801] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:40:0)
[07:48:29.828] Build status: SCHEDULED
[07:48:29.828] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:41:0) after 1m8s
[07:49:37.849] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:41:0)
[07:49:37.898] Build status: SCHEDULED
[07:49:37.899] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:42:0) after 9m37s
[07:59:15.030] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:42:0)
[07:59:15.052] Build status: SCHEDULED
[07:59:15.052] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:43:0) after 6m17s
[08:05:32.238] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:43:0)
[08:05:32.363] Build status: SCHEDULED
[08:05:32.363] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:44:0) after 7m5s
[08:12:37.597] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:44:0)
[08:12:37.635] Build status: SCHEDULED
[08:12:37.635] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:45:0) after 5m52s
[08:18:29.755] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:45:0)
[08:18:29.775] Build status: SCHEDULED
[08:18:29.775] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:46:0) after 1m43s
[08:20:12.816] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:46:0)
[08:20:12.843] Build status: SCHEDULED
[08:20:12.843] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:47:0) after 4m41s
[08:24:54.107] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:47:0)
[08:24:54.612] Build status: SCHEDULED
[08:24:54.612] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:48:0) after 6m34s
[08:31:28.781] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:48:0)
[08:31:28.804] Build status: SCHEDULED
[08:31:28.804] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:49:0) after 9m26s
[08:40:54.845] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:49:0)
[08:40:54.878] Build status: SCHEDULED
[08:40:54.878] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:50:0) after 5m25s
[08:46:19.934] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:50:0)
[08:46:20.033] Build status: SCHEDULED
[08:46:20.033] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:51:0) after 2m57s
[08:49:17.067] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:51:0)
[08:49:17.094] Build status: SCHEDULED
[08:49:17.094] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:52:0) after 2m17s
[08:51:34.127] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:52:0)
[08:51:34.230] Build status: SCHEDULED
[08:51:34.230] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:53:0) after 7m22s
[08:58:56.294] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:53:0)
[08:58:56.370] Build status: SCHEDULED
[08:58:56.370] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:54:0) after 2m20s
[09:01:16.465] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:54:0)
[09:01:16.509] Build status: SCHEDULED
[09:01:16.509] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:55:0) after 6m5s
[09:07:21.617] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:55:0)
[09:07:21.649] Build status: SCHEDULED
[09:07:21.649] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:56:0) after 6m21s
[09:13:42.676] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:56:0)
[09:13:42.781] Build status: SCHEDULED
[09:13:42.781] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:57:0) after 3m42s
[09:17:25.052] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:57:0)
[09:17:25.089] Build status: SCHEDULED
[09:17:25.089] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:58:0) after 1m23s
[09:18:48.289] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:58:0)
[09:18:48.379] Build status: SCHEDULED
[09:18:48.379] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:59:0) after 1m57s
[09:20:45.450] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:59:0)
[09:20:45.481] Build status: SCHEDULED
[09:20:45.481] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:60:0) after 5m54s
[09:26:39.530] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:60:0)
[09:26:39.548] Build status: SCHEDULED
[09:26:39.548] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:61:0) after 5m51s
[09:32:30.579] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:61:0)
[09:32:30.616] Build status: SCHEDULED
[09:32:30.616] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:62:0) after 4m0s
[09:35:54.915] Received PubSub notification, asking Buildbucket for the build status
[09:35:54.957] Build status: STARTED
[09:36:30.808] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:62:0)
[09:36:30.825] Build status: STARTED
[09:36:30.825] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:64:0) after 1m40s
[09:38:11.112] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:64:0)
[09:38:11.131] Build status: STARTED
[09:38:11.131] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:65:0) after 5m26s
[09:43:37.211] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:65:0)
[09:43:37.241] Build status: STARTED
[09:43:37.241] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:66:0) after 8m43s
[09:52:20.558] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:66:0)
[09:52:20.801] Build status: STARTED
[09:52:20.801] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:67:0) after 7m41s
[10:00:01.954] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:67:0)
[10:00:02.090] Build status: STARTED
[10:00:02.090] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:68:0) after 7m37s
[10:07:39.116] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:68:0)
[10:07:39.148] Build status: STARTED
[10:07:39.148] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:69:0) after 4m55s
[10:12:34.237] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:69:0)
[10:12:34.284] Build status: STARTED
[10:12:34.284] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:70:0) after 2m34s
[10:15:08.392] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:70:0)
[10:15:08.412] Build status: STARTED
[10:15:08.412] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.toolchain-toolchain-ci-pigweed-mac-x86:8920944335683377328:71:0) after 7m53s
[10:22:28.720] Received PubSub notification, asking Buildbucket for the build status
[10:22:28.780] Build:
{
  "id": "8755514273560653057",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.toolchain",
    "builder": "toolchain-ci-pigweed-mac-x86"
  },
  "createdBy": "project:pigweed",
  "createTime": "2024-02-21T03:58:52.584001169Z",
  "startTime": "2024-02-21T09:35:09.215352Z",
  "endTime": "2024-02-21T10:22:27.701927Z",
  "updateTime": "2024-02-21T10:22:27.701927Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "llvm.googlesource.com",
      "project": "llvm-project",
      "id": "8603a7b21f301508d3a6af9f2238c7b92ce19617",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[10:22:28.780] Invocation finished in 6h23m37.603432845s with status FAILED