buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-x86-bazel-san-asan"
>

• e0ff413c3c4297b3efc86a326bd7abf5f97e3c07 (2 weeks ago)
• d214608ac080eb9211f2fe6b0729d03580e78727 (2 weeks ago)
• 243a329868f2c0bae9235255f25029e5f1c12013 (2 weeks ago)

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "243a329868f2c0bae9235255f25029e5f1c12013"
}

• buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/243a329868f2c0bae9235255f25029e5f1c12013
• gitiles_ref:refs/heads/main

[23:59:08.386] New invocation is queued and will start shortly
[23:59:09.405] Starting the invocation (attempt 1)
[23:59:09.432] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/243a329868f2c0bae9235255f25029e5f1c12013"
[23:59:09.432] Popped gitiles commit info from properties and tags
[23:59:09.432] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[23:59:09.432] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[23:59:09.432] Buildbucket request:
{
  "requestId": "8877201082374705712",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-asan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8877201082374705712",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "e0ff413c3c4297b3efc86a326bd7abf5f97e3c07"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@e0ff413c3c4297b3efc86a326bd7abf5f97e3c07",
                        "title": "e0ff413c3c4297b3efc86a326bd7abf5f97e3c07",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/e0ff413c3c4297b3efc86a326bd7abf5f97e3c07"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "d214608ac080eb9211f2fe6b0729d03580e78727"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@d214608ac080eb9211f2fe6b0729d03580e78727",
                        "title": "d214608ac080eb9211f2fe6b0729d03580e78727",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/d214608ac080eb9211f2fe6b0729d03580e78727"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "243a329868f2c0bae9235255f25029e5f1c12013"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@243a329868f2c0bae9235255f25029e5f1c12013",
                        "title": "243a329868f2c0bae9235255f25029e5f1c12013",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/243a329868f2c0bae9235255f25029e5f1c12013"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "243a329868f2c0bae9235255f25029e5f1c12013",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8877201082374705712"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[23:59:09.863] Scheduled build:
{
  "id": "8711771020534296641",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-asan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-06-17T23:59:09.512075970Z",
  "updateTime": "2025-06-17T23:59:09.512075970Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "243a329868f2c0bae9235255f25029e5f1c12013",
      "ref": "refs/heads/main"
    }
  }
}
[23:59:09.863] Task URL: https://cr-buildbucket.appspot.com/build/8711771020534296641
[23:59:09.863] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:2:0) after 1m11s
[23:59:10.855] Received PubSub notification, asking Buildbucket for the build status
[23:59:10.883] Build status: SCHEDULED
[00:00:20.928] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:2:0)
[00:00:20.959] Build status: SCHEDULED
[00:00:20.959] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:4:0) after 5m34s
[00:05:54.972] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:4:0)
[00:05:54.991] Build status: SCHEDULED
[00:05:54.992] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:5:0) after 8m17s
[00:14:12.014] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:5:0)
[00:14:12.124] Build status: SCHEDULED
[00:14:12.124] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:6:0) after 6m0s
[00:20:12.201] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:6:0)
[00:20:12.223] Build status: SCHEDULED
[00:20:12.223] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:7:0) after 7m12s
[00:27:24.240] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:7:0)
[00:27:24.257] Build status: SCHEDULED
[00:27:24.257] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:8:0) after 5m47s
[00:33:11.333] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:8:0)
[00:33:11.353] Build status: SCHEDULED
[00:33:11.353] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:9:0) after 1m46s
[00:34:57.427] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:9:0)
[00:34:57.444] Build status: SCHEDULED
[00:34:57.444] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:10:0) after 7m54s
[00:42:51.466] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:10:0)
[00:42:51.524] Build status: SCHEDULED
[00:42:51.524] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:11:0) after 3m21s
[00:46:12.524] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:11:0)
[00:46:12.541] Build status: SCHEDULED
[00:46:12.541] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:12:0) after 4m47s
[00:50:59.628] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:12:0)
[00:50:59.651] Build status: SCHEDULED
[00:50:59.651] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:13:0) after 2m25s
[00:53:24.670] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:13:0)
[00:53:24.748] Build status: SCHEDULED
[00:53:24.749] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:14:0) after 1m16s
[00:54:40.798] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:14:0)
[00:54:40.814] Build status: SCHEDULED
[00:54:40.814] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:15:0) after 8m40s
[01:03:20.835] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:15:0)
[01:03:20.853] Build status: SCHEDULED
[01:03:20.853] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:16:0) after 5m46s
[01:09:06.869] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:16:0)
[01:09:06.954] Build status: SCHEDULED
[01:09:06.954] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:17:0) after 4m36s
[01:13:42.971] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:17:0)
[01:13:42.989] Build status: SCHEDULED
[01:13:42.989] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:18:0) after 5m11s
[01:18:53.963] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:18:0)
[01:18:53.978] Build status: SCHEDULED
[01:18:53.978] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:19:0) after 4m1s
[01:22:54.977] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:19:0)
[01:22:55.004] Build status: SCHEDULED
[01:22:55.004] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:20:0) after 9m21s
[01:32:16.027] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:20:0)
[01:32:16.043] Build status: SCHEDULED
[01:32:16.043] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:21:0) after 3m32s
[01:35:48.115] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:21:0)
[01:35:48.153] Build status: SCHEDULED
[01:35:48.153] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:22:0) after 7m49s
[01:43:37.394] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:22:0)
[01:43:37.416] Build status: SCHEDULED
[01:43:37.416] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:23:0) after 5m47s
[01:49:24.437] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:23:0)
[01:49:24.455] Build status: SCHEDULED
[01:49:24.455] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:24:0) after 7m56s
[01:57:20.480] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:24:0)
[01:57:20.497] Build status: SCHEDULED
[01:57:20.497] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:25:0) after 6m39s
[02:03:59.524] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:25:0)
[02:03:59.543] Build status: SCHEDULED
[02:03:59.543] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:26:0) after 4m10s
[02:08:09.567] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:26:0)
[02:08:09.585] Build status: SCHEDULED
[02:08:09.585] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:27:0) after 2m5s
[02:10:14.655] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:27:0)
[02:10:14.674] Build status: SCHEDULED
[02:10:14.674] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:28:0) after 7m24s
[02:17:39.116] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:28:0)
[02:17:39.135] Build status: SCHEDULED
[02:17:39.135] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:29:0) after 1m23s
[02:19:02.602] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:29:0)
[02:19:02.628] Build status: SCHEDULED
[02:19:02.628] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:30:0) after 5m36s
[02:24:38.947] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:30:0)
[02:24:38.977] Build status: SCHEDULED
[02:24:38.977] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:31:0) after 9m33s
[02:34:12.042] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:31:0)
[02:34:12.061] Build status: SCHEDULED
[02:34:12.061] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:32:0) after 7m12s
[02:41:24.085] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:32:0)
[02:41:24.111] Build status: SCHEDULED
[02:41:24.111] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:33:0) after 9m54s
[02:51:18.133] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:33:0)
[02:51:18.428] Build status: SCHEDULED
[02:51:18.428] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:34:0) after 4m25s
[02:55:43.488] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:34:0)
[02:55:43.510] Build status: SCHEDULED
[02:55:43.510] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:35:0) after 4m0s
[02:59:44.053] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:35:0)
[02:59:44.073] Build status: SCHEDULED
[02:59:44.073] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:36:0) after 3m2s
[03:02:46.149] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:36:0)
[03:02:46.172] Build status: SCHEDULED
[03:02:46.172] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:37:0) after 9m30s
[03:12:16.193] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:37:0)
[03:12:16.213] Build status: SCHEDULED
[03:12:16.213] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:38:0) after 1m54s
[03:14:10.234] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:38:0)
[03:14:10.448] Build status: SCHEDULED
[03:14:10.449] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:39:0) after 1m9s
[03:15:19.434] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:39:0)
[03:15:19.457] Build status: SCHEDULED
[03:15:19.457] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:40:0) after 2m54s
[03:18:13.471] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:40:0)
[03:18:13.491] Build status: SCHEDULED
[03:18:13.491] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:41:0) after 2m42s
[03:20:55.523] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:41:0)
[03:20:55.541] Build status: SCHEDULED
[03:20:55.541] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:42:0) after 8m6s
[03:29:01.893] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:42:0)
[03:29:01.919] Build status: SCHEDULED
[03:29:01.919] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:43:0) after 2m12s
[03:31:13.946] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:43:0)
[03:31:13.989] Build status: SCHEDULED
[03:31:13.989] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:44:0) after 6m31s
[03:37:45.153] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:44:0)
[03:37:45.181] Build status: SCHEDULED
[03:37:45.181] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:45:0) after 1m3s
[03:38:48.204] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:45:0)
[03:38:48.224] Build status: SCHEDULED
[03:38:48.224] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:46:0) after 6m36s
[03:45:24.211] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:46:0)
[03:45:24.230] Build status: SCHEDULED
[03:45:24.231] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:47:0) after 3m44s
[03:49:08.250] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:47:0)
[03:49:08.276] Build status: SCHEDULED
[03:49:08.276] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:48:0) after 6m33s
[03:55:41.494] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:48:0)
[03:55:41.508] Build status: SCHEDULED
[03:55:41.508] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:49:0) after 8m24s
[04:04:05.802] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:49:0)
[04:04:05.821] Build status: SCHEDULED
[04:04:05.821] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:50:0) after 5m36s
[04:09:41.846] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:50:0)
[04:09:41.873] Build status: SCHEDULED
[04:09:41.873] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:51:0) after 9m15s
[04:18:56.891] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:51:0)
[04:18:56.924] Build status: SCHEDULED
[04:18:56.924] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:52:0) after 9m55s
[04:28:51.989] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:52:0)
[04:28:52.008] Build status: SCHEDULED
[04:28:52.008] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:53:0) after 6m10s
[04:35:02.024] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:53:0)
[04:35:02.065] Build status: SCHEDULED
[04:35:02.065] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:54:0) after 3m11s
[04:38:13.124] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:54:0)
[04:38:13.142] Build status: SCHEDULED
[04:38:13.142] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:55:0) after 3m11s
[04:41:24.172] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:55:0)
[04:41:24.206] Build status: SCHEDULED
[04:41:24.206] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:56:0) after 5m47s
[04:47:11.323] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:56:0)
[04:47:11.341] Build status: SCHEDULED
[04:47:11.341] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:57:0) after 6m46s
[04:53:57.777] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:57:0)
[04:53:57.905] Build status: SCHEDULED
[04:53:57.905] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:58:0) after 3m11s
[04:57:08.974] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:58:0)
[04:57:08.993] Build status: SCHEDULED
[04:57:08.993] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:59:0) after 3m45s
[05:00:54.011] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:59:0)
[05:00:54.029] Build status: SCHEDULED
[05:00:54.029] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:60:0) after 9m29s
[05:10:23.016] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:60:0)
[05:10:23.046] Build status: SCHEDULED
[05:10:23.046] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:61:0) after 7m6s
[05:17:29.105] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:61:0)
[05:17:29.128] Build status: SCHEDULED
[05:17:29.128] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:62:0) after 5m21s
[05:22:50.436] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:62:0)
[05:22:50.453] Build status: SCHEDULED
[05:22:50.453] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:63:0) after 5m2s
[05:27:52.475] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:63:0)
[05:27:52.518] Build status: SCHEDULED
[05:27:52.518] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:64:0) after 7m18s
[05:35:10.701] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:64:0)
[05:35:10.721] Build status: SCHEDULED
[05:35:10.721] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:65:0) after 4m34s
[05:39:45.152] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:65:0)
[05:39:45.189] Build status: SCHEDULED
[05:39:45.189] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:66:0) after 8m44s
[05:48:29.256] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:66:0)
[05:48:29.277] Build status: SCHEDULED
[05:48:29.277] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:67:0) after 6m48s
[05:55:17.296] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:67:0)
[05:55:17.316] Build status: SCHEDULED
[05:55:17.316] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:68:0) after 2m53s
[05:58:10.352] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:68:0)
[05:58:10.373] Build status: SCHEDULED
[05:58:10.373] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:69:0) after 5m30s
[06:03:40.399] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:69:0)
[06:03:40.416] Build status: SCHEDULED
[06:03:40.416] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:70:0) after 5m59s
[06:09:39.485] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:70:0)
[06:09:39.523] Build status: SCHEDULED
[06:09:39.523] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:71:0) after 8m36s
[06:18:15.591] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:71:0)
[06:18:15.609] Build status: SCHEDULED
[06:18:15.609] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:72:0) after 1m43s
[06:19:58.700] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:72:0)
[06:19:58.756] Build status: SCHEDULED
[06:19:58.756] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:73:0) after 8m44s
[06:28:42.792] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:73:0)
[06:28:42.827] Build status: SCHEDULED
[06:28:42.827] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:74:0) after 8m26s
[06:37:08.874] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:74:0)
[06:37:08.891] Build status: SCHEDULED
[06:37:08.891] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:75:0) after 9m10s
[06:46:18.907] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:75:0)
[06:46:18.924] Build status: SCHEDULED
[06:46:18.924] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:76:0) after 2m10s
[06:48:28.948] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:76:0)
[06:48:28.970] Build status: SCHEDULED
[06:48:28.970] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:77:0) after 4m56s
[06:53:24.977] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:77:0)
[06:53:24.996] Build status: SCHEDULED
[06:53:24.996] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:78:0) after 2m28s
[06:55:53.170] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:78:0)
[06:55:53.186] Build status: SCHEDULED
[06:55:53.186] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:79:0) after 3m5s
[06:58:58.221] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:79:0)
[06:58:58.296] Build status: SCHEDULED
[06:58:58.297] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:80:0) after 3m9s
[07:02:07.404] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:80:0)
[07:02:07.423] Build status: SCHEDULED
[07:02:07.423] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:81:0) after 3m49s
[07:05:56.489] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:81:0)
[07:05:56.523] Build status: SCHEDULED
[07:05:56.523] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:82:0) after 6m12s
[07:12:08.656] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:82:0)
[07:12:08.674] Build status: SCHEDULED
[07:12:08.674] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:83:0) after 6m56s
[07:19:04.772] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:83:0)
[07:19:04.805] Build status: SCHEDULED
[07:19:04.805] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:84:0) after 9m35s
[07:28:39.837] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:84:0)
[07:28:39.913] Build status: SCHEDULED
[07:28:39.913] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:85:0) after 7m23s
[07:36:02.932] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:85:0)
[07:36:02.947] Build status: SCHEDULED
[07:36:02.947] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:86:0) after 9m47s
[07:45:50.162] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:86:0)
[07:45:50.182] Build status: SCHEDULED
[07:45:50.182] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:87:0) after 9m46s
[07:55:36.331] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:87:0)
[07:55:36.349] Build status: SCHEDULED
[07:55:36.349] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-asan:8877201082374705712:88:0) after 4m42s
[07:59:15.480] Received PubSub notification, asking Buildbucket for the build status
[07:59:15.497] Build:
{
  "id": "8711771020534296641",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-asan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-06-17T23:59:09.512075970Z",
  "endTime": "2025-06-18T07:59:15.194556464Z",
  "updateTime": "2025-06-18T07:59:15.194556464Z",
  "status": "INFRA_FAILURE",
  "statusDetails": {
    "resourceExhaustion": {

    },
    "timeout": {

    }
  },
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "243a329868f2c0bae9235255f25029e5f1c12013",
      "ref": "refs/heads/main"
    }
  }
}
[07:59:15.497] Invocation finished in 8h0m7.124104794s with status FAILED