1. All projects
  2. pigweed
  3. pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan
  4. 8885206802779792944
Status: FAILED
Triggered by: -
Duration: 7 hours
Actions:

Definition (rev d2b949be221fc8dd3acbfdb77921a85f7b1b8e9d) 

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-x86-bazel-san-tsan"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "f9f93dd4a7f4bd8264f4456def8daf966805b75d"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/f9f93dd4a7f4bd8264f4456def8daf966805b75d
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[15:11:38.445] New invocation is queued and will start shortly
[15:11:41.163] Starting the invocation (attempt 1)
[15:11:41.186] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/f9f93dd4a7f4bd8264f4456def8daf966805b75d"
[15:11:41.186] Popped gitiles commit info from properties and tags
[15:11:41.186] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[15:11:41.187] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[15:11:41.187] Buildbucket request:
{
  "requestId": "8885206802779792944",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-tsan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8885206802779792944",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "f9f93dd4a7f4bd8264f4456def8daf966805b75d"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@f9f93dd4a7f4bd8264f4456def8daf966805b75d",
                        "title": "f9f93dd4a7f4bd8264f4456def8daf966805b75d",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/f9f93dd4a7f4bd8264f4456def8daf966805b75d"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "f9f93dd4a7f4bd8264f4456def8daf966805b75d",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8885206802779792944"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[15:11:41.748] Scheduled build:
{
  "id": "8719776739058743345",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-03-21T15:11:41.371683880Z",
  "updateTime": "2025-03-21T15:11:41.371683880Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "f9f93dd4a7f4bd8264f4456def8daf966805b75d",
      "ref": "refs/heads/main"
    }
  }
}
[15:11:41.748] Task URL: https://cr-buildbucket.appspot.com/build/8719776739058743345
[15:11:41.748] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:2:0) after 1m31s
[15:11:42.711] Received PubSub notification, asking Buildbucket for the build status
[15:11:42.735] Build status: SCHEDULED
[15:13:12.763] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:2:0)
[15:13:12.792] Build status: SCHEDULED
[15:13:12.792] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:4:0) after 1m29s
[15:14:41.797] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:4:0)
[15:14:41.824] Build status: SCHEDULED
[15:14:41.825] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:5:0) after 7m28s
[15:22:09.844] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:5:0)
[15:22:09.875] Build status: SCHEDULED
[15:22:09.875] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:6:0) after 6m41s
[15:28:50.944] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:6:0)
[15:28:50.969] Build status: SCHEDULED
[15:28:50.969] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:7:0) after 8m17s
[15:37:07.985] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:7:0)
[15:37:08.012] Build status: SCHEDULED
[15:37:08.012] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:8:0) after 5m11s
[15:42:19.034] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:8:0)
[15:42:19.118] Build status: SCHEDULED
[15:42:19.118] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:9:0) after 2m35s
[15:44:54.106] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:9:0)
[15:44:54.127] Build status: SCHEDULED
[15:44:54.127] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:10:0) after 1m34s
[15:46:28.171] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:10:0)
[15:46:28.193] Build status: SCHEDULED
[15:46:28.193] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:11:0) after 7m47s
[15:54:15.215] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:11:0)
[15:54:15.249] Build status: SCHEDULED
[15:54:15.249] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:12:0) after 6m58s
[16:01:13.314] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:12:0)
[16:01:13.364] Build status: SCHEDULED
[16:01:13.364] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:13:0) after 8m3s
[16:09:16.338] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:13:0)
[16:09:16.414] Build status: SCHEDULED
[16:09:16.415] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:14:0) after 3m19s
[16:12:35.443] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:14:0)
[16:12:35.486] Build status: SCHEDULED
[16:12:35.486] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:15:0) after 5m2s
[16:17:37.494] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:15:0)
[16:17:37.521] Build status: SCHEDULED
[16:17:37.521] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:16:0) after 1m10s
[16:18:47.623] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:16:0)
[16:18:47.643] Build status: SCHEDULED
[16:18:47.643] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:17:0) after 3m35s
[16:22:22.707] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:17:0)
[16:22:22.735] Build status: SCHEDULED
[16:22:22.735] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:18:0) after 6m1s
[16:28:23.824] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:18:0)
[16:28:23.844] Build status: SCHEDULED
[16:28:23.844] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:19:0) after 1m37s
[16:30:00.858] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:19:0)
[16:30:00.875] Build status: SCHEDULED
[16:30:00.875] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:20:0) after 2m27s
[16:32:27.915] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:20:0)
[16:32:27.966] Build status: SCHEDULED
[16:32:27.966] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:21:0) after 1m17s
[16:33:45.086] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:21:0)
[16:33:45.111] Build status: SCHEDULED
[16:33:45.111] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:22:0) after 1m18s
[16:35:03.123] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:22:0)
[16:35:03.146] Build status: SCHEDULED
[16:35:03.146] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:23:0) after 2m12s
[16:37:15.164] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:23:0)
[16:37:15.192] Build status: SCHEDULED
[16:37:15.192] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:24:0) after 1m17s
[16:38:32.191] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:24:0)
[16:38:32.218] Build status: SCHEDULED
[16:38:32.218] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:25:0) after 7m41s
[16:46:13.256] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:25:0)
[16:46:13.273] Build status: SCHEDULED
[16:46:13.273] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:26:0) after 4m1s
[16:50:14.295] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:26:0)
[16:50:14.312] Build status: SCHEDULED
[16:50:14.312] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:27:0) after 5m21s
[16:55:35.414] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:27:0)
[16:55:35.430] Build status: SCHEDULED
[16:55:35.430] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:28:0) after 1m53s
[16:57:28.465] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:28:0)
[16:57:28.484] Build status: SCHEDULED
[16:57:28.484] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:29:0) after 9m47s
[17:07:15.579] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:29:0)
[17:07:15.598] Build status: SCHEDULED
[17:07:15.598] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:30:0) after 2m38s
[17:09:53.624] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:30:0)
[17:09:53.656] Build status: SCHEDULED
[17:09:53.656] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:31:0) after 7m50s
[17:17:43.710] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:31:0)
[17:17:43.734] Build status: SCHEDULED
[17:17:43.734] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:32:0) after 9m9s
[17:26:52.857] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:32:0)
[17:26:52.875] Build status: SCHEDULED
[17:26:52.875] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:33:0) after 5m14s
[17:32:06.916] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:33:0)
[17:32:06.938] Build status: SCHEDULED
[17:32:06.938] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:34:0) after 6m3s
[17:38:09.928] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:34:0)
[17:38:09.948] Build status: SCHEDULED
[17:38:09.948] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:35:0) after 6m31s
[17:44:41.082] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:35:0)
[17:44:41.101] Build status: SCHEDULED
[17:44:41.101] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:36:0) after 2m36s
[17:47:17.065] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:36:0)
[17:47:17.083] Build status: SCHEDULED
[17:47:17.083] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:37:0) after 3m42s
[17:50:59.081] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:37:0)
[17:50:59.097] Build status: SCHEDULED
[17:50:59.097] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:38:0) after 7m17s
[17:58:16.119] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:38:0)
[17:58:16.140] Build status: SCHEDULED
[17:58:16.140] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:39:0) after 1m45s
[18:00:01.122] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:39:0)
[18:00:01.193] Build status: SCHEDULED
[18:00:01.193] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:40:0) after 5m19s
[18:05:20.196] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:40:0)
[18:05:20.214] Build status: SCHEDULED
[18:05:20.214] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:41:0) after 4m33s
[18:09:53.228] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:41:0)
[18:09:53.250] Build status: SCHEDULED
[18:09:53.250] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:42:0) after 2m38s
[18:12:31.274] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:42:0)
[18:12:31.293] Build status: SCHEDULED
[18:12:31.293] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:43:0) after 3m54s
[18:16:25.386] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:43:0)
[18:16:25.427] Build status: SCHEDULED
[18:16:25.427] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:44:0) after 7m58s
[18:24:23.467] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:44:0)
[18:24:23.582] Build status: SCHEDULED
[18:24:23.582] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:45:0) after 6m33s
[18:30:56.744] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:45:0)
[18:30:56.830] Build status: SCHEDULED
[18:30:56.830] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:46:0) after 8m29s
[18:39:25.856] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:46:0)
[18:39:25.887] Build status: SCHEDULED
[18:39:25.887] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:47:0) after 5m33s
[18:44:58.943] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:47:0)
[18:44:58.960] Build status: SCHEDULED
[18:44:58.960] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:48:0) after 2m38s
[18:47:36.983] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:48:0)
[18:47:36.998] Build status: SCHEDULED
[18:47:36.998] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:49:0) after 3m21s
[18:50:58.058] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:49:0)
[18:50:58.110] Build status: SCHEDULED
[18:50:58.110] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:50:0) after 8m33s
[18:59:31.224] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:50:0)
[18:59:31.246] Build status: SCHEDULED
[18:59:31.246] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:51:0) after 9m41s
[19:09:12.428] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:51:0)
[19:09:12.442] Build status: SCHEDULED
[19:09:12.442] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:52:0) after 7m39s
[19:16:51.620] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:52:0)
[19:16:51.646] Build status: SCHEDULED
[19:16:51.646] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:53:0) after 2m15s
[19:19:06.667] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:53:0)
[19:19:06.692] Build status: SCHEDULED
[19:19:06.692] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:54:0) after 1m9s
[19:20:15.673] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:54:0)
[19:20:15.694] Build status: SCHEDULED
[19:20:15.694] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:55:0) after 2m23s
[19:22:38.761] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:55:0)
[19:22:38.780] Build status: SCHEDULED
[19:22:38.780] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:56:0) after 4m45s
[19:27:23.865] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:56:0)
[19:27:23.881] Build status: SCHEDULED
[19:27:23.881] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:57:0) after 2m59s
[19:30:22.946] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:57:0)
[19:30:22.979] Build status: SCHEDULED
[19:30:22.979] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:58:0) after 5m22s
[19:35:45.139] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:58:0)
[19:35:45.155] Build status: SCHEDULED
[19:35:45.155] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:59:0) after 6m33s
[19:42:18.179] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:59:0)
[19:42:18.206] Build status: SCHEDULED
[19:42:18.206] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:60:0) after 4m23s
[19:46:41.247] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:60:0)
[19:46:41.265] Build status: SCHEDULED
[19:46:41.265] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:61:0) after 1m31s
[19:48:12.367] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:61:0)
[19:48:12.390] Build status: SCHEDULED
[19:48:12.390] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:62:0) after 8m0s
[19:56:12.435] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:62:0)
[19:56:12.454] Build status: SCHEDULED
[19:56:12.454] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:63:0) after 8m8s
[20:04:20.528] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:63:0)
[20:04:20.542] Build status: SCHEDULED
[20:04:20.542] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:64:0) after 7m44s
[20:12:04.613] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:64:0)
[20:12:04.633] Build status: SCHEDULED
[20:12:04.633] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:65:0) after 6m33s
[20:18:37.690] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:65:0)
[20:18:37.713] Build status: SCHEDULED
[20:18:37.713] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:66:0) after 6m37s
[20:25:14.740] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:66:0)
[20:25:14.758] Build status: SCHEDULED
[20:25:14.758] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:67:0) after 8m56s
[20:34:10.903] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:67:0)
[20:34:10.919] Build status: SCHEDULED
[20:34:10.919] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:68:0) after 4m49s
[20:38:59.901] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:68:0)
[20:38:59.979] Build status: SCHEDULED
[20:38:59.979] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:69:0) after 5m31s
[20:44:30.975] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:69:0)
[20:44:30.995] Build status: SCHEDULED
[20:44:30.995] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:70:0) after 4m55s
[20:49:26.064] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:70:0)
[20:49:26.081] Build status: SCHEDULED
[20:49:26.082] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:71:0) after 1m50s
[20:51:16.151] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:71:0)
[20:51:16.169] Build status: SCHEDULED
[20:51:16.169] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:72:0) after 8m16s
[20:59:32.190] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:72:0)
[20:59:32.206] Build status: SCHEDULED
[20:59:32.206] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:73:0) after 2m6s
[21:01:38.306] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:73:0)
[21:01:38.324] Build status: SCHEDULED
[21:01:38.324] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:74:0) after 9m58s
[21:11:36.345] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:74:0)
[21:11:36.361] Build status: SCHEDULED
[21:11:36.361] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:75:0) after 6m7s
[21:17:43.332] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:75:0)
[21:17:43.405] Build status: SCHEDULED
[21:17:43.405] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:76:0) after 2m58s
[21:20:41.425] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:76:0)
[21:20:41.447] Build status: SCHEDULED
[21:20:41.447] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:77:0) after 9m49s
[21:23:47.880] Received PubSub notification, asking Buildbucket for the build status
[21:23:47.903] Build status: STARTED
[21:30:30.484] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:77:0)
[21:30:30.506] Build status: STARTED
[21:30:30.506] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:79:0) after 8m47s
[21:39:17.531] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:79:0)
[21:39:17.549] Build status: STARTED
[21:39:17.549] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:80:0) after 1m26s
[21:40:43.585] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:80:0)
[21:40:43.604] Build status: STARTED
[21:40:43.604] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:81:0) after 9m25s
[21:50:08.728] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:81:0)
[21:50:08.744] Build status: STARTED
[21:50:08.744] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:82:0) after 1m13s
[21:51:21.770] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:82:0)
[21:51:21.829] Build status: STARTED
[21:51:21.829] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:83:0) after 5m17s
[21:56:38.900] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:83:0)
[21:56:38.936] Build status: STARTED
[21:56:38.936] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:84:0) after 7m3s
[22:03:41.967] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:84:0)
[22:03:41.987] Build status: STARTED
[22:03:41.987] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:85:0) after 5m32s
[22:09:14.071] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:85:0)
[22:09:14.158] Build status: STARTED
[22:09:14.158] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-san-tsan:8885206802779792944:86:0) after 5m43s
[22:11:42.213] Received PubSub notification, asking Buildbucket for the build status
[22:11:42.235] Build:
{
  "id": "8719776739058743345",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-san-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-03-21T15:11:41.371683880Z",
  "startTime": "2025-03-21T21:23:47.390722899Z",
  "endTime": "2025-03-21T22:11:41.763669099Z",
  "updateTime": "2025-03-21T22:11:41.763669099Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "f9f93dd4a7f4bd8264f4456def8daf966805b75d",
      "ref": "refs/heads/main"
    }
  }
}
[22:11:42.235] Invocation finished in 7h0m3.796671857s with status FAILED