1. All projects
  2. pigweed
  3. pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan
  4. 8865201947343489088
Status: FAILED
Triggered by: -
Duration: 6 hours
Actions:

Definition (rev ed331b29843b850a4bc37ad64e7e3a240af6f486) 

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "pigweed.dev.ci"
  builder: "pigweed-mac-x86-bazel-sanitizers-tsan"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "2bc32d08807c1ead3ef0a8184a3714d14b6a1b76"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/2bc32d08807c1ead3ef0a8184a3714d14b6a1b76
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[10:40:15.279] New invocation is queued and will start shortly
[10:40:16.354] Starting the invocation (attempt 1)
[10:40:16.377] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/2bc32d08807c1ead3ef0a8184a3714d14b6a1b76"
[10:40:16.377] Popped gitiles commit info from properties and tags
[10:40:16.377] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[10:40:16.377] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[10:40:16.377] Buildbucket request:
{
  "requestId": "8865201947343489088",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-sanitizers-tsan"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8865201947343489088",
            "job": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "d89db6589382bfb7a0a7bc08e64719f1e893494f"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@d89db6589382bfb7a0a7bc08e64719f1e893494f",
                        "title": "d89db6589382bfb7a0a7bc08e64719f1e893494f",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/d89db6589382bfb7a0a7bc08e64719f1e893494f"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "2bc32d08807c1ead3ef0a8184a3714d14b6a1b76"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@2bc32d08807c1ead3ef0a8184a3714d14b6a1b76",
                        "title": "2bc32d08807c1ead3ef0a8184a3714d14b6a1b76",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/2bc32d08807c1ead3ef0a8184a3714d14b6a1b76"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "2bc32d08807c1ead3ef0a8184a3714d14b6a1b76",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8865201947343489088"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[10:40:16.761] Scheduled build:
{
  "id": "8699771885488214241",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-sanitizers-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-10-28T10:40:16.424160702Z",
  "updateTime": "2025-10-28T10:40:16.424160702Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "2bc32d08807c1ead3ef0a8184a3714d14b6a1b76",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[10:40:16.761] Task URL: https://cr-buildbucket.appspot.com/build/8699771885488214241
[10:40:16.761] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:2:0) after 3m11s
[10:40:17.689] Received PubSub notification, asking Buildbucket for the build status
[10:40:17.706] Build status: SCHEDULED
[10:43:27.816] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:2:0)
[10:43:27.838] Build status: SCHEDULED
[10:43:27.838] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:4:0) after 6m28s
[10:49:55.851] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:4:0)
[10:49:55.869] Build status: SCHEDULED
[10:49:55.869] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:5:0) after 2m2s
[10:51:57.924] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:5:0)
[10:51:57.941] Build status: SCHEDULED
[10:51:57.941] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:6:0) after 2m12s
[10:54:09.980] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:6:0)
[10:54:10.004] Build status: SCHEDULED
[10:54:10.004] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:7:0) after 2m34s
[10:56:44.067] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:7:0)
[10:56:44.090] Build status: SCHEDULED
[10:56:44.090] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:8:0) after 2m55s
[10:59:39.102] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:8:0)
[10:59:39.124] Build status: SCHEDULED
[10:59:39.124] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:9:0) after 8m19s
[11:07:58.137] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:9:0)
[11:07:58.161] Build status: SCHEDULED
[11:07:58.161] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:10:0) after 4m17s
[11:12:15.173] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:10:0)
[11:12:15.204] Build status: SCHEDULED
[11:12:15.204] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:11:0) after 3m21s
[11:15:36.233] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:11:0)
[11:15:36.284] Build status: SCHEDULED
[11:15:36.284] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:12:0) after 5m3s
[11:20:39.432] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:12:0)
[11:20:39.450] Build status: SCHEDULED
[11:20:39.451] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:13:0) after 8m17s
[11:28:56.465] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:13:0)
[11:28:56.660] Build status: SCHEDULED
[11:28:56.660] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:14:0) after 5m0s
[11:33:56.724] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:14:0)
[11:33:56.737] Build status: SCHEDULED
[11:33:56.737] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:15:0) after 4m53s
[11:38:49.826] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:15:0)
[11:38:49.880] Build status: SCHEDULED
[11:38:49.880] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:16:0) after 5m7s
[11:43:56.892] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:16:0)
[11:43:56.922] Build status: SCHEDULED
[11:43:56.922] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:17:0) after 9m43s
[11:53:39.974] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:17:0)
[11:53:40.029] Build status: SCHEDULED
[11:53:40.029] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:18:0) after 5m30s
[11:59:10.384] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:18:0)
[11:59:10.403] Build status: SCHEDULED
[11:59:10.403] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:19:0) after 2m21s
[12:01:32.873] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:19:0)
[12:01:32.892] Build status: SCHEDULED
[12:01:32.892] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:20:0) after 8m43s
[12:10:15.961] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:20:0)
[12:10:15.979] Build status: SCHEDULED
[12:10:15.979] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:21:0) after 1m51s
[12:12:07.060] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:21:0)
[12:12:07.086] Build status: SCHEDULED
[12:12:07.086] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:22:0) after 4m16s
[12:16:23.108] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:22:0)
[12:16:23.154] Build status: SCHEDULED
[12:16:23.154] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:23:0) after 4m28s
[12:20:51.188] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:23:0)
[12:20:51.276] Build status: SCHEDULED
[12:20:51.277] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:24:0) after 7m55s
[12:28:46.294] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:24:0)
[12:28:46.309] Build status: SCHEDULED
[12:28:46.309] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:25:0) after 1m49s
[12:30:35.322] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:25:0)
[12:30:35.340] Build status: SCHEDULED
[12:30:35.340] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:26:0) after 1m47s
[12:32:22.353] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:26:0)
[12:32:22.378] Build status: SCHEDULED
[12:32:22.378] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:27:0) after 1m5s
[12:33:27.398] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:27:0)
[12:33:27.431] Build status: SCHEDULED
[12:33:27.431] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:28:0) after 4m33s
[12:38:00.485] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:28:0)
[12:38:00.501] Build status: SCHEDULED
[12:38:00.501] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:29:0) after 1m7s
[12:39:07.532] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:29:0)
[12:39:07.555] Build status: SCHEDULED
[12:39:07.555] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:30:0) after 7m29s
[12:46:36.573] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:30:0)
[12:46:36.590] Build status: SCHEDULED
[12:46:36.590] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:31:0) after 2m34s
[12:49:10.653] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:31:0)
[12:49:10.672] Build status: SCHEDULED
[12:49:10.672] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:32:0) after 8m31s
[12:57:41.789] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:32:0)
[12:57:41.827] Build status: SCHEDULED
[12:57:41.827] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:33:0) after 5m20s
[13:03:01.883] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:33:0)
[13:03:01.977] Build status: SCHEDULED
[13:03:01.977] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:34:0) after 1m18s
[13:04:19.961] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:34:0)
[13:04:20.018] Build status: SCHEDULED
[13:04:20.018] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:35:0) after 7m26s
[13:11:46.039] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:35:0)
[13:11:46.066] Build status: SCHEDULED
[13:11:46.066] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:36:0) after 5m1s
[13:16:47.111] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:36:0)
[13:16:47.196] Build status: SCHEDULED
[13:16:47.196] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:37:0) after 5m28s
[13:22:15.212] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:37:0)
[13:22:15.232] Build status: SCHEDULED
[13:22:15.232] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:38:0) after 4m36s
[13:26:51.243] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:38:0)
[13:26:51.387] Build status: SCHEDULED
[13:26:51.387] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:39:0) after 1m22s
[13:28:13.401] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:39:0)
[13:28:13.417] Build status: SCHEDULED
[13:28:13.417] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:40:0) after 2m43s
[13:30:56.451] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:40:0)
[13:30:56.470] Build status: SCHEDULED
[13:30:56.470] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:41:0) after 2m52s
[13:33:48.486] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:41:0)
[13:33:48.511] Build status: SCHEDULED
[13:33:48.511] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:42:0) after 9m40s
[13:43:28.527] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:42:0)
[13:43:28.546] Build status: SCHEDULED
[13:43:28.546] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:43:0) after 2m49s
[13:46:17.567] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:43:0)
[13:46:17.583] Build status: SCHEDULED
[13:46:17.583] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:44:0) after 6m14s
[13:52:31.548] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:44:0)
[13:52:31.566] Build status: SCHEDULED
[13:52:31.566] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:45:0) after 7m0s
[13:59:31.552] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:45:0)
[13:59:31.584] Build status: SCHEDULED
[13:59:31.584] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:46:0) after 1m20s
[14:00:51.614] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:46:0)
[14:00:51.637] Build status: SCHEDULED
[14:00:51.637] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:47:0) after 6m31s
[14:07:22.658] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:47:0)
[14:07:22.685] Build status: SCHEDULED
[14:07:22.685] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:48:0) after 2m20s
[14:09:42.707] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:48:0)
[14:09:42.722] Build status: SCHEDULED
[14:09:42.723] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:49:0) after 2m3s
[14:11:45.737] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:49:0)
[14:11:45.757] Build status: SCHEDULED
[14:11:45.757] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:50:0) after 3m41s
[14:15:26.771] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:50:0)
[14:15:26.787] Build status: SCHEDULED
[14:15:26.787] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:51:0) after 6m37s
[14:22:03.858] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:51:0)
[14:22:03.877] Build status: SCHEDULED
[14:22:03.877] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:52:0) after 6m10s
[14:28:13.948] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:52:0)
[14:28:13.990] Build status: SCHEDULED
[14:28:13.990] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:53:0) after 3m10s
[14:31:24.114] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:53:0)
[14:31:24.130] Build status: SCHEDULED
[14:31:24.130] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:54:0) after 1m29s
[14:32:53.145] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:54:0)
[14:32:53.171] Build status: SCHEDULED
[14:32:53.171] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:55:0) after 3m42s
[14:36:35.214] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:55:0)
[14:36:35.236] Build status: SCHEDULED
[14:36:35.236] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:56:0) after 9m23s
[14:45:58.255] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:56:0)
[14:45:58.363] Build status: SCHEDULED
[14:45:58.363] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:57:0) after 1m39s
[14:47:37.380] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:57:0)
[14:47:37.400] Build status: SCHEDULED
[14:47:37.400] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:58:0) after 8m52s
[14:56:29.475] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:58:0)
[14:56:29.495] Build status: SCHEDULED
[14:56:29.495] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:59:0) after 1m34s
[14:58:03.538] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:59:0)
[14:58:03.556] Build status: SCHEDULED
[14:58:03.556] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:60:0) after 5m29s
[15:03:32.575] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:60:0)
[15:03:32.603] Build status: SCHEDULED
[15:03:32.603] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:61:0) after 9m12s
[15:12:44.858] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:61:0)
[15:12:44.874] Build status: SCHEDULED
[15:12:44.874] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:62:0) after 6m37s
[15:19:22.086] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:62:0)
[15:19:22.138] Build status: SCHEDULED
[15:19:22.138] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:63:0) after 7m20s
[15:26:42.289] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:63:0)
[15:26:42.320] Build status: SCHEDULED
[15:26:42.320] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:64:0) after 5m14s
[15:31:56.337] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:64:0)
[15:31:56.356] Build status: SCHEDULED
[15:31:56.356] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:65:0) after 9m35s
[15:41:31.428] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:65:0)
[15:41:31.450] Build status: SCHEDULED
[15:41:31.450] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:66:0) after 8m16s
[15:49:47.578] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:66:0)
[15:49:47.618] Build status: SCHEDULED
[15:49:47.618] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:67:0) after 3m1s
[15:52:48.587] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:67:0)
[15:52:48.605] Build status: SCHEDULED
[15:52:48.605] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:68:0) after 1m27s
[15:54:15.623] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:68:0)
[15:54:15.662] Build status: SCHEDULED
[15:54:15.662] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:69:0) after 2m1s
[15:56:16.682] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:69:0)
[15:56:16.761] Build status: SCHEDULED
[15:56:16.761] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:70:0) after 8m33s
[16:04:49.775] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:70:0)
[16:04:49.789] Build status: SCHEDULED
[16:04:49.789] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:71:0) after 1m41s
[16:06:30.807] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:71:0)
[16:06:30.872] Build status: SCHEDULED
[16:06:30.872] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:72:0) after 3m47s
[16:10:17.892] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:72:0)
[16:10:17.942] Build status: SCHEDULED
[16:10:17.942] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:73:0) after 8m2s
[16:18:20.004] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:73:0)
[16:18:20.030] Build status: SCHEDULED
[16:18:20.031] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:74:0) after 5m23s
[16:23:43.044] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:74:0)
[16:23:43.059] Build status: SCHEDULED
[16:23:43.059] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:75:0) after 9m48s
[16:33:31.118] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:75:0)
[16:33:31.189] Build status: SCHEDULED
[16:33:31.189] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:76:0) after 6m9s
[16:39:40.259] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:76:0)
[16:39:40.277] Build status: SCHEDULED
[16:39:40.331] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:77:0) after 5m40s
[16:45:20.348] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:77:0)
[16:45:20.366] Build status: SCHEDULED
[16:45:20.366] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:78:0) after 6m5s
[16:51:25.382] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:78:0)
[16:51:25.405] Build status: SCHEDULED
[16:51:25.405] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:79:0) after 8m0s
[16:59:25.407] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:79:0)
[16:59:25.426] Build status: SCHEDULED
[16:59:25.426] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:80:0) after 9m21s
[17:08:46.490] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:80:0)
[17:08:46.507] Build status: SCHEDULED
[17:08:46.508] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:81:0) after 2m23s
[17:11:09.528] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:81:0)
[17:11:09.552] Build status: SCHEDULED
[17:11:09.552] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:82:0) after 5m1s
[17:16:10.669] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:82:0)
[17:16:10.808] Build status: SCHEDULED
[17:16:10.808] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:83:0) after 5m27s
[17:21:37.849] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:83:0)
[17:21:37.959] Build status: SCHEDULED
[17:21:37.959] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-x86-bazel-sanitizers-tsan:8865201947343489088:84:0) after 4m56s
[17:22:20.488] Received PubSub notification, asking Buildbucket for the build status
[17:22:20.504] Build:
{
  "id": "8699771885488214241",
  "builder": {
    "project": "pigweed",
    "bucket": "pigweed.dev.ci",
    "builder": "pigweed-mac-x86-bazel-sanitizers-tsan"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-10-28T10:40:16.424160702Z",
  "endTime": "2025-10-28T17:22:19.968963402Z",
  "updateTime": "2025-10-28T17:22:19.968963402Z",
  "status": "INFRA_FAILURE",
  "statusDetails": {
    "resourceExhaustion": {

    }
  },
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "2bc32d08807c1ead3ef0a8184a3714d14b6a1b76",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[17:22:20.504] Invocation finished in 6h42m5.233445753s with status FAILED