[00:27:32.306] New invocation is queued and will start shortly
[00:27:35.103] Starting the invocation (attempt 1)
[00:27:35.136] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/5ff0d190e0f76626838477f96327e5210c32ee7a"
[00:27:35.136] Popped gitiles commit info from properties and tags
[00:27:35.136] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[00:27:35.136] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[00:27:35.137] Buildbucket request:
{
"requestId": "8839057972821728256",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-asan-gtest"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8839057972821728256",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "9bd6c002202876b6afa09c590ef78d7892d9c6f6"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@9bd6c002202876b6afa09c590ef78d7892d9c6f6",
"title": "9bd6c002202876b6afa09c590ef78d7892d9c6f6",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/9bd6c002202876b6afa09c590ef78d7892d9c6f6"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "9c6a517b398ddb82efe31cd545b782f512463d3e"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@9c6a517b398ddb82efe31cd545b782f512463d3e",
"title": "9c6a517b398ddb82efe31cd545b782f512463d3e",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/9c6a517b398ddb82efe31cd545b782f512463d3e"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "e177c4c9918bfa5ccb033ed475d5fb2248a8e004"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@e177c4c9918bfa5ccb033ed475d5fb2248a8e004",
"title": "e177c4c9918bfa5ccb033ed475d5fb2248a8e004",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/e177c4c9918bfa5ccb033ed475d5fb2248a8e004"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "5ff0d190e0f76626838477f96327e5210c32ee7a"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@5ff0d190e0f76626838477f96327e5210c32ee7a",
"title": "5ff0d190e0f76626838477f96327e5210c32ee7a",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/5ff0d190e0f76626838477f96327e5210c32ee7a"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "5ff0d190e0f76626838477f96327e5210c32ee7a",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8839057972821728256"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[00:27:37.581] Scheduled build:
{
"id": "8673627908330093473",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-asan-gtest"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-13T00:27:35.969290439Z",
"updateTime": "2026-08-13T00:27:35.969290439Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "5ff0d190e0f76626838477f96327e5210c32ee7a",
"ref": "refs/heads/main"
}
}
}
[00:27:37.581] Task URL: https://cr-buildbucket.appspot.com/build/8673627908330093473
[00:27:37.581] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:2:0) after 8m5s
[00:27:38.199] Received PubSub notification, asking Buildbucket for the build status
[00:27:38.223] Build status: SCHEDULED
[00:35:42.599] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:2:0)
[00:35:42.692] Build status: SCHEDULED
[00:35:42.692] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:4:0) after 6m27s
[00:42:09.779] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:4:0)
[00:42:09.798] Build status: SCHEDULED
[00:42:09.798] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:5:0) after 2m50s
[00:44:59.815] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:5:0)
[00:44:59.839] Build status: SCHEDULED
[00:44:59.839] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:6:0) after 1m33s
[00:46:32.976] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:6:0)
[00:46:32.995] Build status: SCHEDULED
[00:46:32.996] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:7:0) after 7m39s
[00:54:12.010] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:7:0)
[00:54:12.026] Build status: SCHEDULED
[00:54:12.026] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:8:0) after 2m21s
[00:56:33.065] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:8:0)
[00:56:33.094] Build status: SCHEDULED
[00:56:33.094] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:9:0) after 6m4s
[01:02:37.108] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:9:0)
[01:02:37.129] Build status: SCHEDULED
[01:02:37.129] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:10:0) after 9m1s
[01:11:38.314] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:10:0)
[01:11:38.363] Build status: SCHEDULED
[01:11:38.363] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:11:0) after 3m47s
[01:15:25.416] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:11:0)
[01:15:25.439] Build status: SCHEDULED
[01:15:25.439] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:12:0) after 5m13s
[01:20:38.503] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:12:0)
[01:20:38.629] Build status: SCHEDULED
[01:20:38.629] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:13:0) after 8m0s
[01:28:38.643] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:13:0)
[01:28:38.678] Build status: SCHEDULED
[01:28:38.678] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:14:0) after 6m18s
[01:34:56.691] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:14:0)
[01:34:56.715] Build status: SCHEDULED
[01:34:56.715] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:15:0) after 3m33s
[01:38:29.727] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:15:0)
[01:38:29.754] Build status: SCHEDULED
[01:38:29.754] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:16:0) after 6m55s
[01:45:24.899] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:16:0)
[01:45:24.923] Build status: SCHEDULED
[01:45:24.923] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:17:0) after 4m35s
[01:49:59.932] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:17:0)
[01:49:59.950] Build status: SCHEDULED
[01:49:59.950] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:18:0) after 5m11s
[01:55:10.975] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:18:0)
[01:55:11.041] Build status: SCHEDULED
[01:55:11.041] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:19:0) after 9m7s
[02:04:18.255] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:19:0)
[02:04:18.281] Build status: SCHEDULED
[02:04:18.281] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:20:0) after 2m8s
[02:06:26.343] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:20:0)
[02:06:26.371] Build status: SCHEDULED
[02:06:26.371] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:21:0) after 9m35s
[02:16:01.587] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:21:0)
[02:16:01.605] Build status: SCHEDULED
[02:16:01.605] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:22:0) after 2m51s
[02:18:52.778] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:22:0)
[02:18:52.801] Build status: SCHEDULED
[02:18:52.801] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:23:0) after 1m50s
[02:20:42.816] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:23:0)
[02:20:42.832] Build status: SCHEDULED
[02:20:42.832] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:24:0) after 3m9s
[02:23:51.847] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:24:0)
[02:23:51.902] Build status: SCHEDULED
[02:23:51.902] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:25:0) after 1m53s
[02:25:44.969] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:25:0)
[02:25:45.000] Build status: SCHEDULED
[02:25:45.001] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:26:0) after 9m37s
[02:35:22.018] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:26:0)
[02:35:22.038] Build status: SCHEDULED
[02:35:22.038] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:27:0) after 5m41s
[02:41:03.054] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:27:0)
[02:41:03.105] Build status: SCHEDULED
[02:41:03.105] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:28:0) after 8m11s
[02:49:14.125] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:28:0)
[02:49:14.149] Build status: SCHEDULED
[02:49:14.149] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:29:0) after 6m44s
[02:55:58.194] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:29:0)
[02:55:58.215] Build status: SCHEDULED
[02:55:58.215] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:30:0) after 9m24s
[03:05:22.234] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:30:0)
[03:05:22.253] Build status: SCHEDULED
[03:05:22.253] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:31:0) after 2m22s
[03:07:44.270] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:31:0)
[03:07:44.314] Build status: SCHEDULED
[03:07:44.314] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:32:0) after 7m35s
[03:15:19.390] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:32:0)
[03:15:19.443] Build status: SCHEDULED
[03:15:19.443] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:33:0) after 3m8s
[03:18:27.509] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:33:0)
[03:18:27.527] Build status: SCHEDULED
[03:18:27.527] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:34:0) after 1m31s
[03:19:58.548] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:34:0)
[03:19:58.610] Build status: SCHEDULED
[03:19:58.610] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:35:0) after 1m5s
[03:21:03.643] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:35:0)
[03:21:03.705] Build status: SCHEDULED
[03:21:03.705] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:36:0) after 9m5s
[03:30:08.727] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:36:0)
[03:30:08.748] Build status: SCHEDULED
[03:30:08.748] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:37:0) after 4m44s
[03:34:52.822] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:37:0)
[03:34:52.841] Build status: SCHEDULED
[03:34:52.841] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:38:0) after 1m19s
[03:36:11.827] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:38:0)
[03:36:11.845] Build status: SCHEDULED
[03:36:11.845] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:39:0) after 5m29s
[03:41:40.913] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:39:0)
[03:41:40.934] Build status: STARTED
[03:41:40.934] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:40:0) after 3m16s
[03:42:09.847] Received PubSub notification, asking Buildbucket for the build status
[03:42:09.867] Build status: STARTED
[03:44:57.084] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:40:0)
[03:44:57.107] Build status: STARTED
[03:44:57.107] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-asan-gtest:8839057972821728256:42:0) after 7m55s
[03:46:11.745] Received PubSub notification, asking Buildbucket for the build status
[03:46:11.763] Build:
{
"id": "8673627908330093473",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-asan-gtest"
},
"createdBy": "project:pigweed",
"createTime": "2026-08-13T00:27:35.969290439Z",
"startTime": "2026-08-13T03:41:32.448314691Z",
"endTime": "2026-08-13T03:45:51.297437814Z",
"updateTime": "2026-08-13T03:45:51.297437814Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "5ff0d190e0f76626838477f96327e5210c32ee7a",
"ref": "refs/heads/main"
}
}
}
[03:46:11.763] Invocation finished in 3h18m39.46173231s with status FAILED