[18:12:27.319] New invocation is queued and will start shortly
[18:12:28.703] Starting the invocation (attempt 1)
[18:12:28.731] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/db63b6f307562b60928a159c1a018e371b6cb345"
[18:12:28.731] Popped gitiles commit info from properties and tags
[18:12:28.731] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[18:12:28.731] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[18:12:28.731] Buildbucket request:
{
"requestId": "8867438421495087408",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8867438421495087408",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "73bda91a1a7ba49ab48f2dd925f70f67064d0e17"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@73bda91a1a7ba49ab48f2dd925f70f67064d0e17",
"title": "73bda91a1a7ba49ab48f2dd925f70f67064d0e17",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/73bda91a1a7ba49ab48f2dd925f70f67064d0e17"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "d2d0b3bcfcde2a23471f16cc3244ea3b9cf19ef3"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@d2d0b3bcfcde2a23471f16cc3244ea3b9cf19ef3",
"title": "d2d0b3bcfcde2a23471f16cc3244ea3b9cf19ef3",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/d2d0b3bcfcde2a23471f16cc3244ea3b9cf19ef3"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "23dca0920f20792eca8b02c256c61e9806597315"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@23dca0920f20792eca8b02c256c61e9806597315",
"title": "23dca0920f20792eca8b02c256c61e9806597315",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/23dca0920f20792eca8b02c256c61e9806597315"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "db63b6f307562b60928a159c1a018e371b6cb345"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@db63b6f307562b60928a159c1a018e371b6cb345",
"title": "db63b6f307562b60928a159c1a018e371b6cb345",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/db63b6f307562b60928a159c1a018e371b6cb345"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "db63b6f307562b60928a159c1a018e371b6cb345",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8867438421495087408"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[18:12:29.285] Scheduled build:
{
"id": "8702008359285464353",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-10-03T18:12:28.804465956Z",
"updateTime": "2025-10-03T18:12:28.804465956Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "db63b6f307562b60928a159c1a018e371b6cb345",
"ref": "refs/heads/main"
}
}
}
[18:12:29.285] Task URL: https://cr-buildbucket.appspot.com/build/8702008359285464353
[18:12:29.285] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:2:0) after 5m56s
[18:12:30.130] Received PubSub notification, asking Buildbucket for the build status
[18:12:30.168] Build status: SCHEDULED
[18:18:25.396] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:2:0)
[18:18:25.451] Build status: SCHEDULED
[18:18:25.451] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:4:0) after 6m20s
[18:24:45.417] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:4:0)
[18:24:45.436] Build status: SCHEDULED
[18:24:45.436] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:5:0) after 4m17s
[18:29:02.454] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:5:0)
[18:29:02.518] Build status: SCHEDULED
[18:29:02.518] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:6:0) after 6m2s
[18:35:04.544] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:6:0)
[18:35:04.620] Build status: SCHEDULED
[18:35:04.620] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:7:0) after 2m5s
[18:37:09.737] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:7:0)
[18:37:09.774] Build status: SCHEDULED
[18:37:09.774] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:8:0) after 7m38s
[18:44:47.851] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:8:0)
[18:44:47.868] Build status: SCHEDULED
[18:44:47.868] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:9:0) after 1m35s
[18:46:22.901] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:9:0)
[18:46:22.928] Build status: SCHEDULED
[18:46:22.928] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:10:0) after 7m50s
[18:54:12.930] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:10:0)
[18:54:13.010] Build status: SCHEDULED
[18:54:13.010] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:11:0) after 9m30s
[19:03:42.998] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:11:0)
[19:03:43.038] Build status: SCHEDULED
[19:03:43.038] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:12:0) after 6m6s
[19:09:49.110] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:12:0)
[19:09:49.203] Build status: SCHEDULED
[19:09:49.203] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:13:0) after 6m59s
[19:16:48.248] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:13:0)
[19:16:48.269] Build status: SCHEDULED
[19:16:48.269] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:14:0) after 5m51s
[19:22:39.383] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:14:0)
[19:22:39.476] Build status: SCHEDULED
[19:22:39.477] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:15:0) after 8m23s
[19:31:02.532] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:15:0)
[19:31:02.547] Build status: SCHEDULED
[19:31:02.547] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:16:0) after 7m3s
[19:38:05.555] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:16:0)
[19:38:05.605] Build status: SCHEDULED
[19:38:05.606] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:17:0) after 1m0s
[19:39:05.621] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:17:0)
[19:39:05.752] Build status: SCHEDULED
[19:39:05.752] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:18:0) after 3m33s
[19:42:38.765] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:18:0)
[19:42:38.783] Build status: SCHEDULED
[19:42:38.783] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:19:0) after 6m48s
[19:49:26.798] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:19:0)
[19:49:26.822] Build status: SCHEDULED
[19:49:26.822] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:20:0) after 6m29s
[19:55:55.871] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:20:0)
[19:55:55.912] Build status: SCHEDULED
[19:55:55.912] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:21:0) after 7m17s
[20:03:12.979] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:21:0)
[20:03:13.017] Build status: SCHEDULED
[20:03:13.018] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:22:0) after 4m30s
[20:07:43.084] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:22:0)
[20:07:43.123] Build status: SCHEDULED
[20:07:43.123] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:23:0) after 2m13s
[20:09:56.231] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:23:0)
[20:09:56.249] Build status: SCHEDULED
[20:09:56.249] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:24:0) after 3m58s
[20:13:54.316] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:24:0)
[20:13:54.333] Build status: SCHEDULED
[20:13:54.333] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:25:0) after 8m32s
[20:22:26.348] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:25:0)
[20:22:26.369] Build status: SCHEDULED
[20:22:26.369] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:26:0) after 1m49s
[20:24:15.428] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:26:0)
[20:24:15.446] Build status: SCHEDULED
[20:24:15.446] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:27:0) after 4m14s
[20:28:29.525] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:27:0)
[20:28:29.548] Build status: SCHEDULED
[20:28:29.548] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:28:0) after 8m28s
[20:36:57.564] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:28:0)
[20:36:57.581] Build status: SCHEDULED
[20:36:57.581] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:29:0) after 8m45s
[20:45:42.610] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:29:0)
[20:45:42.636] Build status: SCHEDULED
[20:45:42.636] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:30:0) after 6m16s
[20:51:58.650] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:30:0)
[20:51:58.673] Build status: SCHEDULED
[20:51:58.673] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-sanitizers-tsan:8867438421495087408:31:0) after 6m35s
[20:54:37.791] Received PubSub notification, asking Buildbucket for the build status
[20:54:37.807] Build status: STARTED
[20:56:50.769] Received PubSub notification, asking Buildbucket for the build status
[20:56:50.786] Build:
{
"id": "8702008359285464353",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-sanitizers-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-10-03T18:12:28.804465956Z",
"startTime": "2025-10-03T20:54:37.637287345Z",
"endTime": "2025-10-03T20:56:50.502449129Z",
"updateTime": "2025-10-03T20:56:50.502449129Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "db63b6f307562b60928a159c1a018e371b6cb345",
"ref": "refs/heads/main"
}
}
}
[20:56:50.786] Invocation finished in 2h44m23.472008107s with status FAILED