[18:04:34.272] New invocation is queued and will start shortly
[18:04:35.300] Starting the invocation (attempt 1)
[18:04:35.328] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/b50149b80cbc0f02fcf5ded428e148dae3586cb8"
[18:04:35.328] Popped gitiles commit info from properties and tags
[18:04:35.328] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[18:04:35.329] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[18:04:35.329] Buildbucket request:
{
"requestId": "8877223389895623536",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-san-tsan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8877223389895623536",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "efc7493df1a5ef8a6ec47b7e9cfff3e5c0f0212f"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@efc7493df1a5ef8a6ec47b7e9cfff3e5c0f0212f",
"title": "efc7493df1a5ef8a6ec47b7e9cfff3e5c0f0212f",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/efc7493df1a5ef8a6ec47b7e9cfff3e5c0f0212f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "3f68bc43f46189c2701e5943740a1e84b4ad8868"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@3f68bc43f46189c2701e5943740a1e84b4ad8868",
"title": "3f68bc43f46189c2701e5943740a1e84b4ad8868",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/3f68bc43f46189c2701e5943740a1e84b4ad8868"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "b50149b80cbc0f02fcf5ded428e148dae3586cb8"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@b50149b80cbc0f02fcf5ded428e148dae3586cb8",
"title": "b50149b80cbc0f02fcf5ded428e148dae3586cb8",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/b50149b80cbc0f02fcf5ded428e148dae3586cb8"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "b50149b80cbc0f02fcf5ded428e148dae3586cb8",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8877223389895623536"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[18:04:35.791] Scheduled build:
{
"id": "8711793328022316529",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-san-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-06-17T18:04:35.433831720Z",
"updateTime": "2025-06-17T18:04:35.433831720Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "b50149b80cbc0f02fcf5ded428e148dae3586cb8",
"ref": "refs/heads/main"
}
}
}
[18:04:35.791] Task URL: https://cr-buildbucket.appspot.com/build/8711793328022316529
[18:04:35.791] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:2:0) after 3m1s
[18:04:36.804] Received PubSub notification, asking Buildbucket for the build status
[18:04:36.832] Build status: SCHEDULED
[18:07:36.872] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:2:0)
[18:07:36.902] Build status: SCHEDULED
[18:07:36.902] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:4:0) after 3m27s
[18:11:04.150] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:4:0)
[18:11:04.198] Build status: SCHEDULED
[18:11:04.198] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:5:0) after 8m41s
[18:19:45.324] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:5:0)
[18:19:45.347] Build status: SCHEDULED
[18:19:45.347] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:6:0) after 8m41s
[18:28:26.371] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:6:0)
[18:28:26.404] Build status: SCHEDULED
[18:28:26.404] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:7:0) after 2m13s
[18:30:39.426] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:7:0)
[18:30:39.453] Build status: SCHEDULED
[18:30:39.453] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:8:0) after 1m20s
[18:31:59.469] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:8:0)
[18:31:59.499] Build status: SCHEDULED
[18:31:59.499] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:9:0) after 7m13s
[18:39:12.552] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:9:0)
[18:39:12.575] Build status: SCHEDULED
[18:39:12.575] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:10:0) after 8m38s
[18:47:50.694] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:10:0)
[18:47:50.712] Build status: SCHEDULED
[18:47:50.713] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:11:0) after 9m12s
[18:57:02.756] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:11:0)
[18:57:02.782] Build status: SCHEDULED
[18:57:02.782] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:12:0) after 3m26s
[19:00:28.795] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:12:0)
[19:00:28.815] Build status: SCHEDULED
[19:00:28.815] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:13:0) after 6m23s
[19:06:51.854] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:13:0)
[19:06:51.940] Build status: SCHEDULED
[19:06:51.940] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:14:0) after 3m6s
[19:09:58.006] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:14:0)
[19:09:58.047] Build status: SCHEDULED
[19:09:58.047] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:15:0) after 3m45s
[19:13:43.072] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:15:0)
[19:13:43.102] Build status: SCHEDULED
[19:13:43.102] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:16:0) after 8m16s
[19:21:59.169] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:16:0)
[19:21:59.193] Build status: SCHEDULED
[19:21:59.193] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:17:0) after 4m7s
[19:26:06.368] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:17:0)
[19:26:06.468] Build status: SCHEDULED
[19:26:06.468] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:18:0) after 3m51s
[19:29:57.532] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:18:0)
[19:29:57.550] Build status: SCHEDULED
[19:29:57.550] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:19:0) after 1m26s
[19:31:23.565] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:19:0)
[19:31:23.581] Build status: SCHEDULED
[19:31:23.581] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:20:0) after 8m26s
[19:39:49.606] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:20:0)
[19:39:49.637] Build status: SCHEDULED
[19:39:49.637] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:21:0) after 9m38s
[19:49:27.661] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:21:0)
[19:49:27.679] Build status: SCHEDULED
[19:49:27.679] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:22:0) after 2m32s
[19:51:59.748] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:22:0)
[19:51:59.782] Build status: SCHEDULED
[19:51:59.782] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:23:0) after 5m4s
[19:57:03.820] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:23:0)
[19:57:03.842] Build status: SCHEDULED
[19:57:03.843] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:24:0) after 1m47s
[19:58:50.857] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:24:0)
[19:58:50.879] Build status: SCHEDULED
[19:58:50.879] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:25:0) after 1m41s
[20:00:31.901] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:25:0)
[20:00:31.926] Build status: SCHEDULED
[20:00:31.926] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:26:0) after 4m56s
[20:05:27.941] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:26:0)
[20:05:27.960] Build status: SCHEDULED
[20:05:27.960] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:27:0) after 3m5s
[20:08:32.994] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:27:0)
[20:08:33.021] Build status: SCHEDULED
[20:08:33.021] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:28:0) after 6m26s
[20:14:59.056] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:28:0)
[20:14:59.117] Build status: SCHEDULED
[20:14:59.117] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:29:0) after 7m19s
[20:22:18.221] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:29:0)
[20:22:18.250] Build status: SCHEDULED
[20:22:18.250] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:30:0) after 6m37s
[20:28:55.277] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:30:0)
[20:28:55.297] Build status: SCHEDULED
[20:28:55.297] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:31:0) after 5m53s
[20:34:48.349] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:31:0)
[20:34:48.367] Build status: SCHEDULED
[20:34:48.367] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:32:0) after 7m24s
[20:42:12.382] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:32:0)
[20:42:12.405] Build status: SCHEDULED
[20:42:12.405] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:33:0) after 9m5s
[20:51:17.468] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:33:0)
[20:51:17.487] Build status: SCHEDULED
[20:51:17.487] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:34:0) after 7m32s
[20:58:49.566] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:34:0)
[20:58:49.589] Build status: SCHEDULED
[20:58:49.589] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:35:0) after 8m50s
[21:07:39.617] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:35:0)
[21:07:39.712] Build status: SCHEDULED
[21:07:39.712] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:36:0) after 3m24s
[21:11:03.930] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:36:0)
[21:11:03.958] Build status: SCHEDULED
[21:11:03.958] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:37:0) after 4m27s
[21:15:31.023] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:37:0)
[21:15:31.038] Build status: SCHEDULED
[21:15:31.038] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:38:0) after 2m51s
[21:18:22.054] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:38:0)
[21:18:22.074] Build status: SCHEDULED
[21:18:22.074] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:39:0) after 8m30s
[21:26:52.210] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:39:0)
[21:26:52.224] Build status: SCHEDULED
[21:26:52.224] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:40:0) after 7m25s
[21:34:17.311] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:40:0)
[21:34:17.339] Build status: SCHEDULED
[21:34:17.339] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:41:0) after 6m32s
[21:40:49.365] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:41:0)
[21:40:49.385] Build status: SCHEDULED
[21:40:49.385] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:42:0) after 8m35s
[21:43:51.492] Received PubSub notification, asking Buildbucket for the build status
[21:43:51.524] Build status: STARTED
[21:49:24.422] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:42:0)
[21:49:24.440] Build status: STARTED
[21:49:24.441] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8877223389895623536:44:0) after 1m17s
[21:49:29.887] Received PubSub notification, asking Buildbucket for the build status
[21:49:29.911] Build:
{
"id": "8711793328022316529",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-san-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-06-17T18:04:35.433831720Z",
"startTime": "2025-06-17T21:43:44.653686417Z",
"endTime": "2025-06-17T21:49:29.494923951Z",
"updateTime": "2025-06-17T21:49:29.494923951Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "b50149b80cbc0f02fcf5ded428e148dae3586cb8",
"ref": "refs/heads/main"
}
}
}
[21:49:29.911] Invocation finished in 3h44m55.648206945s with status FAILED