[23:47:45.419] New invocation is queued and will start shortly
[23:47:46.523] Starting the invocation (attempt 1)
[23:47:46.551] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/d2241c7b1eaa16a566a8f23053eddfe514c953a3"
[23:47:46.551] Popped gitiles commit info from properties and tags
[23:47:46.551] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[23:47:46.551] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[23:47:46.552] Buildbucket request:
{
"requestId": "8884721346725313296",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-san-tsan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8884721346725313296",
"job": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "8ee9b499efa6a3c154a15ce0547392131b7aece8"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@8ee9b499efa6a3c154a15ce0547392131b7aece8",
"title": "8ee9b499efa6a3c154a15ce0547392131b7aece8",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/8ee9b499efa6a3c154a15ce0547392131b7aece8"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "7941b5b7f61317cfbd86f17a6c5703ec4c652baf"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@7941b5b7f61317cfbd86f17a6c5703ec4c652baf",
"title": "7941b5b7f61317cfbd86f17a6c5703ec4c652baf",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/7941b5b7f61317cfbd86f17a6c5703ec4c652baf"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "6be7274b6461d81b623dc618fa7ea2495e2b8d06"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@6be7274b6461d81b623dc618fa7ea2495e2b8d06",
"title": "6be7274b6461d81b623dc618fa7ea2495e2b8d06",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/6be7274b6461d81b623dc618fa7ea2495e2b8d06"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "d2241c7b1eaa16a566a8f23053eddfe514c953a3"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@d2241c7b1eaa16a566a8f23053eddfe514c953a3",
"title": "d2241c7b1eaa16a566a8f23053eddfe514c953a3",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/d2241c7b1eaa16a566a8f23053eddfe514c953a3"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "d2241c7b1eaa16a566a8f23053eddfe514c953a3",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8884721346725313296"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[23:47:46.943] Scheduled build:
{
"id": "8719291284816580513",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-san-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-03-26T23:47:46.613397660Z",
"updateTime": "2025-03-26T23:47:46.613397660Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "d2241c7b1eaa16a566a8f23053eddfe514c953a3",
"ref": "refs/heads/main"
}
}
}
[23:47:46.943] Task URL: https://cr-buildbucket.appspot.com/build/8719291284816580513
[23:47:46.944] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:2:0) after 9m8s
[23:47:47.905] Received PubSub notification, asking Buildbucket for the build status
[23:47:47.926] Build status: SCHEDULED
[23:56:55.063] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:2:0)
[23:56:55.081] Build status: SCHEDULED
[23:56:55.081] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:4:0) after 8m37s
[00:05:32.098] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:4:0)
[00:05:32.162] Build status: SCHEDULED
[00:05:32.162] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:5:0) after 8m40s
[00:14:12.212] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:5:0)
[00:14:12.237] Build status: SCHEDULED
[00:14:12.237] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:6:0) after 7m39s
[00:21:51.255] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:6:0)
[00:21:51.273] Build status: SCHEDULED
[00:21:51.273] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:7:0) after 6m53s
[00:28:44.298] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:7:0)
[00:28:44.322] Build status: SCHEDULED
[00:28:44.322] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:8:0) after 4m11s
[00:32:55.343] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:8:0)
[00:32:55.368] Build status: SCHEDULED
[00:32:55.368] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:9:0) after 8m19s
[00:41:14.391] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:9:0)
[00:41:14.495] Build status: SCHEDULED
[00:41:14.495] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:10:0) after 7m21s
[00:48:35.515] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:10:0)
[00:48:35.544] Build status: SCHEDULED
[00:48:35.544] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:11:0) after 1m39s
[00:50:14.558] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:11:0)
[00:50:14.605] Build status: SCHEDULED
[00:50:14.605] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:12:0) after 9m40s
[00:59:54.581] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:12:0)
[00:59:54.633] Build status: SCHEDULED
[00:59:54.633] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:13:0) after 7m9s
[01:07:03.648] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:13:0)
[01:07:03.669] Build status: SCHEDULED
[01:07:03.670] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:14:0) after 2m15s
[01:09:18.688] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:14:0)
[01:09:18.706] Build status: SCHEDULED
[01:09:18.706] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:15:0) after 7m53s
[01:17:11.729] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:15:0)
[01:17:11.745] Build status: SCHEDULED
[01:17:11.745] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:16:0) after 9m34s
[01:26:45.791] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:16:0)
[01:26:45.879] Build status: SCHEDULED
[01:26:45.879] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:17:0) after 2m22s
[01:29:07.893] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:17:0)
[01:29:07.911] Build status: SCHEDULED
[01:29:07.911] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:18:0) after 1m59s
[01:31:06.963] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:18:0)
[01:31:06.980] Build status: SCHEDULED
[01:31:06.980] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:19:0) after 6m49s
[01:37:56.023] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:19:0)
[01:37:56.050] Build status: SCHEDULED
[01:37:56.050] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:20:0) after 7m39s
[01:45:35.073] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:20:0)
[01:45:35.130] Build status: SCHEDULED
[01:45:35.130] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:21:0) after 9m21s
[01:54:56.147] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:21:0)
[01:54:56.165] Build status: SCHEDULED
[01:54:56.166] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:22:0) after 8m4s
[02:03:00.182] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:22:0)
[02:03:00.203] Build status: SCHEDULED
[02:03:00.203] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:23:0) after 6m33s
[02:09:33.214] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:23:0)
[02:09:33.237] Build status: SCHEDULED
[02:09:33.237] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:24:0) after 1m18s
[02:10:51.257] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:24:0)
[02:10:51.290] Build status: SCHEDULED
[02:10:51.290] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:25:0) after 7m52s
[02:18:43.308] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:25:0)
[02:18:43.323] Build status: SCHEDULED
[02:18:43.323] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:26:0) after 6m10s
[02:24:53.366] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:26:0)
[02:24:53.386] Build status: SCHEDULED
[02:24:53.386] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:27:0) after 1m36s
[02:26:29.390] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:27:0)
[02:26:29.411] Build status: SCHEDULED
[02:26:29.411] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:28:0) after 5m32s
[02:32:01.450] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:28:0)
[02:32:01.470] Build status: SCHEDULED
[02:32:01.470] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:29:0) after 7m18s
[02:39:19.470] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:29:0)
[02:39:19.487] Build status: SCHEDULED
[02:39:19.487] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:30:0) after 9m32s
[02:48:51.512] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:30:0)
[02:48:51.535] Build status: SCHEDULED
[02:48:51.535] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:31:0) after 1m55s
[02:50:46.524] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:31:0)
[02:50:46.544] Build status: SCHEDULED
[02:50:46.544] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:32:0) after 7m24s
[02:58:10.569] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:32:0)
[02:58:10.586] Build status: SCHEDULED
[02:58:10.586] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:33:0) after 2m1s
[03:00:11.609] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:33:0)
[03:00:11.636] Build status: SCHEDULED
[03:00:11.636] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:34:0) after 3m2s
[03:03:13.669] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:34:0)
[03:03:13.698] Build status: SCHEDULED
[03:03:13.699] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:35:0) after 1m58s
[03:05:11.681] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:35:0)
[03:05:11.760] Build status: SCHEDULED
[03:05:11.760] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:36:0) after 6m0s
[03:11:11.784] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:36:0)
[03:11:11.802] Build status: SCHEDULED
[03:11:11.802] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:37:0) after 9m1s
[03:20:12.871] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:37:0)
[03:20:12.887] Build status: SCHEDULED
[03:20:12.887] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:38:0) after 6m27s
[03:26:39.892] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:38:0)
[03:26:39.918] Build status: SCHEDULED
[03:26:39.918] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:39:0) after 4m12s
[03:30:51.935] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:39:0)
[03:30:51.956] Build status: SCHEDULED
[03:30:51.956] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:40:0) after 3m56s
[03:34:47.984] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:40:0)
[03:34:48.003] Build status: SCHEDULED
[03:34:48.003] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:41:0) after 7m54s
[03:37:07.747] Received PubSub notification, asking Buildbucket for the build status
[03:37:07.761] Build status: STARTED
[03:42:42.025] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:41:0)
[03:42:42.051] Build status: STARTED
[03:42:42.051] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-mac-arm-bazel-san-tsan:8884721346725313296:43:0) after 9m38s
[03:50:58.315] Received PubSub notification, asking Buildbucket for the build status
[03:50:58.423] Build:
{
"id": "8719291284816580513",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-mac-arm-bazel-san-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-03-26T23:47:46.613397660Z",
"startTime": "2025-03-27T03:37:07.541538304Z",
"endTime": "2025-03-27T03:50:58.119978865Z",
"updateTime": "2025-03-27T03:50:58.119978865Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "d2241c7b1eaa16a566a8f23053eddfe514c953a3",
"ref": "refs/heads/main"
}
}
}
[03:50:58.424] Invocation finished in 4h3m13.014997592s with status FAILED