[13:45:18.443] New invocation is queued and will start shortly
[13:45:19.606] Starting the invocation (attempt 1)
[13:45:19.639] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/c19046b3d99e0b6bd100ad821a51c50016859985"
[13:45:19.639] Popped gitiles commit info from properties and tags
[13:45:19.639] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[13:45:19.639] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[13:45:19.640] Buildbucket request:
{
"requestId": "8865915080563262368",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8865915080563262368",
"job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "c19046b3d99e0b6bd100ad821a51c50016859985"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@c19046b3d99e0b6bd100ad821a51c50016859985",
"title": "c19046b3d99e0b6bd100ad821a51c50016859985",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/c19046b3d99e0b6bd100ad821a51c50016859985"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "c19046b3d99e0b6bd100ad821a51c50016859985",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8865915080563262368"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[13:45:20.025] Scheduled build:
{
"id": "8700485018597235793",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-10-20T13:45:19.694121543Z",
"updateTime": "2025-10-20T13:45:19.694121543Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "c19046b3d99e0b6bd100ad821a51c50016859985",
"ref": "refs/heads/main"
}
}
}
[13:45:20.025] Task URL: https://cr-buildbucket.appspot.com/build/8700485018597235793
[13:45:20.025] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:2:0) after 8m13s
[13:45:29.912] Received PubSub notification, asking Buildbucket for the build status
[13:45:29.939] Build status: SCHEDULED
[13:53:33.037] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:2:0)
[13:53:33.077] Build status: SCHEDULED
[13:53:33.077] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:4:0) after 7m44s
[14:01:17.138] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:4:0)
[14:01:17.155] Build status: SCHEDULED
[14:01:17.155] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:5:0) after 3m50s
[14:05:07.169] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:5:0)
[14:05:07.189] Build status: SCHEDULED
[14:05:07.189] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:6:0) after 4m3s
[14:09:10.243] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:6:0)
[14:09:10.383] Build status: SCHEDULED
[14:09:10.383] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:7:0) after 2m2s
[14:11:12.389] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:7:0)
[14:11:12.420] Build status: SCHEDULED
[14:11:12.420] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:8:0) after 3m37s
[14:14:49.544] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:8:0)
[14:14:49.567] Build status: SCHEDULED
[14:14:49.567] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:9:0) after 7m44s
[14:22:33.582] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:9:0)
[14:22:33.886] Build status: SCHEDULED
[14:22:33.886] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:10:0) after 2m34s
[14:25:07.899] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:10:0)
[14:25:07.917] Build status: SCHEDULED
[14:25:07.917] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:11:0) after 1m12s
[14:26:19.931] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:11:0)
[14:26:19.960] Build status: SCHEDULED
[14:26:19.960] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:12:0) after 1m40s
[14:27:59.956] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:12:0)
[14:27:59.970] Build status: SCHEDULED
[14:27:59.970] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:13:0) after 6m15s
[14:34:15.109] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:13:0)
[14:34:15.133] Build status: SCHEDULED
[14:34:15.134] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:14:0) after 7m51s
[14:42:06.135] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:14:0)
[14:42:06.157] Build status: SCHEDULED
[14:42:06.157] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:15:0) after 6m14s
[14:48:20.170] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:15:0)
[14:48:20.191] Build status: SCHEDULED
[14:48:20.191] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:16:0) after 6m54s
[14:55:14.301] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:16:0)
[14:55:14.315] Build status: SCHEDULED
[14:55:14.315] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:17:0) after 8m0s
[15:03:14.359] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:17:0)
[15:03:14.375] Build status: SCHEDULED
[15:03:14.375] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:18:0) after 4m14s
[15:07:28.420] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:18:0)
[15:07:28.442] Build status: SCHEDULED
[15:07:28.442] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:19:0) after 3m38s
[15:11:06.455] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:19:0)
[15:11:06.629] Build status: SCHEDULED
[15:11:06.629] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:20:0) after 8m59s
[15:20:05.759] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:20:0)
[15:20:05.793] Build status: SCHEDULED
[15:20:05.793] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:21:0) after 1m4s
[15:21:09.810] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:21:0)
[15:21:09.831] Build status: SCHEDULED
[15:21:09.831] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:22:0) after 2m29s
[15:23:38.846] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:22:0)
[15:23:38.865] Build status: SCHEDULED
[15:23:38.866] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:23:0) after 9m31s
[15:33:09.928] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:23:0)
[15:33:09.950] Build status: SCHEDULED
[15:33:09.950] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:24:0) after 2m6s
[15:35:15.964] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:24:0)
[15:35:15.988] Build status: SCHEDULED
[15:35:15.988] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:25:0) after 9m56s
[15:45:12.214] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:25:0)
[15:45:12.270] Build status: SCHEDULED
[15:45:12.270] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:26:0) after 6m8s
[15:51:20.382] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:26:0)
[15:51:20.405] Build status: SCHEDULED
[15:51:20.405] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:27:0) after 1m20s
[15:52:40.419] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:27:0)
[15:52:40.445] Build status: SCHEDULED
[15:52:40.445] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:28:0) after 5m29s
[15:58:09.459] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:28:0)
[15:58:09.474] Build status: SCHEDULED
[15:58:09.474] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:29:0) after 5m40s
[16:03:49.532] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:29:0)
[16:03:49.549] Build status: SCHEDULED
[16:03:49.549] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:30:0) after 8m5s
[16:11:54.562] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:30:0)
[16:11:54.593] Build status: SCHEDULED
[16:11:54.593] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:31:0) after 8m1s
[16:19:55.657] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:31:0)
[16:19:55.673] Build status: SCHEDULED
[16:19:55.673] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:32:0) after 5m2s
[16:24:57.689] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:32:0)
[16:24:57.722] Build status: SCHEDULED
[16:24:57.722] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:33:0) after 7m19s
[16:32:16.762] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:33:0)
[16:32:16.777] Build status: SCHEDULED
[16:32:16.777] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:34:0) after 6m0s
[16:38:16.840] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:34:0)
[16:38:16.857] Build status: SCHEDULED
[16:38:16.857] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:35:0) after 5m19s
[16:43:35.875] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:35:0)
[16:43:35.894] Build status: SCHEDULED
[16:43:35.894] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:36:0) after 4m23s
[16:47:58.910] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:36:0)
[16:47:58.930] Build status: SCHEDULED
[16:47:58.930] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:37:0) after 5m54s
[16:53:52.961] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:37:0)
[16:53:52.986] Build status: SCHEDULED
[16:53:52.986] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:38:0) after 1m9s
[16:55:02.001] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:38:0)
[16:55:02.056] Build status: SCHEDULED
[16:55:02.056] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:39:0) after 8m29s
[17:03:31.124] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:39:0)
[17:03:31.201] Build status: SCHEDULED
[17:03:31.201] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8865915080563262368:40:0) after 7m51s
[17:06:52.994] Received PubSub notification, asking Buildbucket for the build status
[17:06:53.184] Build status: STARTED
[17:09:38.896] Received PubSub notification, asking Buildbucket for the build status
[17:09:38.931] Build:
{
"id": "8700485018597235793",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-10-20T13:45:19.694121543Z",
"startTime": "2025-10-20T17:06:52.641727558Z",
"endTime": "2025-10-20T17:09:37.857337459Z",
"updateTime": "2025-10-20T17:09:37.857337459Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "c19046b3d99e0b6bd100ad821a51c50016859985",
"ref": "refs/heads/main"
}
}
}
[17:09:38.931] Invocation finished in 3h24m20.496945196s with status FAILED