[03:29:43.628] New invocation is queued and will start shortly
[03:29:44.648] Starting the invocation (attempt 1)
[03:29:44.679] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/83b3862437b4d01ea1fb7104a1e971bebc048cba"
[03:29:44.680] Popped gitiles commit info from properties and tags
[03:29:44.680] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[03:29:44.680] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[03:29:44.680] Buildbucket request:
{
"requestId": "8867493957953513024",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8867493957953513024",
"job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "aa74068fdd8f52ebcc742a66d8f91cda46a372bb"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@aa74068fdd8f52ebcc742a66d8f91cda46a372bb",
"title": "aa74068fdd8f52ebcc742a66d8f91cda46a372bb",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/aa74068fdd8f52ebcc742a66d8f91cda46a372bb"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "83b3862437b4d01ea1fb7104a1e971bebc048cba"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@83b3862437b4d01ea1fb7104a1e971bebc048cba",
"title": "83b3862437b4d01ea1fb7104a1e971bebc048cba",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/83b3862437b4d01ea1fb7104a1e971bebc048cba"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "83b3862437b4d01ea1fb7104a1e971bebc048cba",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8867493957953513024"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[03:29:45.394] Scheduled build:
{
"id": "8702063896061597025",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-10-03T03:29:44.807195685Z",
"updateTime": "2025-10-03T03:29:44.807195685Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "83b3862437b4d01ea1fb7104a1e971bebc048cba",
"ref": "refs/heads/main"
}
}
}
[03:29:45.394] Task URL: https://cr-buildbucket.appspot.com/build/8702063896061597025
[03:29:45.395] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:2:0) after 7m35s
[03:29:46.120] Received PubSub notification, asking Buildbucket for the build status
[03:29:46.136] Build status: SCHEDULED
[03:37:20.408] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:2:0)
[03:37:20.429] Build status: SCHEDULED
[03:37:20.429] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:4:0) after 8m12s
[03:45:32.442] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:4:0)
[03:45:32.465] Build status: SCHEDULED
[03:45:32.465] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:5:0) after 5m45s
[03:51:17.507] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:5:0)
[03:51:17.523] Build status: SCHEDULED
[03:51:17.523] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:6:0) after 1m38s
[03:52:55.571] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:6:0)
[03:52:55.588] Build status: SCHEDULED
[03:52:55.588] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:7:0) after 9m12s
[04:02:07.607] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:7:0)
[04:02:07.627] Build status: SCHEDULED
[04:02:07.643] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:8:0) after 4m4s
[04:06:11.657] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:8:0)
[04:06:11.674] Build status: SCHEDULED
[04:06:11.674] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:9:0) after 4m47s
[04:10:58.690] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:9:0)
[04:10:58.706] Build status: SCHEDULED
[04:10:58.706] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:10:0) after 8m18s
[04:19:16.688] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:10:0)
[04:19:16.706] Build status: SCHEDULED
[04:19:16.706] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:11:0) after 3m32s
[04:22:48.795] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:11:0)
[04:22:48.815] Build status: SCHEDULED
[04:22:48.816] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:12:0) after 2m15s
[04:25:04.230] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:12:0)
[04:25:04.252] Build status: SCHEDULED
[04:25:04.252] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:13:0) after 7m40s
[04:32:44.308] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:13:0)
[04:32:44.379] Build status: SCHEDULED
[04:32:44.379] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:14:0) after 9m31s
[04:42:15.393] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:14:0)
[04:42:15.417] Build status: SCHEDULED
[04:42:15.417] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:15:0) after 3m47s
[04:46:02.441] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:15:0)
[04:46:02.465] Build status: SCHEDULED
[04:46:02.465] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:16:0) after 4m57s
[04:50:59.571] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:16:0)
[04:50:59.592] Build status: SCHEDULED
[04:50:59.592] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:17:0) after 9m56s
[05:00:56.092] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:17:0)
[05:00:56.118] Build status: SCHEDULED
[05:00:56.118] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:18:0) after 6m13s
[05:07:09.249] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:18:0)
[05:07:09.264] Build status: SCHEDULED
[05:07:09.264] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:19:0) after 6m30s
[05:13:39.279] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:19:0)
[05:13:39.300] Build status: SCHEDULED
[05:13:39.300] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:20:0) after 4m1s
[05:17:40.469] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:20:0)
[05:17:40.484] Build status: SCHEDULED
[05:17:40.484] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:21:0) after 7m49s
[05:25:29.552] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:21:0)
[05:25:29.577] Build status: SCHEDULED
[05:25:29.577] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:22:0) after 8m41s
[05:34:10.676] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:22:0)
[05:34:10.694] Build status: SCHEDULED
[05:34:10.694] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:23:0) after 7m30s
[05:41:40.695] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:23:0)
[05:41:40.722] Build status: SCHEDULED
[05:41:40.722] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:24:0) after 6m13s
[05:47:53.736] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:24:0)
[05:47:53.775] Build status: SCHEDULED
[05:47:53.775] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:25:0) after 4m25s
[05:52:18.881] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:25:0)
[05:52:18.898] Build status: SCHEDULED
[05:52:18.898] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:26:0) after 2m17s
[05:54:35.913] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:26:0)
[05:54:35.973] Build status: SCHEDULED
[05:54:35.973] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:27:0) after 4m19s
[05:58:55.061] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:27:0)
[05:58:55.076] Build status: SCHEDULED
[05:58:55.076] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:28:0) after 2m36s
[06:01:31.175] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:28:0)
[06:01:31.214] Build status: SCHEDULED
[06:01:31.214] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:29:0) after 2m40s
[06:04:11.379] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:29:0)
[06:04:11.406] Build status: SCHEDULED
[06:04:11.406] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:30:0) after 8m21s
[06:12:32.440] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:30:0)
[06:12:32.459] Build status: SCHEDULED
[06:12:32.459] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:31:0) after 3m9s
[06:15:41.475] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:31:0)
[06:15:41.491] Build status: SCHEDULED
[06:15:41.491] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:32:0) after 9m6s
[06:24:47.512] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:32:0)
[06:24:47.570] Build status: SCHEDULED
[06:24:47.570] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:33:0) after 9m8s
[06:33:55.588] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:33:0)
[06:33:55.611] Build status: SCHEDULED
[06:33:55.611] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:34:0) after 9m34s
[06:43:29.630] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:34:0)
[06:43:29.645] Build status: SCHEDULED
[06:43:29.645] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan:8867493957953513024:35:0) after 2m36s
[06:43:43.432] Received PubSub notification, asking Buildbucket for the build status
[06:43:43.452] Build status: STARTED
[06:45:00.382] Received PubSub notification, asking Buildbucket for the build status
[06:45:00.398] Build:
{
"id": "8702063896061597025",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan"
},
"createdBy": "project:pigweed",
"createTime": "2025-10-03T03:29:44.807195685Z",
"startTime": "2025-10-03T06:43:39.998471615Z",
"endTime": "2025-10-03T06:44:41.418185591Z",
"updateTime": "2025-10-03T06:44:41.418185591Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "83b3862437b4d01ea1fb7104a1e971bebc048cba",
"ref": "refs/heads/main"
}
}
}
[06:45:00.398] Invocation finished in 3h15m16.778269945s with status FAILED