[19:51:54.301] New invocation is queued and will start shortly
[19:51:55.351] Starting the invocation (attempt 1)
[19:51:55.376] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/ad6e598a99ef81a451019a98d3ea83157ff95dd6"
[19:51:55.376] Popped gitiles commit info from properties and tags
[19:51:55.376] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[19:51:55.378] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[19:51:55.379] Buildbucket request:
{
"requestId": "8841702626201965040",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8841702626201965040",
"job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "420cc372d19ad3ea2f065c62dbdd628659cfe734"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@420cc372d19ad3ea2f065c62dbdd628659cfe734",
"title": "420cc372d19ad3ea2f065c62dbdd628659cfe734",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/420cc372d19ad3ea2f065c62dbdd628659cfe734"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "64258903741ba68cea2c495b2e40bb435962e1a6"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@64258903741ba68cea2c495b2e40bb435962e1a6",
"title": "64258903741ba68cea2c495b2e40bb435962e1a6",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/64258903741ba68cea2c495b2e40bb435962e1a6"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "ad6e598a99ef81a451019a98d3ea83157ff95dd6"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@ad6e598a99ef81a451019a98d3ea83157ff95dd6",
"title": "ad6e598a99ef81a451019a98d3ea83157ff95dd6",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/ad6e598a99ef81a451019a98d3ea83157ff95dd6"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "ad6e598a99ef81a451019a98d3ea83157ff95dd6",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8841702626201965040"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[19:51:55.871] Scheduled build:
{
"id": "8676272564280616433",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-07-14T19:51:55.513093551Z",
"updateTime": "2026-07-14T19:51:55.513093551Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "ad6e598a99ef81a451019a98d3ea83157ff95dd6",
"ref": "refs/heads/main"
}
}
}
[19:51:55.872] Task URL: https://cr-buildbucket.appspot.com/build/8676272564280616433
[19:51:55.872] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:2:0) after 5m29s
[19:51:56.806] Received PubSub notification, asking Buildbucket for the build status
[19:51:56.829] Build status: SCHEDULED
[19:57:25.065] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:2:0)
[19:57:25.096] Build status: SCHEDULED
[19:57:25.096] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:4:0) after 3m7s
[20:00:32.318] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:4:0)
[20:00:32.334] Build status: SCHEDULED
[20:00:32.334] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:5:0) after 2m33s
[20:03:05.359] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:5:0)
[20:03:05.378] Build status: SCHEDULED
[20:03:05.378] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:6:0) after 2m40s
[20:05:45.464] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:6:0)
[20:05:45.483] Build status: SCHEDULED
[20:05:45.483] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:7:0) after 9m48s
[20:15:33.460] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:7:0)
[20:15:33.480] Build status: SCHEDULED
[20:15:33.480] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:8:0) after 2m28s
[20:18:01.506] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:8:0)
[20:18:01.528] Build status: SCHEDULED
[20:18:01.528] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:9:0) after 5m9s
[20:23:10.564] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:9:0)
[20:23:10.589] Build status: SCHEDULED
[20:23:10.589] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:10:0) after 8m8s
[20:31:18.903] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:10:0)
[20:31:18.920] Build status: SCHEDULED
[20:31:18.920] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:11:0) after 6m59s
[20:38:17.986] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:11:0)
[20:38:18.015] Build status: SCHEDULED
[20:38:18.015] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:12:0) after 8m9s
[20:46:27.086] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:12:0)
[20:46:27.150] Build status: SCHEDULED
[20:46:27.150] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:13:0) after 6m31s
[20:52:58.205] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:13:0)
[20:52:58.224] Build status: SCHEDULED
[20:52:58.224] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:14:0) after 9m29s
[21:02:27.319] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:14:0)
[21:02:27.337] Build status: SCHEDULED
[21:02:27.337] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:15:0) after 7m56s
[21:10:23.464] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:15:0)
[21:10:23.483] Build status: SCHEDULED
[21:10:23.483] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:16:0) after 9m30s
[21:19:53.523] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:16:0)
[21:19:53.540] Build status: SCHEDULED
[21:19:53.541] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:17:0) after 6m22s
[21:26:15.563] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:17:0)
[21:26:15.589] Build status: SCHEDULED
[21:26:15.589] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:18:0) after 6m55s
[21:33:10.685] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:18:0)
[21:33:10.703] Build status: SCHEDULED
[21:33:10.703] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:19:0) after 7m23s
[21:40:33.692] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:19:0)
[21:40:33.714] Build status: SCHEDULED
[21:40:33.714] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:20:0) after 8m37s
[21:49:10.815] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:20:0)
[21:49:10.835] Build status: SCHEDULED
[21:49:10.835] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:21:0) after 2m12s
[21:51:22.856] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:21:0)
[21:51:22.880] Build status: SCHEDULED
[21:51:22.880] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:22:0) after 5m24s
[21:56:47.034] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:22:0)
[21:56:47.057] Build status: SCHEDULED
[21:56:47.057] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:23:0) after 7m1s
[22:03:48.109] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:23:0)
[22:03:48.129] Build status: SCHEDULED
[22:03:48.129] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:24:0) after 5m35s
[22:09:23.184] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:24:0)
[22:09:23.199] Build status: SCHEDULED
[22:09:23.199] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:25:0) after 5m4s
[22:14:27.296] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:25:0)
[22:14:27.325] Build status: SCHEDULED
[22:14:27.325] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:26:0) after 5m54s
[22:20:21.495] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:26:0)
[22:20:21.576] Build status: SCHEDULED
[22:20:21.576] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:27:0) after 6m17s
[22:26:38.633] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:27:0)
[22:26:38.693] Build status: SCHEDULED
[22:26:38.693] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:28:0) after 3m4s
[22:29:42.798] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:28:0)
[22:29:42.860] Build status: SCHEDULED
[22:29:42.860] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:29:0) after 1m50s
[22:30:38.955] Received PubSub notification, asking Buildbucket for the build status
[22:30:38.979] Build status: STARTED
[22:31:32.891] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:29:0)
[22:31:32.941] Build status: STARTED
[22:31:32.941] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8841702626201965040:31:0) after 9m42s
[22:34:52.971] Received PubSub notification, asking Buildbucket for the build status
[22:34:53.002] Build:
{
"id": "8676272564280616433",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-07-14T19:51:55.513093551Z",
"startTime": "2026-07-14T22:30:38.747824945Z",
"endTime": "2026-07-14T22:34:52.089589471Z",
"updateTime": "2026-07-14T22:34:52.089589471Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "ad6e598a99ef81a451019a98d3ea83157ff95dd6",
"ref": "refs/heads/main"
}
}
}
[22:34:53.002] Invocation finished in 2h42m58.70498768s with status FAILED