[00:54:16.468] New invocation is queued and will start shortly
[00:54:17.595] Starting the invocation (attempt 1)
[00:54:17.619] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/1e94d08be07c4565b970517a75a461d8718618dc"
[00:54:17.619] Popped gitiles commit info from properties and tags
[00:54:17.619] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[00:54:17.619] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[00:54:17.620] Buildbucket request:
{
"requestId": "8834435845458462288",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8834435845458462288",
"job": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "24c267cee4a0008db399758392a58dae078c656f"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@24c267cee4a0008db399758392a58dae078c656f",
"title": "24c267cee4a0008db399758392a58dae078c656f",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/24c267cee4a0008db399758392a58dae078c656f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "ac6ac563dd48507a1a4f2e9208c0461611b9d12b"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@ac6ac563dd48507a1a4f2e9208c0461611b9d12b",
"title": "ac6ac563dd48507a1a4f2e9208c0461611b9d12b",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/ac6ac563dd48507a1a4f2e9208c0461611b9d12b"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "6ba1b486685569e9bf8356d8869844117ae47f79"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@6ba1b486685569e9bf8356d8869844117ae47f79",
"title": "6ba1b486685569e9bf8356d8869844117ae47f79",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/6ba1b486685569e9bf8356d8869844117ae47f79"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "43885464e9e062a3e441f0f7f4b225115f2370e7"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@43885464e9e062a3e441f0f7f4b225115f2370e7",
"title": "43885464e9e062a3e441f0f7f4b225115f2370e7",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/43885464e9e062a3e441f0f7f4b225115f2370e7"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "346c561d06ce4d73fef72ba9ed930095d650b402"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@346c561d06ce4d73fef72ba9ed930095d650b402",
"title": "346c561d06ce4d73fef72ba9ed930095d650b402",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/346c561d06ce4d73fef72ba9ed930095d650b402"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "3c3f0cc314b5363dc46a5eb2f6d4b24959123a4c"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@3c3f0cc314b5363dc46a5eb2f6d4b24959123a4c",
"title": "3c3f0cc314b5363dc46a5eb2f6d4b24959123a4c",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/3c3f0cc314b5363dc46a5eb2f6d4b24959123a4c"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "3816264cafb210a7936b81cbb1983a4d9b574d49"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@3816264cafb210a7936b81cbb1983a4d9b574d49",
"title": "3816264cafb210a7936b81cbb1983a4d9b574d49",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/3816264cafb210a7936b81cbb1983a4d9b574d49"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "56ca06b88bf620736311c81390c68c9e896bff7e"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@56ca06b88bf620736311c81390c68c9e896bff7e",
"title": "56ca06b88bf620736311c81390c68c9e896bff7e",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/56ca06b88bf620736311c81390c68c9e896bff7e"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "0c4e99dea667cc8c2346e3f359da6de106368828"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@0c4e99dea667cc8c2346e3f359da6de106368828",
"title": "0c4e99dea667cc8c2346e3f359da6de106368828",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/0c4e99dea667cc8c2346e3f359da6de106368828"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "de2d1e080186d627b4470f875a907a29c4c68506"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@de2d1e080186d627b4470f875a907a29c4c68506",
"title": "de2d1e080186d627b4470f875a907a29c4c68506",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/de2d1e080186d627b4470f875a907a29c4c68506"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://pigweed.googlesource.com/pigweed/pigweed",
"revision": "1e94d08be07c4565b970517a75a461d8718618dc"
},
"id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@1e94d08be07c4565b970517a75a461d8718618dc",
"title": "1e94d08be07c4565b970517a75a461d8718618dc",
"url": "https://pigweed.googlesource.com/pigweed/pigweed/+/1e94d08be07c4565b970517a75a461d8718618dc"
}
]
}
},
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "1e94d08be07c4565b970517a75a461d8718618dc",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8834435845458462288"
},
{
"key": "scheduler_job_id",
"value": "pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[00:54:18.476] Scheduled build:
{
"id": "8669005783515322129",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-10-03T00:54:17.692544919Z",
"updateTime": "2026-10-03T00:54:17.692544919Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "1e94d08be07c4565b970517a75a461d8718618dc",
"ref": "refs/heads/main"
}
}
}
[00:54:18.476] Task URL: https://cr-buildbucket.appspot.com/build/8669005783515322129
[00:54:18.476] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:2:0) after 3m1s
[00:54:19.258] Received PubSub notification, asking Buildbucket for the build status
[00:54:19.275] Build status: SCHEDULED
[00:57:19.589] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:2:0)
[00:57:19.609] Build status: SCHEDULED
[00:57:19.609] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:4:0) after 5m8s
[01:02:27.625] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:4:0)
[01:02:27.670] Build status: SCHEDULED
[01:02:27.670] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:5:0) after 1m13s
[01:03:40.772] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:5:0)
[01:03:40.790] Build status: SCHEDULED
[01:03:40.791] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:6:0) after 8m22s
[01:12:02.858] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:6:0)
[01:12:02.887] Build status: SCHEDULED
[01:12:02.887] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:7:0) after 9m39s
[01:21:42.245] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:7:0)
[01:21:42.281] Build status: SCHEDULED
[01:21:42.281] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:8:0) after 3m49s
[01:25:31.397] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:8:0)
[01:25:31.426] Build status: SCHEDULED
[01:25:31.426] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:9:0) after 8m22s
[01:33:53.508] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:9:0)
[01:33:53.528] Build status: SCHEDULED
[01:33:53.528] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:10:0) after 3m55s
[01:37:48.574] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:10:0)
[01:37:48.602] Build status: SCHEDULED
[01:37:48.602] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:11:0) after 9m54s
[01:47:42.649] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:11:0)
[01:47:42.679] Build status: SCHEDULED
[01:47:42.679] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:12:0) after 6m29s
[01:54:11.787] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:12:0)
[01:54:11.813] Build status: SCHEDULED
[01:54:11.813] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:13:0) after 1m46s
[01:55:57.829] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:13:0)
[01:55:57.855] Build status: SCHEDULED
[01:55:57.855] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:14:0) after 5m52s
[02:01:49.872] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:14:0)
[02:01:49.893] Build status: SCHEDULED
[02:01:49.893] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:15:0) after 4m59s
[02:06:48.982] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:15:0)
[02:06:49.014] Build status: SCHEDULED
[02:06:49.014] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:16:0) after 6m43s
[02:13:32.202] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:16:0)
[02:13:32.221] Build status: SCHEDULED
[02:13:32.221] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:17:0) after 4m2s
[02:17:34.330] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:17:0)
[02:17:34.353] Build status: SCHEDULED
[02:17:34.353] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:18:0) after 8m9s
[02:25:43.344] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:18:0)
[02:25:43.395] Build status: SCHEDULED
[02:25:43.395] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:19:0) after 3m48s
[02:29:31.542] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:19:0)
[02:29:31.559] Build status: SCHEDULED
[02:29:31.559] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:20:0) after 4m47s
[02:34:18.673] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:20:0)
[02:34:18.707] Build status: SCHEDULED
[02:34:18.707] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:21:0) after 3m7s
[02:37:25.720] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:21:0)
[02:37:25.742] Build status: SCHEDULED
[02:37:25.742] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:22:0) after 6m52s
[02:44:17.809] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:22:0)
[02:44:17.829] Build status: SCHEDULED
[02:44:17.829] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:23:0) after 7m0s
[02:51:18.057] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:23:0)
[02:51:18.077] Build status: SCHEDULED
[02:51:18.077] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:24:0) after 7m19s
[02:58:37.143] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:24:0)
[02:58:37.167] Build status: SCHEDULED
[02:58:37.167] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:25:0) after 7m47s
[03:06:24.145] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:25:0)
[03:06:24.172] Build status: SCHEDULED
[03:06:24.172] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:26:0) after 9m29s
[03:15:53.197] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:26:0)
[03:15:53.222] Build status: SCHEDULED
[03:15:53.222] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:27:0) after 8m48s
[03:24:41.340] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:27:0)
[03:24:41.401] Build status: SCHEDULED
[03:24:41.401] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:28:0) after 3m9s
[03:27:50.417] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:28:0)
[03:27:50.438] Build status: SCHEDULED
[03:27:50.438] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:29:0) after 6m22s
[03:34:12.552] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:29:0)
[03:34:12.572] Build status: SCHEDULED
[03:34:12.573] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:30:0) after 4m16s
[03:38:28.642] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:30:0)
[03:38:28.662] Build status: SCHEDULED
[03:38:28.662] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:31:0) after 1m21s
[03:39:49.645] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:31:0)
[03:39:49.671] Build status: SCHEDULED
[03:39:49.671] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:32:0) after 4m2s
[03:43:51.689] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:32:0)
[03:43:51.711] Build status: SCHEDULED
[03:43:51.711] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:33:0) after 2m37s
[03:46:28.727] Handling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:33:0)
[03:46:28.744] Build status: SCHEDULED
[03:46:28.744] Scheduling timer "check-buildbucket-build-status" (pigweed/pigweed.dev.ci-pigweed-linux-bazel-sanitizers-tsan-build:8834435845458462288:34:0) after 7m58s
[03:48:48.397] Received PubSub notification, asking Buildbucket for the build status
[03:48:48.432] Build status: STARTED
[03:49:25.952] Received PubSub notification, asking Buildbucket for the build status
[03:49:25.976] Build:
{
"id": "8669005783515322129",
"builder": {
"project": "pigweed",
"bucket": "pigweed.dev.ci",
"builder": "pigweed-linux-bazel-sanitizers-tsan-build"
},
"createdBy": "project:pigweed",
"createTime": "2026-10-03T00:54:17.692544919Z",
"startTime": "2026-10-03T03:48:02.820161035Z",
"endTime": "2026-10-03T03:49:03.224666663Z",
"updateTime": "2026-10-03T03:49:03.224666663Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "pigweed.googlesource.com",
"project": "pigweed/pigweed",
"id": "1e94d08be07c4565b970517a75a461d8718618dc",
"ref": "refs/heads/main"
}
}
}
[03:49:25.976] Invocation finished in 2h55m9.521503174s with status FAILED