[20:53:21.024] New invocation is queued and will start shortly
[20:53:22.122] Starting the invocation (attempt 1)
[20:53:22.149] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/b0beb955806e749ab2e69514e865a291f0ba09df"
[20:53:22.149] Popped gitiles commit info from properties and tags
[20:53:22.149] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[20:53:22.149] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[20:53:22.150] Buildbucket request:
{
"requestId": "8879658889316994064",
"builder": {
"project": "openscreen",
"bucket": "ci",
"builder": "chromium_win_x64"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8879658889316994064",
"job": "openscreen/chromium_win_x64",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "72ef46f75ba26bf70c5a9dc1054eda00c449a270"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@72ef46f75ba26bf70c5a9dc1054eda00c449a270",
"title": "72ef46f75ba26bf70c5a9dc1054eda00c449a270",
"url": "https://chromium.googlesource.com/chromium/src/+/72ef46f75ba26bf70c5a9dc1054eda00c449a270"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "7eea6ed08b00c2086379087a575b49a8eecfd5be"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@7eea6ed08b00c2086379087a575b49a8eecfd5be",
"title": "7eea6ed08b00c2086379087a575b49a8eecfd5be",
"url": "https://chromium.googlesource.com/chromium/src/+/7eea6ed08b00c2086379087a575b49a8eecfd5be"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "6b7b6c377b3b0f942268fe1c131790e6f5deb566"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@6b7b6c377b3b0f942268fe1c131790e6f5deb566",
"title": "6b7b6c377b3b0f942268fe1c131790e6f5deb566",
"url": "https://chromium.googlesource.com/chromium/src/+/6b7b6c377b3b0f942268fe1c131790e6f5deb566"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "2443c3f97a01c6a98bba9e32c25733362b1f43e9"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@2443c3f97a01c6a98bba9e32c25733362b1f43e9",
"title": "2443c3f97a01c6a98bba9e32c25733362b1f43e9",
"url": "https://chromium.googlesource.com/chromium/src/+/2443c3f97a01c6a98bba9e32c25733362b1f43e9"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "2b1e4c93e188831dd93e884feac507bc105d5cf8"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@2b1e4c93e188831dd93e884feac507bc105d5cf8",
"title": "2b1e4c93e188831dd93e884feac507bc105d5cf8",
"url": "https://chromium.googlesource.com/chromium/src/+/2b1e4c93e188831dd93e884feac507bc105d5cf8"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "cfcd0451243d5257dec67e4df9a376c59dcd35a4"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@cfcd0451243d5257dec67e4df9a376c59dcd35a4",
"title": "cfcd0451243d5257dec67e4df9a376c59dcd35a4",
"url": "https://chromium.googlesource.com/chromium/src/+/cfcd0451243d5257dec67e4df9a376c59dcd35a4"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "9a06b54a3ac84d38e3657a3b80b8d9b8327db267"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@9a06b54a3ac84d38e3657a3b80b8d9b8327db267",
"title": "9a06b54a3ac84d38e3657a3b80b8d9b8327db267",
"url": "https://chromium.googlesource.com/chromium/src/+/9a06b54a3ac84d38e3657a3b80b8d9b8327db267"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "c9666cfac827543b724857ba11f4772b7d853e36"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@c9666cfac827543b724857ba11f4772b7d853e36",
"title": "c9666cfac827543b724857ba11f4772b7d853e36",
"url": "https://chromium.googlesource.com/chromium/src/+/c9666cfac827543b724857ba11f4772b7d853e36"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "77ff025bd12e18fb58d144ec57315efdd66f8d4e"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@77ff025bd12e18fb58d144ec57315efdd66f8d4e",
"title": "77ff025bd12e18fb58d144ec57315efdd66f8d4e",
"url": "https://chromium.googlesource.com/chromium/src/+/77ff025bd12e18fb58d144ec57315efdd66f8d4e"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "bd225f95e336022f37cd5ccd2c659aaf59ef4658"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@bd225f95e336022f37cd5ccd2c659aaf59ef4658",
"title": "bd225f95e336022f37cd5ccd2c659aaf59ef4658",
"url": "https://chromium.googlesource.com/chromium/src/+/bd225f95e336022f37cd5ccd2c659aaf59ef4658"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "b4eaf91779d2ee2716974f9d1a1df9453f22d677"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@b4eaf91779d2ee2716974f9d1a1df9453f22d677",
"title": "b4eaf91779d2ee2716974f9d1a1df9453f22d677",
"url": "https://chromium.googlesource.com/chromium/src/+/b4eaf91779d2ee2716974f9d1a1df9453f22d677"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "1350568029a923176f65a907181ef96eea31c82a"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@1350568029a923176f65a907181ef96eea31c82a",
"title": "1350568029a923176f65a907181ef96eea31c82a",
"url": "https://chromium.googlesource.com/chromium/src/+/1350568029a923176f65a907181ef96eea31c82a"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "3175cfc2eb08b30f30cac087311d267cf2caf1b6"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@3175cfc2eb08b30f30cac087311d267cf2caf1b6",
"title": "3175cfc2eb08b30f30cac087311d267cf2caf1b6",
"url": "https://chromium.googlesource.com/chromium/src/+/3175cfc2eb08b30f30cac087311d267cf2caf1b6"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "ef7b8b688398c3806b2988c49d4a9064fb866b79"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@ef7b8b688398c3806b2988c49d4a9064fb866b79",
"title": "ef7b8b688398c3806b2988c49d4a9064fb866b79",
"url": "https://chromium.googlesource.com/chromium/src/+/ef7b8b688398c3806b2988c49d4a9064fb866b79"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "c7aa0874127a9d2f7cfa2b4b88991d8a8450e4ad"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@c7aa0874127a9d2f7cfa2b4b88991d8a8450e4ad",
"title": "c7aa0874127a9d2f7cfa2b4b88991d8a8450e4ad",
"url": "https://chromium.googlesource.com/chromium/src/+/c7aa0874127a9d2f7cfa2b4b88991d8a8450e4ad"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "21b5da5399d6088a0ef2108f92d42b0b258db835"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@21b5da5399d6088a0ef2108f92d42b0b258db835",
"title": "21b5da5399d6088a0ef2108f92d42b0b258db835",
"url": "https://chromium.googlesource.com/chromium/src/+/21b5da5399d6088a0ef2108f92d42b0b258db835"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "ce6ee7feedb18b0af07dbae22ff9a845ad344106"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@ce6ee7feedb18b0af07dbae22ff9a845ad344106",
"title": "ce6ee7feedb18b0af07dbae22ff9a845ad344106",
"url": "https://chromium.googlesource.com/chromium/src/+/ce6ee7feedb18b0af07dbae22ff9a845ad344106"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "d5366e83d62beda0faa82ac020a3ecd8ba977669"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@d5366e83d62beda0faa82ac020a3ecd8ba977669",
"title": "d5366e83d62beda0faa82ac020a3ecd8ba977669",
"url": "https://chromium.googlesource.com/chromium/src/+/d5366e83d62beda0faa82ac020a3ecd8ba977669"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "b1ba844ad0fad0179a636fcd0d0b93699e12144f"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@b1ba844ad0fad0179a636fcd0d0b93699e12144f",
"title": "b1ba844ad0fad0179a636fcd0d0b93699e12144f",
"url": "https://chromium.googlesource.com/chromium/src/+/b1ba844ad0fad0179a636fcd0d0b93699e12144f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "78a2fb0f4036e703262e6c506504cf8ca411ec3d"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@78a2fb0f4036e703262e6c506504cf8ca411ec3d",
"title": "78a2fb0f4036e703262e6c506504cf8ca411ec3d",
"url": "https://chromium.googlesource.com/chromium/src/+/78a2fb0f4036e703262e6c506504cf8ca411ec3d"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "d930b86aa01f9ee51423193f3163a832a1505e30"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@d930b86aa01f9ee51423193f3163a832a1505e30",
"title": "d930b86aa01f9ee51423193f3163a832a1505e30",
"url": "https://chromium.googlesource.com/chromium/src/+/d930b86aa01f9ee51423193f3163a832a1505e30"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "59f864985b2bca273e1a85f2ab6f2bcfc9407eba"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@59f864985b2bca273e1a85f2ab6f2bcfc9407eba",
"title": "59f864985b2bca273e1a85f2ab6f2bcfc9407eba",
"url": "https://chromium.googlesource.com/chromium/src/+/59f864985b2bca273e1a85f2ab6f2bcfc9407eba"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "bc4a83110d435cf9c8b28bd0e0daf7919542583d"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@bc4a83110d435cf9c8b28bd0e0daf7919542583d",
"title": "bc4a83110d435cf9c8b28bd0e0daf7919542583d",
"url": "https://chromium.googlesource.com/chromium/src/+/bc4a83110d435cf9c8b28bd0e0daf7919542583d"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "f8d9a0381a3319ca395ebdb3dd866aaf0df8208d"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@f8d9a0381a3319ca395ebdb3dd866aaf0df8208d",
"title": "f8d9a0381a3319ca395ebdb3dd866aaf0df8208d",
"url": "https://chromium.googlesource.com/chromium/src/+/f8d9a0381a3319ca395ebdb3dd866aaf0df8208d"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "f833b262112de980f12e7ce89f2fed6992416b41"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@f833b262112de980f12e7ce89f2fed6992416b41",
"title": "f833b262112de980f12e7ce89f2fed6992416b41",
"url": "https://chromium.googlesource.com/chromium/src/+/f833b262112de980f12e7ce89f2fed6992416b41"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "6a3b0299f8e6125e88711e4bd4d350c4d3138e7b"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@6a3b0299f8e6125e88711e4bd4d350c4d3138e7b",
"title": "6a3b0299f8e6125e88711e4bd4d350c4d3138e7b",
"url": "https://chromium.googlesource.com/chromium/src/+/6a3b0299f8e6125e88711e4bd4d350c4d3138e7b"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "c74caf8073b0be1fab96480e37f2d55c26217f39"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@c74caf8073b0be1fab96480e37f2d55c26217f39",
"title": "c74caf8073b0be1fab96480e37f2d55c26217f39",
"url": "https://chromium.googlesource.com/chromium/src/+/c74caf8073b0be1fab96480e37f2d55c26217f39"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "06447034f2f6c4d4f273e0dd24dc0aba23c20e20"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@06447034f2f6c4d4f273e0dd24dc0aba23c20e20",
"title": "06447034f2f6c4d4f273e0dd24dc0aba23c20e20",
"url": "https://chromium.googlesource.com/chromium/src/+/06447034f2f6c4d4f273e0dd24dc0aba23c20e20"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "3543577181fa4c81e64305d95ce51089937e0d52"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@3543577181fa4c81e64305d95ce51089937e0d52",
"title": "3543577181fa4c81e64305d95ce51089937e0d52",
"url": "https://chromium.googlesource.com/chromium/src/+/3543577181fa4c81e64305d95ce51089937e0d52"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "82c43d6cdde52a8bafa9dce0a6e3eec77556f4b9"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@82c43d6cdde52a8bafa9dce0a6e3eec77556f4b9",
"title": "82c43d6cdde52a8bafa9dce0a6e3eec77556f4b9",
"url": "https://chromium.googlesource.com/chromium/src/+/82c43d6cdde52a8bafa9dce0a6e3eec77556f4b9"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "b0beb955806e749ab2e69514e865a291f0ba09df"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@b0beb955806e749ab2e69514e865a291f0ba09df",
"title": "b0beb955806e749ab2e69514e865a291f0ba09df",
"url": "https://chromium.googlesource.com/chromium/src/+/b0beb955806e749ab2e69514e865a291f0ba09df"
}
]
}
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "b0beb955806e749ab2e69514e865a291f0ba09df",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8879658889316994064"
},
{
"key": "scheduler_job_id",
"value": "openscreen/chromium_win_x64"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[20:53:22.543] Scheduled build:
{
"id": "8714228827423958641",
"builder": {
"project": "openscreen",
"bucket": "ci",
"builder": "chromium_win_x64"
},
"createdBy": "project:openscreen",
"createTime": "2025-05-21T20:53:22.206536950Z",
"updateTime": "2025-05-21T20:53:22.206536950Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "b0beb955806e749ab2e69514e865a291f0ba09df",
"ref": "refs/heads/main"
}
}
}
[20:53:22.543] Task URL: https://cr-buildbucket.appspot.com/build/8714228827423958641
[20:53:22.544] Scheduling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:2:0) after 6m38s
[20:53:23.551] Received PubSub notification, asking Buildbucket for the build status
[20:53:23.576] Build status: SCHEDULED
[21:00:00.851] Handling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:2:0)
[21:00:00.929] Build status: SCHEDULED
[21:00:00.929] Scheduling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:4:0) after 8m0s
[21:08:01.037] Handling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:4:0)
[21:08:01.065] Build status: SCHEDULED
[21:08:01.065] Scheduling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:5:0) after 2m37s
[21:10:38.177] Handling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:5:0)
[21:10:38.210] Build status: SCHEDULED
[21:10:38.210] Scheduling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:6:0) after 2m9s
[21:12:47.239] Handling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:6:0)
[21:12:47.277] Build status: SCHEDULED
[21:12:47.277] Scheduling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:7:0) after 2m6s
[21:14:53.390] Handling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:7:0)
[21:14:53.433] Build status: SCHEDULED
[21:14:53.433] Scheduling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:8:0) after 9m12s
[21:16:11.229] Received PubSub notification, asking Buildbucket for the build status
[21:16:11.289] Build status: STARTED
[21:24:05.477] Handling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:8:0)
[21:24:05.552] Build status: STARTED
[21:24:05.552] Scheduling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:10:0) after 8m54s
[21:32:59.841] Handling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:10:0)
[21:32:59.873] Build status: STARTED
[21:32:59.873] Scheduling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:11:0) after 6m16s
[21:39:15.867] Handling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:11:0)
[21:39:15.904] Build status: STARTED
[21:39:15.904] Scheduling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:12:0) after 1m10s
[21:40:25.929] Handling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:12:0)
[21:40:25.985] Build status: STARTED
[21:40:25.985] Scheduling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:13:0) after 7m44s
[21:48:10.114] Handling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:13:0)
[21:48:10.175] Build status: STARTED
[21:48:10.175] Scheduling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:14:0) after 4m34s
[21:52:44.340] Handling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:14:0)
[21:52:44.731] Build status: STARTED
[21:52:44.731] Scheduling timer "check-buildbucket-build-status" (openscreen/chromium_win_x64:8879658889316994064:15:0) after 4m49s
[21:56:21.240] Received PubSub notification, asking Buildbucket for the build status
[21:56:21.261] Build:
{
"id": "8714228827423958641",
"builder": {
"project": "openscreen",
"bucket": "ci",
"builder": "chromium_win_x64"
},
"createdBy": "project:openscreen",
"createTime": "2025-05-21T20:53:22.206536950Z",
"startTime": "2025-05-21T21:16:11.002471858Z",
"endTime": "2025-05-21T21:56:21.033601948Z",
"updateTime": "2025-05-21T21:56:21.033601948Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "b0beb955806e749ab2e69514e865a291f0ba09df",
"ref": "refs/heads/main"
}
}
}
[21:56:21.261] Invocation finished in 1h3m0.244981374s with status FAILED