[17:15:41.929] New invocation is queued and will start shortly
[17:15:41.929] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[17:15:43.072] Starting the invocation (attempt 1)
[17:15:43.111] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/79b31ad17919f591bdad46dbac550fe54b213ebb"
[17:15:43.111] Popped gitiles commit info from properties and tags
[17:15:43.111] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[17:15:43.111] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[17:15:43.112] Buildbucket request:
{
"requestId": "8836548426762531632",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux MSan Tests"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "79b31ad17919f591bdad46dbac550fe54b213ebb",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "79b31ad17919f591bdad46dbac550fe54b213ebb"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8671121120055306145"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8671121120055306145"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8671121120055306145"
}
],
"parent_build_id": "8671121120055306145",
"parent_builder_group": "chromium.memory",
"parent_buildername": "Linux MSan Builder",
"parent_got_angle_revision": "6ab2d16c77f8a4f09e2d28d45c2329c07c261fa6",
"parent_got_dawn_revision": "94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655",
"parent_got_revision": "79b31ad17919f591bdad46dbac550fe54b213ebb",
"parent_got_revision_cp": "refs/heads/main@{#1694652}",
"parent_got_swiftshader_revision": "5b0479bd2d15058aaa9eb490e364f920ff824a8c",
"parent_got_v8_revision": "42f91abd5e765bd4e006853c84b055c0f5f864e6",
"parent_got_v8_revision_cp": "refs/heads/15.5.26@{#1}",
"parent_got_webrtc_revision": "0520f29091f6c2e4848076e3ad13dfd51d1a3cff",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#48543}",
"swarm_hashes": {
"absl_hardening_tests": "ec2ab8a9aafd8174207fd6aaf5e4e3980a0e14464924a5f38f05de4b073b5145/489",
"accessibility_unittests": "744c801b6fcf96919069b55da6756718919a39790a304a095ff765f957d19062/489",
"angle_unittests": "6f2471b8220dfcba1c0c2b3ebeba1ea3d7909f1a61d06bfd0c7ac915284f965d/489",
"aura_unittests": "079b5df5b77e29dcc73ca8838419828fca5af7ed42ff540fb61e5b0677584963/489",
"base_unittests": "f8c3e7aa78498561a09702daa354be476d942cf6bf1a9ab03ad247b4a23e0e01/567",
"blink_common_unittests": "58503aa459670d915bd08ed01de9e0aa5eb43da32f1141583615c4b25aefa1ea/566",
"blink_fuzzer_unittests": "5b82f1352d0c16ccbf90b66ec48e3ed1c6a131dcfdf59891c2d5aec31edce9d3/489",
"blink_heap_unittests": "8f587d6b7ab7b79e6a6d281c71b96508b7a3bc63b8f22a078c9439e48173b884/489",
"blink_platform_unittests": "80288555a4377abe11494307d0e61955954fc68a737590354df9fd6a20738748/645",
"blink_unittests": "fb76a6649aa9cf1cf339b872bf1171ce9e78546f61bcda6e48307377508d6366/729",
"boringssl_crypto_tests": "efce8be41ab331eaf0f90e6981180115f93eb702dd1fde763a6df98eb2324def/489",
"boringssl_ssl_tests": "7866c41953eb3f4190c482edf3b31d977e8a74ebe7b50c890ab836bd877521a6/489",
"browser_tests": "a188a05b5c3160a316ccd0784bae51a3b3996007fb129055474cae25868794bc/1220",
"capture_unittests": "b3b4ba6e9587298cd7c32116d871a474e28b9a5886dad11d9a11bb855d884127/568",
"cast_unittests": "44ae211d824fcb41552ced0ea708cbd47c8fd480c3781274bcf8ba51188832b2/489",
"cc_unittests": "232f9517c97a4ad0defdccd15adcb6bb878346b1483d1dff39bff008b30f534d/649",
"chrome_app_unittests": "659198219db9e5d0c9e7ab33e52a0f0ea78222621522f87c70c9d5fb139fa5f7/566",
"chromedriver_unittests": "52458af61a70f82c5dce57e5b38b84986f865c4b1a53108eb555e3349253a698/646",
"components_browsertests": "6c7a4f13109dc6f84c36beb951a2a692c0a7818123b86b9db967f1526f2f6b40/732",
"components_unittests": "9597344f1b743de360762a2bf6915ea6b99867eb55445c25d91be447e1aef261/811",
"compositor_unittests": "b30cbc3b1e73466eaf2d889ffcde26eb15d353c043b780431df899fcac8dff6a/565",
"content_browsertests": "446671882c25e68858aa5f8397e39987b5447cc982b4ce4cf189c11babe02e9d/805",
"content_unittests": "1501c5ec96b2146f3e88f7a98f746dac2b6ca9aa16f21f8dfae9f55b05d0281b/726",
"crashpad_tests": "2d72bd702b2d6e70522b73241ffab9a19be98427faf09b5f3286ada805bfec4f/489",
"crypto_unittests": "6652053ca380b665207c3592e375cbfc841d24d6830ee74648d793ae2d984aca/569",
"dbus_unittests": "b2cccfe5f51adfef9d659b2d0051a55796b3b66237b9f0c36a7d71bd579c6211/489",
"device_unittests": "639da5f18c5401d7c758f85504702c7260650184780b28b58252139d8c7e556b/566",
"display_unittests": "fcab026b45155eec3b82de491c4809d190dfd7a90eb802663130a613ef66cf2f/489",
"env_chromium_unittests": "f7f4135bf33ccd0f018375875e2b4dd7d98429dabcd6735fb75d87609d8b7475/489",
"events_unittests": "632ddc8f2480f663260d04a8e05fb1eabd9bcdbabdfa5ba7094f38d52b66e95e/489",
"extensions_unittests": "41cec69dcef35c6450e44e2e6e713b9648f759d53aa3166a14c242f7ca5d17eb/731",
"filesystem_service_unittests": "921516e036e3700818d8bde82566d370a6be3a061809b99ada2eaaaf503b7357/489",
"fuzzing_unittests": "3e41ad05a2fdd087fc173477791bbe69e3c5b3747e63ddd364b018a21269e0df/489",
"gcm_unit_tests": "a1450f7bef7d6f590814ed3d789df90bf933c690ef86e17141f7919b2276741a/566",
"gfx_unittests": "6d327f37e1f585e97e5631c6277b3fc3c2980cc40017b566257f165ec017370e/565",
"gin_unittests": "c6ce85cff5b364ff208a864c6f8aa06f96610776505f558fea3dbf11d99434d5/566",
"google_apis_unittests": "33ba441e6c48dbd4f39449dcddb99f1744f0e8c2efeac14332aaeeee0aa9b9a4/651",
"gpu_unittests": "abef0f14d1624ac33b6454b884d493633b2c064233934cccb71d53eeb7c4b40e/566",
"gtk_unittests": "39bdbf65c843cd0973d9e009e316f0f5bae08949685561f76035a70d36007f9a/489",
"gwp_asan_unittests": "e85edc9b275ed50cfb2cdb985de03788ec79890a0138145de46a9eb718c1aa6d/489",
"headless_browsertests": "581bbfe56e640a5d9f2e8fc1524cf6262f053d2e91f3399ee81afdb87b9a8f8c/733",
"headless_unittests": "837557d699bf49f1f271f3b0726cdb280563f06aa3e623e3f5dfe7317c5d03ab/489",
"interactive_ui_tests": "153b9c099ae2f4736934509a952877ccc8fe730a1add6472b1a3b3afc1015070/1059",
"ipc_tests": "1967339466b016e9ff4e666872accb0df92593ba07333d133e0e7db5fc817c57/489",
"latency_unittests": "884c6187b0ea0e90254b0f07dbd343c22e30fc74f8b143a18178e530eb8e1b58/489",
"leveldb_unittests": "a6cdc256f6a8b752cd3b73f4928af4ebb5effd469c3511f078c8ea68e7cc1867/489",
"libjingle_xmpp_unittests": "43fd1774cc8717e378c0520cc4eb6b325c6ad1b2b58cc3bcc46f9ee09bc13cf5/489",
"liburlpattern_unittests": "fd2cec3edaa55b74831b1f3d87fc535aa235a7bdbe8f4dc411d6321160de7262/489",
"media_unittests": "bfa0878efb875babc5afbaed0bd2043ef6b93a56fb0be14bdd3ae91fd2db5a48/569",
"message_center_unittests": "2498b6607c470dcc5cb5d9cb37c6b8d855fb50fa6699e3bc154c65af6159db48/489",
"midi_unittests": "f38199828e4f8a896fd0040f5c43f8740cdec9ce1fe7f4709a3f4036ed1daab6/489",
"mojo_unittests": "eb2ebfab90056fdde69a87e01d69924ab77e924f89822929ea74ec932388c1c1/567",
"native_theme_unittests": "1dfaeb00bd2a28eba26a265eb14a52af24060d2d5f673d55bb53dd3d29e86bc5/489",
"net_unittests": "b9e50980d97ac59a9e411669eb401fc3aff3b3c7e0bbafe8f50670f762980b9a/567",
"openscreen_unittests": "a7b3c6ade8e067ac5283194803d4f4937d5232090c7d75f641e476cc0ac58e10/489",
"ozone_unittests": "6c137b27e198db24049aeda8e4748b3ae60e70e695f449e91a741a3e185e8e67/489",
"ozone_x11_unittests": "418d0125406ee116a4aad2663c410ca37795fced1422c6dabaf48410f4cda061/489",
"pdf_unittests": "669d40eef8ce7bb4cdd9e182d7fae4c5c822e57ce28b21f01436b7dda8c6e2cc/566",
"perfetto_unittests": "cb3d2d10c362c7f5adfaf931634e920d2a553476ba6b9fc84e6428cf596881a0/489",
"printing_unittests": "8b0115ff7294d115e469e54fcac87e84fe3d82bbe16a701555675895300a2724/489",
"pthreadpool_unittests": "25ca10f552c5549d1fa1b94aa444c258e625167be19cd5114c6e04fe3de72c14/489",
"puffin_unittests": "f376d8f5d6d9e7a5cc201cbb4282654a29ed0bd20a96fbfb55ad05e89ebaf4e7/573",
"remoting_unittests": "f312c1bb37560b5d6cd6ab6af0cc6b9f1c2bed483d9f208676624bc50f719da5/566",
"sandbox_linux_unittests": "03724b6ea1095b9f69ad44073dd35c44cab9f0093e14f3238e8d7a0e9710d34b/489",
"shell_dialogs_unittests": "1ddb9e7e51f44188d8bd2a70ce9b1849ce2917de5345ac59736270618dd9e2c6/489",
"skia_unittests": "771979bc226952a78cf460a5c567c0398fb984cd2180252477eaca817e76620f/489",
"snapshot_unittests": "d9771e60f9b40c88d0839789cb87cd05fb6d6a56c8a14916cefb9baf92091265/489",
"sql_unittests": "32276ce671c164ad67976446daecf100662ca65afb2080626c94ebc3271d1b1d/566",
"storage_unittests": "504241b7258614e3a7adcf5e3823dbee229404c2c221235071dbdc31dc00cc24/647",
"sync_integration_tests": "5ee5474b13c839d2756dc8b33c48e84ff2f39521b3e3cfde8355b46c7dd28b5d/647",
"ui_base_unittests": "a244feaa0d4ffd4c77c93a2a6e29fbf9a4f46866041abe2bfea8a5c0aff80066/642",
"ui_touch_selection_unittests": "f43cdb972bb991b2329393f42f010e2f9f52947fdb2ee8215d3dd1c501e2b133/489",
"ui_unittests": "f4a755a9068d0de51bd6e00f7acc67724aeb9f10ad09dfa53ab9e03f90881a30/489",
"unit_tests": "06eee9ac06136bc6601ac37e7b24bab2b4c5c4db1a68a1bcef7229d9a43fa672/1140",
"url_unittests": "c6f084e097d393e175fc2056d823d0e7f291b9472c0923ac1607e2def3223979/489",
"views_unittests": "7dd05b86b208ea065a36f847f22135e13a19d3ed23161f32c22a526efe458f94/489",
"viz_unittests": "ad9b73546782b52a47f12e85dd0ad4911992353a0d7446574339af3c51434e27/573",
"vr_common_unittests": "0907bca2800dfed45b761eddf456a970a59e2be6eaa545936c46ed3cdecf2331/566",
"wm_unittests": "ebff246ea69962d59cdc95b9aff5ab4028656e386b65cce2af3504326104ca77/489",
"wtf_unittests": "cc9e038da5946c95d0a3e31fe063bcba3be5d351fa254032c8a244b5a2e3f6ef/489",
"xr_browser_tests": "6bbaed86dd821c98d94b16ba417849aa58dae2c117ce26a874356f4f90414a26/646",
"zlib_unittests": "90db8607cb0a9ed3611f2fdf6bc1f84824706916b15b7a3c6f94fd763d08dccb/489"
},
"swarming_command_lines_cwd": "out/1415-Linux_MSan_Buil",
"swarming_command_lines_digest": "ca9ec5df1ec6a68a8c4e68c791d44d0c8d6b00a2d8f50efae91433712e9f60f9/94"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "79b31ad17919f591bdad46dbac550fe54b213ebb",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8836548426762531632"
},
{
"key": "scheduler_job_id",
"value": "chromium/Linux MSan Tests"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Linux MSan Builder"
},
{
"key": "parent_buildnumber",
"value": "128496"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[17:15:43.684] Scheduled build:
{
"id": "8671118364774107537",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux MSan Tests"
},
"number": 64869,
"createdBy": "project:chromium",
"createTime": "2026-09-09T17:15:43.204595259Z",
"updateTime": "2026-09-09T17:15:43.204595259Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "79b31ad17919f591bdad46dbac550fe54b213ebb",
"ref": "refs/heads/main"
}
}
}
[17:15:43.684] Task URL: https://cr-buildbucket.appspot.com/build/8671118364774107537
[17:15:43.685] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:2:0) after 9m22s
[17:15:44.623] Received PubSub notification, asking Buildbucket for the build status
[17:15:44.643] Build status: SCHEDULED
[17:16:36.649] Received PubSub notification, asking Buildbucket for the build status
[17:16:36.672] Build status: STARTED
[17:25:05.715] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:2:0)
[17:25:05.733] Build status: STARTED
[17:25:05.733] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:5:0) after 1m1s
[17:26:06.920] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:5:0)
[17:26:06.948] Build status: STARTED
[17:26:06.949] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:6:0) after 8m58s
[17:35:04.964] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:6:0)
[17:35:04.988] Build status: STARTED
[17:35:04.988] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:7:0) after 3m18s
[17:38:23.057] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:7:0)
[17:38:23.075] Build status: STARTED
[17:38:23.075] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:8:0) after 6m56s
[17:45:19.173] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:8:0)
[17:45:19.193] Build status: STARTED
[17:45:19.193] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:9:0) after 7m50s
[17:53:09.244] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:9:0)
[17:53:09.268] Build status: STARTED
[17:53:09.268] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:10:0) after 1m32s
[17:54:41.276] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:10:0)
[17:54:41.316] Build status: STARTED
[17:54:41.316] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:11:0) after 3m47s
[17:58:28.395] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:11:0)
[17:58:28.467] Build status: STARTED
[17:58:28.467] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:12:0) after 6m30s
[18:04:58.526] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:12:0)
[18:04:58.591] Build status: STARTED
[18:04:58.592] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:13:0) after 3m14s
[18:08:12.610] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:13:0)
[18:08:12.653] Build status: STARTED
[18:08:12.653] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:14:0) after 6m46s
[18:14:58.731] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:14:0)
[18:14:58.749] Build status: STARTED
[18:14:58.749] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:15:0) after 4m36s
[18:19:34.904] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:15:0)
[18:19:34.995] Build status: STARTED
[18:19:34.995] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:16:0) after 7m50s
[18:27:25.086] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:16:0)
[18:27:25.112] Build status: STARTED
[18:27:25.112] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:17:0) after 7m41s
[18:35:06.167] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:17:0)
[18:35:06.187] Build status: STARTED
[18:35:06.187] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:18:0) after 5m25s
[18:40:31.193] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:18:0)
[18:40:31.237] Build status: STARTED
[18:40:31.237] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:19:0) after 3m47s
[18:44:18.291] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:19:0)
[18:44:18.307] Build status: STARTED
[18:44:18.307] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:20:0) after 1m33s
[18:45:51.338] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:20:0)
[18:45:51.360] Build status: STARTED
[18:45:51.360] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:21:0) after 6m16s
[18:52:07.415] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:21:0)
[18:52:07.434] Build status: STARTED
[18:52:07.434] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:22:0) after 6m0s
[18:58:07.460] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:22:0)
[18:58:07.479] Build status: STARTED
[18:58:07.479] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:23:0) after 1m15s
[18:59:22.516] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:23:0)
[18:59:22.558] Build status: STARTED
[18:59:22.558] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:24:0) after 5m8s
[19:04:30.612] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:24:0)
[19:04:30.647] Build status: STARTED
[19:04:30.647] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:25:0) after 7m20s
[19:11:50.730] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:25:0)
[19:11:50.750] Build status: STARTED
[19:11:50.750] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:26:0) after 9m43s
[19:21:33.805] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:26:0)
[19:21:33.834] Build status: STARTED
[19:21:33.834] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:27:0) after 5m8s
[19:26:41.874] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:27:0)
[19:26:41.893] Build status: STARTED
[19:26:41.894] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8836548426762531632:28:0) after 7m6s
[19:30:50.249] Received PubSub notification, asking Buildbucket for the build status
[19:30:50.268] Build:
{
"id": "8671118364774107537",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux MSan Tests"
},
"number": 64869,
"createdBy": "project:chromium",
"createTime": "2026-09-09T17:15:43.204595259Z",
"startTime": "2026-09-09T17:16:36.475157790Z",
"endTime": "2026-09-09T19:30:49.833286970Z",
"updateTime": "2026-09-09T19:30:49.833286970Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "79b31ad17919f591bdad46dbac550fe54b213ebb",
"ref": "refs/heads/main"
}
}
}
[19:30:50.268] Invocation finished in 2h15m8.343519488s with status FAILED