[20:29:58.183] New invocation is queued and will start shortly
[20:29:58.183] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[20:29:59.258] Starting the invocation (attempt 1)
[20:29:59.290] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ca17ee4bacc9010208d55280becf88e2fb6115f8"
[20:29:59.290] Popped gitiles commit info from properties and tags
[20:29:59.290] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[20:29:59.290] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[20:29:59.291] Buildbucket request:
{
"requestId": "8878844987607378800",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux ChromiumOS MSan Tests"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "ca17ee4bacc9010208d55280becf88e2fb6115f8",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "ca17ee4bacc9010208d55280becf88e2fb6115f8"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8713418257257595521"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8713418257257595521"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8713418257257595521"
}
],
"parent_build_id": "8713418257257595521",
"parent_builder_group": "chromium.memory",
"parent_buildername": "Linux ChromiumOS MSan Builder",
"parent_got_angle_revision": "ef29d42e08b3065fcec842516591a8f4045ca9a9",
"parent_got_dawn_revision": "56a4aec006181ed29853e5cd2bdd6a237798c958",
"parent_got_nacl_revision": "e3fce84f253bc1e77bb239185c0fbff23dc8e3ee",
"parent_got_revision": "ca17ee4bacc9010208d55280becf88e2fb6115f8",
"parent_got_revision_cp": "refs/heads/main@{#1467615}",
"parent_got_swiftshader_revision": "a8133cbb3c8969e3c1e6b3cea2c02ec8312ef9ca",
"parent_got_v8_revision": "e7a316250a09280945c50da90226195292c1bcaf",
"parent_got_v8_revision_cp": "refs/heads/13.9.57@{#1}",
"parent_got_webrtc_revision": "c711fd51bd7c66a0fa42a7e3bc15cc5b56b13431",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#44795}",
"swarm_hashes": {
"absl_hardening_tests": "863bb183c8eea7230fae6ed8e9ff175fd2208b5df7dd8ba9f21001a88d1b854b/489",
"accessibility_unittests": "e404fe27bf9493e6b84e0c8cbb07c0897b4dfc59019ad3229c437fcd3c0de89c/489",
"angle_unittests": "5784701b786178376aaacb078f703890861ad8c018301952ad716f045bc7d812/489",
"app_shell_unittests": "32d8d4ab5b17d2aaaf16034ac92c8c5a0b1505ce783258de073411e73cf84fa9/650",
"ash_components_unittests": "8bec7c0c2a663040c073b6870b4ebd13cce33fb2b43dd791e9fa7715f6bd9df2/728",
"ash_unittests": "c62b17e815859de49a7db1675eb39adf1bc50fa93b50cc799354553ab649a328/801",
"ash_webui_unittests": "f63ee08fcdfd9e73e58e4db29317fd87d71347cf36ffc323cb4f1dd7afc5032b/727",
"aura_unittests": "9eb64518860ea92a32d3438fb929cd87c4dceb352a367ce1a23d45c864bcd9ce/489",
"base_unittests": "8ed9684bad741572a64bf564e5704f4c1f8a177d8f99537b113c5e1c34326caf/567",
"blink_common_unittests": "4925973b8895adba67db1a90bbd87753c4d672042f8d1c59c3a5c99c76427d92/566",
"blink_fuzzer_unittests": "0bfaf02a8cb7c77c1a44014228b85552c79fe5247edcdb713308c535f019c9f7/489",
"blink_heap_unittests": "84b67fe3ba39bfc8e5ed8e50dd639c392b1deb1258a0ebf08550bd8ee862236c/489",
"blink_platform_unittests": "96eb7317e72504608e40ed970e378f2c8d040c6af91831179008acf375d708e6/645",
"blink_unittests": "88073ce29c9877077af41bbd9c8028727a912a3742d0c0865778d61ca4ef5bab/729",
"boringssl_crypto_tests": "5e5a9977faf77d76b0994dd05a22dedec9c4ed7b1ce41e7adb6616a13c751b7d/489",
"boringssl_ssl_tests": "65b76c34a3f09b0f2b47b6453948530ed5a33d55dc6527836baec3d59f405103/489",
"browser_tests": "323c4a5734166d4294644de33cc9b07cb885ba45029f805452d1dc407b85a297/1378",
"capture_unittests": "175b69b9998d704ab2607cc91a5172da20902d3b06ee1458d047734c970aa73d/568",
"cast_unittests": "0be3e0e348a9228c2790a261dd3179f4aa187419e67a8fb50b926d3dee200262/489",
"cc_unittests": "7c918f53f11af378b18e345f87221a02cc028e6eae7e24926276c19e422b1699/649",
"chrome_app_unittests": "e090679a1495b8ac103c44815c8ac0602425cdb285573db8f011b51aebc72025/727",
"chromedriver_unittests": "49e5e0c7f5ef41962dfa5c56b3fc9f90da210565cbcd08934d8a5bbdbb9fdec9/646",
"chromeos_components_unittests": "2249babbc684306d20c43a69a62593c879498f914f01690f2239e840bf40be83/648",
"chromeos_unittests": "2eeb6e9a800db0319919feced39efd4b956ee69be31e51860ed1cf8bcf2768f2/886",
"components_browsertests": "33e5595e5d6eb7c6dda7ddb3327b32e001650c61c9b5f76ad85bb8c1fce01dd1/732",
"components_unittests": "d15adb4f17acd8b31444575092018fc497ec00eb5f559c5f1645fb733b4ab583/894",
"compositor_unittests": "695b1e8fd5d48d091d701f54611f5ac76cddcd2ab923f24472c47b50175328d6/565",
"content_browsertests": "107a5ed09f0f149c632d1f91eca661937e7d41eeb3bb9beba635803c4d0d3dc6/884",
"content_unittests": "87ab4fe18fe8b60bf952fb88590e0cf8903afade92e68475d6a6ca7e39309109/726",
"crashpad_tests": "705737ad7dd57ed30f4ee8e2cf5978eb1a91f88fc9ab3589ee8338037e804e55/489",
"crypto_unittests": "56580a516ec476130623424e49946cd91d9cf31a2aac9d16dfac219fff66598e/489",
"dbus_unittests": "3968404b25f53922ef0029f6d6ad2654b393d753a04a20ec22a8e444619b94f2/489",
"device_unittests": "16a9b8ca255f58ae9a067882d8eedf74a418816cc995881e919f81a4c3c1226e/566",
"display_unittests": "177049cad606cb923b8635bbb85a746ccfdc8f8470cc803fd082f421b397fa37/489",
"env_chromium_unittests": "e7e43c9cbe71758d73576cf1e4db47cb60a71d301da9254375b70003ed5eed4a/489",
"events_unittests": "d9dbcfdc5a84c1785ff3b215ed31fd317a7ba76e4ced74b920605c91b74765df/489",
"exo_unittests": "e72cc08b64be3539ba726b3f4cd83fb2609f81edbf408e4f38ea82463b444173/648",
"extensions_browsertests": "125ed10fe8c62c06c39769cfe5065a5b4953d298aab1ac3a5d9e67eca254ddf9/651",
"extensions_unittests": "eb372e693e6da209e5581fdc1dc32c1413be51d389865c5b26ff846554e24ac4/813",
"filesystem_service_unittests": "f086e0bcd9aeb75ba2a6b795bd74fc9c5ccf447e7072fff3b821179d28d10896/489",
"fuzzing_unittests": "1241797f003494bae21905c38bb16c94f125b52fc07e96fb2a621b8bcb9970aa/489",
"gcm_unit_tests": "c808b59ce5d8d95b8d0e1884c3e034f4d68baaa49c17a627836258e336d7a3a7/566",
"gfx_unittests": "ca68d353ad2904a35628ebeda93acc6f399c067b4a36e8b79c31b56cdbdcacfd/565",
"gin_unittests": "7941d1bccadf6c44f832b273574ab0b3496bfc8627d53d9022f6504f1394fca2/566",
"google_apis_unittests": "f2d077190b6349902393b7918264a7e06ccc16040ab1a96a20819c767584c7f6/651",
"gpu_unittests": "7e6d23ef48850288d6c9b291f4a2c42682be20859ece705235e76f6ad98ab387/566",
"gwp_asan_unittests": "725468fc74fd096bc28a61b38df82beda43495e5615cf02e21513d98bf8daf86/489",
"interactive_ui_tests": "7cfa4c24dd3351f7d74a0a03bd718933b12b98701d2deaec2dabfa05b0e44604/1135",
"ipc_tests": "84438f9caf4d9fa8cc17383f87ba4d4813064b87de92109ee86a4e9f82793d98/489",
"keyboard_unittests": "8f808f2e654fd50357d42b55e9073a03d475ed48662413644b8f2959536eb715/489",
"latency_unittests": "0ff5df51aa0b4a8b49c75066cb7e02ef80b96fa76b62d9ed8bc86301780e08fa/489",
"leveldb_unittests": "9254b97cb30206b75afbe6766a4011e303bc85a67d7f05c1c6301ca8a8521617/489",
"libjingle_xmpp_unittests": "7f9367371732260589e21498171263e8b18ab3a680dab6ca8ce9375315481916/489",
"liburlpattern_unittests": "3ef2d23ddf7b098da73355f88062d16964d2d7b632f0ec09e5c515e49ebbffa9/489",
"media_unittests": "053c57e9718c80e29e16b6349d6f22c1ca2e34256bd12e62d50d1f496e9c3cf0/569",
"message_center_unittests": "503ba967d9508058aa59d2616edc73fd8948266654ad55dba1e06d98d57150f1/489",
"midi_unittests": "2d85f69ec3fc1e5679227888d0d396690f1edfc6c566e262e4a36ed2973c9450/489",
"mojo_unittests": "c3333e288aef6e1dd56879d9c4191cf54f3b60e0d26433426d996d379ae63b11/567",
"nacl_loader_unittests": "39a8160b182ae694123568c915709e1397c082de75337e5fdda9cc3c535f4413/489",
"native_theme_unittests": "c32c3d5945f78dba127a665a5835f9dd68baa87d04d5da2b29ecbea5b27030b8/489",
"net_unittests": "b637f48a684335eedd629a68d2eb18fb2388b590384dc86dee95fdf70d4d5f1c/567",
"ozone_gl_unittests": "a7ca47c7bed6af9b81686089584c9ca63c38789f5f816d702770bc54967e97fb/489",
"ozone_unittests": "e429ba33e1c70e8f8a07a35e66b1b011590b4da5410d16411f09cbc4a10c7855/489",
"ozone_x11_unittests": "2a1af004006c206a66497076b9df9f427df3d4ab4bfaccf58d7bc9792b3e9c83/489",
"pdf_unittests": "0e083a5a42d581bc04453c15d4a3e48eae0113aaf9aa776fd312f63669c55085/566",
"perfetto_unittests": "fd5e570aa5908d46733fe87b7775dd174e58418ff3aec1877cb6440867f99138/489",
"ppapi_unittests": "71a592091437a8aea16b4b5148af97a72d436ba80199354b2d656a903f1d7751/489",
"printing_unittests": "b43a3db70141971371383df04a82816caf0c73e7bc5f7c4b25b49e3ceef4fe2e/489",
"pthreadpool_unittests": "ef57f7ccd27da5c0eed3971b94bf031860444c28ae1e61c2f3bad66677bebc8a/489",
"remoting_unittests": "f496fedffc5c6181093b87df4033d28622b8d3f28c7d65061b9482567b881b63/648",
"sandbox_linux_unittests": "ad9e20f8af6511770cf13f668d725aa48377e83fc9fec33392699d8c55fbd4b7/489",
"shell_dialogs_unittests": "ae3bac33cf75bba526e0682763dd6de59ed830e78eeeae82c7d18faf91430e4f/489",
"shell_encryption_unittests": "ae5f822e97adcea540e78142dbc52603d1eabcc3b7d43432634e39b36d61be5e/489",
"skia_unittests": "cdee15637c1912887b43e9310b5f67521b1e1bacfd11033729fb4b4fd32b211c/489",
"snapshot_unittests": "835a247aebf97c76c3f50cadf1c331e7ca126f85b927b0865f808442ba3f884c/489",
"sql_unittests": "06bcc142fa2c61fa92007ce8e0c312616851501c9c4f62e3fe624a0f77319a25/566",
"storage_unittests": "e47923acff75f73e9a4232bc7d2bb833564a7e968696cb4e901a4802f91cac03/647",
"sync_integration_tests": "c9169d3c9090f246bc9becfb2b94dbb496e6d940f8582731c2480c105a92c1c4/809",
"ui_base_unittests": "2b01046aa5060816d878b8d47fcfa3ea8eb6cd41efc60b81ea7d81263dbb6068/642",
"ui_chromeos_unittests": "dca2547615b5b63734c65889a6ab9bfc27d3fc25ec7a29c6cfa92f7fd4db7e55/489",
"ui_touch_selection_unittests": "1caae1d4028dc008f0a535f5309cf9ca04859b1d95b87f9375ce4e6d3e883b81/489",
"ui_unittests": "77de8fcb0bc5cd99c67e1a4a83ad895c472fa687596e5cb96e321524b32b9d27/489",
"unit_tests": "aed591bba7c4a31b2e5c134f5f93d80d0f60a04d393b74a6c3d1aae4de68130d/1141",
"url_unittests": "62dce1765ded74c96e8affce15bccc7ded8d641467dbd20b701be19e483887d4/489",
"usage_time_limit_unittests": "72da4784ad811271886e36a870ba508353d5d336af0692f0be52f816afb31d0d/569",
"views_unittests": "95e1acb1553767191892c3e11f140a349c91a4c5cf3187ffd7b69d3a47115805/489",
"viz_unittests": "45ee39fcfb941d5f2553d329b4ad69d8b2e53d4a05804f0f188963c1d0e10678/573",
"wayland_client_perftests": "40cbcf1959becef1be6b69bb81a1344dc5efdd4737136f267a7dc28498fbcf5d/648",
"wayland_client_tests": "9065fe3b45fc341c5bb3fe3d1e90f657ee6d41b18d3d7d69214ea355c3d393b7/648",
"wm_unittests": "940ba267ce3dc4a5590aa4bc992f9e9c0d8d3ab2959de768657aeb4730ec4b44/489",
"wtf_unittests": "cc8d4525de6225c1494ca45b6be88316c922eb64f043ae450e249dd40415e30d/489",
"zlib_unittests": "bdcf346894e0dc1233970f5e8e0a0f2776441284ce6c5c6426a2d9cea5220ad4/489"
},
"swarming_command_lines_cwd": "out/fc85-Linux_ChromiumO",
"swarming_command_lines_digest": "044841b95c1246203d0fdc5fbd35beba71e36ca1a0e8f26d4231580ded8f981f/94",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "ca17ee4bacc9010208d55280becf88e2fb6115f8",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8878844987607378800"
},
{
"key": "scheduler_job_id",
"value": "chromium/Linux ChromiumOS MSan Tests"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Linux ChromiumOS MSan Builder"
},
{
"key": "parent_buildnumber",
"value": "92918"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[20:29:59.865] Scheduled build:
{
"id": "8713414925646860721",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux ChromiumOS MSan Tests"
},
"number": 52555,
"createdBy": "project:chromium",
"createTime": "2025-05-30T20:29:59.427838147Z",
"updateTime": "2025-05-30T20:29:59.427838147Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "ca17ee4bacc9010208d55280becf88e2fb6115f8",
"ref": "refs/heads/main"
}
}
}
[20:29:59.865] Task URL: https://cr-buildbucket.appspot.com/build/8713414925646860721
[20:29:59.865] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:2:0) after 2m45s
[20:30:00.868] Received PubSub notification, asking Buildbucket for the build status
[20:30:00.892] Build status: SCHEDULED
[20:30:04.060] Received PubSub notification, asking Buildbucket for the build status
[20:30:04.225] Build status: STARTED
[20:32:44.891] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:2:0)
[20:32:44.944] Build status: STARTED
[20:32:44.944] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:5:0) after 9m13s
[20:41:57.974] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:5:0)
[20:41:58.114] Build status: STARTED
[20:41:58.114] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:6:0) after 4m36s
[20:46:34.238] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:6:0)
[20:46:34.258] Build status: STARTED
[20:46:34.258] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:7:0) after 9m51s
[20:56:25.357] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:7:0)
[20:56:25.370] Build status: STARTED
[20:56:25.370] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:8:0) after 9m19s
[21:05:44.440] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:8:0)
[21:05:44.475] Build status: STARTED
[21:05:44.475] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:9:0) after 5m52s
[21:11:36.465] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:9:0)
[21:11:36.503] Build status: STARTED
[21:11:36.503] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:10:0) after 8m0s
[21:19:36.579] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:10:0)
[21:19:36.595] Build status: STARTED
[21:19:36.595] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:11:0) after 8m2s
[21:27:38.625] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:11:0)
[21:27:38.646] Build status: STARTED
[21:27:38.646] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:12:0) after 4m44s
[21:32:22.665] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:12:0)
[21:32:22.680] Build status: STARTED
[21:32:22.680] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:13:0) after 3m4s
[21:35:26.709] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:13:0)
[21:35:26.751] Build status: STARTED
[21:35:26.751] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:14:0) after 9m0s
[21:44:26.851] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:14:0)
[21:44:26.866] Build status: STARTED
[21:44:26.866] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:15:0) after 9m17s
[21:53:43.999] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:15:0)
[21:53:44.061] Build status: STARTED
[21:53:44.061] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:16:0) after 7m15s
[22:00:59.243] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:16:0)
[22:00:59.331] Build status: STARTED
[22:00:59.331] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:17:0) after 6m30s
[22:07:29.581] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:17:0)
[22:07:29.608] Build status: STARTED
[22:07:29.608] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:18:0) after 1m27s
[22:08:56.635] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:18:0)
[22:08:56.664] Build status: STARTED
[22:08:56.664] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:19:0) after 7m8s
[22:16:04.647] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:19:0)
[22:16:04.662] Build status: STARTED
[22:16:04.662] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:20:0) after 3m44s
[22:19:48.704] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:20:0)
[22:19:48.731] Build status: STARTED
[22:19:48.731] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8878844987607378800:21:0) after 4m40s
[22:19:54.205] Received PubSub notification, asking Buildbucket for the build status
[22:19:54.221] Build:
{
"id": "8713414925646860721",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux ChromiumOS MSan Tests"
},
"number": 52555,
"createdBy": "project:chromium",
"createTime": "2025-05-30T20:29:59.427838147Z",
"startTime": "2025-05-30T20:30:03.592599365Z",
"endTime": "2025-05-30T22:19:53.824265822Z",
"updateTime": "2025-05-30T22:19:53.824265822Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "ca17ee4bacc9010208d55280becf88e2fb6115f8",
"ref": "refs/heads/main"
}
}
}
[22:19:54.221] Invocation finished in 1h49m56.047267735s with status FAILED