[04:00:46.472] New invocation is queued and will start shortly
[04:00:46.472] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[04:00:47.527] Starting the invocation (attempt 1)
[04:00:47.553] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/da07f3d346c1c3880da7a4c89bd2b8edc326ba67"
[04:00:47.554] Popped gitiles commit info from properties and tags
[04:00:47.554] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[04:00:47.554] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[04:00:47.554] Buildbucket request:
{
"requestId": "8899110345896933456",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "win11-arm64-rel-tests"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "da07f3d346c1c3880da7a4c89bd2b8edc326ba67",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "da07f3d346c1c3880da7a4c89bd2b8edc326ba67"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8733692673598189185"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8733692673598189185"
}
],
"parent_build_id": "8733692673598189185",
"parent_builder_group": "chromium.win",
"parent_buildername": "win-arm64-rel",
"parent_got_angle_revision": "4aa12e9e17f5ab6b140febd83185583e8b7ac9d4",
"parent_got_dawn_revision": "ce207a7782c4ac883ed464b884ca19967b90b9de",
"parent_got_revision": "da07f3d346c1c3880da7a4c89bd2b8edc326ba67",
"parent_got_revision_cp": "refs/heads/main@{#1370916}",
"parent_got_swiftshader_revision": "0afe6a306dd264f05239bed7c5ff085b9078118b",
"parent_got_v8_revision": "30135a401fa29d97176bf0d809fdda572f3c7355",
"parent_got_v8_revision_cp": "refs/heads/13.2.23@{#1}",
"parent_got_webrtc_revision": "cecee51bc40852824a28941d5e037d1bbe0d09a9",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#43262}",
"swarm_hashes": {
"absl_hardening_tests": "b93d4e519e9f898be20204df5f36739ffb30fa8ebe0c447fc4b15008fd401a38/323",
"accessibility_unittests": "bad33924c0f38f8b4e6991eed8eebdca54281d34180d274fcaf1ce5883617b53/323",
"angle_unittests": "5173b2ab20c6055c7452b0540a032f1695799b993ce5d4e1cc0ee00a25df50ba/408",
"app_shell_unittests": "c0e76b7e1a9ceb5fa62ec870855159dcbf45ffbceba05d3106cea12f02c8aa81/484",
"aura_unittests": "809f8a39c75a999cc70bd4fd9bfbd8e62e38618e38e6705ab45bbeb0c75e7d48/323",
"base_unittests": "2e58e93f505d41cb99ef3f03b3c23c985816ba6c2b7e55877081ca711dc5c43c/480",
"blink_common_unittests": "21d1cc609753e0d4ce0e3fc10979aee39347f4076407f806a1e24d69a7c6ea4f/485",
"blink_fuzzer_unittests": "9f1a6dba169eea90a74e5787651f68e3a3b6a23a5474a2ae6a2ce51cba15f77f/323",
"blink_heap_unittests": "88dd2118f0f1daa33341a370b8e7540690cf5d98a630737b6c83748878b400cd/323",
"blink_platform_unittests": "053619ab23b544bf92c079ca1e2de6e81d750f8dfd7446bd6783849552e39e2b/643",
"blink_python_tests": "fe15a7541087371861c0ee31a21c5a30b919355351b4b9d8711067bef7504da8/574",
"blink_unittests": "37a5e742cc5443ae663f79020b0c3fcefbe85c9348ddfc0481cf279d9ee66008/649",
"blink_web_tests": "2a7748be34c2ec682a6f0f84ec1ab0cb78f60ab10dd8c949560cae00f01ec04d/566",
"blink_wpt_tests": "1c2e673f8f782dfac67fe1f0bcf5303f50abb59d7045132cbc74b43912010ff7/566",
"boringssl_crypto_tests": "f5f94f96948392b31cad8c4f314b7b7cf39120bcf40ceb07c8726c47c6ebb5e3/408",
"boringssl_ssl_tests": "6475d2f9916e63a7f9d2cbad006693c919b960d17403ba5079c64cc30be2dbcf/323",
"browser_tests": "719cf35a17466b8913a05d021ec193c773e83ea6182fd515b9405bc05b8dbb94/1215",
"capture_unittests": "9b1b8bdd691acb095da32baae43882574b22738dfdf08595a34d111f2a38722f/402",
"cast_unittests": "afa89c09bf79adc32f64ad9c061183984c53202be088116e99b4977e1c39dc99/323",
"cc_unittests": "c997d36e2179fde300587dbdcd3935429db49724ce7211f6efc000025dfde760/483",
"chrome_app_unittests": "b8846787cccec82a6512d0a0fd7c32a5837e7dfaf483b48d456bc543a0c5e93a/564",
"chrome_elf_unittests": "6f8da669b1432939723df6e1a0f4bcc90a7663c992193a01ef4a49fed5102166/323",
"chrome_wpt_tests": "0ea2e61b2dc30fa577e52b632e2c8fa2944bfa0e213267db0bf376e9b5477be2/566",
"chromedriver_py_tests": "ab738c7869a2a47b07a3ca7046d2266af9b438b1fb9b4a8fefa7303112ef29f4/489",
"chromedriver_py_tests_headless_shell": "b20066e8ad2262b86c2e6c884c6c2c780cbe1bae75b966dee96cc6a67239161a/489",
"chromedriver_replay_unittests": "e15a05c27bdd75d6ba1e71f6812e91ae824a38a8e8d4429ad2b76b24ba2b67e8/326",
"chromedriver_unittests": "60b0d1c6f35f8951a2b278c043a0b3477a812853d2af0814f6f5817a9f521e9b/480",
"components_browsertests": "18ca6f58b32c5fd217a10a4183fada60cba7ac211e00d430662b65ce402a4466/652",
"components_perftests": "c4919cedc26f38132e88a3d0b93cd09b7c7e0e7442b8354cfeda623ab0dd994e/649",
"components_unittests": "8892c7e975c646eb4a2858c1d30b747efd4b5972ad1060d4c83a1eda1eccb99e/731",
"compositor_unittests": "7e2d89a48ba2f6408e927cbf77a181f4fc021d6f4c979f31ea037cf3a657b6b6/399",
"content_browsertests": "8533cc1fae55c39eaea8d2c9425d4c2f9e382fcee14cf0c71a19592b868b0acd/646",
"content_shell_crash_test": "92ade768914fa52417d951788077ebb208850ff98ca48886789fe84e428c1040/481",
"content_unittests": "820b8ba3788a04599a2fc1d58a4f9166438cff56f473243e196cbe99e11bb6d3/560",
"crashpad_tests": "08ef8dcf9cb9602d65751d822f507a693a95fc72e6f704a51c76a75ce94b59dc/408",
"cronet_tests": "65e017ede4ec8ec38c8fd659540a9645a4290fd4feaae1ff62dae99cfda1a4ea/484",
"cronet_unittests": "299ce2fd61101992bf712c8694bd9a12f7e376478af3a7bb4f820e99393775da/400",
"crypto_unittests": "8edf69cd4ba3bccfc4c8c38cc50a7234b6352033522fbee75883d4c03db78727/323",
"delayloads_unittests": "715b7936d0f5c0bafba86a9440bb6e65256699a61bce9475f01c539b576cc01e/323",
"device_unittests": "4edabb18bb4ab2c0926da8f2535aad02918593ee385525ae9bb4c1b01e55b8f3/480",
"display_unittests": "fbc72b4d17749a2f6eaca2812b393f2ee29e1ba61543f544abd4128e996fbd48/323",
"elevated_tracing_service_unittests": "b2dea1b94b293af35d26b7a32fcdedd8c509a9c12af48214f5865ec7d1e61596/323",
"elevation_service_unittests": "19f079fb79a69dbf302cfd3fd22c7eeca6ded02e3e60c0daedc372e5474f72d9/323",
"env_chromium_unittests": "39c01dad2d2ff829ad191fcddce4e1a1efea6a2c014a17cb58703050e09032ef/323",
"events_unittests": "83efa5468c16d5eff50e58f5d07ace6204c3eb2f2199deca6fbcfb8f2a06d06f/323",
"extensions_browsertests": "d299bef633e1c99fdec4f2df925a49273992e7dd0d44356efed9f6788f5792c1/570",
"extensions_unittests": "65a13e64d8609db538d4bd42bcbb7f74683e8fc3d02a0094d6a44079e65d4294/644",
"filesystem_service_unittests": "adf3c868d4a3ca6b8b38155db4511a9b52c543a7485eef74f6c70e49bdba6e58/323",
"flatbuffers_unittests": "a25bb316879a52bd3e0ba67522436a748d9573abfa84556679802f8185f74453/408",
"fuzzing_unittests": "39b4ed7fd833d6bfed9753b0649024ba5c409777c3d1256b538906b305e35080/323",
"gcm_unit_tests": "c9448f1760f771cab4583c68c056da141d74f1cb9cbf520f44a8a263dd970ec5/400",
"gcp_unittests": "e1cae8cbb1b6c8fc40d37304d92f91f00189b6bd0d207dbde16fc8b20a6b0cf3/486",
"gfx_unittests": "954d7b3ebe1d25f418dcd8a82dbe3f9d73d13b04b761a61132d845e3f8df4477/399",
"gin_unittests": "8c9b0f9b1506c60933c9ea79ca3b4f2033337d576ab47e6afc1cd655db8f4c3d/400",
"google_apis_unittests": "c7cc34ad538ee9655ed28bce0cff19a57e38b0ee65fe05b267094ec361e61f02/485",
"gpu_unittests": "98763ce3442fa8d3a64f84c2a19b050bc32d69b0e64232763f4df74c33c24dae/400",
"grit_python_unittests": "46357bcc21bcb564053d20dbb3f677836215106708c2b1337d6f33468fd84b38/411",
"gwp_asan_unittests": "9724eb486cfa420ea78c319c0fa04deb01af8445829ecfd46922c2b676a98a1a/408",
"headless_browsertests": "0cb5868f8d689d9516c250bbe522cf1ebb8d5f4d24bb68d98456072e31918c7f/569",
"headless_shell_wpt": "dab9e6a294bf1c15b302b91b654248db46a97b338614936201be7e9c687b6b64/566",
"headless_unittests": "2b9b664468f9f8f390794f98de1af24dde8071a40cebd270a54c20de45603764/323",
"install_static_unittests": "5040cab6cc75caff267b8618bf205ea283869acb8f9d6bb1cf3f2ca7fbe9eaae/323",
"installer_util_unittests": "c2c146112140f3145c1dec1cddfdb5b3c9146e3326fa96bb1b631255737f1c71/403",
"interactive_ui_tests": "3065d1be1f9a348584fd14e8c4de977782cac1fb7a7082632f742e1f69f717fe/973",
"ipc_tests": "2e87291b567f224f2fd0985169fded5a79139a812e121f10ec0fc7deb5744371/323",
"latency_unittests": "88510e8742543098053336c179a874adfbd17fcfb361b48692204444c43ff254/323",
"leveldb_unittests": "773952253c71a42704f9f9bbda5fdccd9d3e12fdcc91afdfc1217836d01a5e6d/323",
"libjingle_xmpp_unittests": "f8f8b2b9907f7c8443f441748d69c154554817634f7a9b20ad35290c98ce3d64/323",
"liburlpattern_unittests": "915958cea6439f61db78c91d3efbd69adb911ca144ffb844ead086b096af4d2a/323",
"media_unittests": "c4443a96fefa7079e66851e12ab58f17b590f292d08d62fd263a346c443b4410/403",
"message_center_unittests": "76a0ad2ad7cd8d0350d112b4a2d0883754a5c8ac97f05f02ded1ecb2e9856bc9/323",
"midi_unittests": "b5681575f262b025ccb6c0f900119a1d764205bff7f415c617baba8f0e6df513/323",
"mini_installer_tests": "8b7304ee8e7d05d629153533252d0a0bbfda74b06286f708407c0b4ef080cb65/489",
"mojo_python_unittests": "e85f5df368c5096984a0c7859102df6bec1ac67e6a3f936984b7b699fb4d503d/490",
"mojo_unittests": "a7f8b83f11f5e482c23fb9bdddc5d2f929ccac4663229d26d8277333cb26804d/401",
"native_theme_unittests": "edcf10775750b0664dc76cf7524cbf600fa2c2b3a429c157cd514025fba70840/323",
"net_unittests": "025ceb22bc34015d785f26b73861ccf990a3918d05bb41074042f4c3c3d6f824/487",
"notification_helper_unittests": "8bba38e3a88cf8afcb7ec81432d4b870f308ca88bb7d8ff659cf07fd01416887/323",
"pdf_unittests": "23d8f4fdb03c94c47a7f836bb8fc07d0e8eda360d3ab044f12713f817dc07c8a/400",
"perfetto_unittests": "339a7ea0b97862e9b91316ecfe72ee1801bfaae61472d8459aa783c99108566d/323",
"printing_unittests": "5b68147f73f1a8c0dfa07cdd1b6c2a1c3e72fcf03266ab507979173cc7452364/405",
"pthreadpool_unittests": "6e504487af7a75252e49982a72ea74691743497aead478aaa1d81ba4cfb5ad6a/323",
"remoting_unittests": "15c269028c661d96b9de6738abfefa14eb342e40b2a8743f26ae7c995c8d80b6/400",
"sbox_integration_tests": "ad2ddd59201f22d2735fdc304a3074a5bbcc6d73f62f026f3f6c0bb378d02ebe/401",
"sbox_unittests": "e2d157ae4db29a48c485a6da79f00c11a1157b9bf84a39b526ee5fd4f4470ded/323",
"sbox_validation_tests": "91bfc63588eeee60819ccdd98e5a4aa00d97a0b77172738bbbc47bb33be55631/323",
"services_unittests": "2dc89661a564520c5ffc7621807cfaffa8ad8f8087af2584aa4726363d297264/653",
"setup_unittests": "8e9f460b510d81ef9968cc57d02d9e22225efad965a82eeb639caa05878fc437/404",
"shell_dialogs_unittests": "6459d6eb2f4aa098c8f8e085aaae4c200b55b1bd82b1c71039fea601417b3ee9/323",
"skia_unittests": "4a9ba9d7243506d4575fee4285f03e90148e05804760da5523edc9b3b1fb73b0/323",
"snapshot_unittests": "347c811c56c728a5e9c2b69f42095e602e8b99eff105f359d7560fe25c6a6763/323",
"sql_unittests": "17bb65a3fd6865380c74492c55450d1a5d12741e6086ee35c56ad192185d389d/400",
"storage_unittests": "66af68e69d20be36aa3c01083982f7cc8dfb27882bb701416e2f0dc0767a52dd/481",
"sync_integration_tests": "693bf8c14ae33c3e3fa0cd1915b95e3349e7a40f82debe19ac0cc3b28216ee01/646",
"telemetry_gpu_unittests": "9e672f2fa31661a00bb379cc8eb2d32058b7646cafe0ffdd929e59039a6a0440/736",
"telemetry_perf_unittests": "f011bb31e4343501a2ec41e94d8c48637039c9f0c73a5e48da96e92b5328fe9d/655",
"test_env_py_unittests": "785b8a7fbeb01b6f29ab644316b5617fd2972f8052fcb64aab4529856b446ae8/323",
"ui_base_unittests": "6599476cd6c183822f6d2539ad56ce3b99b4e7c79b7d3b57fcebecda9b77f095/555",
"ui_touch_selection_unittests": "fbe173eae5a28f3fc6009cc8a5435b046c1383b4a803da9067658ee4b0e7c4b9/323",
"ui_unittests": "6ff400d32d8a4d18a5a2e7364d6246ce30a0b0780c8e7c07233be3f9882be6ce/323",
"unit_tests": "ea93fc4718df5bf0dc05cc6f754b77bb57f64e2eefcfb04a47d7cb86910fdc76/1056",
"updater_tests": "74d3b59725b408127c76df6c474eba2cd45a485b25ab00852d598181c1fc6fa7/566",
"updater_tests_system": "ba836bac89278f5552aac78e8c6155ab0686519bc071330e2bd1612319b8adca/566",
"url_unittests": "fe0c275b23a26cdc716ae9bd28d2be76410f94624453dea7041d8c9d801639d1/323",
"views_examples_unittests": "7893b24482d2d73bf5f89adc2539d1bf05a8a99c4e5a8def68d671fb65d765e7/402",
"views_perftests": "671a706cf80c8fa4140064ec1b28b464eae45d43c3790a0048222fa28a36f115/487",
"views_unittests": "cac1bb75f565a30070b9eefeaa5bf0f988fd40fe073de990ebcf638dcd108e8f/402",
"viz_unittests": "1e090482b905d001c36eaa4a1c3cf568608be90243198d4d71c7c2766ece38d1/407",
"vr_common_unittests": "32c4e35f2cbc896fba7c34f1ce186a4b9787ce30d6b1308e9e84f53b5ce9a141/564",
"wm_unittests": "ac8b138aff9486f3ebe4db4b7883bd2e12898129c6359c460499515f2a187b8d/323",
"wtf_unittests": "e887228629f942e11ec99163d8ecf9b4746264343c6f682cdc8e2eba1afdc266/323",
"xvfb_py_unittests": "afaf4a3488d9848618b8c8528b9dcbac5904e178298b9a34313424e728be4c88/323",
"zlib_unittests": "1bc094e5a3bbe80696babc88d9d0c474de06c3bc3e533985b88a0432b0b13b2c/408",
"zucchini_unittests": "a134e86cb8834497cf98e85949ae8005cfe83c201c1d43d42e87e745caec4504/323"
},
"swarming_command_lines_cwd": "out\\Release",
"swarming_command_lines_digest": "d9d8da29f284e230f02a4ac1ffd121f33f7a7a456b58fed04c2a0fabc7f594a6/94"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "da07f3d346c1c3880da7a4c89bd2b8edc326ba67",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8899110345896933456"
},
{
"key": "scheduler_job_id",
"value": "chromium/win11-arm64-rel-tests"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "win-arm64-rel"
},
{
"key": "parent_buildnumber",
"value": "2857"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[04:00:48.157] Scheduled build:
{
"id": "8733680284039348625",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "win11-arm64-rel-tests"
},
"number": 2738,
"createdBy": "project:chromium",
"createTime": "2024-10-19T04:00:47.616510624Z",
"updateTime": "2024-10-19T04:00:47.616510624Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "da07f3d346c1c3880da7a4c89bd2b8edc326ba67",
"ref": "refs/heads/main"
}
}
}
[04:00:48.157] Task URL: https://cr-buildbucket.appspot.com/build/8733680284039348625
[04:00:48.157] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:2:0) after 2m45s
[04:00:49.191] Received PubSub notification, asking Buildbucket for the build status
[04:00:49.318] Build status: SCHEDULED
[04:03:33.178] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:2:0)
[04:03:33.244] Build status: SCHEDULED
[04:03:33.244] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:4:0) after 1m35s
[04:05:08.452] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:4:0)
[04:05:08.470] Build status: SCHEDULED
[04:05:08.470] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:5:0) after 2m49s
[04:07:41.838] Received PubSub notification, asking Buildbucket for the build status
[04:07:41.922] Build status: STARTED
[04:07:57.580] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:5:0)
[04:07:57.595] Build status: STARTED
[04:07:57.596] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:7:0) after 3m21s
[04:11:18.723] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:7:0)
[04:11:18.749] Build status: STARTED
[04:11:18.749] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:8:0) after 6m59s
[04:18:17.878] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:8:0)
[04:18:17.912] Build status: STARTED
[04:18:17.912] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:9:0) after 7m39s
[04:25:56.960] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:9:0)
[04:25:56.984] Build status: STARTED
[04:25:56.984] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:10:0) after 9m23s
[04:35:20.081] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:10:0)
[04:35:20.105] Build status: STARTED
[04:35:20.105] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:11:0) after 8m15s
[04:43:35.135] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:11:0)
[04:43:35.152] Build status: STARTED
[04:43:35.153] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:12:0) after 5m22s
[04:48:57.177] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:12:0)
[04:48:57.193] Build status: STARTED
[04:48:57.193] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:13:0) after 2m57s
[04:51:54.334] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:13:0)
[04:51:54.354] Build status: STARTED
[04:51:54.354] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-rel-tests:8899110345896933456:14:0) after 3m26s
[04:54:32.713] Received PubSub notification, asking Buildbucket for the build status
[04:54:32.748] Build:
{
"id": "8733680284039348625",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "win11-arm64-rel-tests"
},
"number": 2738,
"createdBy": "project:chromium",
"createTime": "2024-10-19T04:00:47.616510624Z",
"startTime": "2024-10-19T04:07:28.426498736Z",
"endTime": "2024-10-19T04:54:32.385624289Z",
"updateTime": "2024-10-19T04:54:32.385624289Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "da07f3d346c1c3880da7a4c89bd2b8edc326ba67",
"ref": "refs/heads/main"
}
}
}
[04:54:32.748] Invocation finished in 53m46.288100722s with status FAILED