[05:59:31.767] New invocation is queued and will start shortly
[05:59:31.767] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[05:59:32.878] Starting the invocation (attempt 1)
[05:59:32.908] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ecbe1f35d0359987b721728f68e256cb4b06dcf8"
[05:59:32.908] Popped gitiles commit info from properties and tags
[05:59:32.908] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[05:59:32.908] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[05:59:32.908] Buildbucket request:
{
"requestId": "8893938847399162096",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "win11-arm64-dbg-tests"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "ecbe1f35d0359987b721728f68e256cb4b06dcf8",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "ecbe1f35d0359987b721728f68e256cb4b06dcf8"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8728516667450569441"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8728516667450569441"
}
],
"parent_build_id": "8728516667450569441",
"parent_builder_group": "chromium.win",
"parent_buildername": "win-arm64-dbg",
"parent_got_angle_revision": "00d9ef8f182ca015ca6ec6a0fa4bd76512a9bca2",
"parent_got_dawn_revision": "028acef1bf52513385ce1d0673f2a290b1c21f95",
"parent_got_revision": "ecbe1f35d0359987b721728f68e256cb4b06dcf8",
"parent_got_revision_cp": "refs/heads/main@{#1396431}",
"parent_got_swiftshader_revision": "23b93c7cfe27e205e178a135a6ea24d1dcba1068",
"parent_got_v8_revision": "b501500458758d56822daf931a088cfd22ddec7e",
"parent_got_v8_revision_cp": "refs/heads/13.3.339@{#1}",
"parent_got_webrtc_revision": "021cf5ac3ef6c5ffd8b65c7365d0f6d7ff456b9c",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#43560}",
"swarm_hashes": {
"absl_hardening_tests": "16f9c46a55cd17d48daab56793912d6fc0f61a687edc2e4c4212a36d746deef7/323",
"accessibility_unittests": "2048163f735bde0e4ba766a128b27f8d29115eb8ebfc82e6b59c48218aa8c0a5/323",
"angle_unittests": "d7b3f282be5d4eb91f9ceb37cda29ed130d5517c85dfc55d5ff01594befa2528/408",
"app_shell_unittests": "342c39860de7ece3c14d815ae2e1231d4d3ee0562ddb430ac23518a48f04dddb/484",
"aura_unittests": "e35fa9ea395362a9a30f79bf5bd11f82625f12b81ed40a4aea7f46bb62e97a5b/323",
"base_unittests": "797d48713975f391a4afefb172f722f281ba6b041981f6067a5cf2eee333cfc3/480",
"blink_common_unittests": "bf872ccf40139f3c1bd3898dbe94bc3503e565fa2c84782def95645896573f6f/485",
"blink_fuzzer_unittests": "6a99c8e4bf49fa12605fc4010220e3bca91a6e2cdd06b31b2533a90943d0eaa1/323",
"blink_heap_unittests": "cc49a2d0e9c73c73956e02603900f30063e217ae807d48db6ce866686c0b6d54/323",
"blink_platform_unittests": "e9286feabc0c6e2d29848c836908074ae03e5a387f6143de4723730033c4f438/643",
"blink_python_tests": "5270acfe46ce206b3e0abdcb5ab8b01c5846e617cddc6707286ab893fd600094/574",
"blink_unittests": "90cbd7dc4ee0f39a02acd9af7d278d89f55f16b081c18fcdc179f890c6dbc65c/649",
"boringssl_crypto_tests": "5be633b1a3f0369c4ec942adc034de48147411b2933e903d9cba2df5695da911/408",
"boringssl_ssl_tests": "c93f66d71d444031f09bb5d4bc23e0ee35559cb7ada9e47cd772cfebc7b6b83a/323",
"capture_unittests": "c93a301c760eaa2831e3269e60eac2f1f55e020faf09d45429d551f82b31b544/402",
"cast_unittests": "6a998f026d74fd4e7442c0ffc66b4905515b3973db06b1f4b58e9c518c1f2440/323",
"cc_unittests": "4df60cab2d3bbda4fc3b79bdc132610e57893f624e8bda64aa29ecdac4fbdb0b/483",
"chrome_app_unittests": "b5c36fe36e113573706c0f7b4d75082d7944529c0b65739e81fb1ecdea2fcd8c/564",
"chrome_elf_unittests": "b9ad59ba4da3eff9bce630dc96502b7bb9f7c7a80792179409e48e435905abfa/323",
"chromedriver_py_tests": "c83526079136ed50a76ab079b88db9f3fb7646a57d1645a63d646fcbc4ae084b/489",
"chromedriver_py_tests_headless_shell": "27c39a235b49bbeb94a483f2bbb111cae613319e32a4153e1e7361c727f2e954/489",
"chromedriver_replay_unittests": "b6b11bb59eb9705ff25d02d2d1dff90f013d81d3e57f9dbef1ad60ca764f667e/326",
"chromedriver_unittests": "bddb363e8de1f914c5dd6013980db836125495c77e8ebdcffe0c9e8cff00b585/480",
"components_browsertests": "52b14db717c10808a1e10a6d31d22291570c8698c524fbc2bd43a8ada18a28e2/652",
"components_perftests": "d7b16b56a1cde335ca2eb7f4a16f171af04f70da0dadeac62195e703ed8c290a/649",
"components_unittests": "df1c12b0d3d222f140904341fbc46c7988a37cdddf38179a0e8698eca7b9bba1/731",
"compositor_unittests": "77bbb93aa34522af82709a004b51922fa07c9edb89694815a091b6234f1f8b2f/399",
"content_browsertests": "a0e5ee1e6b143098d23c2a1d9d06a8e78b0ee0b55b8ab78a1c8755942f47c6ad/724",
"content_shell_crash_test": "9017c99038dd58adaa6f5ca7db46c1fb3706491862b730a5beb1c2d4a81037f2/481",
"content_unittests": "eb7b175d9d605857e851ee20b97bc04803034ece45090508b852844757cef11e/560",
"crashpad_tests": "6b8285deec02971819be6928798e6d9418b6389143f53a9f70227a2bb29e434d/408",
"cronet_tests": "ca927d53b9547853a2148c474191d654ce640e252747831adcc69164f637965e/484",
"cronet_unittests": "65753e899f71ef565e20740f751074a970bca991cd430eef8f94e3807eac60c4/400",
"crypto_unittests": "f878cac5a3a6dff210703a11d39ed0ede22fde35d04b7ef713914cd1d8953797/323",
"delayloads_unittests": "8bb175c993c15d7f0ed9f8ddf09165caec55308e47a6435ad57211b652cccef7/323",
"device_unittests": "93d9fd371fa25e22e83b2390612b6ca20cc1dab08295d8b761ad4cc05c34b094/480",
"display_unittests": "f10f0e6f4679e5fae5b35d31b4512b5b6b7ec56959396fff80ca48bcc3cc0603/323",
"elevated_tracing_service_unittests": "729f9f833e4bab149c75d9d7bcca6710d3bb4bfdda87b9aae38b62b84a5fb1a6/323",
"elevation_service_unittests": "55ded9a3b94aee5ec8429ea6c63c1b83e3c6ae86b6724c2f724b7e01d91ed894/323",
"env_chromium_unittests": "b92db2cb8729c832acd8c666e8b21b8bd532b25c120c82118d98adc0887dcb97/323",
"events_unittests": "59bdb077b63ce54393e996382f1cfc67db819798f1be09b7932a0f08bceb0933/323",
"extensions_browsertests": "fcdf3f3230f133b396437a6106b1ffc8d8d966291cd8e91b53737e1509094269/570",
"extensions_unittests": "c09a8dc697757553aff3833a3ae057f94fc77f59a1e2d2b805a963ef7870ac0b/644",
"filesystem_service_unittests": "577e56b5dccc9d8e6658b2876174dcd555df8bae6c28f38b3779f8d498954df1/323",
"flatbuffers_unittests": "915c54cc389b8b4975612c5600b405f102ab20abdb7f03b9f4638f0ab467a287/408",
"fuzzing_unittests": "06fd698b73581116d62a088283ca9c4ea578109f368b4c66a294c0a1c25d8d28/323",
"gcm_unit_tests": "a95f2d1ca381e69ae24dfb17b0ecfbaf308923001f47a9e94ccc42153e86c500/400",
"gcp_unittests": "3d7e022dddcb203d87d0a1b090348619f967aa63f5882e7f52c77c78aa9f0460/486",
"gfx_unittests": "516abcf79adfd1c5b7192ea1485c28229dd6f171095410464522dc20c16c26fc/399",
"gin_unittests": "af66e7ff1c319145c82ba123516ecd5d3bb4b37a25076df34dd1547815c9af0d/400",
"google_apis_unittests": "31d7c8059541550b6a5ac43cb7e3b6e58bc3469b314c0d2d0c314e5e5e39a858/485",
"gpu_unittests": "09376e4741f8dec1123c23c792e406454a726f3958556d5910d873971117878c/400",
"grit_python_unittests": "cddff5d767604d91ec1295f0178a49047ba5064e12aff993cd7356dec008730c/411",
"gwp_asan_unittests": "24233ee4531d06bf1ee014b84c856c30fb9395e17e8218faf85c7e4c9256b23a/408",
"headless_browsertests": "46fc5d1d2db88241ae9d8de7d7d3b430d7af127b49874ec059110abb7844555f/569",
"headless_unittests": "348a1d72a2ae05eed483fd485f60f67f8e9166bebd75990daf0d668a3ee3f01b/323",
"install_static_unittests": "ded13fddfb48e9fe812e078112a2fa0a5d1ea975a3c4aaa6eb73cdf603e9c864/323",
"installer_util_unittests": "88ce7adecfe4810c8ec88712f479fcbea74833853f15d79a1437667f4be0d58e/403",
"interactive_ui_tests": "0ab589c084a8539e025f1c13f3189b7be1a071f39378ab44b7a37d0c440145c6/973",
"ipc_tests": "8a9a95a8a06065b337dd6252c771d95d48e43ff3b1f11a2f82886bb98902af47/323",
"latency_unittests": "9716c065fe6906eb2bea2ec0bea06a328c20a87adf71f3400be887f8ba5b080b/323",
"leveldb_unittests": "27cb434cec6f819c74c709a67dfb4ca2b14ada5d413c16bad2041c43172db536/323",
"libjingle_xmpp_unittests": "d32613f94d5a559719cec18e070647f3ff0b05c8c6754b04d60490ec21694a1f/323",
"liburlpattern_unittests": "a24842d38d861f1b5a93cfe01cf6ce92a9bb3b57106a805141ce4fc249dd1fbb/323",
"media_unittests": "00966e20c237ae714af7e1e40610376414d37aca68129467c5f81ceefdedfafa/403",
"message_center_unittests": "9963ca6101bc6ac3997f060caeb37cfca2a5ebe516d1288a842322280d8bfaed/323",
"midi_unittests": "d7ae3d4cbf9370b30e404b94a822c46b5c11bd04aa4aaad986216c3110649213/323",
"mojo_unittests": "f479e215ecfa6aa23dcd95b963222ce249bd606ac06dc9726d1a33e80f828c6a/401",
"native_theme_unittests": "8533eb023b35bb7fc2f75b72a7b24fc9e4c6c8e75b21a957c07b7d5b106a572f/323",
"net_unittests": "28278a7e73f6a8582afa6ef707bdb72f37e54315923b413ee001949f0b4b8937/487",
"notification_helper_unittests": "90c993feead947ab46d615263b84ac9a964780f09b87bcb22457cc914800b6bb/323",
"pdf_unittests": "911ba7519037b08e9ec02480f764b9f74f168f7207338bc71d0cc2dbe8513e3d/400",
"perfetto_unittests": "2db7a02a655635a986ce11d7d873459dda9b3e4d52253b35959f484db01d7e2d/323",
"performance_test_suite": "c9dc19497e77ca52b3df92a0df6534ea6e406e84a0a40c684f56fe02d2c39469/732",
"printing_unittests": "b9efaf5d718ec969abedb37f890009c0af3130ae17f253d2d4b169875c6e88d5/405",
"pthreadpool_unittests": "9dfd8674ecc8f781a1aec566b5178893650242ff19e6d0b620a10203686b3f5e/323",
"remoting_unittests": "939d38a27a94501a349dda8d887d98515e20b7a496ce557334ae515f39c4f200/400",
"sbox_integration_tests": "dd7f7b34dce871bd6a786cc6bf8f7b5906ca222b3d53fc7f2a52bfe96e2d445c/401",
"sbox_unittests": "d7465c09d502d4e3550a231e3d4805f42ae6e7f11f9ed87fbd7d7a9eefbd32b8/323",
"sbox_validation_tests": "4f135c720174f71aa79b88249b04be3d5c2c6c20dad4c9aae4c39a3050134dd9/323",
"services_unittests": "cdabb661913f51979d14ef89051baf2bc7f85ab40108cb8ae3bac92709badf3a/653",
"setup_unittests": "90d0ffaccc4d3747935245103681b547fc0e4d7dd74cce1239a41df12c1f66a8/404",
"shell_dialogs_unittests": "e23f2ae494242ed1c52a52b27802a64ec34f37e4b8ad3268247f969dec09d2d4/323",
"skia_unittests": "e6ce1a07cad2a83153a79ea126806e3e91d7f358ff5455b0dec1b61960613998/323",
"snapshot_unittests": "75b72eb72b14d8f55d0bfa7fbbc366634137fd567dabf364cea746bddc4c86b2/323",
"sql_unittests": "ee4a7583d22f2cafe0f4784c3771f75b483529bfaba47608533f005be4dcda73/400",
"storage_unittests": "323b11dfd27fd928a3ecf47a58ee07814fa5c15adfd1b4d53a7554454cda91e5/481",
"sync_integration_tests": "07c95e2c8fe491ea7ecb7699877b27f0eaaf3ff7bc8311263e443c45d8c60906/646",
"telemetry_gpu_unittests": "2740b6633a3ab3ebeab696cc47c1d9e082970b9656f1dca66218022322e7af1a/736",
"ui_base_unittests": "cea84ac2edcbb32b2ed9077fbe8c5eb8fdd0827459517797b8d320073b0558d5/555",
"ui_touch_selection_unittests": "6f368b32c91802d56ee484744b625078ac5d089400df4a38ee637a1609386dd2/323",
"ui_unittests": "92a822e4a682e2c7fde8ef9cde52ae2ea24a6498db1044d3a72be658ee7509c4/323",
"unit_tests": "ba8ce4237cf3b1e5eb70d080075f8a05dc326d8e2b10eb5159b5c4e448d7eae8/1056",
"updater_tests": "b0969be17adb9f04dad86cd8b688cac2e4e049f9080a0f7be11d48b4f4524304/566",
"updater_tests_system": "034ceb01cbdb13c07f794fc16caaa9fd6919a251e95e78f8c5de6cb6f828c1e7/566",
"url_unittests": "fecc0380115f666ce3e1556bbcb6b05e4d07ca3ea1d0669a61dc9ce517d0e203/323",
"views_perftests": "8ae7dbbf785ee330bed888d692e208b4ce918047950a6cb67ec23b2a2bdfa043/487",
"views_unittests": "1f8efe4187271c343c0493c71a811516b355423239b5092e13207061c3386e3d/402",
"viz_unittests": "e3cbeb69717c93f518069a19a3d740a9d0d4298fe0c24e8d8448e9b7207a9694/407",
"vr_common_unittests": "db6617c80dca464336f56cb2bcb4ea1f10304290c9662f60b336f706d5115133/564",
"wm_unittests": "a3b92544c7273a0a71d08192a17eb16bf30a8ce59382e7d8fd1aa34d0f04781a/323",
"wtf_unittests": "dbda6eb5e76c2524c3298afbc42cbddea4878e40178a65c245f67aa382f2d4a4/323",
"zlib_unittests": "535d6a03e56e7461728ce618bd8b7d9f940959436fb9022c72c565861f21c54e/408",
"zucchini_unittests": "90114efb41f3777dcc7f42b62b3a6ee0a07db4618967f2ed069c9a89350f52a0/323"
},
"swarming_command_lines_cwd": "out\\5c6a-win-arm64-dbg",
"swarming_command_lines_digest": "3761d4eaee38ff00d7c911895cc592c4f74635d4e50524bc18f7981f7e1c405f/94"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "ecbe1f35d0359987b721728f68e256cb4b06dcf8",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8893938847399162096"
},
{
"key": "scheduler_job_id",
"value": "chromium/win11-arm64-dbg-tests"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "win-arm64-dbg"
},
{
"key": "parent_buildnumber",
"value": "4374"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[05:59:33.669] Scheduled build:
{
"id": "8728508785481756385",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "win11-arm64-dbg-tests"
},
"number": 3468,
"createdBy": "project:chromium",
"createTime": "2024-12-15T05:59:32.967923396Z",
"updateTime": "2024-12-15T05:59:32.967923396Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "ecbe1f35d0359987b721728f68e256cb4b06dcf8",
"ref": "refs/heads/main"
}
}
}
[05:59:33.669] Task URL: https://cr-buildbucket.appspot.com/build/8728508785481756385
[05:59:33.669] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:2:0) after 9m51s
[05:59:34.162] Received PubSub notification, asking Buildbucket for the build status
[05:59:34.175] Build status: SCHEDULED
[06:08:39.851] Received PubSub notification, asking Buildbucket for the build status
[06:08:39.869] Build status: STARTED
[06:09:24.638] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:2:0)
[06:09:24.654] Build status: STARTED
[06:09:24.654] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:5:0) after 7m20s
[06:16:44.744] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:5:0)
[06:16:44.764] Build status: STARTED
[06:16:44.764] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:6:0) after 6m49s
[06:23:33.800] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:6:0)
[06:23:33.818] Build status: STARTED
[06:23:33.818] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:7:0) after 1m6s
[06:24:39.837] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:7:0)
[06:24:39.862] Build status: STARTED
[06:24:39.862] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:8:0) after 4m24s
[06:29:03.884] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:8:0)
[06:29:03.902] Build status: STARTED
[06:29:03.902] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:9:0) after 2m35s
[06:31:38.918] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:9:0)
[06:31:38.938] Build status: STARTED
[06:31:38.938] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:10:0) after 7m17s
[06:38:55.958] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:10:0)
[06:38:55.974] Build status: STARTED
[06:38:55.975] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:11:0) after 6m0s
[06:44:55.993] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:11:0)
[06:44:56.015] Build status: STARTED
[06:44:56.015] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:12:0) after 6m31s
[06:51:27.034] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:12:0)
[06:51:27.050] Build status: STARTED
[06:51:27.050] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:13:0) after 2m25s
[06:53:52.069] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:13:0)
[06:53:52.085] Build status: STARTED
[06:53:52.085] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:14:0) after 5m17s
[06:59:09.168] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:14:0)
[06:59:09.182] Build status: STARTED
[06:59:09.183] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:15:0) after 7m14s
[07:06:23.366] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:15:0)
[07:06:23.384] Build status: STARTED
[07:06:23.384] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:16:0) after 7m57s
[07:14:20.491] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:16:0)
[07:14:20.571] Build status: STARTED
[07:14:20.571] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:17:0) after 6m2s
[07:20:22.589] Handling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:17:0)
[07:20:22.607] Build status: STARTED
[07:20:22.607] Scheduling timer "check-buildbucket-build-status" (chromium/win11-arm64-dbg-tests:8893938847399162096:18:0) after 6m18s
[07:24:57.319] Received PubSub notification, asking Buildbucket for the build status
[07:24:57.366] Build:
{
"id": "8728508785481756385",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "win11-arm64-dbg-tests"
},
"number": 3468,
"createdBy": "project:chromium",
"createTime": "2024-12-15T05:59:32.967923396Z",
"startTime": "2024-12-15T06:08:39.538621789Z",
"endTime": "2024-12-15T07:24:56.870233363Z",
"updateTime": "2024-12-15T07:24:56.870233363Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "ecbe1f35d0359987b721728f68e256cb4b06dcf8",
"ref": "refs/heads/main"
}
}
}
[07:24:57.366] Invocation finished in 1h25m25.611112113s with status FAILED