[23:04:06.902] New invocation is queued and will start shortly
[23:04:06.902] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[23:04:11.056] Starting the invocation (attempt 1)
[23:04:11.088] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/7fb257ccdc15a2a3b27dc7f6be3dafdf5e13b741"
[23:04:11.088] Popped gitiles commit info from properties and tags
[23:04:11.088] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[23:04:11.088] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[23:04:11.089] Buildbucket request:
{
"requestId": "8864520968933156592",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux ASan LSan Tests (1)"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "7fb257ccdc15a2a3b27dc7f6be3dafdf5e13b741",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "7fb257ccdc15a2a3b27dc7f6be3dafdf5e13b741"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8699095272660739569"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8699095272660739569"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8699095272660739569"
}
],
"parent_build_id": "8699095272660739569",
"parent_builder_group": "chromium.memory",
"parent_buildername": "Linux ASan LSan Builder",
"parent_got_angle_revision": "6140e10434d9a676c8d282fc0b2ea3807bd3517c",
"parent_got_dawn_revision": "89872eaabc3dade72f429eb25b9053609d1bdfd1",
"parent_got_revision": "7fb257ccdc15a2a3b27dc7f6be3dafdf5e13b741",
"parent_got_revision_cp": "refs/heads/main@{#1540316}",
"parent_got_swiftshader_revision": "7ae85e6f6694ba7b1801d7bf1801b7fbcdb38925",
"parent_got_v8_revision": "9645fdb1e43c8edd3852071890c827a626f97011",
"parent_got_v8_revision_cp": "refs/heads/14.4.54@{#1}",
"parent_got_webrtc_revision": "c443b287624bf023684783b534b8058bd4e317a4",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#46120}",
"swarm_hashes": {
"absl_hardening_tests": "e90c6a51733eec3a9779c84f1ffef314e9f750cf0efb9d25ed3381d47bb389ea/488",
"accessibility_unittests": "e91163e86b09dc2095eee80fa34c7e09ee0be7f7b23e4bcdda542c6527fe6268/488",
"angle_unittests": "1152c12cdce8850994f94ae1392e94cb8a643852956086d94b5a6748af4b3dcc/489",
"app_shell_unittests": "c78f7a0357d5493ba91fef6810eec81e0dbe7c1f48e1135a3e4731dd62adeb3e/649",
"aura_unittests": "a05f284ba639fe8325d411ad0f1c01f38cb2c61fcd7eecdfa77eedb452a15ecf/488",
"base_unittests": "f7c8603cb060f4a941e526df535f0b253ade5a7aab40af2d355b690b93db8265/566",
"blink_common_unittests": "db64d970d4110ead05575260b7e20db0ddef57b8a5426bb5f2b56f044148bec5/566",
"blink_fuzzer_unittests": "5349664e841c1cf168b5c1d4fde7a247f7bd921b6c58ce413cdd4f9237a612cc/488",
"blink_heap_unittests": "c97d3aac9e8ca528cd382e6fa13f946066477d9985d93341f7bdf7d70e931d06/488",
"blink_platform_unittests": "aec1b6141b99cea6c3af8d98bfcacad2a319686937c7700afb734d34ea3bd908/645",
"blink_unittests": "907899216e38f8c97bad29636a963b5c088f0fcb8c5f828583b96de88bfebefb/729",
"boringssl_crypto_tests": "890d571ec6f97860b68c15cffba2e66cf8639e53a61fa9f7eaa49c50d45f550c/489",
"boringssl_ssl_tests": "afc666fbdb55d7f95c90e93eae54f1c29be451a51dc433a3c7675289bd044be3/488",
"browser_tests": "facae91d82f367e011fb7b2cc3ae7c9fc8453655a498355c7f3236c5b4ef39b8/1220",
"capture_unittests": "30745b7eb7fafd5add5f4afd4a2e1f9792888cefa9f1162fd2cce805522b88fa/567",
"cast_unittests": "c18cf8188e74f18876ea168a1e9ebb56a3784cdb159021c3cf8ce3cc2d271aef/488",
"cc_unittests": "2b1b5af299ced92e6aa938a563020229bc40e37746b82cee96647f70488b2f17/648",
"chrome_app_unittests": "5a72c132e91bf8b42e1ad90e5a51b2f9a742ed0db9a42117bfa3b817b815d89d/566",
"chromedriver_unittests": "2920ebbe76c633d4c252a166c6e7ccba1ad745b8ac167c30b0846f8ab34646fe/645",
"components_browsertests": "a72f83dbfa0eb131147a4aef3630c75888afbb1bfed32d946ef3e116c9396794/732",
"components_unittests": "1bbc80c49a40f74fc06c2e4a7ac724b979576de02c776af97a42c47efcb31615/811",
"compositor_unittests": "452fcd548f983444b2794f3623a00041409f11d4c4ceb7071f6d3cee6933aa27/564",
"content_browsertests": "69dac82afc8c2974e29edd9d3b64a76f44cb789ff4948b7c8c62580d2a7ced03/805",
"content_unittests": "750ac61f0545ffad44b4b1a1c758c0e15f8ecf61266762e23833fe3cead1f986/725",
"crashpad_tests": "97694387cccd00a1550b8f81f4ce64d8a47903beaf15f307f232f8b4248b2636/489",
"cronet_tests": "81a4691b2a03317d250eda47ded2812a1a7b628c3b4daf03a3bb5f17c1c0030a/565",
"cronet_unittests": "93b2ded9cc68f5ad409e80f0d86606e5a3627c05964bfe839d25b6c3281a2404/565",
"crypto_unittests": "746727550c5f7649ce30647f1c8e1f427c6453575151d9b86a9d8bf7b0867a07/568",
"dbus_unittests": "3cfb5f5b0bcbda2dd8f00aa586b96fb0c21b7d5489e4f8454f30f2535627b511/488",
"device_unittests": "e355a0c6cdb0bc894baa3b34c0ceda815f81b14c160350cbfaf92dd1fe739791/565",
"display_unittests": "4f909cd4133bbb29bed879f9ff2d72b192ddf18e7a928225d3224433f1a45ddb/488",
"env_chromium_unittests": "34e31ca165b670b903602168c408fe4d8d31b8e9ff80b2dcb5857b8f83b8c3f0/488",
"events_unittests": "553d48a429e06bb2715b0ccab092e718696d1aca38c525f5c6cc744f9621406c/488",
"extensions_browsertests": "9a544b1aa83d78765b3641b459c37786f1ded7a04fe8235e145bdfb54ae680ef/651",
"extensions_unittests": "75c3588b509702f5dfb99be420a058fa8880cc58d9a931381e9750fabbae4e49/730",
"filesystem_service_unittests": "fb66704fb0d5e12eb1102d6d750c5271070c25d1d4dc3ed68d0c57d4f8632a3e/488",
"fuzzing_unittests": "b74f321a0ef6facc39fc231892b3ebbed324d85f8a1f93836ef99ebbb0f298c0/488",
"gcm_unit_tests": "b0a20ff7994fb40290240433b3e1f5e3f8a453855a7823923746b93a6c881db8/565",
"gfx_unittests": "68e65f7dbf5bc1f4532262200c5218ef95d1fecfa563076b18dad67c182423f0/564",
"gl_tests": "8dfa6fe924cd641d37f17ac55e19ffec9beb490a61b2f86e2d8f49f17de1f732/488",
"gl_unittests": "51874e3e82ae3149b353e3011b0a11e770b1617771525fa5a25a21f2046782ce/488",
"google_apis_unittests": "403c25711206977eb433232f918d1f4b40df961cae2f7cf9d157ab0b0e06ff17/650",
"gpu_unittests": "6193f1bd52d910e89c5a254d9f18c0e7b2e3bc419b7626cfd58e92a6977ef3ef/565",
"gwp_asan_unittests": "ad814dde0e2e784479ecc3c3f6aaec300e4382f3d9e84e46fd25292e1b5efa7d/489",
"headless_browsertests": "df87058c6911af863ec2a325cc57813e1b6a0d9d655eec2df28ed71afced41c7/733",
"headless_unittests": "a66966602141c72e8fdbb605025a193adaad5e71ab9f87d78f51bba83920b2cf/488",
"interactive_ui_tests": "4a94f51898e4ebcd44f39b7fb0f678f2cac958c37b56af879952752f60a0a1be/1059",
"ipc_tests": "11408462abffa3105436ed62d9f5b3f0d50a432552215170444ba047f0c37834/488",
"latency_unittests": "c660c5895554aa7690c157d8f068ec01306b6de12298f04b89f7f1cb2d9fe7cc/488",
"leveldb_unittests": "c1baa98f1c8a7e732b45ec8983f147ff6900258b8d306a4bc4976b8ea1029a31/488",
"libjingle_xmpp_unittests": "f318f92a5239255ffbf33a2742577df9b2d966d80cd6c55f99fce846b4131902/488",
"liburlpattern_unittests": "370c586e2ed0568eeed6456cb128e7efbab8e21b16007fa91399fb3d1e5ab48e/488",
"media_unittests": "1e65372e83177c76d9e1d4ba924fefcb30c1eb4f28580e508bf661f95bff7115/568",
"message_center_unittests": "1843f168471ca215350d7805d0857af60df53fae63c5b04120b36f885a9309b2/488",
"midi_unittests": "1a9642c2296ae55b2fe2fde9c28716e4b0adde6029d25479bb488cda88a358f4/488",
"mojo_unittests": "9b771505d1c6798d14d392cf000ee6b4d141971b038e918e897af13c249a4748/566",
"native_theme_unittests": "b906951c3b911e5cb1433097cd5130865c3f4628aa9a99df612a4344a3f1d1da/488",
"net_unittests": "c2ada0f59eac29fe64579886617f8aba92bdfd5c4a62e0ca18b5c62e9362cd1b/567",
"openscreen_unittests": "45c490af0a2432c5192a782d2dddf89347d588e1cd13cd2272470fa717ae8f6f/489",
"ozone_unittests": "7a8959d9bfd50325f433ab0815024a0f640ca928d7e3d71b0e18f96dc0be4ef9/488",
"ozone_x11_unittests": "40ce78a63f12739407adba5cbef779519d029b748e8cd93a5673219b3ceeef3d/488",
"pdf_unittests": "45af9f3f2ff7ea6fc46b39546df15d92dd100df2fb1171d1fa7c9aba525c35ea/565",
"perfetto_unittests": "13453ec6618fa17730d72b89283b868cb30772ace643e8a98668bb8ad0286aff/488",
"printing_unittests": "2e1ae22d86ee8abaeadf840a8cb4323454e0782420ce6af7f60fae21ec175307/488",
"pthreadpool_unittests": "c0b5716fa42f104605187dc73bb008288800ed9fd51fb102db9ff5f3f7861fcf/488",
"remoting_unittests": "0996b45b6cb24c548b463f3f061d73ab3908b4db6357585caf7d99d7b001cfee/565",
"sandbox_linux_unittests": "c18bc1f252f3e2ef5d39015fa8405d1d683af5c3597158984152a01239f3f54c/488",
"services_unittests": "c376e9644e428c22308c3140917b431cc5c277682177692323929fe027ca9dd7/734",
"shell_dialogs_unittests": "b0b9dc587677073618116fb53d04c429c767cdbb0a6e45431ddfb2b50ce9c85d/488",
"skia_unittests": "316eae07de6b61ce7c5cd29b9c172df872c474e818e706f4ec59344f4accb62f/488",
"snapshot_unittests": "a36a7c0f8d0054dbdcbb0e040012680ddf18144bdd4757a8295513bcf074ee6d/488",
"sql_unittests": "b5c9df775db1e0bca23a337beaf5fdf47a6383b5160f1afd02356fbb1e203586/565",
"storage_unittests": "4292a841be6343de4350548f4bf72ed082be877771125e4c341b681bfc58d79a/646",
"sync_integration_tests": "1d797dd42670f8ef124d77442e16da99f558ae2a15b0c1bd5f40aa6234917c3f/647",
"ui_base_unittests": "767efcc4f74822c74c99c242c5dcee2b5927909e43779ed03a008493955eb915/641",
"ui_touch_selection_unittests": "806f24482b80cdef9813c2da507dc069696932cb2c4d02834572a3b436c1a60f/488",
"ui_unittests": "fccd698b6e091b2c2ba83d59992a9516fac49849ba6057208db814b4b8b53db0/488",
"unit_tests": "db16c0a53edf300e77370790e5d5c877285ed61f4d9a50218d3b5934ae8460ab/1061",
"url_unittests": "c135974ada9d6ca6f8d6dbdd49fe2b9d969160b8443436fa7bbea7b961d81f53/488",
"views_unittests": "e646438d7b9b71b91a380c12aadf594357b3ff130cc3f9c4aa8356321bee897e/488",
"viz_unittests": "9e887594b6ed94ea1e751280df67f421f8d4c0a18a0e80e177549a6baeabe506/572",
"vr_common_unittests": "5b237c5b813fc03d611b51aabc1d296cddf9cbd18432eb9fa8afa818102f3214/566",
"wm_unittests": "a5cc5c51d5540fc53e625f3aed5110c3a97f23deb26bc6fb00ef1a00d99c3940/488",
"wtf_unittests": "35ad3d7a2d63ee1513397fe2810756dfe7f3638752e45b6673d59d75f7a4ed44/488",
"xr_browser_tests": "02b0786579394e04cc15ead81d2af027dddd7ba663ba6bb6952381c711f6dec2/646",
"zlib_unittests": "589ce20dc7043fe507478e9d2e1e3b94ca95da3fa7a117f7258b563ab714d341/489"
},
"swarming_command_lines_cwd": "out/2734-Linux_ASan_LSan",
"swarming_command_lines_digest": "2028f3d4b3066efda85facc8fe5391766d5423d7d1d5939f0b815621bd0d9820/94",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "7fb257ccdc15a2a3b27dc7f6be3dafdf5e13b741",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8864520968933156592"
},
{
"key": "scheduler_job_id",
"value": "chromium/Linux ASan LSan Tests (1)"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Linux ASan LSan Builder"
},
{
"key": "parent_buildnumber",
"value": "177844"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[23:04:11.597] Scheduled build:
{
"id": "8699090903808669393",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux ASan LSan Tests (1)"
},
"number": 137237,
"createdBy": "project:chromium",
"createTime": "2025-11-04T23:04:11.162557875Z",
"updateTime": "2025-11-04T23:04:11.162557875Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "7fb257ccdc15a2a3b27dc7f6be3dafdf5e13b741",
"ref": "refs/heads/main"
}
}
}
[23:04:11.597] Task URL: https://cr-buildbucket.appspot.com/build/8699090903808669393
[23:04:11.597] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:2:0) after 9m46s
[23:04:14.197] Received PubSub notification, asking Buildbucket for the build status
[23:04:14.222] Build status: SCHEDULED
[23:04:50.613] Received PubSub notification, asking Buildbucket for the build status
[23:04:50.626] Build status: STARTED
[23:13:57.720] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:2:0)
[23:13:57.744] Build status: STARTED
[23:13:57.744] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:5:0) after 9m8s
[23:23:05.717] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:5:0)
[23:23:05.733] Build status: STARTED
[23:23:05.733] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:6:0) after 1m43s
[23:24:48.752] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:6:0)
[23:24:48.789] Build status: STARTED
[23:24:48.789] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:7:0) after 9m41s
[23:34:29.807] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:7:0)
[23:34:29.825] Build status: STARTED
[23:34:29.825] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:8:0) after 3m47s
[23:38:16.893] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:8:0)
[23:38:16.906] Build status: STARTED
[23:38:16.906] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:9:0) after 3m54s
[23:42:11.021] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:9:0)
[23:42:11.052] Build status: STARTED
[23:42:11.052] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:10:0) after 9m52s
[23:52:03.134] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:10:0)
[23:52:03.156] Build status: STARTED
[23:52:03.156] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:11:0) after 4m9s
[23:56:12.130] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:11:0)
[23:56:12.144] Build status: STARTED
[23:56:12.144] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:12:0) after 8m11s
[00:04:23.163] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:12:0)
[00:04:23.185] Build status: STARTED
[00:04:23.185] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:13:0) after 4m8s
[00:08:31.162] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:13:0)
[00:08:31.209] Build status: STARTED
[00:08:31.209] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:14:0) after 6m9s
[00:14:40.274] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:14:0)
[00:14:40.291] Build status: STARTED
[00:14:40.291] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:15:0) after 4m36s
[00:19:16.313] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:15:0)
[00:19:16.330] Build status: STARTED
[00:19:16.331] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:16:0) after 9m31s
[00:28:47.489] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:16:0)
[00:28:47.513] Build status: STARTED
[00:28:47.513] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:17:0) after 8m35s
[00:37:22.532] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:17:0)
[00:37:22.551] Build status: STARTED
[00:37:22.551] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:18:0) after 2m52s
[00:40:14.703] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:18:0)
[00:40:14.717] Build status: STARTED
[00:40:14.717] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:19:0) after 4m34s
[00:44:48.736] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:19:0)
[00:44:48.753] Build status: STARTED
[00:44:48.753] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:20:0) after 6m57s
[00:51:45.827] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:20:0)
[00:51:45.845] Build status: STARTED
[00:51:45.845] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:21:0) after 3m40s
[00:55:25.943] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:21:0)
[00:55:25.968] Build status: STARTED
[00:55:25.968] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:22:0) after 9m13s
[01:04:38.985] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:22:0)
[01:04:38.999] Build status: STARTED
[01:04:38.999] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:23:0) after 4m55s
[01:09:34.090] Handling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:23:0)
[01:09:34.108] Build status: STARTED
[01:09:34.108] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ASan LSan Tests (1):8864520968933156592:24:0) after 5m30s
[01:11:05.135] Received PubSub notification, asking Buildbucket for the build status
[01:11:05.153] Build:
{
"id": "8699090903808669393",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux ASan LSan Tests (1)"
},
"number": 137237,
"createdBy": "project:chromium",
"createTime": "2025-11-04T23:04:11.162557875Z",
"startTime": "2025-11-04T23:04:50.329190161Z",
"endTime": "2025-11-05T01:11:04.504080216Z",
"updateTime": "2025-11-05T01:11:04.504080216Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "7fb257ccdc15a2a3b27dc7f6be3dafdf5e13b741",
"ref": "refs/heads/main"
}
}
}
[01:11:05.153] Invocation finished in 2h6m58.26227944s with status FAILED