[21:19:52.769] New invocation is queued and will start shortly
[21:19:52.769] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[21:19:54.338] Starting the invocation (attempt 1)
[21:19:54.368] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/e9967da5cf52bcfad4e086df23749e7a053e8cc5"
[21:19:54.368] Popped gitiles commit info from properties and tags
[21:19:54.368] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[21:19:54.369] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[21:19:54.369] Buildbucket request:
{
"requestId": "8843780821603097584",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux Chromium OS ASan LSan Tests (1)"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "e9967da5cf52bcfad4e086df23749e7a053e8cc5",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "e9967da5cf52bcfad4e086df23749e7a053e8cc5"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8678352744201980849"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8678352744201980849"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8678352744201980849"
}
],
"parent_build_id": "8678352744201980849",
"parent_builder_group": "chromium.memory",
"parent_buildername": "Linux Chromium OS ASan LSan Builder",
"parent_got_angle_revision": "28262f144abb132a2ba06f39de48410d6547c12a",
"parent_got_dawn_revision": "e247c6101ae17400e803eae1717e8500677e8cfc",
"parent_got_revision": "e9967da5cf52bcfad4e086df23749e7a053e8cc5",
"parent_got_revision_cp": "refs/heads/main@{#1650086}",
"parent_got_swiftshader_revision": "5b0479bd2d15058aaa9eb490e364f920ff824a8c",
"parent_got_v8_revision": "9e6390af87fb05e2efcbf36f4c4942f6a33958f2",
"parent_got_v8_revision_cp": "refs/heads/15.1.159@{#1}",
"parent_got_webrtc_revision": "a5ddff6086d96fd2356ad6c521525ea2803f7988",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#48050}",
"swarm_hashes": {
"absl_hardening_tests": "ae11f8458196cae477febfb161112056a31cb6bc1fa5a0976ca47783120163a9/488",
"accessibility_unittests": "55f8d440a32b89341f62a99aad22393a9714297fd67400b299963174643109ef/488",
"angle_unittests": "94bd9a608b724c726f5ebe0fe81a99ad4a88d866d6525391e9b1563bc581dca5/489",
"ash_components_unittests": "0ba6d3fec4863101539cc3840c53cfce5f66d549cab9add14efcab7b8105fb45/649",
"ash_unittests": "1d810f0603a2a009eb159583462f009b14b8b6ea7ee8ac0b4fffcc1e7fe62a59/800",
"ash_webui_unittests": "0f354905cb834ebea2528e3493eb8a1d30123e80201ee34650338ca4acadca25/648",
"aura_unittests": "fa79400cdcbabafae367ba462d8628e58ce6667db9949404e8809b287d8ce95e/488",
"base_unittests": "2985ef65657d4380d6e8b104735bca6e5adc5a34b8bc8c94dece2961888a20b9/566",
"blink_common_unittests": "3d999b86575be92d0a47096fd8fa8cce2db291409c94fca115e5726307887867/566",
"blink_fuzzer_unittests": "11be00cb10e9030d9ddc9a3f9478f0d8c3c9dd90fcee8639e4cd362b5029a7d3/488",
"blink_heap_unittests": "b220b82e8d1bb5c2508799330fa6c0b2f9b07f598f9a4e6cc33c6d13477262f2/488",
"blink_platform_unittests": "36d942d659cc4b3291672b5559e9fd443c7de0aecd43c641da7964e707dc91ac/645",
"blink_unittests": "4ee5112fb2a9b8b061dc107b1799f790e156b760b4c030275077231e11a10a4a/729",
"boringssl_crypto_tests": "7c89a311a7e3d2d770e9f80443732a75bed82c4243d64b820c8a0af064754317/489",
"boringssl_ssl_tests": "9f160e64774a952d1ebff05a32808eae61d9eedfd16de4059de24bb4a6361b75/488",
"browser_tests": "e4e83f481fac854749eedf4c2170848f41589ff9ee13a16828fbaf4aa43fdf99/1299",
"capture_unittests": "c303d661cbffc7491583c9c456bc106f52916cf44b2cb5ffa7ed8e29d100a21e/567",
"cast_unittests": "68d73ee8ee537a46946146f4feef6da18b52b0eaff98b73e1dbc9a58189d2f39/488",
"cc_unittests": "fb4635740539aeecc5ec53d8b4d598e3cc677943b895bc92678f069e1fbab8ed/648",
"chrome_app_unittests": "974708c27d0eda2992c76ff861909b93389450204b44d4c8c23207cd2f1d2d74/648",
"chromedriver_unittests": "fe742988df0c73d0bdb9a3d7217217e7387fb04ce274ada30115b8b068afb5d4/645",
"chromeos_components_unittests": "6a265b6ad5e949a6ebde41afacf081e01f60250583a0c751debe30290bbfe6f9/647",
"chromeos_unittests": "7cecad9743642c838fbabd7c254d7a2e8f88597f0f6265b1d8cb72113135f765/649",
"components_browsertests": "311180e820333e9255e9bd2360f7369778ea966c7c8fafc008a02c5fbe57a390/732",
"components_unittests": "65b5a1959f8e75ffec8a9ab0768e83a39335d54223b4f32e81b75adc692e237a/894",
"compositor_unittests": "5d26e6c8a269d1b30911a899d9c912abaced61f3ac0ad83132b41a00a37ea755/564",
"content_browsertests": "de3896237c2e6f7265931473f0a8baad4e5f81542497a571a70097630c030736/805",
"content_unittests": "ce8d05341904b0461368b88ed888403687b3180f44f4d0f224ae1a350fdff5cf/725",
"crashpad_tests": "b394ea85ba569b529ee2230797191d79cbdf9687df354f29e2b903039ffb3293/489",
"crypto_unittests": "da3b29e8e017337387cda464fc73236e34c34506d903677035fef3114fd4a781/568",
"dbus_unittests": "8060e34b9f40cf2cdd6837e9c151a98bc506466d44aa77996b6abe7bf7eb3d2f/488",
"device_unittests": "73f1915cd5fc431af66b490e9d636d6c9f1bdff55c72271e4455c6db0d95c27e/565",
"display_unittests": "2596ea17952d78b885888c126ef585afb32ff37699be55ce4b539b6675e96610/488",
"env_chromium_unittests": "b27f62a217d80058ef1ea6d88d8506a26afb3d1cfda255051333086846b9e2e9/488",
"events_unittests": "9fd4f98e130f58c79a5bad06af0a8bf8f5c97ea4253ce38218f0db3017897b5a/488",
"exo_unittests": "b0fcee3e3681e7d011fb5d397c7495a9b26ae4b235cba88a3903f08e8057f4bf/647",
"extensions_unittests": "0f6224802eacb80e73a10af0f0e23d568b341d373d51aab7749d9662a335756f/812",
"filesystem_service_unittests": "1d4d156cfd34cee170e1f82e3cdfbb2bcc2b5353d2bd6f00ee546bf94b0af203/488",
"fuzzing_unittests": "a4e67aee8aceb46b142f56841948b229362560edbb049c1e6d2722200c18e074/488",
"gcm_unit_tests": "ef432cda6a5806cc0b7f3dba13bfcaa1073f2bf1f8e7f8f029eba30f1da17b76/565",
"gfx_unittests": "984e0e72e749473ea79550cbb5e5027561ab1dd227f18233eaceeca1969b389d/564",
"gl_unittests_ozone": "adee0dbe6a5e3da9f1743615616ad4d8e876312bbbc81e2ff4f4248bad6b0e43/488",
"google_apis_unittests": "12d0a7d7dcdd94301862a168e051f75c611e28bb435730d402451585c9cc7581/650",
"gpu_unittests": "0a17de851d26e90eac223353a9a3c150551bd5f788dfdde3d9ecf40b36b87cb9/565",
"gwp_asan_unittests": "10f22b47f30710cd90ba22be85de670894a75f7c8bd774e2f13d1081659ead04/489",
"interactive_ui_tests": "cb8615736e1e778a7c43e99811930c267b61826c2c3ba620f15eebed8998639e/1142",
"ipc_tests": "b4232856e2a9e3420d3b6d473983c92806d9be875eea702a34b2cf63f354d726/488",
"keyboard_unittests": "32b730ae81b6bcf7e7b3ae9825595fa7deb66e0a2797bc3480ec8e88a1ba1e67/488",
"latency_unittests": "5fdf714edc819133b7a7425bd747c243bcd79054ffcaa27e63631a480938abbe/488",
"leveldb_unittests": "cb5dc959ec7b47c4c3e42b7327b48295e70638e12a6407e1e1d3360e1261cba1/488",
"libjingle_xmpp_unittests": "e36a2c6ecc0449a9a5ace383d56156e91fb226e0f46175f526e2edd3d256566d/488",
"liburlpattern_unittests": "107828f3e45d2c8405eb460d6d372daacc27542a04d0a17611ea76460a3e1d24/488",
"media_unittests": "b2dbd4041fcc6e1fc1da4abf1817ce37d58c5e5975a0668e565965c70afc190d/568",
"message_center_unittests": "0beae9122a6b2ad5b0917ef96f0be3d546b48b51f65afcb2dd5fa415f1f26421/488",
"midi_unittests": "afee70d9bbd6fd5b8118ed328c534e2c4f4154d8bbe62740fc7c1cf49989c7dc/488",
"mojo_unittests": "03d075ce0d857c9a47a0604252cac5c9ea5c31a9d7a6958bf1ead87b1536d835/566",
"native_theme_unittests": "9819a782a3b7fb3bf588653982783e3240f51b9d70daceeb74272f0b532367e2/488",
"net_unittests": "fa6f0a085135eb3f0b5d0adf18cd37a554eeb127ce9716b302f68b792b169536/567",
"ozone_gl_unittests": "184cf965d2a6c383235565d43aa809bd44e8e0cc8840d5a8f838b1263af17556/488",
"ozone_unittests": "33aa4fd356c282ca742a57fc45895708187da5062c5b2f56af72b01c78fd9b77/488",
"ozone_x11_unittests": "0fef0fb7749b810bfe0463b75ef148c6d3f71abecdefc30d9e253c3ae4ecdb3c/488",
"pdf_unittests": "02e25337a0ff9ab6fc8de9bf3f4ebe7cc92bccffc7f533a785632d03ab9722e7/565",
"perfetto_unittests": "06aa244c38305bd7993117453a9ab3a62b95992958f97ba3d34c48103ce6ec14/488",
"printing_unittests": "b50588f71db75d617245fd8150c07cca8450de05e54962f7de2419eca3a969fa/488",
"pthreadpool_unittests": "060dbedcf358833009527573d44e9b924ab6c284174fd6e4d31c4bee675cc232/488",
"puffin_unittests": "68dff237aa71f3744f88d1ed9aae0e94a8b86f629e9d8cffb429b9bf58111beb/572",
"remoting_unittests": "50f40915dc6754806e4bf6560f23f634f8d6c333575d3126fdef270133ce53d2/647",
"sandbox_linux_unittests": "e44d90c19e637a84f41df8a072029190e6a3362a958ff7220b50e339580837c2/488",
"services_unittests": "31375f319f8cfbabde5b96275844f5cb82c302f70bca9630b93c3e0f32947ba3/895",
"shell_dialogs_unittests": "c7c47520965fe33f062b7d39f9c494a99bd5f0238963069c50d12a08233e0432/488",
"shell_encryption_unittests": "a8eeba0b736d6bd99b5de0b591de700d3f9c199ab779e04ef8ebf5123bd1b642/488",
"skia_unittests": "03e6085fecc99261c8d11e983db600cce269fc16acac1e71c85bc190845d55e4/488",
"snapshot_unittests": "4594c57e1622c857fa4bd9fcc5af38c682698590f5a4092011fa8079a0e3a6d5/488",
"sql_unittests": "ccf898e23581ce339eb20373d32d183b223974abfb0880bcbfd6d1389d49582c/565",
"storage_unittests": "4f4d43be9576a05f5f15ced57999d3b6535e999bd8d9963021d0c06994a80769/646",
"sync_integration_tests": "73b28c38a773efadd62102270487dcd0710b572c6b8e41245471f0c1605e70e8/730",
"ui_base_unittests": "2a9a9842d21c8c744175744c740b778a04ce9b20252aa12e3e9cda8ccd9e72f8/641",
"ui_chromeos_unittests": "97254d8c8a0b09e1aa1b50529eacc79a7f4f93d7cbea2309e3d38cb1df19f5ce/488",
"ui_touch_selection_unittests": "6c1d6d8a7cc6eae85e18c92ffce669a963ed7dcc181a6d3239972396b7539bce/488",
"ui_unittests": "30e8e083380cd00258e41a737429f53d2dd369789c401ae4b35e52dfaf887be6/488",
"unit_tests": "d9e9a4a5df1f69b70d3033e845e3e34e27555337b23ca9e52cca1cdd0f33e96c/1062",
"url_unittests": "22b028ca84b97a78f3375d42a0e0f8f2aaf69e6a31346daa319b49377b386ffd/488",
"usage_time_limit_unittests": "02b2a9d2d5b4e68cd7c55df87045c95c2adcb363d8f5b405a88fdd4ecbfb40da/568",
"views_unittests": "4e2543f203e0da2dcf009efe515764fc101644d06de7c24ba37075c52cf22501/488",
"viz_unittests": "4e363137e05f0ab6a72824beb140e7a7e8097e1591ce2df3dc960260aa69569b/572",
"wayland_client_perftests": "ffd5b2bd6a01bf38ec3ca3df247b386a452a23afb005616fa2a7605ce2bd4e19/647",
"wayland_client_tests": "ea52ec1763943cffa4233b2737578a29dfd17aaf7b9a151cac8af16e8482191e/647",
"wm_unittests": "3931f57a745fb5e3c1829a5fa389e4f67746f661d526d959bf7c0b05326b753a/488",
"wtf_unittests": "6849804963cce7d6fda481718ee42be0b0d614975a260474af5f27b5fe7de8a5/488",
"zlib_unittests": "d1b6669d9014db5e11afe39f03242e58aab16aa9cdb697455ad6fcc143dc4c85/489"
},
"swarming_command_lines_cwd": "out/8916-Linux_Chromium_",
"swarming_command_lines_digest": "4cd1da415ab443858f55853479ab3587949e9d55deaf53d8f41b055d9e00a405/94",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "e9967da5cf52bcfad4e086df23749e7a053e8cc5",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8843780821603097584"
},
{
"key": "scheduler_job_id",
"value": "chromium/Linux Chromium OS ASan LSan Tests (1)"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Linux Chromium OS ASan LSan Builder"
},
{
"key": "parent_buildnumber",
"value": "127580"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[21:19:54.898] Scheduled build:
{
"id": "8678350759141456609",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux Chromium OS ASan LSan Tests (1)"
},
"number": 69900,
"createdBy": "project:chromium",
"createTime": "2026-06-21T21:19:54.450311902Z",
"updateTime": "2026-06-21T21:19:54.450311902Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "e9967da5cf52bcfad4e086df23749e7a053e8cc5",
"ref": "refs/heads/main"
}
},
"canary": true
}
[21:19:54.898] Task URL: https://cr-buildbucket.appspot.com/build/8678350759141456609
[21:19:54.898] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:2:0) after 1m59s
[21:19:55.562] Received PubSub notification, asking Buildbucket for the build status
[21:19:55.599] Build status: SCHEDULED
[21:20:47.790] Received PubSub notification, asking Buildbucket for the build status
[21:20:47.808] Build status: STARTED
[21:21:53.920] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:2:0)
[21:21:53.938] Build status: STARTED
[21:21:53.938] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:5:0) after 3m7s
[21:25:00.969] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:5:0)
[21:25:01.022] Build status: STARTED
[21:25:01.022] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:6:0) after 3m28s
[21:28:29.088] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:6:0)
[21:28:29.108] Build status: STARTED
[21:28:29.108] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:7:0) after 9m13s
[21:37:42.129] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:7:0)
[21:37:42.160] Build status: STARTED
[21:37:42.160] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:8:0) after 5m59s
[21:43:41.212] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:8:0)
[21:43:41.293] Build status: STARTED
[21:43:41.293] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:9:0) after 8m7s
[21:51:48.315] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:9:0)
[21:51:48.333] Build status: STARTED
[21:51:48.333] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:10:0) after 4m21s
[21:56:09.386] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:10:0)
[21:56:09.417] Build status: STARTED
[21:56:09.417] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:11:0) after 8m37s
[22:04:46.441] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:11:0)
[22:04:46.483] Build status: STARTED
[22:04:46.483] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:12:0) after 8m22s
[22:13:08.605] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:12:0)
[22:13:08.620] Build status: STARTED
[22:13:08.620] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:13:0) after 7m12s
[22:20:20.718] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:13:0)
[22:20:20.761] Build status: STARTED
[22:20:20.761] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:14:0) after 1m17s
[22:21:37.785] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:14:0)
[22:21:37.801] Build status: STARTED
[22:21:37.801] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:15:0) after 1m31s
[22:23:08.923] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:15:0)
[22:23:08.993] Build status: STARTED
[22:23:08.993] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:16:0) after 2m59s
[22:26:07.969] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:16:0)
[22:26:08.005] Build status: STARTED
[22:26:08.005] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:17:0) after 8m38s
[22:34:46.218] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:17:0)
[22:34:46.236] Build status: STARTED
[22:34:46.236] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:18:0) after 3m53s
[22:38:39.371] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:18:0)
[22:38:39.463] Build status: STARTED
[22:38:39.463] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:19:0) after 7m51s
[22:46:30.483] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:19:0)
[22:46:30.503] Build status: STARTED
[22:46:30.503] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:20:0) after 9m10s
[22:55:40.638] Handling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:20:0)
[22:55:40.655] Build status: STARTED
[22:55:40.655] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Chromium OS ASan LSan Tests (1):8843780821603097584:21:0) after 9m18s
[23:02:44.482] Received PubSub notification, asking Buildbucket for the build status
[23:02:44.542] Build:
{
"id": "8678350759141456609",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux Chromium OS ASan LSan Tests (1)"
},
"number": 69900,
"createdBy": "project:chromium",
"createTime": "2026-06-21T21:19:54.450311902Z",
"startTime": "2026-06-21T21:20:47.631889402Z",
"endTime": "2026-06-21T23:02:43.626592207Z",
"updateTime": "2026-06-21T23:02:43.626592207Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "e9967da5cf52bcfad4e086df23749e7a053e8cc5",
"ref": "refs/heads/main"
}
},
"canary": true
}
[23:02:44.542] Invocation finished in 1h42m51.82445527s with status FAILED