[18:59:32.318] New invocation is queued and will start shortly
[18:59:32.318] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[18:59:33.557] Starting the invocation (attempt 1)
[18:59:33.589] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/626d3f74ebe96e155bd32285d18a216df7151053"
[18:59:33.589] Popped gitiles commit info from properties and tags
[18:59:33.589] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[18:59:33.589] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[18:59:33.590] Buildbucket request:
{
"requestId": "8842340099577121904",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux MSan Tests"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "626d3f74ebe96e155bd32285d18a216df7151053",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "626d3f74ebe96e155bd32285d18a216df7151053"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8676913167958441793"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8676913167958441793"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8676913167958441793"
}
],
"parent_build_id": "8676913167958441793",
"parent_builder_group": "chromium.memory",
"parent_buildername": "Linux MSan Builder",
"parent_got_angle_revision": "4e0fcf14911c55b9f304c0cb36c7d6bf76a3a7b2",
"parent_got_dawn_revision": "71b50302352d0e8c072d79cae8860620e5bc3f87",
"parent_got_revision": "626d3f74ebe96e155bd32285d18a216df7151053",
"parent_got_revision_cp": "refs/heads/main@{#1658108}",
"parent_got_swiftshader_revision": "5b0479bd2d15058aaa9eb490e364f920ff824a8c",
"parent_got_v8_revision": "47f4c771e4710b5123b43abc416b610ef29b2b23",
"parent_got_v8_revision_cp": "refs/heads/15.2.38@{#1}",
"parent_got_webrtc_revision": "ba2543d33c88a5df970378f7e49e84c637a26272",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#48165}",
"swarm_hashes": {
"absl_hardening_tests": "cbd15bab169c04eb4f4028bd08d13c708c9a43d159ada245d58c3bbaa5752391/489",
"accessibility_unittests": "23c1a115d33d39953ccedc6750a5b2300e7754db0152996a809b254be35af532/489",
"angle_unittests": "01efc17dea7c77b4114d58341eaf48197a45b3f95568202538ba7bf80b751115/489",
"aura_unittests": "31f248a88a4c831cd3e1979665a9f463f316cea7bbcc67f615aa8366f55a928e/489",
"base_unittests": "b214340a14e58f733ba52b691fba443fe6f9eba755e1ea8b270f6efd6442a662/567",
"blink_common_unittests": "600e659f3bbb663e4a345d03091a3e9ce0f54b6903dd9aca341b907bcf5e6f74/566",
"blink_fuzzer_unittests": "6a3cfd4e4d77493f3c1afc201a4e93a0bc33c8a0e823f19c7367dbff33351ccf/489",
"blink_heap_unittests": "085314a60bd0a2eab54070877325e10a5009a67783b02019357242783ec5a34e/489",
"blink_platform_unittests": "1abbede25f20ff56b0148519035fae0a63281ba6a19b11b7d1bf382f1316a177/645",
"blink_unittests": "9eaba6b66409f8490f8e024ab2f3a8b0af60de34711f9bc74dc2f431eddc0d2a/729",
"boringssl_crypto_tests": "07d53996dee2aad4ca6960e2ee8061b3af91223a7ef2fd5a2dbe85f8cefbac22/489",
"boringssl_ssl_tests": "ae34e04019ea79f92f1c6036fbf7bdfd7243e80a1ceba17ac5a310a2494d0152/489",
"browser_tests": "003bab8e24dd4e9e1bfcb6dfbb073f7c1dabfb9e0e765a9c3f8b39975e26ba64/1220",
"capture_unittests": "1de11c24b0aa6313f5037d8c4569be8ac444664de48ab6ad55daef4481ea43ee/568",
"cast_unittests": "50221f4b2b571ee4d9c27b5d55a635c2983031751505df0257daf0b9393d65e0/489",
"cc_unittests": "4015425b52bdc80f6bcc0a9191298780b6c0d7186184ce279bb2aab8c695dda5/649",
"chrome_app_unittests": "f0c513a74381bb26d82875e61a5d1ed3bcc0314dfef50d4e5822d4fe51c1e3aa/566",
"chromedriver_unittests": "304bb166c9c1996ed274c3edd86772382186f11be02c0e2ce9b0af02daac49c1/646",
"components_browsertests": "41bc36a4855892bd26cebff86e03c4c733b10695f9f217c6c2f8f0a405d8e239/732",
"components_unittests": "05333ba36cdceb845cfe0dc3307a38a20d6c36d11ab46a86387107ce534d6e3b/811",
"compositor_unittests": "2d8de40d4cf578a4b3971027ee3823f9e69ec8c0402ac9f264599d3d8400ce61/565",
"content_browsertests": "5052c2ae4e8550a3d6dfe0f0a2bcbd885845a7d251baae30000637073cb7e0bf/805",
"content_unittests": "43f1723f75a57d08dcfcf5a4a34d3d622d7dfc20f7572e94a383ac0efa1108b9/726",
"crashpad_tests": "5bf92b6a2da4063dfdb58d39eff1fd7aeb24ad528dad86c3052dc65adb4e162b/489",
"crypto_unittests": "523f06653de785494fa970cfd308e4b18f06d2f1274631fc8d132db0992aaaaa/569",
"dbus_unittests": "2766353fa2aea82236a55c88e0e70645402a809aaec4ed436dda5f12772267df/489",
"device_unittests": "280628519dd1010fc5e44a1a1319250441be5e776d7ecdd66ad22706edd58b82/566",
"display_unittests": "1d8bd51279de1a4663a2b5ae5951fe2ec0f4549e929b31206427fa98bcd21b31/489",
"env_chromium_unittests": "cb71ed6e1fd9f8c821ec101dfabeb3ef771d452a35cc2a49151d5796100a8e50/489",
"events_unittests": "afe006ad85c08d5f3e2503878e56ff02343e942a0f0ddde9dae073a283941d7d/489",
"extensions_unittests": "238f66e08ee5f1b59b55cf467f54cc6f8854f0be6707b147a7029279b99f45bf/731",
"filesystem_service_unittests": "b8f879b4039851a575dd62e875025eb3769d3c35685b5e8270d9599cb37b841b/489",
"fuzzing_unittests": "7e179f248c7fe2ba01e12895618a1ff16239735e3e15c0fd469ea2d85899dc36/489",
"gcm_unit_tests": "5ffee3e422b8bb60a5382fb7c7a256795e8b48dfb810e2b8bb246b36ba490a94/566",
"gfx_unittests": "d98b74f7fdc6bd0d86b26b3d4f5fb5757b68a01915b8f5497ce511b4e96a82de/565",
"gin_unittests": "70f06efa79ce77b9b6b52cdd51c3769c972c8d3b4ceb43f613b207a524b6f10f/566",
"google_apis_unittests": "00473a7a3f842757fb9af0d5b313613d6ac5cf0f907f6e726570c70e11f8301a/651",
"gpu_unittests": "bea3e53e9782b2ef1131df39e01a38a5e25476d9204b033a64cfffd4c5a0fbb1/566",
"gtk_unittests": "d647e6a530c6c42f250751411a38a0dfaf848f8a5d200e0346ef1ebab7c91043/489",
"gwp_asan_unittests": "166c2c2df17aa99a1dd1d0e97b928d09e7f6998ce7ebfb5d584e7be1c9621176/489",
"headless_browsertests": "5379d497ef37130af6848bc228ec8e9fe6df5362717f6be3a1eed2e267f96a41/733",
"headless_unittests": "904a4fdb97462dfd62a4276a790727fb3dcfed2833b8f6a6486c6326b962b518/489",
"interactive_ui_tests": "81ee28d3480e4db0423dbb9bcae2e8236fd2ba31e0cfd5546ee53d648af3cb03/1059",
"ipc_tests": "c4bca7eb802016a7898e754da888a535121ce0a70a8db899e294841167b5133d/489",
"latency_unittests": "fa9934514a4601e00cbba732196db1f7f47925c32f2f9159585174cc276e488d/489",
"leveldb_unittests": "a3d0da1c5d3208cee6775ab3124dba54d6b987c32add46d5739da863f7ae3bdb/489",
"libjingle_xmpp_unittests": "8fbae08d728da2904f4c52d807a90a62df07d60dcd5cd3dc58f76a2fbd08d05d/489",
"liburlpattern_unittests": "104ea78a2db7e5d26adbe863a5818b1bd494782297d4eab843ab496a692cb11f/489",
"media_unittests": "3b5a2f2cdf930230e5e23b88bda8b2c25b81354a734fb1354d4a887b67f19d87/569",
"message_center_unittests": "384307215d2136001c93e844fc5157eb2654f93859077f5e394dc4a6adc308c4/489",
"midi_unittests": "ab220aaae1c0427958bbfc5ddd974bf2f0c0ce7f0873b1c304b4bbde0ea54468/489",
"mojo_unittests": "9e8bc2189d19c36a932e0cfb1bc15bc6d7f3023d7c5372054bc531129bf4ba15/567",
"native_theme_unittests": "67aa7f4d5505f7b62c09342d446ed0d7af648d053ba519c17a1fa54e397f7183/489",
"net_unittests": "2aec39fce40024b7159379010fb727c96f33746e7daf5d379088ebb0b4ab92d8/567",
"openscreen_unittests": "2ecb143fefb9ae170f608619db0fd7fb1c9385524ec703b9333aebe7503c3db7/489",
"ozone_unittests": "b780eb0b67e6d2e73e313ec789649967b4023fc2d8fa2a85c357974d0fe3aaf6/489",
"ozone_x11_unittests": "cb094ebeeaeb4c8bfcc928ec736c698b4ff8fd16c8e66c0235b0fc497a81756c/489",
"pdf_unittests": "b72d0fdf3322010ed1cef3edf12533ce74f6c431cf50680659d3c616427fc01a/566",
"perfetto_unittests": "5b3e6d4eb51479db4019a487d3a46535450eaba29212c63e260ca748c952db57/489",
"printing_unittests": "637e18e40be768f7d98f47322fe2c58606b33eb4a1fb899ebbd10c7448814d08/489",
"pthreadpool_unittests": "5865409a377ac1b4d7ac19ae65e0b5d9652493a72d959d327fa9c652c905d6d2/489",
"puffin_unittests": "64903625697e981c1c7f4be47ec02e0cacf1da28ac747c18dee981425e6d129d/573",
"remoting_unittests": "e90b50bd0656cbcc7b13152b4f308184cee9c83bd23e34ccebe91209facb7966/566",
"sandbox_linux_unittests": "8ef0f5058c90ddcf1e6587ec5b78b25c1e21334867c15b60eb8403cb9666781b/489",
"shell_dialogs_unittests": "de809ab99a8b14f072a092b60887320c697f50257930304a2c6454048c900adf/489",
"skia_unittests": "e911178670d2c94e4e0e280d4f639a6fe422096236ada3eda938eac7b5e8070b/489",
"snapshot_unittests": "4609eca59145079d14ce136508288150048ed124998be06240b2f8a3967f5115/489",
"sql_unittests": "8f10e602179e2391eb039386cb2034666bdf253132eb601a47f86c45b7cc31c3/566",
"storage_unittests": "1d96d4dfe135f7c1c96fac5ec189eae7baf142cba912398e984a5c48c09713b3/647",
"sync_integration_tests": "8dbaa24bf97b7e0278af4362384c566da09114f6c2e2e81e0a7a6f43244f9fa4/647",
"ui_base_unittests": "0b167a35d56e5b6f1be7f1e5c54e7e875e18a1ac6cf2119caba58c989815b1fd/642",
"ui_touch_selection_unittests": "0137389aa90dfc240b77b18193507f1ac8241164cf04e69c86d6b26b1ee8246e/489",
"ui_unittests": "12135feffb0d245a92525e3142fa0aa62b2418230e382741672a5a19f04d22b7/489",
"unit_tests": "7ccdf6a1b279fc90e5112bc5577cae10bba735057b9f5214ab55b7e8cbf49ca3/1061",
"url_unittests": "ecad6a1880e896048cb753565cb4c282e4a7cf61793b254b4dd265cfd3a14612/489",
"views_unittests": "3150cff039261d9811c1a11078734a7d262a87aa9601bc62a11d46dd91acbafb/489",
"viz_unittests": "1da30c767490c0a8417ef8bc6016aa1bb089b2fee17a8d00d76489f64474f4a9/573",
"vr_common_unittests": "25a6f138b145899c5ffc4664a794d7ad27dba78be742ab3d75c49a1525c5ad70/566",
"wm_unittests": "24c7c9f42d4717cc1b58dc32d4293dbebf87924201477878cef31bef4be5046d/489",
"wtf_unittests": "e66a41f4548c28efd4fe7dd1063f6b14725ca29b503682c29b1a9fe389c0faea/489",
"xr_browser_tests": "516948aa139e54089dafc80956405efd062d4214fed3d22a67ab12b0bac5fa32/646",
"zlib_unittests": "ddf9356950ac1ba7653f3a3cbd68d6f17aa15baf0c8a64aca4e697a269f7f61c/489"
},
"swarming_command_lines_cwd": "out/1415-Linux_MSan_Buil",
"swarming_command_lines_digest": "ca9ec5df1ec6a68a8c4e68c791d44d0c8d6b00a2d8f50efae91433712e9f60f9/94"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "626d3f74ebe96e155bd32285d18a216df7151053",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8842340099577121904"
},
{
"key": "scheduler_job_id",
"value": "chromium/Linux MSan Tests"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Linux MSan Builder"
},
{
"key": "parent_buildnumber",
"value": "125011"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[18:59:34.169] Scheduled build:
{
"id": "8676910037504187025",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux MSan Tests"
},
"number": 64018,
"createdBy": "project:chromium",
"createTime": "2026-07-07T18:59:33.672014604Z",
"updateTime": "2026-07-07T18:59:33.672014604Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "626d3f74ebe96e155bd32285d18a216df7151053",
"ref": "refs/heads/main"
}
}
}
[18:59:34.169] Task URL: https://cr-buildbucket.appspot.com/build/8676910037504187025
[18:59:34.169] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:2:0) after 2m18s
[18:59:35.158] Received PubSub notification, asking Buildbucket for the build status
[18:59:35.179] Build status: SCHEDULED
[19:00:27.673] Received PubSub notification, asking Buildbucket for the build status
[19:00:27.691] Build status: STARTED
[19:01:52.237] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:2:0)
[19:01:52.259] Build status: STARTED
[19:01:52.259] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:5:0) after 7m12s
[19:09:04.387] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:5:0)
[19:09:04.411] Build status: STARTED
[19:09:04.411] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:6:0) after 2m17s
[19:11:21.529] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:6:0)
[19:11:21.590] Build status: STARTED
[19:11:21.590] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:7:0) after 4m13s
[19:15:34.663] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:7:0)
[19:15:34.684] Build status: STARTED
[19:15:34.684] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:8:0) after 1m54s
[19:17:28.702] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:8:0)
[19:17:28.720] Build status: STARTED
[19:17:28.720] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:9:0) after 5m14s
[19:22:42.766] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:9:0)
[19:22:42.783] Build status: STARTED
[19:22:42.783] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:10:0) after 6m3s
[19:28:45.833] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:10:0)
[19:28:45.850] Build status: STARTED
[19:28:45.850] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:11:0) after 5m7s
[19:33:52.993] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:11:0)
[19:33:53.021] Build status: STARTED
[19:33:53.021] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:12:0) after 3m33s
[19:37:26.040] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:12:0)
[19:37:26.070] Build status: STARTED
[19:37:26.070] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:13:0) after 6m47s
[19:44:13.108] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:13:0)
[19:44:13.128] Build status: STARTED
[19:44:13.128] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:14:0) after 3m50s
[19:48:03.231] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:14:0)
[19:48:03.254] Build status: STARTED
[19:48:03.254] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:15:0) after 7m25s
[19:55:28.294] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:15:0)
[19:55:28.372] Build status: STARTED
[19:55:28.372] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:16:0) after 5m29s
[20:00:57.434] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:16:0)
[20:00:57.485] Build status: STARTED
[20:00:57.485] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:17:0) after 8m11s
[20:09:08.539] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:17:0)
[20:09:08.562] Build status: STARTED
[20:09:08.562] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:18:0) after 9m50s
[20:18:58.602] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:18:0)
[20:18:58.626] Build status: STARTED
[20:18:58.626] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:19:0) after 7m26s
[20:26:24.746] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:19:0)
[20:26:24.759] Build status: STARTED
[20:26:24.759] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:20:0) after 7m21s
[20:33:45.767] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:20:0)
[20:33:45.787] Build status: STARTED
[20:33:45.787] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:21:0) after 3m24s
[20:37:09.806] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:21:0)
[20:37:09.832] Build status: STARTED
[20:37:09.832] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:22:0) after 5m31s
[20:42:40.853] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:22:0)
[20:42:40.885] Build status: STARTED
[20:42:40.885] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:23:0) after 3m51s
[20:46:31.904] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:23:0)
[20:46:31.925] Build status: STARTED
[20:46:31.925] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:24:0) after 9m13s
[20:55:44.956] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:24:0)
[20:55:44.974] Build status: STARTED
[20:55:44.974] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:25:0) after 2m16s
[20:58:01.114] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:25:0)
[20:58:01.139] Build status: STARTED
[20:58:01.139] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:26:0) after 2m49s
[21:00:50.163] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:26:0)
[21:00:50.178] Build status: STARTED
[21:00:50.178] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:27:0) after 7m30s
[21:08:20.270] Handling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:27:0)
[21:08:20.296] Build status: STARTED
[21:08:20.296] Scheduling timer "check-buildbucket-build-status" (chromium/Linux MSan Tests:8842340099577121904:28:0) after 3m0s
[21:09:56.782] Received PubSub notification, asking Buildbucket for the build status
[21:09:56.803] Build:
{
"id": "8676910037504187025",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux MSan Tests"
},
"number": 64018,
"createdBy": "project:chromium",
"createTime": "2026-07-07T18:59:33.672014604Z",
"startTime": "2026-07-07T19:00:24.313643344Z",
"endTime": "2026-07-07T21:09:56.399157450Z",
"updateTime": "2026-07-07T21:09:56.399157450Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "626d3f74ebe96e155bd32285d18a216df7151053",
"ref": "refs/heads/main"
}
}
}
[21:09:56.803] Invocation finished in 2h10m24.491287679s with status FAILED