[22:21:52.874] New invocation is queued and will start shortly
[22:21:52.874] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[22:21:54.061] Starting the invocation (attempt 1)
[22:21:54.097] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/e2fb4a2b118541818be9b8c53a0639438e079f3c"
[22:21:54.097] Popped gitiles commit info from properties and tags
[22:21:54.097] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[22:21:54.097] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[22:21:54.098] Buildbucket request:
{
"requestId": "8873764516631790688",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "mac15-tests-dbg"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "e2fb4a2b118541818be9b8c53a0639438e079f3c",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "e2fb4a2b118541818be9b8c53a0639438e079f3c"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8708336193279827745"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8708336193279827745"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8708336193279827745"
}
],
"parent_build_id": "8708336193279827745",
"parent_builder_group": "chromium.mac",
"parent_buildername": "Mac Builder (dbg)",
"parent_got_angle_revision": "e4007fff3335258e97e5e8b9569d8d698df820c9",
"parent_got_dawn_revision": "fb4d36177feeea4831be2d0ba5d845149ac4f105",
"parent_got_revision": "e2fb4a2b118541818be9b8c53a0639438e079f3c",
"parent_got_revision_cp": "refs/heads/main@{#1492305}",
"parent_got_swiftshader_revision": "8802b4ec130bc6f23f34ca0cda4690f3e5408aa3",
"parent_got_v8_revision": "5d83ccf80ddd9959325fde53ce0b9869e76505b8",
"parent_got_v8_revision_cp": "refs/heads/14.0.314@{#1}",
"parent_got_webrtc_revision": "10a1bbbf89dd5a037f99dcce418e0709604eb21a",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#45212}",
"swarm_hashes": {
"absl_hardening_tests": "d1762e42c1d2d88ad4fadf8b959213f7315617e6f67ae9cd014fb2487b7314e4/323",
"accessibility_unittests": "eed4c58d4848f71bc2fc601023f3d69fe6f5356c51c6b23053d7f3f70a967a4a/323",
"angle_unittests": "9ceb3d9fb3dcb4439af49baa1f561c3810556c015c3ac1b945f66e3e8bc6b4a1/408",
"app_shell_unittests": "6accf65f41cf46cf6c36dd51f5b26db013718e9b18606cef6fd5e242cd8af941/484",
"base_unittests": "729a5384d2b48606038cd80858217969542be85e47f287a16ebc637226da3399/480",
"blink_common_unittests": "328035590d57e4dffb0c5cb750b4647fe8cdf0ec0fdd8f1d3ce526bf3f7d140b/485",
"blink_fuzzer_unittests": "48bcc126585679aaaa543394e449b727af6bbc7f8944f72806c2d7c554f7b747/323",
"blink_heap_unittests": "b3d89a82f7c49b88bb05898d71a04daef7625d9ad0f32cf1b231444cd7f32606/323",
"blink_platform_unittests": "5afc39d858df8afd9cc61c543d72c6944a1af5500d47fbcc0206fba8052ab29e/643",
"blink_python_tests": "1afbee3af9dbbcec09bc42943f9608f48a9f4718419a0da61b45705fef06afa6/574",
"blink_unittests": "b8fe80623bf003a1b8b06e835fee4ff463e875cd79274f07720c70ea9b90e1b3/649",
"blink_web_tests": "00e2206f232362dcc78b82f3e21310f27afe06a3155c3c656ca057b1aa62591f/566",
"blink_wpt_tests": "7d9af1db0de9d97185b2dddea3f5307d4dc3a5863fe35dfdf1fd6d7b07a6f735/566",
"boringssl_crypto_tests": "fd993e9681e42f1e6d26a703582d79b9ced6a2574476c95edcf4501a269f84b2/408",
"boringssl_ssl_tests": "f7cf9ba18b36913e7b130ec95dbb6b9ce1c5c89259741448f909dfaa436af495/323",
"capture_unittests": "fffd8c2762191eea309eae7dc8e0671610d86218e6237199952976527dd3f9af/402",
"cast_unittests": "ebfa6ad84eb79ee748b4115cc22f418f393e46ff2e6b5d22b893fdccf7f8aafb/323",
"cc_unittests": "d2ac79e513bdd4e84dcb521ca02600c68a39348aeec82834427072bd3d3a5840/483",
"chrome_app_unittests": "5cacae05b6d3bd446a1e9ae6844bdc04cab1965ba7612dc638c0a6f10a338466/564",
"chrome_wpt_tests": "c051ca4dcbc4c749e0751c3ca294450a37740f435230a7443957793bab9e891a/566",
"chromedriver_unittests": "de62669cef5e1986facac86149ecfbe10b6abe0b75dcc0c502543022c522076c/480",
"components_browsertests": "203febc8fa054b3a2d148395b824c5339e6832cf7c9b62f141c823c3530735f0/652",
"components_unittests": "d42ccfd05be40f614ef06b17d8e1be6c19dcf42ea00769c4d24437ab86849278/810",
"content_browsertests": "6cf98f3d7b61957e10e75abe4ca2d27e60dad4541dd43af60dd50b1c887846b0/725",
"content_shell_crash_test": "e58d9f46a6d693bf1d152f6d2cfb85656e60c33c70ce1cfafaf584b140f16229/566",
"content_unittests": "a0644785a3f63b35aee062c7cecfdbc65058f142914c5323a341486dfcb3f842/560",
"crashpad_tests": "5eb6b5132ba30cb4efab8ec38598b2d6b72137d8cc7f52a95f375b580a7693be/408",
"cronet_tests": "dfecbff83a875d8ae12b16919a0a93a26d32bfec120ffd99210374757ab8ddf2/400",
"cronet_unittests": "8d4e23b3c37ee4bb3f97eb42e6e94fc07b88c7e24a53738823e63f5e22ca4a61/400",
"crypto_unittests": "8ab630bd8a55e89d11adf46188c520032eebcd1bea5433727c32ca1a2a95c58b/323",
"device_unittests": "6ff932c2037304374b184ea98693f95864653eebfae0fa329e748e91a1369233/400",
"display_unittests": "1e646bd541a04406118fe7f59a03504fe7a9b436016a918d52d15a379bc87bce/323",
"env_chromium_unittests": "92a349c21900c72fdfabfe0041d3ab6318f160062a193f6506585676a0177729/323",
"events_unittests": "398c93b42b39c3328ebf431c5ce6318a2efbb20b2502b20dc95a3417721a90c7/323",
"extensions_browsertests": "18810ea83a26778b0e8cbf3c006c487635eb567db1497f18d41409cf0408f6cb/570",
"extensions_unittests": "eb41c77b8e8f10c3723ead4871e8de1c1de8033d391b4bd4ce59a0892e918a53/644",
"filesystem_service_unittests": "fa0b58aefbad0516a9e8059ab89a93b52d9dbeab191dcfdbe431af1838c2dc21/323",
"flatbuffers_unittests": "cfa8e88644e5e18a6f17ab34ef1cec3da6874db9040175a9bca937c02e193de9/408",
"fuzzing_unittests": "ac9439c907a2f470fb59c412eb30c70d08609613f97631ea324c4972cb6e8acd/323",
"gcm_unit_tests": "3b44436db809e27dc387d431734c56e66e23d0180184636dd05bc24bcf00dff2/400",
"gfx_unittests": "1e00b41194c0b889f5a33782a84be43741374f6d6e819b0c191dd3f81792a40f/399",
"gin_unittests": "3be0899293ab1d40449fab179d5810c1597a32db98c4749c2272c2a146519314/400",
"google_apis_unittests": "2ab37d9ddc1dfe392fececfef48d75582d46d5c9360dd114bb6a35bd858ef2b7/485",
"gpu_unittests": "0076481b70ab54af49997ab94ff36001f6ed0b2bc3fa0363b0444a9b1ef03d7b/400",
"grit_python_unittests": "c8786863a115ecab444302fc5dba46ec3026d7a798994ac08efadca2a5c8ddd3/411",
"gwp_asan_unittests": "6aff559b58430e42b006b7f3238295ec4261bfcea2ae0f5dec69cfd0c39d8fbc/408",
"headless_browsertests": "71cc4259792a4967c09bc423ad75903de5ed3726c37f1f6ed12120b61a0e32d7/653",
"headless_shell_wpt": "31938b55becb6536ab8a2d5f74d2552a3ef0759d70dd5c7cb51be08ef444e801/566",
"headless_unittests": "f83a6e9683e342aee4d73a67d5c24ea531035fd5925f817d2c0accde0fce5994/323",
"interactive_ui_tests": "b5c0504f0a65b477595a7b466a5ab21560872b895baae1a37a8eee632ca78ce0/974",
"ipc_tests": "63d66b1c60034dec991aa2dff27ef77133747866a09a3223d3ec940e1cf95590/323",
"latency_unittests": "8226e9cc8ec0e3f867ea757cf1bcae0870aba23da0e5c2169acd41b8a061f320/323",
"leveldb_unittests": "d7d3a7751acf47f0bafdcf4a43fa012b7a06baaa0ed56b946ed0416c3b5bd07a/323",
"libjingle_xmpp_unittests": "d40eac04636b9a0fd21619802a3e8d463d2b7a030b9cd7a1742c6acab3c2cb64/323",
"liburlpattern_unittests": "22ed8397d3fa681d224626447e4bd7d396f3772ded356833f35149079e791308/323",
"media_unittests": "c1bb5188eac97d7d458e72e56d3ec7c2e537984c8bda1285def1b3b5373b5782/403",
"message_center_unittests": "1c3d1e7c3c2e0b84907af4ab7eff501217867e717b97cd55e6cb01a9318cc09c/323",
"midi_unittests": "e40c2a16fe4e661a07bffc49d0f175323306980f78fb206a9dc0b2399b948bdf/323",
"mojo_unittests": "2d8827c624c4579663c8faad230cc3085d33325ef4308bda1db8777644fcc019/401",
"native_theme_unittests": "5bf91834c8ec365a631f579479a97d8bf87342ccac605fb7f3faeb77f7687d9c/323",
"net_unittests": "5fea24d12794913013252101bde5ec5317f69b2622b4e381be1b07462f73914f/486",
"openscreen_unittests": "df818b549bc0bb47af6602df3db15e1c2b726f1e91cf5b2b2799864b85a7405e/408",
"pdf_unittests": "9c02c21cf663d442f657aaa37074aaec2b4324ae740a0f92240210940abb4a15/400",
"perfetto_unittests": "c894a03f054bd5d7b2347fb87a8b2d49eb82cd833025c9dc7bfe09ef29a4e938/323",
"performance_test_suite": "b84d6637e256138c05ef53ddb9726fb6bb27c5c82af10dc533e2fe9c3babfeef/732",
"power_sampler_unittests": "ebcdf620cb49995b76e05bd38efe925dc9024da636deda841b9862e4cfb07ec2/323",
"printing_unittests": "a3cf4291a4bb4954c73f03d1f2f4ed0c83333d550bcfc49b00717c17108ef0d5/482",
"remoting_unittests": "7c6a4ad032c77189d060d12407af40c2f77e71bb693a547f35d0ac66c506e997/400",
"sandbox_unittests": "e36f9d4dbbb302a99f5fcb5f1fe71b20aef655e6595ff91c48377175a46d5bed/323",
"services_unittests": "1476763aab649f46e5064eceff630099eec0de84a94c5681251c97db4e3f5a79/653",
"shell_dialogs_unittests": "e4b261abe06024f139c69da3b51e2b12fa193b200976c611852aa04fe0a7564d/323",
"skia_unittests": "a9200272b297ce4bc9d55be88c29718af35138c157da1e9b76d101bc4815e805/323",
"snapshot_unittests": "a8cb9fe7b90143b145ac2ffdfced673706565162f5f793ee63ce66077599ba00/323",
"sql_unittests": "54cc73ac9f78e175c0b0463835d366abf97537e5a73d22f44c37738a4b6f46fa/400",
"storage_unittests": "2d872fa01bd8e47fdb00469e457a2b4c300a49f912a5610d047853d5c79c8566/481",
"sync_integration_tests": "f05bea7bf63d921bb33f6cc8d30ff4e2b20ac443aeb8413d5a044758e07fc512/646",
"telemetry_gpu_unittests": "16e0fed120565fec4ff69294360ee89ed3ca08eba10a67fd29663566c8e09461/736",
"telemetry_unittests": "8e4a7a13804830e559b97b75e7326c23e3750e0eb15bc95500d0ca27553f0972/575",
"ui_base_unittests": "029bd0dd8593428af9af3c2efd41255c921cddc098e45b32358b9b1e1d83008b/555",
"ui_touch_selection_unittests": "a8cf5e26abab73da183566212714709fdd51e983d3d1faead50b4a6a26a818c9/323",
"ui_unittests": "85a3edaf010dc3566e014ae4e47184843aa4f69aaaf616b55f48c3eeeca392db/323",
"unit_tests": "47bf7d6b616b4858525f5035a016b2f8e30f4132c210230f8afe7e03d15d3e46/1061",
"updater_tests": "91058171012312775bff4324b6d2d54b9fd0502c82cbf51deb6b24250d7604c1/566",
"url_unittests": "a522fb8346a3a40d7b6b84cd534dbf298bc2d84c1f74f027833de32d13ec7727/323",
"views_perftests": "7fb7c72adf8ab6583ef612fde0e875d52636d727b09c111fb5b4b7189e8b1a17/487",
"views_unittests": "b5557b42df8f78b09b5a7d7741635600c561f8bf135ad63ee0d400f7a136ee9e/402",
"viz_unittests": "40d06c5ad6e8af123f496b624e4c080b3ebb53ee969169bedd23877b5d8b36fe/407",
"wtf_unittests": "4eb80e0c5a67d4ce05c934916dcacf1bb36bce13c898c3673eb716a4611add28/323",
"xr_browser_tests": "c8e956b0b4495557c7cf83f6c959874f80620ed820816010279b0514ea40cc0f/645",
"zlib_unittests": "58e227c22b0955bee94ae93fc43ce16773a292609e6d1a8332e096a29810777e/408"
},
"swarming_command_lines_cwd": "out/38ad-Mac_Builder_(db",
"swarming_command_lines_digest": "07e6b3479b3c11c2c9a1d3696e884c2d2ab29c264c793cc2c8cd325ce244da36/94",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "e2fb4a2b118541818be9b8c53a0639438e079f3c",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8873764516631790688"
},
{
"key": "scheduler_job_id",
"value": "chromium/mac15-tests-dbg"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Mac Builder (dbg)"
},
{
"key": "parent_buildnumber",
"value": "310536"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[22:21:54.634] Scheduled build:
{
"id": "8708334454617853905",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "mac15-tests-dbg"
},
"number": 1409,
"createdBy": "project:chromium",
"createTime": "2025-07-25T22:21:54.163342136Z",
"updateTime": "2025-07-25T22:21:54.163342136Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "e2fb4a2b118541818be9b8c53a0639438e079f3c",
"ref": "refs/heads/main"
}
}
}
[22:21:54.634] Task URL: https://cr-buildbucket.appspot.com/build/8708334454617853905
[22:21:54.634] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:2:0) after 3m52s
[22:21:54.948] Received PubSub notification, asking Buildbucket for the build status
[22:21:54.970] Build status: SCHEDULED
[22:22:45.803] Received PubSub notification, asking Buildbucket for the build status
[22:22:45.830] Build status: STARTED
[22:25:46.667] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:2:0)
[22:25:46.686] Build status: STARTED
[22:25:46.686] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:5:0) after 3m53s
[22:29:39.714] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:5:0)
[22:29:39.740] Build status: STARTED
[22:29:39.740] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:6:0) after 5m44s
[22:35:23.787] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:6:0)
[22:35:23.808] Build status: STARTED
[22:35:23.808] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:7:0) after 9m39s
[22:45:02.876] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:7:0)
[22:45:02.940] Build status: STARTED
[22:45:02.940] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:8:0) after 2m32s
[22:47:35.006] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:8:0)
[22:47:35.025] Build status: STARTED
[22:47:35.025] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:9:0) after 6m24s
[22:53:59.069] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:9:0)
[22:53:59.107] Build status: STARTED
[22:53:59.107] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:10:0) after 8m28s
[23:02:27.225] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:10:0)
[23:02:27.261] Build status: STARTED
[23:02:27.261] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:11:0) after 2m5s
[23:04:32.305] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:11:0)
[23:04:32.326] Build status: STARTED
[23:04:32.326] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:12:0) after 8m54s
[23:13:26.421] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:12:0)
[23:13:26.444] Build status: STARTED
[23:13:26.444] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:13:0) after 5m17s
[23:18:43.488] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:13:0)
[23:18:43.513] Build status: STARTED
[23:18:43.513] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:14:0) after 8m22s
[23:27:05.542] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:14:0)
[23:27:05.559] Build status: STARTED
[23:27:05.559] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:15:0) after 4m41s
[23:31:46.680] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:15:0)
[23:31:46.696] Build status: STARTED
[23:31:46.696] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:16:0) after 6m17s
[23:38:04.764] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:16:0)
[23:38:04.790] Build status: STARTED
[23:38:04.790] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:17:0) after 3m31s
[23:41:36.084] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:17:0)
[23:41:36.101] Build status: STARTED
[23:41:36.101] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:18:0) after 2m56s
[23:44:32.175] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:18:0)
[23:44:32.197] Build status: STARTED
[23:44:32.197] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:19:0) after 5m4s
[23:49:36.678] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:19:0)
[23:49:36.700] Build status: STARTED
[23:49:36.700] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:20:0) after 3m34s
[23:53:10.835] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:20:0)
[23:53:10.851] Build status: STARTED
[23:53:10.851] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:21:0) after 8m13s
[00:01:23.881] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:21:0)
[00:01:23.904] Build status: STARTED
[00:01:23.904] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:22:0) after 3m39s
[00:05:03.136] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:22:0)
[00:05:03.160] Build status: STARTED
[00:05:03.160] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:23:0) after 9m13s
[00:14:16.253] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:23:0)
[00:14:16.280] Build status: STARTED
[00:14:16.280] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8873764516631790688:24:0) after 9m9s
[00:17:10.503] Received PubSub notification, asking Buildbucket for the build status
[00:17:10.525] Build:
{
"id": "8708334454617853905",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "mac15-tests-dbg"
},
"number": 1409,
"createdBy": "project:chromium",
"createTime": "2025-07-25T22:21:54.163342136Z",
"startTime": "2025-07-25T22:22:45.557353832Z",
"endTime": "2025-07-26T00:17:09.837654948Z",
"updateTime": "2025-07-26T00:17:09.837654948Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "e2fb4a2b118541818be9b8c53a0639438e079f3c",
"ref": "refs/heads/main"
}
}
}
[00:17:10.525] Invocation finished in 1h55m17.665885237s with status FAILED