[16:18:33.319] New invocation is queued and will start shortly
[16:18:33.319] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[16:18:34.386] Starting the invocation (attempt 1)
[16:18:34.413] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/4119561eac402f47bb9be431db2994f1c3a6ef38"
[16:18:34.413] Popped gitiles commit info from properties and tags
[16:18:34.413] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[16:18:34.413] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[16:18:34.413] Buildbucket request:
{
"requestId": "8833834112928961280",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Mac13 Tests"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "4119561eac402f47bb9be431db2994f1c3a6ef38",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "4119561eac402f47bb9be431db2994f1c3a6ef38"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8668405600249654945"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8668405600249654945"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8668405600249654945"
}
],
"parent_build_id": "8668405600249654945",
"parent_builder_group": "chromium.mac",
"parent_buildername": "Mac Builder",
"parent_got_angle_revision": "a408a647ed0f31a1c53815c4bc101af3193d1efb",
"parent_got_dawn_revision": "6d1897130dd7fda56934e6d7c0c3d328dfa8b88f",
"parent_got_revision": "4119561eac402f47bb9be431db2994f1c3a6ef38",
"parent_got_revision_cp": "refs/heads/main@{#1715626}",
"parent_got_swiftshader_revision": "0f87bb2d3742c2992fd1a7dcb2f051eebf79299c",
"parent_got_v8_revision": "108dca91c740a0fe27bc16d3df79e211159404d1",
"parent_got_v8_revision_cp": "refs/heads/15.7.99@{#1}",
"parent_got_webrtc_revision": "3f3b784c3c28c5a25730234b7726e917f9e44533",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#48837}",
"swarm_hashes": {
"absl_hardening_tests": "59131afbd094980b426a8541679bd5d232d8a84292ad920e240def49d028784a/421",
"accessibility_unittests": "3b1528ce446488615794d419b1f642ffaec0ebe2a24582721bcb5a87503c2399/421",
"angle_unittests": "39692a045d03a62a14cf11647c1d4dcff7084846ddd885a47273ad1025264959/506",
"base_unittests": "abed5a756942421d96c0bdb1ead6f537915d2057733a712ed756c8695e6fe3ed/578",
"blink_common_unittests": "f4b58a85870ced949673af04560c55cd83d25ae54055c773f8df280c218848d9/583",
"blink_fuzzer_unittests": "742c2393314515f506aee4d601c7e7343a914a1cfc77d7a1e37ace2f0684cf12/421",
"blink_heap_unittests": "57225ace3ac5f380bfeed20cc24a34ff5cfb6efd45344c6d056dd8e3e06c12b2/421",
"blink_platform_unittests": "ab696dba13c195fdf55a0ea3d886ce6cd77e9375a8d840ec1d9287992779e383/741",
"blink_python_tests": "504ab9a6c37b761c5cab69b5abe3fee414c242cc608ddff9c7c2cfe947f087bf/671",
"blink_unittests": "ed97401f111083e3bc512d88340aeaca92295db30fc5ce90ac8f575516998b68/747",
"blink_web_tests": "efa269787c657c8f18f77f9c014cea058b9ba2a6835f0bc7c67d8ac4981a1711/584",
"blink_wpt_tests": "b68e3f281b66f98c7a1c015491e83ee1b7c1f64e9afcf9ff40d7228a41ef7594/584",
"boringssl_crypto_tests": "156c000128029dd0a004229952701ffa2f82a1b4be402dd198205b09c8eb774e/506",
"boringssl_ssl_tests": "0432590ab11683c77982db2a0e14455f54eb55063b75bd3c0a2e283505bd4120/421",
"browser_tests": "ffdcbb7a29be06b3b1494006093313798f14fd1a79d74c292d6c96f12a8cadbd/1318",
"capture_unittests": "e5917daa8859e587dc6ca89eb7975cfba1fcfe976afdb665a19f4b133104c8a7/500",
"cast_unittests": "a83a9f9ba5a2751113721a7c8fb8d26e4f45b4c29116176607765f1ec35d33c0/421",
"cc_unittests": "01a6ab5faaa6ca34482f6ac3fe6c586e71cdf236eb584eb937f2a6972cc51a19/581",
"chrome_app_unittests": "da12d1bf9528e9e32fa5287029e2d0547415701f943aa29f698150fab335f1ef/662",
"chrome_wpt_tests": "6c18dccc49af720e8e35e071aab661bf57ef2cc7bf0df587391dd6da85d2d996/584",
"chromedriver_py_tests": "25bdbe8cc8472536b0eeb4b06a21a5f964411768166696abaf34d28a2c7aa7dd/587",
"chromedriver_py_tests_headless_shell": "59a5f4a2b4dedaec311c791f6eb12599263b74184ee26b79c167bf521127707e/587",
"chromedriver_replay_unittests": "be2403549ccd893d1ae20ed0f95eff634b5528759db29ab7b9a149817f6fa173/424",
"chromedriver_unittests": "2790122e3c148bb8349db7ed1197689cae63a55c08f8961f844defb58e7d4856/578",
"components_browsertests": "3af6e36ba299fe40f015013236b799a49f9c7eeffc31d588dbcda1012357b17a/750",
"components_perftests": "e8ba8475dff699e9fc49d0e9beb5947938faf176f8618b660b3561e68112f8b7/748",
"components_unittests": "092e53ff1c1d39814dea15233b737e4e7de74ecec314cddca17e5945c00f8209/908",
"content_browsertests": "d96c6954a9ac4a8b3ec5672655360065447d24e673e268bb7bb3fee56a53e9de/823",
"content_shell_crash_test": "d3c621ff291ea92686f9c837a32a2a6f41886d12eb64fe9dab2a91fbbdec854a/664",
"content_unittests": "bcd8ff65cbd5d994b41661b74d1f8e214bb159f247f91c808c2a270c7a48367d/658",
"crashpad_tests": "c3f76e7ced0ede8772431cbf7a451b41300066a127ca9286cc3fe19f338e4dd8/506",
"crypto_unittests": "79f884059aa6353ea069746bdac10be94750b2763e7a293e562175ce015094c6/501",
"device_unittests": "f104d3a55c3b01a4f822c91415f683793b23efa32f4c39a191331be77ffb9634/498",
"display_unittests": "f4119a7b7bf9a30c31893be7f4740c743a3d2a6d5cea0f5209b09f53fa76524d/421",
"env_chromium_unittests": "b4270fd4c8d2fc8efb890cdf0626bec3774d4d2303128832800d523a0d89615d/421",
"events_unittests": "007f110d29e93fd78bf47a136065909c32a4f89c7cd339ee54e0c97c9b6a20e1/421",
"extensions_unittests": "5c988265469ca9a430c213ef6be1a7df274285bcba388bc6b75f5590c544f86f/742",
"filesystem_service_unittests": "c7ebdb079f68658f16b475eefda0fb92024069af2790f18d687de320bea6e2d4/421",
"flatbuffers_unittests": "cbcf9df33c191715d1531bdd24fb5f8081b07eee8be2ec4073cbafabe8f03da3/506",
"fuzzing_unittests": "8e51653105ad7cca3b0ba7b01bd6dd02dba6797bc4dcf1e175479c9c04a96181/421",
"gcm_unit_tests": "0e4b0a089139c6b8b56034c3d6d8d4e70f7aa42eded6c8ff1ecf3f5b4b6b6e8f/498",
"gfx_unittests": "80176218cc58ff6c7a22b75bb572518bd7bfc0e2f8a47b36b5d5be42af2ec8d5/497",
"gin_unittests": "fb13b342daee6834620cfdc4ad0cbde7fc851a00c0719e2a2cdf71bec64bc319/498",
"google_apis_unittests": "8e9d681610bf1ba8db86f234f316889d4891a4a0b02ad83ca24a99bba52a55b6/583",
"gpu_unittests": "1675ddc0c0a44937d1bc52e8788224bb4c24bb5b0420e567b54c6fac7bd318d9/498",
"grit_python_unittests": "da0b78dfd7cbdf5ca478e8bcd4ca5965b8f066861af452dd2559a44f54c45497/509",
"gwp_asan_unittests": "ae16ccb523b3dec7ba7400a4ac3b2e2a5d2a538ff52e60039f2489144dc669e4/506",
"headless_browsertests": "a36f7d67e90f23eeeb801ef4cd24e4c3ba065866e66d1395dc8e8c11460b586f/751",
"headless_shell_wpt": "e9ec03e30e0d36e5333ebdb1f39d739ad39e72ee34f9d702b12bc655d240c021/584",
"headless_unittests": "a2b42357fbc340b71a52dd9dade10103424483cff91415fb75090588bc3c3b1e/421",
"interactive_ui_tests": "2e5b60b5b4af9b5022647cd64eefc139772c501864f39ad45d5ede5fa73c005f/1158",
"ipc_tests": "c9d95e18ddf1e29e58a08ca7bec5e411c01d15c1e1a42f16143090f6c0c90b57/421",
"latency_unittests": "8d1d8889a58a619e614d82047da59fb01691a21f27e6cae5bc47d9285654d4e2/421",
"leveldb_unittests": "19ba6cffbe6aed85fe1d7a24880a189f71dd3a49c66bbcd5bb4751ade18aef98/421",
"libjingle_xmpp_unittests": "ab78d468c3da00dff1dcef62e1ccac312b7e02da10c87920661935f87e7e6192/421",
"liburlpattern_unittests": "752f3ad96b32fb3527eff46164056e9982ee4fad6379cc2ff81bd710a2ab6d45/421",
"mac_signing_tests": "13d9a9fd92f0ff1adab26eb5764321a4ffd1984ddf97ddd36176fa4b961f0c78/586",
"media_unittests": "57d651a101df0944aaef3767662104afd5438e2a3175a82539163a5172d8c091/501",
"message_center_unittests": "0e1043f611ad9b7c0fe9953ebbf750b37d94ad9794f1184bde827bcc3194214c/421",
"midi_unittests": "c84dce76807b64b0c5adaeaac0233a0955ea18e428efdd122b388a113b154870/421",
"mojo_python_unittests": "b42ec3890ff9c8a2f9cbc5971be8f3bdf476739162facd4e5e3b6788eac046ed/509",
"mojo_unittests": "da08233df836254233eadd6b394b81095c11efaea825040f77384eec5c3f701e/499",
"native_theme_unittests": "a8ea836dcd25da0ec6c4d8dad35afd2d8323d5c7b90415014a9c1abb9ad07ef5/421",
"net_unittests": "5bfd99a8ed07ec16a96645a6926d4a493b567b8fe25dfdea5a03ca8f6c51b38b/584",
"openscreen_unittests": "9ab6ddeaf7e7367e80d075a3350bbda183e8700c431bebd0bbd18f0e777e57a9/506",
"pdf_unittests": "ec1ccfdd8451032a97ef13c2bdaa0b1dcc6bf6ae45ce3413428c4625910bc619/498",
"perfetto_unittests": "31ec025aae6b3bb8772b17c46ee9f3eb2679cd8ed1a43e85e631ee318fb6ef9b/421",
"power_sampler_unittests": "4f80fee5b680b539aaf1ed3ea0485f3dac83b248c0a00debbbd11d08abf8683b/421",
"printing_unittests": "62ef5872ccfd02b2d161451f7681c572ecde7509901d927c80444b15a52b8d34/580",
"puffin_unittests": "1da5a33ca7a98ea333ebdb0bd1ea9b9949494ffa992669b2e78c40f9ae057fbf/505",
"remoting_unittests": "d20838fc185b3a4046b94fc5686b1b8ad1cc0ac971e7f9dc7e770565203a6f54/498",
"sandbox_unittests": "df5078b6476b7cf6192dbae7518a0a7ff9e9a3e028300552b58caaabbb1d7308/421",
"services_unittests": "1789ad84719b2b918558d09c417f19da03d502b82346aa6324546688a532b85f/747",
"shell_dialogs_unittests": "31bf85036d2d12a83dfd56c1863e6b7c4e783b217c9bcd42fbc15d2eaa5d145b/421",
"skia_unittests": "01cc0b10d7ce15279ec64d00bdf5fb362762737122fea96f95278fa32e29cc56/506",
"snapshot_unittests": "cce269c6383127f988335af6aa6428d566c965bae6c5e4fc58d093968e4deaa1/421",
"sql_unittests": "814851176351f632c227e4df210b4b560b2c324c4565a810c0fefd651b500403/498",
"storage_unittests": "08f4b86f106df112d765c5770e929e79d8e5a1d7729d56652781f52092f93d75/579",
"sync_integration_tests": "ec2b026f2ba8d2fb1a8e579ea047de62d7ef72fa2c84dc5d307133c5e0b365df/744",
"telemetry_gpu_unittests": "8e684e3673d71cf298b9135210ed3ddd4c10ace4430421baa99f60321470794f/834",
"telemetry_perf_unittests": "010185c53dee5ac9e0142cd81d57c472132b865faa519c68a9a5753dc8054cad/852",
"telemetry_unittests": "5105bd1827555eef928249c93e809e08890602e79c09585dcdb54438d8ec2754/673",
"test_env_py_unittests": "eb8de1f649951b3b8e9ad2cdc45bb604b2c95ec5192f083106ee1944eb47a2cc/421",
"ui_base_unittests": "93a790b98daae621d882da003c42931cbe8c1ac311593a33019c71e76a6d1c44/653",
"ui_touch_selection_unittests": "62c0248f7fc9abc3dc32710b9979b37675d3b03b5094365c28a5ce15deaacd7c/421",
"ui_unittests": "fa51cfc383e75d8703d32aa9305c42584a0b358e4c577b72a3b48afd59eaf42e/421",
"unit_tests": "5920ba0d28d041bd6007cc02e533170a60ccac3bbfe94c5c36e404dc115fc32d/1238",
"updater_tests": "790204541b1f90947e0c5bc1edd3f810d941f053c5dfd26b8acdf0e13afe482a/664",
"url_unittests": "35fbdefd3cb095b44a532a69297dde3252961b315e94e446aaef24e04386ae25/421",
"views_perftests": "05053a651df010006ebb821692ad5fb66823ea3932c0d4c4ecbbfe27de2d072f/585",
"views_unittests": "cc61e2e144edc3641abfeef63db2db4db40a67c0841b3417788259ed4fc721bb/500",
"viz_unittests": "7053f22a4030937662091ead51fe307797216d4e23ac59bacd5312ff6985a495/505",
"webui_resources_tools_python_unittests": "241c326b695b5e2f6c9879408fb4444720682742ee671615488d5bef398cfda0/747",
"wtf_unittests": "60d953beffc1adb0607336f1d8ec491349e95cbe86b55fc3f72ec5e3700ebb8b/421",
"xr_browser_tests": "d282a3e87b6ed638c46ac70d90dec7580fdbeb4fbe536687467b59aeddc24675/743",
"xvfb_py_unittests": "9ea6f476fc1659bac8109fbfa6372216228e742af08e5e7a72a16e647b861659/421",
"zlib_unittests": "493fc0f357d88d1911fd806be271557ae41f20612a2e1fd0792782543383802e/506"
},
"swarming_command_lines_cwd": "out/ea77-Mac_Builder",
"swarming_command_lines_digest": "db9a98e44e682c4e918a4d750045cd9bb894bb1d679893dbfe9f61b709fde95d/94"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "4119561eac402f47bb9be431db2994f1c3a6ef38",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8833834112928961280"
},
{
"key": "scheduler_job_id",
"value": "chromium/Mac13 Tests"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Mac Builder"
},
{
"key": "parent_buildnumber",
"value": "283927"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[16:18:35.207] Scheduled build:
{
"id": "8668404050950406961",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Mac13 Tests"
},
"number": 43167,
"createdBy": "project:chromium",
"createTime": "2026-10-09T16:18:34.584644623Z",
"updateTime": "2026-10-09T16:18:34.584644623Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "4119561eac402f47bb9be431db2994f1c3a6ef38",
"ref": "refs/heads/main"
}
},
"canary": true
}
[16:18:35.207] Task URL: https://cr-buildbucket.appspot.com/build/8668404050950406961
[16:18:35.208] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:2:0) after 6m19s
[16:18:36.198] Received PubSub notification, asking Buildbucket for the build status
[16:18:36.241] Build status: SCHEDULED
[16:18:52.723] Received PubSub notification, asking Buildbucket for the build status
[16:18:52.747] Build status: STARTED
[16:24:54.238] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:2:0)
[16:24:54.367] Build status: STARTED
[16:24:54.367] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:5:0) after 4m22s
[16:29:16.427] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:5:0)
[16:29:16.456] Build status: STARTED
[16:29:16.456] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:6:0) after 9m3s
[16:38:19.439] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:6:0)
[16:38:19.462] Build status: STARTED
[16:38:19.462] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:7:0) after 3m39s
[16:41:58.580] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:7:0)
[16:41:58.618] Build status: STARTED
[16:41:58.618] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:8:0) after 1m50s
[16:43:48.683] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:8:0)
[16:43:48.708] Build status: STARTED
[16:43:48.708] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:9:0) after 9m13s
[16:53:01.815] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:9:0)
[16:53:01.833] Build status: STARTED
[16:53:01.833] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:10:0) after 4m40s
[16:57:41.928] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:10:0)
[16:57:41.950] Build status: STARTED
[16:57:41.950] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:11:0) after 2m52s
[17:00:33.986] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:11:0)
[17:00:34.009] Build status: STARTED
[17:00:34.009] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:12:0) after 7m56s
[17:08:30.138] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:12:0)
[17:08:30.180] Build status: STARTED
[17:08:30.180] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:13:0) after 7m46s
[17:16:16.696] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:13:0)
[17:16:16.828] Build status: STARTED
[17:16:16.829] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:14:0) after 3m22s
[17:19:38.908] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:14:0)
[17:19:38.929] Build status: STARTED
[17:19:38.930] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:15:0) after 1m34s
[17:21:12.961] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:15:0)
[17:21:12.982] Build status: STARTED
[17:21:12.982] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:16:0) after 6m57s
[17:28:10.024] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:16:0)
[17:28:10.042] Build status: STARTED
[17:28:10.043] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:17:0) after 7m52s
[17:36:02.069] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:17:0)
[17:36:02.097] Build status: STARTED
[17:36:02.097] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:18:0) after 1m4s
[17:37:06.132] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:18:0)
[17:37:06.152] Build status: STARTED
[17:37:06.152] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:19:0) after 7m38s
[17:44:44.357] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:19:0)
[17:44:44.377] Build status: STARTED
[17:44:44.377] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:20:0) after 6m24s
[17:51:08.457] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:20:0)
[17:51:08.485] Build status: STARTED
[17:51:08.485] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:21:0) after 9m47s
[18:00:55.630] Handling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:21:0)
[18:00:56.013] Build status: STARTED
[18:00:56.013] Scheduling timer "check-buildbucket-build-status" (chromium/Mac13 Tests:8833834112928961280:22:0) after 5m13s
[18:01:55.108] Received PubSub notification, asking Buildbucket for the build status
[18:01:55.423] Build:
{
"id": "8668404050950406961",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Mac13 Tests"
},
"number": 43167,
"createdBy": "project:chromium",
"createTime": "2026-10-09T16:18:34.584644623Z",
"startTime": "2026-10-09T16:18:51.497607076Z",
"endTime": "2026-10-09T18:01:45.263964509Z",
"updateTime": "2026-10-09T18:01:45.263964509Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "4119561eac402f47bb9be431db2994f1c3a6ef38",
"ref": "refs/heads/main"
}
},
"canary": true
}
[18:01:55.423] Invocation finished in 1h43m22.10963753s with status FAILED