[07:25:42.611] New invocation is queued and will start shortly
[07:25:42.611] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[07:25:43.781] Starting the invocation (attempt 1)
[07:25:43.815] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/8268294e50b8c631543f490ac579ac1fb3994034"
[07:25:43.815] Popped gitiles commit info from properties and tags
[07:25:43.815] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[07:25:43.815] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[07:25:43.816] Buildbucket request:
{
"requestId": "8890218949747447600",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Win10 Tests x64"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "8268294e50b8c631543f490ac579ac1fb3994034",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "8268294e50b8c631543f490ac579ac1fb3994034"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8724790597094196417"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8724790597094196417"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8724790597094196417"
}
],
"parent_build_id": "8724790597094196417",
"parent_builder_group": "chromium.win",
"parent_buildername": "Win x64 Builder",
"parent_got_angle_revision": "8c3179402f7f317aac45e0ea02cce49b9fc13144",
"parent_got_dawn_revision": "4d01a63f72fa410355c599905db256015be4039d",
"parent_got_revision": "8268294e50b8c631543f490ac579ac1fb3994034",
"parent_got_revision_cp": "refs/heads/main@{#1411282}",
"parent_got_swiftshader_revision": "a87fa39f64462d4c3ef607d1055366370f7bddd6",
"parent_got_v8_revision": "83ad4c3cdfbbe766e4556f604f27cbddb2cb15f1",
"parent_got_v8_revision_cp": "refs/heads/13.4.79@{#1}",
"parent_got_webrtc_revision": "f2ecdd7ea3aefa9d65b3fe6b983cf584c7465237",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#43796}",
"swarm_hashes": {
"absl_hardening_tests": "b1e64a185f85afb21eb99c98bf54480a4963911d703a1df0dac45b88d1c899e3/323",
"accessibility_unittests": "2fd6f0d9e8cf742c1cec3800446be22ddec1bae4cc917fe920dd819097e23c1d/323",
"angle_unittests": "4a6318d3cdd213648c34b7f7e5cc87421099b4b55adbe236696dd4efdb08a506/408",
"app_shell_unittests": "88c55ea698a032640a0c8b24cf4284bb12f78e81d08404ca5f475f8f1e52fd92/484",
"aura_unittests": "75bd78759bb4e8cdcc7aec28a1c5ed0bbbd833db5fb767dd69fca2dda34a23fd/323",
"base_unittests": "c1e253fac9a45cc99c3db03f35a7150890cae65605e9393dcdb8a461c5b70c75/480",
"blink_common_unittests": "6a534c01707057fced313ebc0461982e32ae77f1ffa35e73e1d3ef24395ab4ff/485",
"blink_fuzzer_unittests": "0a8eabdeb4d875db667d9f81801e1e6794b18257f62e647cbe11a2f353c4729f/323",
"blink_heap_unittests": "51a4dee697879d1a420415da2b4adbb2b2a6f6d4c1e0fdc5fd0262763208e300/323",
"blink_platform_unittests": "17843f4287d9fb41b90f409bbbde264ade3b2a37740258e5183451daeadaa907/643",
"blink_python_tests": "dd4b404efc149d379cd5d7b0b7ae55ecb1ac719a4b6f35582b66defac7ce1d3a/574",
"blink_unittests": "4775af90934de3c3ce514d9451a1f439c0b3453a56ec29e72c70efcb07aa357a/649",
"blink_web_tests": "6ae59ba2b590649b58dbf9dc629fa5ae75caa685850081b473bd7493710a5d8e/566",
"blink_wpt_tests": "53fcb86620bfd5c797da95373731e49c69aef50a8978a701549e0e30fb848c34/566",
"boringssl_crypto_tests": "8171e63605d7552ea9ee3c27cada2c7738d9d64a2b9a85cd1e8c630c210ec3db/408",
"boringssl_ssl_tests": "4758c1a834f09421ff3891efe5caeabbf778c0bd4f65d539537c864704591fd3/323",
"browser_tests": "4897700100e3a4b3ed6a04f1caec3f0b2f1b50e478aaf31624cb7c65c875197c/1220",
"capture_unittests": "0937515a3360d4a3c0d432fef13f9383f86193330551acf22ee944f585c28933/402",
"cast_unittests": "1752db722af17fa42cb6b08fe20e7dced647df8da3210bacbf8b8a5292c6c388/323",
"cc_unittests": "3ad8a3258a1bbf1b69a41d516fa4d0e9ea2a2d327d7733f3ac5152970b462d88/483",
"chrome_app_unittests": "7d3cddb8ed8bd6457057537717b45855a995eed3c7a5381f320acb3e68205193/564",
"chrome_elf_unittests": "3b5befebbb82d625328ba25e6f829b409978bd872c3ac1a94efb9f363ceb5bec/323",
"chrome_wpt_tests": "28b2ae4ce0bf5ddea49738d41355a7aa44b6f33a716036bec892d345575b8eda/566",
"chromedriver_py_tests": "d434de021832029b837844395fbc9d5c6c647213a108c51f421083d6a0995696/489",
"chromedriver_py_tests_headless_shell": "bdd0e3bfe1bbb1b36b3e974cb1a0fc834ce53a34b57461c4ce742ce820b70998/489",
"chromedriver_replay_unittests": "d0f350e6ee16abc5a85e4058bbe951f0c4720724d531e4d641659272d55a6aa9/326",
"chromedriver_unittests": "f09177f4a2a7b38e28e23d117e07bc2eab1e31ff45e3007a690d7abbaf60eb5a/480",
"components_browsertests": "a355581daa2712b8087d5d9a8bb39e9a7f221e081df3466590a87aaec5955931/652",
"components_perftests": "a0f31bffcb2d49bdcb41a7ae1d2eca53e4a182329f351591798913b00c37f2c1/649",
"components_unittests": "c6411b0501d897bc0efb869ae9def0fb6a4cb7a93bbbb98c6f35bcb15921d91c/731",
"compositor_unittests": "10faecb797cec39c2e60bed3a0a78b34e243550d241504960c2b9e23e5544f6e/399",
"content_browsertests": "f019935a3c961173652247f23fe773feebaa68cedbd0044d65f680b2da373849/724",
"content_shell_crash_test": "31aaadf05857d7ce40420f6130a840917d862b4d44c683d633224a986d9d6494/481",
"content_unittests": "c18ff2efd767978f9c3098dbd447d9e5b78dec1421c24042f0114994385ae72c/560",
"crashpad_tests": "53dfba260e60c393e818a2d8d7e9bd3d308f96f1518a8527674c88dcbaab059c/408",
"cronet_tests": "f6af59adb4e17ae5dbaf9bd8e0e8bba9df69f08409b518f9407102a10e5cf211/484",
"cronet_unittests": "284dcd036a91e6cac9b4722da84965732d4a6134edb7f60f1901ed89a7f4e23c/400",
"crypto_unittests": "8973d7bce7dbbf8532636a52af948cca21313e98628cd368ee42ffaa3a008d44/323",
"delayloads_unittests": "7355dbae0b521c7d5d4d08c7f1bb1b9203cdf78d404f733f8ea80411885f6740/323",
"device_unittests": "edd545b15c23952a73af45854553c86f53ab5cf351dead4e45b363579baece82/400",
"display_unittests": "a8c45248f886b4366a114659356062537cee5970f5e6fb8989ff3356382abfd5/323",
"elevated_tracing_service_unittests": "c66c53b20a766b1eaf2744265a75829d30c93da41187bafa10dcd612e237b2ea/323",
"elevation_service_unittests": "043e990afead1f266df42e9543df8da891ca576d4c33b5db3b90137ac5420e21/323",
"env_chromium_unittests": "41fb9c36da409a02a4e083937535951b4d9569a903451c706e407a27e1dd4a9a/323",
"events_unittests": "644fc404fefd38a6d935db1b6b83f59bd02b77734da50e8bdcb40ea7233beaf4/323",
"extensions_browsertests": "20085c4f8f1f9e8edd4e43afce1bd729174189cad78365a9f217054d54443285/570",
"extensions_unittests": "a68fd5fbdc4c66681118b8e6c04cb7193881dce2137af60ebf5e96228c8d8861/644",
"filesystem_service_unittests": "2996ac203225b6d03dd747485f1267e2a934c72c07ed48b665985f384e250521/323",
"flatbuffers_unittests": "40da30ff4cd6b1a2d1b5c1ea1610d12055d2da1675754ac259eb310abcfc5b33/408",
"fuzzing_unittests": "09bc8db681ce6420c0b5d95280e6212d02533ad859d86a286bcda147bd92aa72/323",
"gcm_unit_tests": "e84bcd254ccc9620abe3d5a03e7e8b7feb54471118b03bbd56a9bfaf43e9813d/400",
"gcp_unittests": "fa68bf55421b0c4e8f9a3a8ed0bdbabe8ca9de178b1ad1fb5d813b4d3f5703d4/486",
"gfx_unittests": "3087139b00046aa3c1615437f6a36486f67566340a8d3960e3be18cae59ba5d2/399",
"gin_unittests": "29eff6f8cd02943c1ae9ac8f8315839e2fd5f47b57cc153cd70e01bb246d2a1c/400",
"google_apis_unittests": "9e0650c6f9b54980b9f4e890ec04ca15d4a1ee1ff122d8b1db89015649cd02ed/485",
"gpu_unittests": "25775023d57fa492326cc5ec34fedbffcafbf5595c52f821b1f1b68336666780/400",
"grit_python_unittests": "6f3c00bf29f646559658a4265613946cba41d0a8de6c639dbac5cc224ba825e9/411",
"gwp_asan_unittests": "3be91e5412943b651f0d8af810bb2279955d63086f36dd3f402aa890ecf6aa6f/408",
"headless_browsertests": "38f7910325ae50286336ac028a2833182f7d4b4b0f0d45b23aaa3d9edb8605b9/569",
"headless_shell_wpt": "5a2b11132d95ba633c6afaaae076505dfb70f52d6c77b947163e0fa0e6e0f427/566",
"headless_unittests": "09bf1f5f757376a649607e3bfc9a5dc03dba6355f59b5a84a90fb183cf49371f/323",
"install_static_unittests": "01264b759bd9c573a8c9694ae91aa667215be0bbee6217f098ddf2159b4c00c6/323",
"installer_util_unittests": "8ef4c55e5ec4cf153d093626d3a481ae665ee4c8cc075ee3fe7f35f5ed189cca/403",
"interactive_ui_tests": "6c388e6d89a5564d8345c77fd6044e5aa5b75023a8b77be12ebf9b0c2bdf03bc/973",
"ipc_tests": "15181191cf1da3caf5c279378d4c79e92c217f488cf63d19b0cf07da010dd9b7/323",
"latency_unittests": "0bf31e0c58388cfcafa9d8bf0075500c8c4eb50c54eaea2a4a10f8248a4167d9/323",
"leveldb_unittests": "c46d14a6925d91ec2258cf491acb9d4c7a19c8aa3be7c3b339a63d9f62b51233/323",
"libjingle_xmpp_unittests": "3913946be60e997b51be210b047600c2d9db54b56461825ed3e328d9b007538f/323",
"liburlpattern_unittests": "17c71f743d36f665ca41c6f35eb1eb79e1d16fd6dc893ab7b447af70845638b1/323",
"media_unittests": "e753ccad53a494dfc93c12146d5266b02ad77ac2d0a76342d2c0195412c3a013/403",
"message_center_unittests": "0372c7c35ebd4b049b62fbb8628fecbfa30206760b69c78964a690d1b58fd0df/323",
"midi_unittests": "908e7bcac27888d85f4ebec1c2ae4871d90751ab026d8e108cb45e195593f1bf/323",
"mini_installer_tests": "aa60f99a7fa703e048363b6b8bfbc736c80f7d74fa3c371ea412816f232530ca/489",
"mojo_python_unittests": "f1b011cd6f548ef35c8c3efe9b3bc367317cfb4c4927163ae10c0526f7c6a4e7/490",
"mojo_unittests": "2177bd140dfb10532d0b55ec9ee076dad4abbea7dee901b52f0ab3a260ef6c2b/401",
"native_theme_unittests": "1054855bb38c4f4432c73056cb0a7c1ccaf714033f7ed9cc702d537697fba26f/323",
"net_unittests": "7591304e8f15803aca7d7746b1cc5b9300a32706349face6d28c99970536dc41/487",
"notification_helper_unittests": "440f183c04827eb1079867f777fb090f3185ca7c246557855b5bde39c01cd799/323",
"pdf_unittests": "e7c73487f169cd619acabb509ea0eaeca80282d3e70d2f00f7375022bab9caf0/400",
"perfetto_unittests": "f172c7feccab32e1b6f55b189f771d8c281d3ade7c7fd20766342502f2957b8c/323",
"printing_unittests": "f22d9bc2fa40b78d834e13b785d9b9d3622c33a3c991afed3b9f37931b67be71/405",
"pthreadpool_unittests": "ed898ba9f57d92fdd4ee637602ee715236b55895da9e7d34d816233b536c8544/323",
"remoting_unittests": "da7c902fa93b9ff7d7955c097ee275564a00cf6ba8b0c2ec56fcb10f370ad319/400",
"sbox_integration_tests": "798ec30fe748885e80019b66a66fa72179f1ef177c930adfd8937b1f2cd4ab22/401",
"sbox_unittests": "1f799c20e25ee1497638ca7e6c1642fd9ff871c812574e76891a0d04d3d14466/323",
"sbox_validation_tests": "77ae8d6b1d433c5c2596942722f737d9bae2dd98587b55001727e8e861ec4c30/323",
"services_unittests": "24729dc3375d88fda86130a397aa634677d527b1723f0ce47c0a844ea3009c32/653",
"setup_unittests": "dc1a22d595d58d5ede32edbb1b1c5c7c3d977a8468576381b91dd6f509238cb2/404",
"shell_dialogs_unittests": "1d3b11b82c87a7c25a5ec48f12d3e60fbad1f7db72cff0fb65ce563dd4af0bb6/323",
"skia_unittests": "40f1d8925b49006582904a2f838e55ef420ba82429718aeee46d2d67c4f723d7/323",
"snapshot_unittests": "ad4c6f875af786d7b677d0f37f17c28cc22c91aa9cbd2db4026d362a2f6bd0a7/323",
"sql_unittests": "cd8a75282f7fc749e374903d426a354e1cb5dcf2b90f2f6a2d19b55c19e16add/400",
"storage_unittests": "0ae1a8229b245f813907f7bdaf63bd93b65f19effb68e34d26a57be77a596de0/481",
"sync_integration_tests": "9804dbdfb6128598efd3353afbfee65cea5dddd78f3e1c18508893465e39bd85/646",
"telemetry_gpu_unittests": "8f7529c57e52d245d2e34266f21cea3e54c83cfa64eb2d5c35f38dd30b2c4286/736",
"telemetry_perf_unittests": "4627cb7570c6b619266b52cdad78a9b5437d3d7eae845c659976afb3a5df7b9f/655",
"test_env_py_unittests": "99b876fffc8f274139573f80c3c628971f715ecba7bdce0c9fbb8998908ded59/323",
"ui_base_unittests": "13393119c57752748d5752b3a397fbb53175786b09a2f4057dfe3e0b15cb2bf6/555",
"ui_touch_selection_unittests": "be4526063b9f5074b1c12d59a4a7bd9c54af785d2f3da9248107be6b44061cab/323",
"ui_unittests": "cfef7b8f4294d13fd4c6ce8656d4e3c2bad0b6d46ac8c5ad7143286080e6d2e8/323",
"unit_tests": "deb1f9f637e07a8f54500a7d9c8ca1d25ffcd56ba7f7d97d30877f0f39d736d9/1061",
"updater_tests": "a97dc2762ffcade8e044682149c8796b5057bc21fb08b4d83eb6966448b51c5f/566",
"updater_tests_system": "f1a5348ff1e83f47a920906cf718d0c6c7a4e5c36d7f9515510ca1f2b1a655ed/566",
"url_unittests": "474b375c8c2a72ac20115d5f774b6f247b2666da64909297a4bf7a69f355b2f1/323",
"views_examples_unittests": "172372dec3ba50169ab1c63574c97a39617b3e283c593d22db4efe9b0fb03877/402",
"views_perftests": "cc14ecace689ea13dbe33d1edb009eed1f9cf8b565e482678760018770dd7873/487",
"views_unittests": "d1f813a9287c5a373b9fc6f3f717b9d4b13793ce5d969aa1fda645b61bf698a5/402",
"viz_unittests": "8408aa4773c5686bedf003056ae6fd841479d543542a09393a8be331d90b8cfb/407",
"vr_common_unittests": "98cfc8a2be6ed6f8d7bdc72f5e04a42b52c13a9b5a4fe2490dd63c6cbeec9336/564",
"wm_unittests": "ee6945b5a8ee209312992454d0eb576ea64fca9a43997b5ee899c034bb45c9ed/323",
"wtf_unittests": "f31b5c1ffced32e9b86d3e127e3bb97387adb10edc0e1aaee0a70c6dfa811500/323",
"xvfb_py_unittests": "8d329e8ae05717e539060b315c8776e5844ed23045a02d3f084d575392243798/323",
"zlib_unittests": "fba94b7f8f0e02fc13088407d0ccdfba79a5ae901c3c5def8e2c88a789797209/408",
"zucchini_unittests": "3dcbe16aa59db832adb247988d3d6d17655629f127ce6d299e68724a601f541b/323"
},
"swarming_command_lines_cwd": "out\\28e3-Win_x64_Builder",
"swarming_command_lines_digest": "d9d8da29f284e230f02a4ac1ffd121f33f7a7a456b58fed04c2a0fabc7f594a6/94",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "8268294e50b8c631543f490ac579ac1fb3994034",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8890218949747447600"
},
{
"key": "scheduler_job_id",
"value": "chromium/Win10 Tests x64"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Win x64 Builder"
},
{
"key": "parent_buildnumber",
"value": "155920"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[07:25:44.793] Scheduled build:
{
"id": "8724788887686043089",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Win10 Tests x64"
},
"number": 97406,
"createdBy": "project:chromium",
"createTime": "2025-01-25T07:25:43.955181559Z",
"updateTime": "2025-01-25T07:25:43.955181559Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "8268294e50b8c631543f490ac579ac1fb3994034",
"ref": "refs/heads/main"
}
},
"canary": true
}
[07:25:44.793] Task URL: https://cr-buildbucket.appspot.com/build/8724788887686043089
[07:25:44.794] Scheduling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:2:0) after 7m40s
[07:25:44.958] Received PubSub notification, asking Buildbucket for the build status
[07:25:45.012] Build status: SCHEDULED
[07:27:17.243] Received PubSub notification, asking Buildbucket for the build status
[07:27:17.346] Build status: STARTED
[07:33:24.916] Handling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:2:0)
[07:33:24.935] Build status: STARTED
[07:33:24.935] Scheduling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:5:0) after 5m6s
[07:38:30.957] Handling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:5:0)
[07:38:31.113] Build status: STARTED
[07:38:31.113] Scheduling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:6:0) after 2m54s
[07:41:25.131] Handling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:6:0)
[07:41:25.159] Build status: STARTED
[07:41:25.159] Scheduling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:7:0) after 6m16s
[07:47:41.197] Handling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:7:0)
[07:47:41.251] Build status: STARTED
[07:47:41.251] Scheduling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:8:0) after 8m16s
[07:55:57.384] Handling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:8:0)
[07:55:57.414] Build status: STARTED
[07:55:57.414] Scheduling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:9:0) after 6m6s
[08:02:03.404] Handling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:9:0)
[08:02:03.429] Build status: STARTED
[08:02:03.429] Scheduling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:10:0) after 5m50s
[08:07:53.420] Handling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:10:0)
[08:07:53.441] Build status: STARTED
[08:07:53.441] Scheduling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:11:0) after 2m23s
[08:10:16.485] Handling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:11:0)
[08:10:16.508] Build status: STARTED
[08:10:16.508] Scheduling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:12:0) after 8m10s
[08:18:26.583] Handling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:12:0)
[08:18:26.607] Build status: STARTED
[08:18:26.607] Scheduling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:13:0) after 7m27s
[08:25:53.638] Handling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:13:0)
[08:25:53.662] Build status: STARTED
[08:25:53.662] Scheduling timer "check-buildbucket-build-status" (chromium/Win10 Tests x64:8890218949747447600:14:0) after 9m26s
[08:26:30.747] Received PubSub notification, asking Buildbucket for the build status
[08:26:30.774] Build:
{
"id": "8724788887686043089",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Win10 Tests x64"
},
"number": 97406,
"createdBy": "project:chromium",
"createTime": "2025-01-25T07:25:43.955181559Z",
"startTime": "2025-01-25T07:27:16.938621362Z",
"endTime": "2025-01-25T08:26:30.370587340Z",
"updateTime": "2025-01-25T08:26:30.370587340Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "8268294e50b8c631543f490ac579ac1fb3994034",
"ref": "refs/heads/main"
}
},
"canary": true
}
[08:26:30.774] Invocation finished in 1h0m48.169170878s with status FAILED