[03:37:28.547] New invocation is queued and will start shortly
[03:37:28.547] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[03:37:29.631] Starting the invocation (attempt 1)
[03:37:29.656] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/e895d900fc96929a1d46ec8b572d81bcb031f0a2"
[03:37:29.657] Popped gitiles commit info from properties and tags
[03:37:29.657] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[03:37:29.657] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[03:37:29.657] Buildbucket request:
{
"requestId": "8902101511614615232",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Win11 Tests x64"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "e895d900fc96929a1d46ec8b572d81bcb031f0a2",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "e895d900fc96929a1d46ec8b572d81bcb031f0a2"
},
"parent_build_id": "8736675400904621009",
"parent_builder_group": "chromium.win",
"parent_buildername": "Win x64 Builder",
"parent_got_angle_revision": "25c693c96d87507b942b56ccaad843d404351d28",
"parent_got_dawn_revision": "1d203d2ca022a0a5acfa9f65ef7e9ea51b09e78b",
"parent_got_revision": "e895d900fc96929a1d46ec8b572d81bcb031f0a2",
"parent_got_revision_cp": "refs/heads/main@{#1355698}",
"parent_got_swiftshader_revision": "3239872f9c5f1f7a1d029733dd1036d1247cfb4f",
"parent_got_v8_revision": "4be854bd71ea878a25b236a27afcecffa2e29360",
"parent_got_v8_revision_cp": "refs/heads/13.0.245@{#1}",
"parent_got_webrtc_revision": "13e377b804f68aa9c20ea5e449666ea5248e3286",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#43019}",
"swarm_hashes": {
"absl_hardening_tests": "86bc6b72d67d3cd9d8ba2bc72dc20eebcf0de0ebfb140163dff602ebd972a5f8/322",
"accessibility_unittests": "3ed5c1f4db57b5741f535aa7c297e472a08a93bb959fb1326dbadaf38ebedbb6/322",
"angle_unittests": "2847542659d3c46135d7de782014f3e0780fc2f368dc776a00a7fbf5a23edf5e/407",
"app_shell_unittests": "3a2a8d5d8f887f5e0e654562d345993677ead68922335277f28d2c6fd5efe6f4/483",
"aura_unittests": "d3a345353d082bbaa41b2bd150f538bdecae641ba2cbae7b4f5abf774197b796/322",
"base_unittests": "dd119b18de8dcac99bb97e95f95c7fd8440a1666a54d276d4042bac6dc08d456/479",
"blink_common_unittests": "ca4e885e9110895b7dccb444a43e3fe50c6f780baa30e75593dfd4cb7f973551/484",
"blink_fuzzer_unittests": "2a5d0377f9c67c64f43eae9c025fe09c46751afa55e582d67dd0f5527501930d/322",
"blink_heap_unittests": "409681d50508ca4c08778436fc9619ecaded7a7097a4c730bfa90100474e239c/322",
"blink_platform_unittests": "bb840bd3b147ab93be043491b4c8a9fa4df4c787d52d02524019e6524788278a/642",
"blink_python_tests": "5f61cb218969c3a8c4b282f856ce46a9d75cf6dbece419f432b8702ce8bb2651/573",
"blink_unittests": "e1c1e5b8f85a5f517b303f4ec64067a9caffb5f8b763b3274ffe9e9e358b7aab/648",
"blink_web_tests": "4703a80de1246c252dee8e4811bd78e36cd52b679794df9b95c42dcdf88691c0/565",
"blink_wpt_tests": "194405af608991dc495c13246c00dd2ebed2164e9002701c52e4ea56c331174d/565",
"boringssl_crypto_tests": "8924fda89b72eaca59ddcd67f3347bc85f166a4936faca37c6c5cd2ffc5ea632/407",
"boringssl_ssl_tests": "f694469395aac41eea5018fa931959123541df3910e5032faa8f027c75a37897/322",
"browser_tests": "eb5b8f9eb53c311eeecc4768d0ef05035f6ac6b065e27261e9d04448da217c5d/1214",
"capture_unittests": "734decd69b52aa13eb239a4d7c993118802588de1dd975eadef49c87541a5d1e/401",
"cast_unittests": "4c3d9d08a7711d070202f97a50cadbf32dd60db67959f6ab5b08f19f3d2ea40f/322",
"cc_unittests": "9aa9296c90b372441f9210fb6e0f74631972b87a2eb0cb12b1f4207dd247619b/482",
"chrome_app_unittests": "ec0463c8b93ac23ed9cda218302fc6f9cd317d7b1f0641c4063ac8bbbc597924/563",
"chrome_elf_unittests": "72231918ca6aebcfc0d0b72d7354497e6157b575407f96a9573af2c9c6990785/322",
"chrome_wpt_tests": "787e12a3016677694a60ecd86a07b896475ce27fa1e64db3373fa9ee10d7d09f/565",
"chromedriver_py_tests": "e20ba2e0b7e5dd5db5d751be523ca30b6563edee4f765073706b6fb7ebadb17e/488",
"chromedriver_py_tests_headless_shell": "a1fcf0a90b73f84838b5d7f192745996015631fd0f5a9f41748294fb7f3ad8e3/488",
"chromedriver_replay_unittests": "f39f34aae35ab618e7c61ca3b8028354026dafaa6bdbe1e79a0d3c266f772589/325",
"chromedriver_unittests": "c3210fe7444804f970acea2e45efca1c8a5618af287d657d87cb3ee471322ebd/479",
"components_browsertests": "cbcba4eb3893e47dce535a5bb12e935708a83cd3637b11fb509d3fa1ce0cee15/651",
"components_perftests": "d9dbde98d193f8cca39d4d872b32fcfd6a8822071643308d38d5d0f2f264abda/648",
"components_unittests": "03f1c29714e01140db4c049ff9316397bd365bb9bb7ea05ad9cde8d8d59ee2e0/730",
"compositor_unittests": "fbb17046bfb99a8ed3dac04cc83b684c341eb53dd85a613e0d52bfadfa3b0f42/398",
"content_browsertests": "a29e1c52a05c9cc1997da5ce5d49c89275b6769927d24d8f001a591369fa1475/645",
"content_shell_crash_test": "dddea62561cd58f2c58c442c188354f032fb689428b5bb550a82cd434ee1df6f/480",
"content_unittests": "c5e9c95bd5c094f88833d1ea1d686d359ade1fa7dda3a6aea800685cfa76f4f5/559",
"crashpad_tests": "09141ab7cbef2d84b14bc322dad9dcb229c4ae027df7b230fc61ade1ee1f0048/407",
"cronet_tests": "c93e9c7090204839e455ccb3b318055e7073ca520446b3c15d14b9e71a5dd251/483",
"cronet_unittests": "89d75546b47a6d34d3c6e16d9dc6c3b07a11f6f89ef8d2a53ab24f0a6ee07f6d/399",
"crypto_unittests": "f286fa7c494f0a90c1b4b4d60f7cae92b96a22772776303e9dfa11926d0e684d/322",
"delayloads_unittests": "f75eeeee2beb71772d8fb7f36164b4310213e42d3c21e189562eacda7aaf9ceb/322",
"device_unittests": "b4856bfff543e5e68ba3e28b4535abb21c7a2c1b1faa621ad3e3343252d7cc87/479",
"display_unittests": "e52852c6860b80d8794d8465b3ecf26cc33953a06cf841e058bb35a995f8f2e4/322",
"elevation_service_unittests": "c8862b9311dfb4a1a9aaf1cdf01a4c99b6f5cad44ebcebba8460a31ba38908a5/322",
"env_chromium_unittests": "bb7858cf944d94f96bbd86016aedf7a7d1858b560bcb61bee1cac2eef2bf5efc/322",
"events_unittests": "68e0184b06cb0f02fc1fae2a29c33e8ee758b825255d7d7d5b700221009702f2/322",
"extensions_browsertests": "e2291440511d0bc7c4e8660101a61cdbc71b91f9a6ccf5a9b3932a6c9c4bbc29/569",
"extensions_unittests": "e569bafb5ebc62cf91a89354327deeaef0101d2a9c994967218142dcd20e4e25/643",
"filesystem_service_unittests": "92522b10120bf54d9e811105b20775db11728ea78669a2777af30c1c089b06d9/322",
"flatbuffers_unittests": "aaad7f48b11a34485a96f6d8bd1fcc9229dac014a92c3b9a49b87746a267cb4d/407",
"fuzzing_unittests": "a70c114d5c5c9553a953a97c4eaeddaffd4da104b7ee8f4ff6cadc09da66e106/322",
"gcm_unit_tests": "2256ef5345a79d9baae9ec135dbd54db4c91066a8e494fdd48bb9480bd173a13/399",
"gcp_unittests": "464495f3eab9296a64af24283465661e70ec6cb828cba14546566abc4ff7f1ac/485",
"gfx_unittests": "79b21b37a031f1315c7ed39cf962307dce467fb0cf79e1eed7babfe6bd33f434/398",
"gin_unittests": "5d32a87432e1b59fbed1095972be7f4603927449eb1c5742bd6c46350cfc01b2/399",
"google_apis_unittests": "1fb4bd5945b44f47c3b5ced951addd12504f86048731c8da821edbad2bbd9b4b/484",
"gpu_unittests": "7d5c5058f22f1aad896f6cb09d39866f3c75a2574b02c82049f28e9a73c105f3/399",
"grit_python_unittests": "423475cfb1f32a1a8d0e76a3d2df05f72231aea3c43d6e0894224ce8e909fc97/410",
"gwp_asan_unittests": "f108000f6bc98984c369f48935439de91b85e886064b23f2695885e1cac12715/407",
"headless_browsertests": "eab7a4c09377f115516bd35292820b97644fd4b5071d7de0324871d6d8ebb712/568",
"headless_shell_wpt": "eac3250f443b6b6d0f4b11660263ab789c9121dee87cbbe44ea44f670fcac6e8/565",
"headless_unittests": "06e9f9144a02d2d577c47f1e862d1fd381aa7e02a4768886b267a7202d23e377/322",
"install_static_unittests": "94cc9669798ad758ca05d24e6eb283c6b29084cdeecffdbae588a8312272111b/322",
"installer_util_unittests": "e78089b09f62edaf6f1636fa5e21f655f4b78e5de95e6b485ef8b664c0d64dc1/402",
"interactive_ui_tests": "9bf5b58fc4241c3b1392cc99ce0529a14527cd6d3fe1a237dfc872c61f351e15/972",
"ipc_tests": "e06ffa81b80a3be68494e1a91ee63f471564f7f485eecc01b87e196a54073b1f/322",
"latency_unittests": "511952a4d7783ca9a4c50bd4395858f8ad7da029598b717b1abdb4d6106c431f/322",
"leveldb_unittests": "b76b908c5e34eb1debbe04002cb2ceee47b3eba75b4c56c71a6beb9a10d61452/322",
"libjingle_xmpp_unittests": "f9516d469a096b947c19f1b8e18ec27f8505ae0b5414f0d6f99426e6d117df6e/322",
"liburlpattern_unittests": "c887e40fed580ff136fa91005748eaeac1cf20c43631224dee2ff7853fa17be2/322",
"media_unittests": "3131b024b231564d0d4d811561ffd28a96604f526e9a07e8a8bf3662d81a32d2/402",
"message_center_unittests": "338929023794101bcb7ed72a7f7a19d2e5eb434a29fa7b546348508fac979f88/322",
"midi_unittests": "2163726c746f751c9118d6495ff01198ffead4e8bac9026ba11e2cc67b98abf9/322",
"mini_installer_tests": "deacfb06f4fe5269539cc3750462cefe08915a31487c7702b565012880b68b3d/488",
"mojo_python_unittests": "91d6dca352aea587fd0e3b3a4d4e8bd2d5f47d79f1dde97a3a83180373cb5c0a/489",
"mojo_unittests": "54e05482b7fbe8144352d1005bbc75668cd794c603a30df63080f01ec99f2c97/400",
"native_theme_unittests": "f84a37340f52f5a787b82ec8c6f34bed54b51c97608101484ad59c8d492f49be/322",
"net_unittests": "25274ef3dab406af1dc0085eb7b55d39718445ac1b037602cfc6201a9ce95498/486",
"notification_helper_unittests": "d0b6d6c4e9b9f30a54c038467162e808175fb061e173704584ca3e9ff3fe8cd1/322",
"pdf_unittests": "72fae8da3cbf20a94f3ea2ae659caffc8369801b651c6c7cf8667415d1ebb017/399",
"perfetto_unittests": "b0c594e207a92fe605952070dec91ca38a8ca653a8d659d1af115b20623ffe4a/322",
"printing_unittests": "12219397c7243a678341d09975ae3389bfde296f78e7e4dd006a3ccfea35dc0b/404",
"pthreadpool_unittests": "51a41063201d2b77c74dacf7715081bd79249f88251e12d979e479533fea6dcf/322",
"remoting_unittests": "633d09a5b2e0fa21f0bcd6af87e76cf4eaf4f440b055b41b1f8719b9e435e5df/399",
"sbox_integration_tests": "014f1768a8701b83e36f80507d6ea2e00e86159ebdab64794e96abc920f3f368/400",
"sbox_unittests": "dec45898159b8753cc273085df3cfeae4a49d0395ad6bd07bfb34775fc4a975f/322",
"sbox_validation_tests": "d94d270579e7c762f9a5c2bce9f7f8cf92c0e4a5435736c8aec8f32ec7b1de2d/322",
"services_unittests": "8068aedd7da479bfebd565a0e00a142486e4b4560d9b30ada92f1eb3cfe88f64/652",
"setup_unittests": "1276b3d146cb08aac430a23105bb95a13b7b673ee20e655870e3778ed6092fba/403",
"shell_dialogs_unittests": "c3b174edcba7bb58d9df5539387586bec25daa1c4411ae8329e171af37a08e6b/322",
"skia_unittests": "146c45fc1ba63f213e11576f87f19db19b321ce1d54c7bf4209b598f4a8dab58/322",
"snapshot_unittests": "ede2efd2d2a5f1a74287d802589622e032311fe3ee043612cd22d310acc85e24/322",
"sql_unittests": "a83be78cec467e7977786635209fce9f7877135f27ec58682bcea00bd0843ed0/399",
"storage_unittests": "ce73fe973a43cb79b1fc385d2dd6eac9c3740a2df1312e17ccedf5286d20c692/480",
"sync_integration_tests": "b91682abd00a7e1a7fc67456b7774885fd56dc25cca77e4f7316ebc336b70569/645",
"telemetry_gpu_unittests": "4bc15fb4f11093f3ea005adf06ab9536e67087ff7494fdaade937b050455112e/735",
"telemetry_perf_unittests": "3490e4aa8874a3fbbda81db0e1e85bb48d3ddf4f6549b7fca242687c9a997209/654",
"test_env_py_unittests": "adc3a8e1055e95cbe2e09e1a83343c7abb19715adc05d7472fb8addf71aa8d50/322",
"ui_base_unittests": "4f2e5fe44e935ce057cd1713498a18080f6cbf7dd22e61119250d0e0dd5b6605/554",
"ui_touch_selection_unittests": "f66c157ab297826619d6aee6b37ee59216a5fa4ecafd2b6a8dfc9343272804c1/322",
"ui_unittests": "34cda8413f7d9f9f356f42c900c12b42c7693195f80c4dce1e48316e92c7a51b/322",
"unit_tests": "43b61235d6efe1c3c4c4f1af230297e95602c0bfc2185bdc00eb84734002a014/1055",
"updater_tests": "e1e29ae73fd5e03750be8be0478f4191ce9584693e27e820aa224d109e9f8989/565",
"updater_tests_system": "10343022819e47349e37b61a4c5f2875533cd56b98a20c4691a440d10910ab3f/565",
"url_unittests": "93e439544f44b6e3a84221d64100af79c4c5fe94faea0bd017f4f2cc6292c68a/322",
"views_examples_unittests": "a2596083259d8ba576b72a68500aae72ae4bd47ce621e7d566af16fba7356819/401",
"views_perftests": "af5e6f547a93e3298f136c164d6d0e85be75c321e3673c523a10882c9906062e/486",
"views_unittests": "7b4d6316afe76ab86f894b17fa8277fbe8bdc7c717cfed8e1fcbf3ebae929458/401",
"viz_unittests": "4d4b8382cf77ca2af93812b8803cf38d9e9277fb915f598fac21fd5c9c974f19/406",
"vr_common_unittests": "4c44566a1772df72a535210ba147012ace031d49cbe540a3b63d08e1bb7790fc/563",
"wm_unittests": "867354533592776e7ebab4d0d48c318d643a818a1ea6580a0e4f000ff9d9a83d/322",
"wtf_unittests": "b7ec37ba5d0c74b97285e0fc5b6f2b35c26d498917c2eb8ad72d81120cf5d17b/322",
"xvfb_py_unittests": "3d1fa217646c63f44259797b435a9ac908fbbfd5cee28b23fdfa50a3d111a52a/322",
"zlib_unittests": "cb552fdd7b6c46f7e5bfc1d739aaa56471e91b8b206bdc6d57120c5d95004cab/407",
"zucchini_unittests": "214b566c3806e71e7c698981355484f95b6f0126e262b2ebba912fc0f6f5eb46/322"
},
"swarming_command_lines_cwd": "out\\28e3-Win_x64_Builder",
"swarming_command_lines_digest": "46919f6d52df9be824f477b87f03ed5579e38a806cdb6056e56236b0b9680c9a/94"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "e895d900fc96929a1d46ec8b572d81bcb031f0a2",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8902101511614615232"
},
{
"key": "scheduler_job_id",
"value": "chromium/Win11 Tests x64"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Win x64 Builder"
},
{
"key": "parent_buildnumber",
"value": "150605"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[03:37:30.346] Scheduled build:
{
"id": "8736671449721081057",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Win11 Tests x64"
},
"number": 24872,
"createdBy": "project:chromium",
"createTime": "2024-09-16T03:37:29.729107306Z",
"updateTime": "2024-09-16T03:37:29.729107306Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "e895d900fc96929a1d46ec8b572d81bcb031f0a2",
"ref": "refs/heads/main"
}
}
}
[03:37:30.346] Task URL: https://cr-buildbucket.appspot.com/build/8736671449721081057
[03:37:30.346] Scheduling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:2:0) after 5m3s
[03:38:35.259] Received PubSub notification, asking Buildbucket for the build status
[03:38:35.285] Build status: SCHEDULED
[03:39:11.820] Received PubSub notification, asking Buildbucket for the build status
[03:39:11.849] Build status: STARTED
[03:42:33.437] Handling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:2:0)
[03:42:33.453] Build status: STARTED
[03:42:33.453] Scheduling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:5:0) after 1m13s
[03:43:46.628] Handling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:5:0)
[03:43:46.654] Build status: STARTED
[03:43:46.654] Scheduling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:6:0) after 4m24s
[03:48:10.881] Handling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:6:0)
[03:48:10.901] Build status: STARTED
[03:48:10.901] Scheduling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:7:0) after 5m8s
[03:53:18.927] Handling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:7:0)
[03:53:18.949] Build status: STARTED
[03:53:18.949] Scheduling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:8:0) after 6m31s
[03:59:50.077] Handling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:8:0)
[03:59:50.099] Build status: STARTED
[03:59:50.099] Scheduling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:9:0) after 1m35s
[04:01:25.122] Handling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:9:0)
[04:01:25.187] Build status: STARTED
[04:01:25.187] Scheduling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:10:0) after 2m30s
[04:03:55.303] Handling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:10:0)
[04:03:55.320] Build status: STARTED
[04:03:55.320] Scheduling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:11:0) after 7m6s
[04:11:01.548] Handling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:11:0)
[04:11:01.565] Build status: STARTED
[04:11:01.565] Scheduling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:12:0) after 2m57s
[04:13:58.855] Handling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:12:0)
[04:13:58.872] Build status: STARTED
[04:13:58.872] Scheduling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:13:0) after 4m46s
[04:18:44.891] Handling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:13:0)
[04:18:44.909] Build status: STARTED
[04:18:44.909] Scheduling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:14:0) after 1m15s
[04:20:00.106] Handling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:14:0)
[04:20:00.132] Build status: STARTED
[04:20:00.132] Scheduling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:15:0) after 1m21s
[04:21:21.161] Handling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:15:0)
[04:21:21.182] Build status: STARTED
[04:21:21.182] Scheduling timer "check-buildbucket-build-status" (chromium/Win11 Tests x64:8902101511614615232:16:0) after 8m56s
[04:23:49.952] Received PubSub notification, asking Buildbucket for the build status
[04:23:49.971] Build:
{
"id": "8736671449721081057",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Win11 Tests x64"
},
"number": 24872,
"createdBy": "project:chromium",
"createTime": "2024-09-16T03:37:29.729107306Z",
"startTime": "2024-09-16T03:38:49.015200796Z",
"endTime": "2024-09-16T04:23:49.642847606Z",
"updateTime": "2024-09-16T04:23:49.642847606Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "e895d900fc96929a1d46ec8b572d81bcb031f0a2",
"ref": "refs/heads/main"
}
}
}
[04:23:49.971] Invocation finished in 46m21.432408081s with status FAILED