[07:19:20.842] New invocation is queued and will start shortly
[07:19:20.842] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[07:19:21.937] Starting the invocation (attempt 1)
[07:19:21.962] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/165301a71e0a9c837bd7a662ae74db5b08492fd0"
[07:19:21.962] Popped gitiles commit info from properties and tags
[07:19:21.962] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[07:19:21.963] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[07:19:21.963] Buildbucket request:
{
"requestId": "8938960517993557056",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux Tests (dbg)(1)"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "165301a71e0a9c837bd7a662ae74db5b08492fd0",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "165301a71e0a9c837bd7a662ae74db5b08492fd0"
},
"parent_builder_group": "chromium.linux",
"parent_buildername": "Linux Builder (dbg)",
"parent_got_angle_revision": "135a24fc3706c5e09460356beb115f3624fe8c51",
"parent_got_dawn_revision": "aeab5ac7d6ad6fe8d05eea4e98413182e103633e",
"parent_got_nacl_revision": "f9cb132c419b9e019853103e88c518026b9f5083",
"parent_got_revision": "165301a71e0a9c837bd7a662ae74db5b08492fd0",
"parent_got_revision_cp": "refs/heads/main@{#1180034}",
"parent_got_swiftshader_revision": "729e92f8ae07d7b695bdcf346318dec4d11d899e",
"parent_got_v8_revision": "dbb21fc02cc5b72af3190a88bb7b05e25c7a1be7",
"parent_got_v8_revision_cp": "refs/heads/11.7.422@{#1}",
"parent_got_webrtc_revision": "e2e04513e7ed6a43bf854403fedf3812ac3d2991",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#40514}",
"swarm_hashes": {
"absl_hardening_tests": "e8d1841f6283c8a8da62e38785019f7ea870466caae4ee68ec47c6efac95c271/322",
"accessibility_unittests": "e95185a99a60a5710e68b002df7fa96b46e903dd50fb6c5bb6ef50c2f4efc36d/322",
"angle_unittests": "485edfcb3b7e507ec25cb07db8be0b0aa9b6a487216e7fc4f22eaa7babfac694/407",
"app_shell_unittests": "3151ae057ddf6bcca32d0bda1edbc8f85140178af00197d0485116fadf301bfb/483",
"aura_unittests": "4c712d1832f9de1ce362bd2635f689526a06bef877faf67c2d9d1a2348df8f86/322",
"base_unittests": "b30a7e24ce5f8664346d5e4ca8b1c68439d7350e984e17055be1512bb45181d6/479",
"blink_common_unittests": "1504861e0d9cd22fcc17f0d1415f32c05830a8c4b6fc2a3c25c86d4dea57db3b/484",
"blink_fuzzer_unittests": "14c09ca5250bbed37cbe881a355c433cf0e3b43cd3d277b0c3d58cd7dfb03a2f/322",
"blink_heap_unittests": "ed43db2edb5ea2fce9255d9dedb75c866e3d02a30cf13e99fc922be509e2edfc/322",
"blink_platform_unittests": "ef286c108d7cd2a56e2f2f166a7539cac319f52363bd66d7b44cc935382ff5de/563",
"blink_python_tests": "a4e1e15be7fb54e66671baae1b0a1d4c58c5da2a4e2f7afa47e97a20ccb83fb5/573",
"blink_unittests": "3e2371bbb9f520a02c84926b69c01b0541a197eddc9737bee9f3faaf65704f8d/648",
"blink_web_tests": "95d3ea16844b812263711b2237d610281cbf8a00d0021ef3d89334175d258cf0/565",
"blink_wpt_tests": "8ca29facd56f08b3449a8276c7b1c51805ce1d9decb73c64ce518f2b5dc8b71e/565",
"boringssl_crypto_tests": "1c50bfafab5e4b0b9580bf78245664800ea8338fa6fe22e10e8294e1551f4320/407",
"boringssl_ssl_tests": "182ba91cff8829e798c3611336be36cc8a10a02a5b2e1bdd1c9fb355a942fb38/322",
"browser_tests": "b62d0436f761a4b15b709b2602c3c86d8b1d9612465e82c5aceaf44fc848a405/1293",
"capture_unittests": "3583d2eceac1738217e9cb066d3099885cfdac975d126068c6c2eeec6c2bc0ff/401",
"cast_unittests": "f608d47075d8d9aa9aa27cdf497b4fda8deff0cafa0cddbfd91bc866f5cc27b0/322",
"cc_unittests": "dcb76ac3ad5678fd5f90c4df170951909a4fa9c0bc3fb00c281b5d9398d26e73/482",
"chrome_app_unittests": "180988548bfeaac90b27b138a19abf10161da7e7f900d388af5b7feffde85d25/647",
"chromedriver_unittests": "69ae3b1bf640a0dfdb720eec36796ff02b4af761bc5c2427502253493cc79868/479",
"color_unittests": "7d80948adfadbd2183e4628f98788487ba59050319924f86022461bfa397a247/322",
"components_browsertests": "d44b5ba48a390c4b2a23d85369ff6e2d53b51ba8ec23d63d9e9b666cc71abc05/651",
"components_unittests": "aebf86af6ae3fe73d679cd381a0011257ce619fa3a5d00aa9ecf86c402425e91/730",
"compositor_unittests": "99db45ab671fa93697f99c8ddab5ef3879d30bbb48a9c3524f61e4b0653076f3/398",
"content_browsertests": "f615de3761ba370b7e15ca3aeedd9504b8c3593a102e42cee220772e497ea389/724",
"content_shell_crash_test": "49db0f65bd697f6498f4c224066f79615cb4e2089c7e8035e8e967ccf651b516/564",
"content_unittests": "7beb210c957a679120b1810450d1ba83edf491a21c8f10fe9be11e824ac9b11d/559",
"crashpad_tests": "347af8f76257044636a71ec794ecbfb5b56f44f763059d4b5edcad7cf7a2de17/407",
"cronet_tests": "7ff4cf9284fb09ec2b626dea16dad820b9aa1e9548786f6c4346c64e7ae6820f/483",
"cronet_unittests": "02d9b188922fbaa42a00c0993a77c579682a8b0ca50500b80054e15901666be5/399",
"crypto_unittests": "8a5261b432b7f3e20594590bbf067df4a55c6792c45fd55f62407f737944111e/322",
"dbus_unittests": "9c65da4339e3fa36083351df72fe0bdbf4a6c6e252b8bb97b8645926a9276200/322",
"device_unittests": "a10b696d5b03fb5894274cab588ff65ee21c0b398773fecc25800520951ecbdb/399",
"display_unittests": "de59438ab0f306e928a81099e2c7cfb6da2e765147bab1cb97cd987ffd6f2ab5/322",
"env_chromium_unittests": "5289bdec5ea7086f521fd7500bd5717bd5f66943c698d1da137a0f27c928a104/322",
"events_unittests": "da057ff6748fbaea59c3d6c9938fad6a00837024a4112ffdcce4ed636e692f8a/322",
"extensions_browsertests": "415666e5152965e1d2b0b842ae3723b53a99331ccde20eadbfbb9d48186f4759/569",
"extensions_unittests": "89db78b92cdf3b2c98a79571fc065448f0a88ecf198deb154ed005125082f9ef/643",
"filesystem_service_unittests": "1d7a9ff0a6ef2be6b11c54839439f6d7af7e2896c2a5145a519ec1262eaf0e3d/322",
"flatbuffers_unittests": "760268d6b8040f7fe2965c5c7b8fad3113ef056f24e9a09ee16a9a79a0356f10/407",
"gcm_unit_tests": "e96b08c639f4cae42c409ac6a1b01a90f963f2e82758980be747ee7d9f1af193/399",
"gfx_unittests": "f848c764b4ea45eed68c5467531862ee8eeeae9d93147c719858f8fb6a422936/398",
"gin_unittests": "57ef48e3ef462655199c8523fed45442e6c5976710f06fc36ee5c0b78eb7735e/399",
"google_apis_unittests": "2a59e8fc92f08c38d264ab1af236c365a5e1c72837104eb6a6726318aa5816b4/484",
"gpu_unittests": "2d36c4af9b1e54be149701ac9a7057ac7f6ee622d73c30bcb9339ae34ad6170c/399",
"grit_python_unittests": "8fd0317b52cbc5db87d3370520598e0d5c04bc8b2d7c831120b9d36f617422f3/410",
"gwp_asan_unittests": "3f8f335cc10c62d3a0f36a4845e840953e3e5c7679e5b4958db41521a3c514d6/407",
"headless_browsertests": "641c5a842683114acdc9f3cd7d3a460a609372a4e57150e2e74d595031009a38/568",
"headless_unittests": "b037a1cd9347eca64bcd0b540c8393b2462c5ef35c4e55544cfbeacc17aceabf/322",
"interactive_ui_tests": "994efb4a06c4d9c7cb611dab8cf3706cad8ae3909b320204578e9f50249ffc8c/967",
"ipc_tests": "28a5aaa159c1c4d08664ea23eb62fbe38bd163bd9c06ac81978329406f4f1d49/322",
"latency_unittests": "3f189301ec294c4c8f8caf68773d598849e9e1bb709e31cc83ce98719e742314/322",
"leveldb_unittests": "7a95589d920ce0638d2a7e7900c1e579733fced5c00a7af873f445611af6f978/322",
"libjingle_xmpp_unittests": "4814e844f887733fc6b76af85db9276617b4974ed829e1593a5925fa8ff4073c/322",
"liburlpattern_unittests": "e854e0f38689ec0b25635351ac51b99a276b18764719fc5ded755df3f979ea6c/322",
"media_unittests": "fde2df4117a00c87889f1aa976ba297fd98c6bcb95bc6d513b169087ab4b4abb/402",
"message_center_unittests": "30832290b807e5a5b15fdfcf9cf427f4e419e20b99b548258101470ad7f1d008/322",
"midi_unittests": "2963bc419390e425fb2a865893ba6ffbedc6530f6e530d6491455f56cb71aaf5/322",
"mojo_unittests": "3c4f3a11706fc91cdb0e3a4047e5dbb090ba681f16847a284306cef889ca306b/400",
"nacl_loader_unittests": "cbb7a65a8264e073562e1f4acbf28e45cae65feff7c60e4ff287ea00a8495c03/322",
"native_theme_unittests": "2e217206a9468e47d728c1a85c2a778a05bb7dd3c18706de98df8c984aec7a87/322",
"net_unittests": "951ff691e78ab77c7155a1786910f3119ea2fe5a9c8b41a75db47c54fa2caa91/485",
"openscreen_unittests": "fbf7a0b62828f8cd94ec2f0cae1915f52e5cef19c04811df3684fdee3d89778d/407",
"ozone_unittests": "ef5134214c272f381f326da9d83e3c1c1f21fe9b21abcfb6046f39a51f27382f/322",
"ozone_x11_unittests": "6c15a22dc2969ddcd21a22f71879757bcfcbeb927dc3bdd6ba008d858fe325da/322",
"pdf_unittests": "b4ad335d3f5c66f568c086cb4f152973878dedf542bbda0a7464bf98d7d1e739/399",
"perfetto_unittests": "eaa233ccf4528a61c488925eb46146a1951dbb153bbb269b4b63f184c2e71700/322",
"ppapi_unittests": "6dec342815ee7a602ca7bd9a3c12a5c23acad1701a267b777378e69550740c83/322",
"printing_unittests": "fb2ec171e2a505f6dc34231d0299229c5cc72cecaf735919b95e77b00ee2153f/322",
"pthreadpool_unittests": "a32bd92455136ba4626597ec6fae3dbd64aaded7224a74d2ba9f8821f1c3ac36/322",
"remoting_unittests": "e6b8f12ccfd07e78518a6eca22ea3fbbdb7d34ea7ca6062fddd2cecf17915552/483",
"sandbox_linux_unittests": "d6c734c21f7714e092ed0b9172d1ff7bd1e859d4001bb71f246a7fc1905b8111/322",
"service_manager_unittests": "73bf8a981def5dec412f828874f3fed9a302488cda700556e00599a488acdacb/322",
"services_unittests": "46e7b47708cd15107cf8939e054c078b50836ec3b440b4a8825fc7b1f7191ad4/652",
"shell_dialogs_unittests": "4be9301c37fd59d01614423077213bb9c67050a9148737a07fc5173572215c1b/322",
"skia_unittests": "a26ee90fcedccba81fec1d015c33395b9c61b9ccfbed3b6dcbb8ea9cfa930479/322",
"snapshot_unittests": "7bc5a508b1a79a07c185cb03bbb4f60b575ac7791b61fd0c839db28e9eacc57e/322",
"sql_unittests": "f616a890fff718e3a4e8abb8caeae8be33413dc86da23463c0e4982db5e1f151/399",
"storage_unittests": "18647283333ffe2945b5bbc0c1a48354b759ef754f140193ba1f7770cbc6e22d/480",
"sync_integration_tests": "ac42c4cafbdd1dbef083984137aa8128c60bc2947ff59d33f0d09d7acde05f5c/729",
"telemetry_gpu_unittests": "ef6f79a27599e0d61ea83e711424f94f09befec80accc46c7ca384021a7f0741/655",
"telemetry_perf_unittests": "1121e7458846063adc76452e572c9fc616180cd0b13808e2561993bfd3aeaa2e/654",
"telemetry_unittests": "a15203eff63d999eb6893a5576602a71cb68b1eee224f12e61094ee2a592525e/574",
"ui_base_unittests": "3048acd6c6d4b45345ded2a01edc6eb4b26c434092cf0c02b1514eafa97e41d8/554",
"ui_touch_selection_unittests": "12948e67b55d4267520b36a9811026af972076fccaed2ea3065debf99c1ac334/322",
"unit_tests": "1f599837fdbe953a7c24ce4a7b23b796da452cef9907609f8b1489f9fd6dd0b5/1056",
"url_unittests": "ddfe73bf60ed53d6070dbc4c652279188e3ad0f83d29851d5e432f99c32fe5ba/322",
"views_perftests": "1cecdd05d8c9b18a83f7f884db22ddeb24c7e4bd64cbbf731c97ee2238973a93/486",
"views_unittests": "60f6bcc26dca4e714f17e2312da61fd977717d6cd1b95c4a1f3ebf2ae29f2d7a/401",
"viz_unittests": "78e8a7d441a386d77e94cd93da9acde75528dcad4f6caab10a7e031c4036f2e5/406",
"vr_common_unittests": "f966b8d371fa01333b682274c4e2318ec3a3e30a8470fe212abe91d5555142d6/647",
"webdriver_wpt_tests": "9a7623e084415640e9cde6a4efd3faec8b1fed3f40dd1e440b3fdbc0aa8d6b1c/408",
"wm_unittests": "76cb5e0ac2b8570ed0bac24d1e607a73a9fd5202cf68b68b146b6c764f011e57/322",
"wtf_unittests": "93cef1204b3598ce582d040bf794f25cdd3f7ecce362db76f3944472e24ff9b5/322",
"xr_browser_tests": "230ace34bced054bdc1118e56e9e43f6e070641edbbccea26e9d0f5eaf2d8f24/728",
"zlib_unittests": "19df36ba5cf24e7ea5f7c16eddbfa4eb00e14e756e3d3d0171b896352376bc3d/407"
},
"swarming_command_lines_cwd": "out/Debug",
"swarming_command_lines_digest": "409f83e9d09fe5818061fb89dc3ff7f61cebdd488ba154f78ea2f8e57c3400f6/94",
"swarming_inverted_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "165301a71e0a9c837bd7a662ae74db5b08492fd0",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8938960517993557056"
},
{
"key": "scheduler_job_id",
"value": "chromium/Linux Tests (dbg)(1)"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Linux Builder (dbg)"
},
{
"key": "parent_buildnumber",
"value": "203890"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[07:19:22.681] Scheduled build:
{
"id": "8773530456077208273",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux Tests (dbg)(1)"
},
"number": 115058,
"createdBy": "project:chromium",
"createTime": "2023-08-06T07:19:22.039356348Z",
"updateTime": "2023-08-06T07:19:22.039356348Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "165301a71e0a9c837bd7a662ae74db5b08492fd0",
"ref": "refs/heads/main"
}
}
}
[07:19:22.681] Task URL: https://cr-buildbucket.appspot.com/build/8773530456077208273
[07:19:22.681] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:2:0) after 3m6s
[07:19:23.664] Received PubSub notification, asking Buildbucket for the build status
[07:19:23.687] Build status: SCHEDULED
[07:22:28.698] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:2:0)
[07:22:28.715] Build status: SCHEDULED
[07:22:28.715] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:4:0) after 4m23s
[07:24:49.419] Received PubSub notification, asking Buildbucket for the build status
[07:24:49.440] Build status: STARTED
[07:26:51.736] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:4:0)
[07:26:51.759] Build status: STARTED
[07:26:51.759] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:6:0) after 6m28s
[07:33:19.787] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:6:0)
[07:33:19.807] Build status: STARTED
[07:33:19.807] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:7:0) after 4m21s
[07:37:40.791] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:7:0)
[07:37:40.811] Build status: STARTED
[07:37:40.811] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:8:0) after 2m22s
[07:40:02.828] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:8:0)
[07:40:02.911] Build status: STARTED
[07:40:02.911] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:9:0) after 2m47s
[07:42:49.929] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:9:0)
[07:42:49.955] Build status: STARTED
[07:42:49.955] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:10:0) after 4m54s
[07:47:43.972] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:10:0)
[07:47:43.987] Build status: STARTED
[07:47:43.987] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:11:0) after 4m50s
[07:52:34.012] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:11:0)
[07:52:34.032] Build status: STARTED
[07:52:34.032] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:12:0) after 1m29s
[07:54:03.059] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:12:0)
[07:54:03.074] Build status: STARTED
[07:54:03.074] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:13:0) after 1m44s
[07:55:47.141] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:13:0)
[07:55:47.159] Build status: STARTED
[07:55:47.159] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:14:0) after 2m2s
[07:57:49.278] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:14:0)
[07:57:49.297] Build status: STARTED
[07:57:49.297] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:15:0) after 8m4s
[08:05:53.314] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:15:0)
[08:05:53.329] Build status: STARTED
[08:05:53.329] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:16:0) after 3m23s
[08:09:16.388] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:16:0)
[08:09:16.413] Build status: STARTED
[08:09:16.413] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:17:0) after 2m34s
[08:11:50.435] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:17:0)
[08:11:50.452] Build status: STARTED
[08:11:50.453] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:18:0) after 3m33s
[08:15:23.481] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:18:0)
[08:15:23.499] Build status: STARTED
[08:15:23.499] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:19:0) after 6m13s
[08:21:36.515] Handling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:19:0)
[08:21:36.568] Build status: STARTED
[08:21:36.568] Scheduling timer "check-buildbucket-build-status" (chromium/Linux Tests (dbg)(1):8938960517993557056:20:0) after 8m55s
[08:28:30.773] Received PubSub notification, asking Buildbucket for the build status
[08:28:30.875] Build:
{
"id": "8773530456077208273",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux Tests (dbg)(1)"
},
"number": 115058,
"createdBy": "project:chromium",
"createTime": "2023-08-06T07:19:22.039356348Z",
"startTime": "2023-08-06T07:24:48.903413Z",
"endTime": "2023-08-06T08:28:30.457416997Z",
"updateTime": "2023-08-06T08:28:30.457416997Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "165301a71e0a9c837bd7a662ae74db5b08492fd0",
"ref": "refs/heads/main"
}
}
}
[08:28:30.875] Invocation finished in 1h9m10.046510614s with status FAILED