[05:26:12.631] New invocation is queued and will start shortly
[05:26:12.631] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[05:26:13.801] Starting the invocation (attempt 1)
[05:26:13.832] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/7d46148dcabf6a4d263014dde9d3ac2239da4c12"
[05:26:13.832] Popped gitiles commit info from properties and tags
[05:26:13.832] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[05:26:13.832] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[05:26:13.833] Buildbucket request:
{
"requestId": "8944494050891744256",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux ChromiumOS MSan Tests"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "7d46148dcabf6a4d263014dde9d3ac2239da4c12",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "7d46148dcabf6a4d263014dde9d3ac2239da4c12"
},
"parent_builder_group": "chromium.memory",
"parent_buildername": "Linux ChromiumOS MSan Builder",
"parent_got_angle_revision": "721c15efaa705ba6b377ffc4d44ade8717331023",
"parent_got_dawn_revision": "cff18f3ec89141714c1e93ab325a290f21b7bc3e",
"parent_got_nacl_revision": "f9cb132c419b9e019853103e88c518026b9f5083",
"parent_got_revision": "7d46148dcabf6a4d263014dde9d3ac2239da4c12",
"parent_got_revision_cp": "refs/heads/main@{#1153667}",
"parent_got_swiftshader_revision": "ae667fe96db9b7f76edea242015d61f293c7210e",
"parent_got_v8_revision": "83b1daa99a9fa17f46d87806991fe6ae9d691a46",
"parent_got_v8_revision_cp": "refs/heads/11.6.72@{#1}",
"parent_got_webrtc_revision": "e00a12fb30fc492f833bebc31339d48c5b5f3c6d",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#40227}",
"swarm_hashes": {
"absl_hardening_tests": "7b5c3cd66e803195fb6f0876ff6c8ab6c5ee2464f6108bc29725fcd783d53f05/486",
"accessibility_unittests": "6b0f47cecea85fd0590bd413077a63ad4a62626ede4c9e881e33df8bddc41148/486",
"angle_unittests": "4f13ce3309cf32c0538dea66096e6a4100db38d2b9c56c0c8aced811f0db6f54/487",
"app_shell_unittests": "ea8a912b1e84d27b597cff3bd6eea8e849d12fa5849a7bc918d03a3c1632c850/647",
"ash_components_unittests": "29cf991354447c026ced5d98d1a8af1bd51fd7bf040ee71eb1a42ac713312e21/723",
"ash_unittests": "c39e8126bc53beb3c1e1c0488ff20c75e7eea8e446e64f512187d33ef7356a77/799",
"ash_webui_unittests": "777fa3512e0258f2c17e2c536e234b9764d1a172d2a743374a4457f424df27ae/646",
"aura_unittests": "7d950efd821afb876d5ca4c7f6770a87fe473844a43953eee9d5246443e42b50/486",
"base_unittests": "ffa9bc4d24a2d46d008f285fdb9bff21ee82afea054d407c8abe497b994e7d08/565",
"blink_common_unittests": "c68bd1f6a77012cde9f4973d02b1f6bf73dd25d511befa78a8104914a635adf9/564",
"blink_fuzzer_unittests": "6c9ac791d116365c2adf6c9ddf3717cedfd027026071cf7737a59c6115e5d95c/486",
"blink_heap_unittests": "006cbe2d6849cd6d8fcdcaabb3126eeba76ac55455e9ff0fd00cf81e275d4d73/486",
"blink_platform_unittests": "6b58ac647f5ffda9e082ac43baeb3a4bbd6e994a609fdb326f982b0dae34019c/643",
"blink_unittests": "d34aaac261fa97d2c387a066c72db3d35f39be9da0878967d317f67bb6152b13/727",
"boringssl_crypto_tests": "407b086201646fb7526756ee4e61b27e96b5adf37d3bf01ebcce3fc6c2d68b09/487",
"boringssl_ssl_tests": "e5e812d772c21599c64bc459ad0938173ee543115eb779e96edec68ed2f936d5/486",
"browser_tests": "d33ceb7d62184de65e32aedf289de6a10509a09cf5bd9349549a5a6c1d36bcca/1378",
"capture_unittests": "5353a4290be0714adb627ca31e897c455e9b818d3c1473f6ac66c35638037cb3/565",
"cast_unittests": "fde0bc8971dbb0641a2090df53da922591a44dab084fe80fe44300e17d27bf7c/486",
"cc_unittests": "bfb5f1ada5e29bd419bc26fd6547fb82825504d8e6ec6f2a25782939f4f4d303/646",
"chrome_app_unittests": "271f2c2b1147d1b048d563624cfee00fa1077948fa00a629fef2192690d8844a/731",
"chromedriver_unittests": "a2020424de5bc029678e1d84a422ce7e26ee8848476c7587a2bb186b1aaffec7/643",
"chromeos_components_unittests": "238b1dca0700c63c97ceb9fa954800c2c6025ec934e1b1f355fef5dbd5b06428/645",
"chromeos_unittests": "245539554f7758d09036162ce94bbdac67cc7932524e5fed5c18009ee5ff2b3f/732",
"color_unittests": "404266d350da4c31ff9d2f672b6220840727012bd55ea791db0632b9f87cf9ae/486",
"components_browsertests": "b5f6afbe7b20e837042774c9fb515d641ac148a6717b8e53db01ed74e91b6e51/730",
"components_unittests": "c5061d99f1eb3fb8ae4c535705998b3a834a0ffd5fd2d8a0ac87c8d795cd798b/890",
"compositor_unittests": "8a13603cbc7cc890a51add705f728eea604b85ffb737f0d281e7dc1cc58c5d32/562",
"content_browsertests": "8d12209495b3cf70998962c93a0e807b5466db24e5b098025af97533c6592ee2/804",
"content_unittests": "53fab73c3ed1e4c505437497ccdac94ee9b3949c7d2db454e690d46e881f7872/723",
"crashpad_tests": "1214d598cc18fea3a1688f1e3090bbd36a9307caad999a5d4ec28d8230d6a7e9/487",
"crypto_unittests": "7a348e5cbb37f6a282dc20248fa94bf05f3064fdb6c67ee504a0e82def0121a8/486",
"dbus_unittests": "2d85aebacc4c8ee4eb93ec01241ab5c8bb76aa15f20a256e33896f3601f35847/486",
"device_unittests": "1276221e6ac109f9d8144032ace71226e8b18fd79abfb51e5f4e14e716abbafe/563",
"display_unittests": "6d2f0de36281aac0f35cf387a2344a86390e84e51798891326045f1835d141f0/486",
"env_chromium_unittests": "5d14b0003353fe22e6512fcb0efa48d2fa95ac4df66d35d7346313ccc94e5af5/486",
"events_unittests": "dfb624b47f149a8d2aeea0a02ee887aa24dcc2ef12017576f314b0f79ac4c23a/486",
"exo_unittests": "aeabe8c70276ea4b4fbf4507f153cfc5022a072bf9ef31f9025f902bff6599ed/646",
"extensions_browsertests": "cb5f1eca3e15fd438987899bfa133bf1b17d62cb582d1b29527a4d6afe6081fa/649",
"extensions_unittests": "692168891ae5e22fc44d6274e31a47c9e44d636606bc1bc891347f815ffb4cdf/811",
"filesystem_service_unittests": "a6159bee794059c971fc12fab111105cc6668263e1611984e2c8a0ab940d4d84/486",
"gcm_unit_tests": "8ea40a69a1ca47fe111bdb3f7fd120680d4c2601fa8b9072d28d7f2786c1989e/563",
"gfx_unittests": "b8d570fa0875434bcb3ae4361cc1721375a65225ef841120fdfd2ed3bd5e05d1/562",
"gin_unittests": "b0c029d2cfab3d2390ea3d763edca65b2b7d9627635222c23195458c90c034de/563",
"google_apis_unittests": "33810244eab0c17711e12961ea3570f1486b00e5274c1aeb8c77655458a6aa1f/648",
"gpu_unittests": "e047c555f4ed6bbcf67cc9cfd52c444cbcb609251e91d7f4231a21c64e229944/563",
"gwp_asan_unittests": "2e8d3b2d2fc8af04c8297b935bf6cd479a6a1ec79b88863b189e72f2170d5d61/487",
"interactive_ui_tests": "0f81d4d8beac8362fa5bfd53e57ea402a6e6923ed46d8970a5ad33f616c3c089/1126",
"ipc_tests": "423d67b1c8a419b8d86482b5fdc57ec2d27d16bac4e973932062726a86e8cc7b/486",
"keyboard_unittests": "5e0d805117d04cf61b06e0ee15e152d0d385888771caff15c69d63fd51bb09f3/486",
"latency_unittests": "2701565a8bedfe2b0d54c8d61fc8191f4d31a4d5fce55a7d5a474b2b5b21937a/486",
"leveldb_unittests": "694009d48d84f029329455b5138612a1744ca39d2c40a3f15397df4ef11a5e36/486",
"libjingle_xmpp_unittests": "2dbd773e01bf33751f25bc23b920079acbe756008ed2f18edc29fb378e9499c5/486",
"liburlpattern_unittests": "72c08972608527673ef23bfe07d68a33532badd660a21a5360854e7909ce2a33/486",
"media_unittests": "b06b75d5ca978bdb06d8076e5b8e1087ad77974430c73e33867d868770f14904/566",
"message_center_unittests": "004efd1a36bf297394f662dc5119fcb5e17c57c4f3898463d7e1ae3aa2686c4f/486",
"midi_unittests": "d56f1648a428f3e7ae5af87fd42597952387ef8f8173699b802f95e02b115329/486",
"mojo_unittests": "dc18b0bb064e7d693203f5d886cb27edcf5816cbe7846e2bcfe21824d847a039/564",
"nacl_loader_unittests": "647e28e32b425fe6b2ad2c29461f5ebad158f88a16e06a9fab077b1241e52fb8/486",
"native_theme_unittests": "b6e4146f38c6e48fc2e5cac9fa09e239138a323c12fc7f44388ef0a003a22955/486",
"net_unittests": "f78779a9e466159ace106bf879da8b8325498e95a10d47680c98d3d868453822/565",
"ozone_gl_unittests": "373fcc294877bbb17dde20ccc2e4f2b8d1a5596d91a96348a459d608786a915a/486",
"ozone_unittests": "3d2b97666c7786d9c9312167f64c59e29a7b6e1b34480cff27869b37742c07a4/486",
"ozone_x11_unittests": "c34f9e9b1139564c727ddd7ab2664169b89533829ec48c4f2f6ef2acfbec8eb6/486",
"pdf_unittests": "b7e7d51beb729bcee3db8cc45a37702fb43acc1092fdd8daa9c2a9a8b7cf43ca/563",
"perfetto_unittests": "1f2578a78631af553ff30ceb046314427fb672f989fea1ae459fd965e602c27d/486",
"ppapi_unittests": "c22b5b3fcad25803e0932804b9d03d5ceb8151354a7a44d7e20ca4951965a47f/486",
"printing_unittests": "a06f43e075602ab51a422850d39b7c197794511018bdd51b3b634edf894d6885/486",
"pthreadpool_unittests": "ea1e20b57913ccad720ff9abec05e1343d3a6d37908a4c2619fef8dc6a1ebc5e/486",
"remoting_unittests": "020b3656bdb5f058b52f8afff1de59007709edb04b6c6cfe709ff3f7234abf36/647",
"sandbox_linux_unittests": "c3973c73b9af9e1e07b9937224d1e0b28387e190adadacbdff21c9307a1db692/486",
"service_manager_unittests": "9b79c75f1a375fa6819c3deb5cb5435b7cba355ebb3a908671350991c2a62da5/486",
"shell_dialogs_unittests": "bccf3ef8aff4b2d11af8bb5d416883b83184dc68233ef19b3feac9f77de3204a/486",
"shell_encryption_unittests": "d74c0658c063a3e4c18059ae85d3ae8783ab55a98fc2fc2286d453ed1f678b77/486",
"skia_unittests": "5f32602d9b076929f62b0d2570ecb56a360cc534f22713bd9161a58057802110/486",
"snapshot_unittests": "34c7cda9f49be29be71f26f76e2977f973471157d03a47e691afe83ecb0c3076/486",
"sql_unittests": "8a54ed152755a49d0a9302b2273a2cfd6bc3746d55425f0dc7ca54be6de0d86d/563",
"storage_unittests": "ee070d989460842cafb742c92d9bc83e7506068bfa3b78cf862ec7f3b9778ad2/644",
"sync_integration_tests": "b626927061a1e762f71ffc46109ad7a860d8e0ebd07a9918245266fcc9d96821/889",
"ui_base_unittests": "02f5d7ca2142740c25421dabd9150de21d1e6aa3ff48768bc114519e397aafc2/639",
"ui_chromeos_unittests": "8cfa1fd880eef70dd9f96bca14bc9e55871618f9f6893f06a35696ee35e2cc6a/486",
"ui_touch_selection_unittests": "8cdbce2e7b546c2f9169df823fe832ce907ef56c852a44d08aaadb9d7705ea8f/486",
"unit_tests": "fbefa971158532442cc370a552300637603519647a5a37969fcaa0d2bdba07cc/1139",
"url_unittests": "4275dbbb3851d8b69e656e6a6cb6756f348549eee91305248ed0b9297cc2761b/486",
"usage_time_limit_unittests": "e1be5527625dbef3a93002aae7263c063936721b4509e9a2d9800fdb6d871283/889",
"views_unittests": "381a5c297199afcfbc011f76412e0e86e791c97ca4e8b1244ab69fb1a863ff7e/487",
"viz_unittests": "b7f7ecd9204cf38b65bd99cb2634fb03b3f157346388cad62f75daf738ee14ca/570",
"wayland_client_perftests": "035704ab33faf048d1384a8dd94b86e6d4ca6bb68df273e4e2a1ddef308055d3/646",
"wayland_client_tests": "daaa31995d4e6378885587f6e099cb03d016b1370432ba7ce1d32ab6938e4d9e/646",
"wm_unittests": "b4f3761799199380c1bf14327022d207805ef67ff299684ca612d12881c3e1f0/486",
"wtf_unittests": "6f30d74e783f0299a2be42c75ca4b08e5eeaaf20f7260c5e1711918627df7960/486",
"zlib_unittests": "41d75be3062f8aaea940c86ffb74d1b0b03ba71ee7bc7a79f4b48aaf0d1a1edc/487"
},
"swarming_command_lines_cwd": "out/Release",
"swarming_command_lines_digest": "c4670b336c60f749425c43c43df1822d26d9df7c654222412d954d12de0c94d2/94",
"swarming_inverted_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "7d46148dcabf6a4d263014dde9d3ac2239da4c12",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8944494050891744256"
},
{
"key": "scheduler_job_id",
"value": "chromium/Linux ChromiumOS MSan Tests"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Linux ChromiumOS MSan Builder"
},
{
"key": "parent_buildnumber",
"value": "53925"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[05:26:14.291] Scheduled build:
{
"id": "8779063988878951617",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux ChromiumOS MSan Tests"
},
"number": 36293,
"createdBy": "project:chromium",
"createTime": "2023-06-06T05:26:13.927467403Z",
"updateTime": "2023-06-06T05:26:13.927467403Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "7d46148dcabf6a4d263014dde9d3ac2239da4c12",
"ref": "refs/heads/main"
}
}
}
[05:26:14.291] Task URL: https://cr-buildbucket.appspot.com/build/8779063988878951617
[05:26:14.291] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:2:0) after 2m19s
[05:28:33.344] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:2:0)
[05:28:33.376] Build status: SCHEDULED
[05:28:33.376] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:3:0) after 6m51s
[05:31:19.097] Received PubSub notification, asking Buildbucket for the build status
[05:31:19.123] Build status: STARTED
[05:35:24.467] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:3:0)
[05:35:24.486] Build status: STARTED
[05:35:24.486] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:5:0) after 1m28s
[05:36:52.601] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:5:0)
[05:36:52.759] Build status: STARTED
[05:36:52.759] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:6:0) after 6m51s
[05:43:43.788] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:6:0)
[05:43:43.815] Build status: STARTED
[05:43:43.815] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:7:0) after 6m36s
[05:50:20.018] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:7:0)
[05:50:20.048] Build status: STARTED
[05:50:20.048] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:8:0) after 3m42s
[05:54:02.175] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:8:0)
[05:54:02.198] Build status: STARTED
[05:54:02.198] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:9:0) after 5m7s
[05:59:09.241] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:9:0)
[05:59:09.259] Build status: STARTED
[05:59:09.260] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:10:0) after 3m42s
[06:02:51.381] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:10:0)
[06:02:51.400] Build status: STARTED
[06:02:51.400] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:11:0) after 7m39s
[06:10:30.432] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:11:0)
[06:10:30.464] Build status: STARTED
[06:10:30.464] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:12:0) after 6m39s
[06:17:09.507] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:12:0)
[06:17:09.525] Build status: STARTED
[06:17:09.525] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:13:0) after 1m26s
[06:18:35.522] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:13:0)
[06:18:35.537] Build status: STARTED
[06:18:35.538] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:14:0) after 8m30s
[06:27:05.570] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:14:0)
[06:27:05.591] Build status: STARTED
[06:27:05.591] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:15:0) after 7m23s
[06:34:28.941] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:15:0)
[06:34:28.962] Build status: STARTED
[06:34:28.962] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:16:0) after 4m23s
[06:38:52.110] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:16:0)
[06:38:52.194] Build status: STARTED
[06:38:52.194] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:17:0) after 5m13s
[06:44:05.276] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:17:0)
[06:44:05.295] Build status: STARTED
[06:44:05.295] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:18:0) after 5m23s
[06:49:28.317] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:18:0)
[06:49:28.333] Build status: STARTED
[06:49:28.333] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:19:0) after 2m25s
[06:51:53.499] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:19:0)
[06:51:53.517] Build status: STARTED
[06:51:53.517] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:20:0) after 5m55s
[06:57:48.658] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:20:0)
[06:57:48.680] Build status: STARTED
[06:57:48.680] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8944494050891744256:21:0) after 2m25s
[06:59:40.504] Received PubSub notification, asking Buildbucket for the build status
[06:59:40.603] Build:
{
"id": "8779063988878951617",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux ChromiumOS MSan Tests"
},
"number": 36293,
"createdBy": "project:chromium",
"createTime": "2023-06-06T05:26:13.927467403Z",
"startTime": "2023-06-06T05:31:18.591097Z",
"endTime": "2023-06-06T06:59:40.015812894Z",
"updateTime": "2023-06-06T06:59:40.015812894Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "7d46148dcabf6a4d263014dde9d3ac2239da4c12",
"ref": "refs/heads/main"
}
}
}
[06:59:40.603] Invocation finished in 1h33m27.978692392s with status FAILED