[07:15:05.420] New invocation is queued and will start shortly
[07:15:05.420] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[07:15:06.604] Starting the invocation (attempt 1)
[07:15:06.643] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/2ab4a8e0540e8bad9801aa3997740d2377e522b6"
[07:15:06.643] Popped gitiles commit info from properties and tags
[07:15:06.643] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[07:15:06.644] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[07:15:06.644] Buildbucket request:
{
"requestId": "8902178417465304672",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux ChromiumOS MSan Tests"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "2ab4a8e0540e8bad9801aa3997740d2377e522b6",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "2ab4a8e0540e8bad9801aa3997740d2377e522b6"
},
"parent_build_id": "8736749494776018545",
"parent_builder_group": "chromium.memory",
"parent_buildername": "Linux ChromiumOS MSan Builder",
"parent_got_angle_revision": "25c693c96d87507b942b56ccaad843d404351d28",
"parent_got_dawn_revision": "dcb6f7ec881c40cbea339d7665604ac23a6af745",
"parent_got_nacl_revision": "6944e6b79dbd1b9776681c025bd4f4c281bb4791",
"parent_got_revision": "2ab4a8e0540e8bad9801aa3997740d2377e522b6",
"parent_got_revision_cp": "refs/heads/main@{#1355640}",
"parent_got_swiftshader_revision": "3239872f9c5f1f7a1d029733dd1036d1247cfb4f",
"parent_got_v8_revision": "1cb227d3ee187090e93dfeb224efcae9e4b90bda",
"parent_got_v8_revision_cp": "refs/heads/13.0.244@{#1}",
"parent_got_webrtc_revision": "13e377b804f68aa9c20ea5e449666ea5248e3286",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#43019}",
"swarm_hashes": {
"absl_hardening_tests": "6b68922b9415256c04e5bfc0011e4a8bd556d4f6adda09242d3a48522c93b882/488",
"accessibility_unittests": "c138f37d08a47e6f06471e9fc2516f8e60e669828d3b285d0dbcdc81096b7fd4/488",
"angle_unittests": "0b3b7d7c7ec9cc60653d5f54fa48ad3e97edf9e377702bb0edcb40ef3382b1b3/488",
"app_shell_unittests": "c6d1ca4cc46e9ff770b97a0d2327f0af8b8abb249a489c07ceb889ca541f4f09/649",
"ash_components_unittests": "c87725b74a75870bb7f7a25a0534f5736bb13a55e665cd8fb25897648ce66e34/724",
"ash_unittests": "1b6fb9480f2f8a0e311a2c3b13690d403bda0f756668b81d6a9e1f052ba5ca73/800",
"ash_webui_unittests": "ed140f9b60fe7dc17dbd55809486060dd1b2c793f7189d1f0a8134d83a5e6f8f/726",
"aura_unittests": "91280460d1078c374b93d3e360da4814e15baa6260d394b17012801be1162125/488",
"base_unittests": "e95d04a7ca2bf0443ea46370d09616475cf51410a2b15eef2e630724a8747bb9/566",
"blink_common_unittests": "95d2e99c8d2386b05068886562f1c20665a012e4a238f45ba6400ca499f75f0e/565",
"blink_fuzzer_unittests": "6a9926845cad766b974d417db983769712c741d28128f0cff3c6b325dd3c636c/488",
"blink_heap_unittests": "411dcf3f86489d7c683fbae0210526a4bb932530f08f75a3d850c61c0792a563/488",
"blink_platform_unittests": "9320a4d42359b8c6ae5fe8baa130435cda3aba4bb9b9ca5b0bd9ee6d2d89686d/644",
"blink_unittests": "98eaff06aed0e548bddb7b8bfb2a684c1aebe63dfe3156f1523194c89d9084f9/728",
"boringssl_crypto_tests": "7860f745488553db8c6cab418a7a2b20e26a9a2bdf96ca6c6e29ee20f00673d6/488",
"boringssl_ssl_tests": "0409afe66b5cce8b5378889ba00bf2f130c44833f6ae58a774c12b03e232f5f4/488",
"browser_tests": "5813411260922eab3554ef8d33f7b3253282c42bdb824d705f2bfff7c2bf79fa/1378",
"capture_unittests": "2b6b1cbd642d0568864c000c4f6771d882431663cfa7135eddfa3e4aee35f821/567",
"cast_unittests": "d23cf161932546ff3d8d284a6135cd7283721948077fcdddeb7f8cdf032279b7/488",
"cc_unittests": "13ad51198f838a43c2d8ea6cfc2a2ed993285cfc3af0c42a90ce13e225fa376d/648",
"chrome_app_unittests": "b2656719e42a6fca3811cc0b75474b5e1a173dfc0ef87ee45c29990a20ecaec8/726",
"chromedriver_unittests": "05d82153dac98ad09f62c91ba63e95977dda5ffca779bc28f61e310485d0b77e/645",
"chromeos_components_unittests": "0d6de9d0799d882b862250dc6d6b7f45c6c17d1b57f143bf588b89619a43dc4a/647",
"chromeos_unittests": "31f77bf1b1250d77e02e39a183bbd1db64d505a948906885194a8a27a7e0ca3a/806",
"components_browsertests": "d0620ae92bbf8bc3c4a30d0959d89371364b912ed401c0ae12775a85cf1db1ec/731",
"components_unittests": "21243d554a1cbbab152887c6cd6751d6e661209a6e1c1a7a4275972cfff5375d/890",
"compositor_unittests": "af0cf6d4f6f51f68e31c1d9b92ded01cd790c522191aa055a0d0b4fb3a30784e/564",
"content_browsertests": "5627f3443f5016289c6c246d6c0cc6ffb6070774ae5151679dcb5fe6475bdc30/805",
"content_unittests": "5042bfbe3e8ea709570da8efa9c4ec8badcfa88a53f5fdf65208ddd644b719b4/725",
"crashpad_tests": "9cfe0bed146ee9c9e54989cdcec1753c56d9275246e46a00c454c76b2286b2df/488",
"crypto_unittests": "397aaffcab8cc19cd550b77aaaf56b65d07a207f18e846c10ab29874ce63692a/488",
"dbus_unittests": "e8aeb28ea36ceda24ffa68d4dc12d46f0b6ed1b7a61b389f01c7bd1f961faf59/488",
"device_unittests": "50afb38c642ad87c01d94ade529b4c0d1266bfe827686c818246a4ac65ac60bf/645",
"display_unittests": "ac421402f23837f4d1f50f79bd100d216b73ce4cb50bf3caa6d940e988ee75db/488",
"env_chromium_unittests": "df03e0289e51c05189a50cfc7bd2435924e8aeb0a315b0b3032e5fdcb0976c36/488",
"events_unittests": "2725725c7e7b4c7a665329207db17e1f774e6fbf6f375837caf23d50b1b6b3ef/488",
"exo_unittests": "a458fc919991bac6e1dded4059c097c0fd374fad90331458ea42e8416af5d564/647",
"extensions_browsertests": "a3252ff20dc59ba35c9313f67d212c3c7c27bc9b70197203fce30967406725c9/650",
"extensions_unittests": "a4a2188e3aef977bd75f52fc7a3a9ed782339b58107bdda5af7d828e9e540c2b/812",
"filesystem_service_unittests": "91e2b3ddfe89cd334a76d40b0bae0ec0c48d26e166acc46b14e781289639fe71/488",
"fuzzing_unittests": "955786e9276515d319334c9d0d4d456f44c040e8d649cf4c5d4bc5b89b913755/488",
"gcm_unit_tests": "f90970ef2da4adabdd9c41aca96beb7e30dffa1f021f074e149439ba6c4fca0d/565",
"gfx_unittests": "462ffbe3f30dfdf40c2ddf22ec17b59ff628ccb3e0bced32031770e54d5b1b07/564",
"gin_unittests": "be97135e25798fe186ad2bfcca3464b0ce6412834089afddc8e1c87abb0e48b2/565",
"google_apis_unittests": "02b2ffc15ce4166291d465eaa0f41b586300f0e5cc3cb3442e6eaee0d5fc00fd/650",
"gpu_unittests": "4d48a01878ea9d5c13ee0ea428742182876d0d38cd9bd0ddcde653c236d4b5b2/565",
"gwp_asan_unittests": "9798d6906d2df198248b6fa65e72e97cfb0842fe35b754ba1a76b066a61f9881/488",
"interactive_ui_tests": "77696abd6087d87960bb3f4cf7ab65f3b1a963f506c948a925664abc926ebdec/1210",
"ipc_tests": "1913a38e35ebaf9d71c2f527edc325da6d5e32a006a9aceee14b79e8d62f674a/488",
"keyboard_unittests": "e522cfebe419a8835c6ef3b351f00805012ebb84815769ec88bfe23f84e064a7/488",
"latency_unittests": "de58f414b8e4b91eed30620eae76da531eac1d98f2274356110400fb43cc1942/488",
"leveldb_unittests": "218db60e2d38812a1f777ce6be8f77c30c25bc55e756b17f6839d5af1da096e0/488",
"libjingle_xmpp_unittests": "998c5c07459047d893ca25de2a6375213c6cd358011ba85275da28e90c8677e4/488",
"liburlpattern_unittests": "72460d798c8a8129d095851c823fa43a7a22bd58f6e9fee3c120ecfbf0b83a13/488",
"media_unittests": "8a4d881bac4ccaa2e29f9abc2a224da26eede6a8cc118d9e800eb0dd37c696a8/568",
"message_center_unittests": "b232f28fcadbb392552a558fcb0aaf35af8a7f08ba724d53558aa8a09b6bf9bc/488",
"midi_unittests": "a011973f0aa4c5491b0904286b9515ebaa267d52d4bef19708e9760655ed962a/488",
"mojo_unittests": "c7ca47ff502679e96abab98a9b9a887f79625edb93bbb88ba790fca756ac34ea/566",
"nacl_loader_unittests": "ac8ebf908a929a87a8f9701bc44289c23199b7ba04202e4428ef738ee2102a11/488",
"native_theme_unittests": "7cf6075dc71f47fd558ff77d31d4e6bddbdba0c6e4397dff0fad76e8e3d9d812/488",
"net_unittests": "c0bbda318d6cb6e959d00e1d2f4095d681c241ab28aff71ed6aae74d32c2201d/566",
"ozone_gl_unittests": "a39d3bced2eb07f99b145387e5257fda82262758a8ec360637a6ecb70404bea7/488",
"ozone_unittests": "ef899cf54b39d6547a45aea99ffa67452aa1b430d3db03be53611d578fd0cdbf/488",
"ozone_x11_unittests": "60dc66ad1acdf8d6e7cc5a8ee60d48da5c1321e44cd44413d4f451240a933978/488",
"pdf_unittests": "00e9106ef9c9694067a03b398ce46050888995b472df296becd95dd07152ac46/565",
"perfetto_unittests": "7e04c21b3d55dcc7c0abf50f98e447d74aa61c4999e18c9cbe8bed1d3b84ae7c/488",
"ppapi_unittests": "294c1e8e428987555199ed017aa41dc7a16a53c5c04649e38bb296e264358f06/488",
"printing_unittests": "aa2ce816c09bc798570f01db64b66991d3420e64aa374b0df534c94e83267f4d/488",
"pthreadpool_unittests": "bc9f8c3efa3c35d5cea293a9ba62e987c85973a34f5eb50433817b53f9f0b1b5/488",
"remoting_unittests": "485430ea28a5837993621b74a1d2bc01bbb834c4a8972eabf883dad16b18a998/647",
"sandbox_linux_unittests": "60fe3d76e45a490a488cbbff3f29ad549c9da5b3b66f9b802eff32ab889597cd/488",
"shell_dialogs_unittests": "e99b9dfd6625e07c4f81f99da9917d15db6952dfcb3fd50287f9f00140f41194/488",
"shell_encryption_unittests": "9431f90263368345e5a9186615a8956b2f0dc6111df14b95a47f6634a21af66a/488",
"skia_unittests": "7da3101601aaf399b670c752cf32340cfb33cd0af60eabfa480574291d999fe8/488",
"snapshot_unittests": "b7ce07c22b777b9cc222bde5f7f9fcac5e65fe0c1124e1a86e7cd74b0d028b1e/488",
"sql_unittests": "0a296132f7a8dc28cc9d3f97b931836f1560e7f7f854d19f94d4b7494403e139/565",
"storage_unittests": "cccc56e6398c1a5dd5bf759cef84434970b84f76cf4c7d8e332b726816cbe729/646",
"sync_integration_tests": "865f198e3a53a62210dda24b0ac3115ee09e12a8dd4dbdfa4fe77203987f6f00/884",
"ui_base_unittests": "47fec83e9fcec9ad6ec57f38f2dcfd186a60b20ae7af678caa3d6ceb724cfdc7/641",
"ui_chromeos_unittests": "2d7c50badb02d1450f5490d1f7d00b284721268ffc7d6b27ec2d5fa1cde7f230/488",
"ui_touch_selection_unittests": "25af24592b1bd8a8c0b3c6437c53fe40a4d16d0d0655205784d9a1df154a73c1/488",
"ui_unittests": "0217b310e76f8f0eb2e02dae02db2bb3d173f5033188b0466bdf317e9dd1d030/488",
"unit_tests": "3e9c7f97f71d8d2fdd0ee54351876b6b2536ec681583ef5028dd9e8f221bf15f/1217",
"url_unittests": "a97484ea4d201fc44cadfc080cc7fe8a8f9f113cdcff1158e8da50c0149270c4/488",
"usage_time_limit_unittests": "29bac9a17c67977717bea218fc7f8cc55a52147f51f82e69cc8651b036cac3ee/568",
"views_unittests": "6edf87cd209d3602291dfdec185f7a0eff4a268ce738e892700e7ed317aa8366/488",
"viz_unittests": "3e5f5ff78cb6792faba44865ec2b1c80c7474eed235ea368d6215b0fe42edf5a/572",
"wayland_client_perftests": "a92f578f2d37f98e0826b5067a794977344f3954cff499cde982551cc1f3be7b/647",
"wayland_client_tests": "8ec2659b3bee3aa396e6c16683cb10ffd529532759e1b54ca8a83c239c033355/647",
"wm_unittests": "4118e28170a7321d047ddb8170a332e4b312e7a3f08cd3f2a00a849929ba46be/488",
"wtf_unittests": "0bd69627cf601aaa39d15231b3d3085255cbb5b169e07602e57dbbca58e117c2/488",
"zlib_unittests": "e7888fa101ad7292a0a11161520b28ae9ad40134eb334b6d5358b6e6161968c4/488"
},
"swarming_command_lines_cwd": "out/Release",
"swarming_command_lines_digest": "044841b95c1246203d0fdc5fbd35beba71e36ca1a0e8f26d4231580ded8f981f/94"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "2ab4a8e0540e8bad9801aa3997740d2377e522b6",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8902178417465304672"
},
{
"key": "scheduler_job_id",
"value": "chromium/Linux ChromiumOS MSan Tests"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Linux ChromiumOS MSan Builder"
},
{
"key": "parent_buildnumber",
"value": "77440"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[07:15:07.285] Scheduled build:
{
"id": "8736748355367424881",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux ChromiumOS MSan Tests"
},
"number": 46165,
"createdBy": "project:chromium",
"createTime": "2024-09-15T07:15:06.789469903Z",
"updateTime": "2024-09-15T07:15:06.789469903Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "2ab4a8e0540e8bad9801aa3997740d2377e522b6",
"ref": "refs/heads/main"
}
}
}
[07:15:07.285] Task URL: https://cr-buildbucket.appspot.com/build/8736748355367424881
[07:15:07.285] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:2:0) after 8m47s
[07:15:08.222] Received PubSub notification, asking Buildbucket for the build status
[07:15:08.307] Build status: SCHEDULED
[07:16:22.748] Received PubSub notification, asking Buildbucket for the build status
[07:16:22.790] Build status: STARTED
[07:23:54.320] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:2:0)
[07:23:54.353] Build status: STARTED
[07:23:54.353] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:5:0) after 6m9s
[07:30:03.409] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:5:0)
[07:30:03.428] Build status: STARTED
[07:30:03.428] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:6:0) after 4m31s
[07:34:34.550] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:6:0)
[07:34:34.567] Build status: STARTED
[07:34:34.567] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:7:0) after 2m22s
[07:36:56.599] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:7:0)
[07:36:56.624] Build status: STARTED
[07:36:56.624] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:8:0) after 2m54s
[07:39:50.652] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:8:0)
[07:39:50.728] Build status: STARTED
[07:39:50.728] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:9:0) after 1m9s
[07:40:59.841] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:9:0)
[07:40:59.862] Build status: STARTED
[07:40:59.862] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:10:0) after 3m18s
[07:44:17.883] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:10:0)
[07:44:17.923] Build status: STARTED
[07:44:17.923] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:11:0) after 1m2s
[07:45:19.945] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:11:0)
[07:45:19.963] Build status: STARTED
[07:45:19.963] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:12:0) after 3m39s
[07:48:59.393] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:12:0)
[07:48:59.409] Build status: STARTED
[07:48:59.409] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:13:0) after 7m53s
[07:56:52.527] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:13:0)
[07:56:52.547] Build status: STARTED
[07:56:52.547] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:14:0) after 7m9s
[08:04:01.663] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:14:0)
[08:04:01.680] Build status: STARTED
[08:04:01.680] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:15:0) after 1m31s
[08:05:32.859] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:15:0)
[08:05:32.962] Build status: STARTED
[08:05:32.962] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:16:0) after 7m9s
[08:12:41.964] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:16:0)
[08:12:41.986] Build status: STARTED
[08:12:41.986] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:17:0) after 6m43s
[08:19:25.230] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:17:0)
[08:19:25.250] Build status: STARTED
[08:19:25.250] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:18:0) after 7m17s
[08:26:42.270] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:18:0)
[08:26:42.291] Build status: STARTED
[08:26:42.291] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:19:0) after 1m0s
[08:27:42.315] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:19:0)
[08:27:42.334] Build status: STARTED
[08:27:42.334] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:20:0) after 2m46s
[08:30:28.356] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:20:0)
[08:30:28.381] Build status: STARTED
[08:30:28.381] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:21:0) after 8m2s
[08:38:30.422] Handling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:21:0)
[08:38:30.440] Build status: STARTED
[08:38:30.440] Scheduling timer "check-buildbucket-build-status" (chromium/Linux ChromiumOS MSan Tests:8902178417465304672:22:0) after 1m44s
[08:39:44.293] Received PubSub notification, asking Buildbucket for the build status
[08:39:44.317] Build:
{
"id": "8736748355367424881",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Linux ChromiumOS MSan Tests"
},
"number": 46165,
"createdBy": "project:chromium",
"createTime": "2024-09-15T07:15:06.789469903Z",
"startTime": "2024-09-15T07:16:22.332131066Z",
"endTime": "2024-09-15T08:39:43.383656040Z",
"updateTime": "2024-09-15T08:39:43.383656040Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "2ab4a8e0540e8bad9801aa3997740d2377e522b6",
"ref": "refs/heads/main"
}
}
}
[08:39:44.317] Invocation finished in 1h24m38.912354511s with status FAILED