[15:22:23.602] New invocation is queued and will start shortly
[15:22:23.602] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[15:22:24.698] Starting the invocation (attempt 1)
[15:22:24.731] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/0f36b6ec450075ef8b2d591672b05f44336d3d4b"
[15:22:24.731] Popped gitiles commit info from properties and tags
[15:22:24.731] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[15:22:24.731] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[15:22:24.732] Buildbucket request:
{
"requestId": "8868083299638026272",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "mac15-tests-dbg"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "0f36b6ec450075ef8b2d591672b05f44336d3d4b",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "0f36b6ec450075ef8b2d591672b05f44336d3d4b"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8702655620733153409"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8702655620733153409"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8702655620733153409"
}
],
"parent_build_id": "8702655620733153409",
"parent_builder_group": "chromium.mac",
"parent_buildername": "Mac Builder (dbg)",
"parent_got_angle_revision": "5e51d8e71c08a84ad81080036c6b25cc36b53b1b",
"parent_got_dawn_revision": "f653beb667734196a8d784e93f55c1ba7c2c8210",
"parent_got_revision": "0f36b6ec450075ef8b2d591672b05f44336d3d4b",
"parent_got_revision_cp": "refs/heads/main@{#1521278}",
"parent_got_swiftshader_revision": "5f1c459a11bb4899301e89609cbf7547f9f31e20",
"parent_got_v8_revision": "ca4e6e3b342f30687ca4d36fda90be00be782331",
"parent_got_v8_revision_cp": "refs/heads/14.2.222@{#1}",
"parent_got_webrtc_revision": "ba391e6893da350a48a7895a0bff8f033369a729",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#45749}",
"swarm_hashes": {
"absl_hardening_tests": "af4b862fcaf72b4e788b772db58d3cd5080cbf3c914884617cd4dd1d5c7d3a60/323",
"accessibility_unittests": "1e7b6d825e9a5b95a363a3aaaad123b49b62df401f8280fe19b132b96bd931cf/323",
"angle_unittests": "08948863df4354ccdfda8433add400574f1e6407572e589bce4db2fe8b4bf1ba/408",
"app_shell_unittests": "3b317ddeea7e0d0b34bb971c30880dcff95de40927867c1253347e12b262c462/484",
"base_unittests": "d30dc49b5780709ecd0261657583df5812f85c0a87d092e7cb58ee836b602611/480",
"blink_common_unittests": "042531f1aed7287c043419f90f56985ad71001e32d678bd662daacbcc9a1a3cc/485",
"blink_fuzzer_unittests": "cc27eb56673ddc76115ddd682f415e87dab7f8c4649cd70f7e39984dac47e720/323",
"blink_heap_unittests": "833bc8e0e9efecbe8c413aaf3a42cbccfdfed594f704d47ab1d2e2416c0be2ee/323",
"blink_platform_unittests": "65f001b94f4e840e77453e32a87a76a0624e8b400879e4be8953ecdea28c9716/643",
"blink_python_tests": "40b20e8af3256fef62d7838b30669f898799d70ee83a196306845e9b398bcc0f/574",
"blink_unittests": "0bc0cea986faf6e55fafe7c92854e634f0388c6e3cd8c0544054404b2d9aea58/649",
"blink_web_tests": "20ee64945dc028565928476d315f66dd6eedda3b648e8e3a34d62ec52d8025a8/566",
"blink_wpt_tests": "02f17ea9c14f6ba2bafab9369ee63a18124c4f8382c7f26a93c19a7a743091dd/566",
"boringssl_crypto_tests": "7399d2d576573eb7368dac6ca4c7b069c9e987781cda1ba32c9c7ab995a095de/408",
"boringssl_ssl_tests": "d55d62700c3b2fa8eb400220e0b4f1ccd946943f6c537535cf36184dcfd0a96d/323",
"capture_unittests": "10ac95155180ac0db5d2cc93f202c824d6c16423d7176210ded850baf27da620/402",
"cast_unittests": "1a77baec9f817f81e59800a0d106e3f77ca3e9457d8620a3b0edab0abe67674f/323",
"cc_unittests": "6355b9ff5b811bd69a21af7b1251e331e9bc72db374384194f65c8a24f4ed03a/483",
"chrome_app_unittests": "bd54126d84157aee9fd170717bdae76cfe883fcbf3cf9f9cdc8fb1693ff48470/564",
"chrome_wpt_tests": "86fdba344915bdd3ccdc2f122d1a3902d1f9e57ff95cf86efdec9e16604ab43a/566",
"chromedriver_unittests": "f01adca67ebc9e3a425f1067abf2fdfdd78fa111d52dbe2308d0fa93be3e4142/480",
"components_browsertests": "d1d5aa172e96227e64c87ac34d7e6c258020ae604f6d18eed10b22ab08cbac7a/652",
"components_unittests": "dac87760763915ac23c5ba9c92ce93a9d625164ce83c179d429390b6257a919c/810",
"content_browsertests": "ca09270d00f5816b75998e2ff798a89d545a035d888b75de4d16145cfeea634c/725",
"content_shell_crash_test": "0f59edb305394d8e8be3a19d4138dac301524f25edd7fecdecc6a1ae3a40bd88/566",
"content_unittests": "40be350e59eea037dc4095a95449e4b43e0036e0ccf370e47d46dfc66999927a/560",
"crashpad_tests": "83cc5b83b401054fe37069b6726c035369716b339dd3cc53fbf7e7cd45a12c32/408",
"cronet_tests": "5f48b93d9dd0fc8de46bf851d6076210465c725ed42d3568fae4e185688dcc90/400",
"cronet_unittests": "c579d5af26eb3660a06dc0cb97bb2c9fe8b120a4321ce9c0eff17b2ce88aac4b/400",
"crypto_unittests": "c246af8f8df85296bddc8d2a25c41a9766f15c88ad30561d51c07fd7d7053d24/403",
"device_unittests": "f69c16a0319d058c4b90894c555fc6035c025baf083df3f8384d8dc9378c66bf/400",
"display_unittests": "323d226bf3cdef259ae6ee3051f21a2ab0beed2b0670a60c7cff591ef6ade9d3/323",
"env_chromium_unittests": "8499c41a97f2dfc7211a656573bc98aab5e2a0e163f8687d8aff8108ecad053d/323",
"events_unittests": "c26e9f4c044394c9b28959d880bbe8e3dbc12af72e4d45b2e2f49168a5894589/323",
"extensions_browsertests": "3414e154bb51c388efdd8d3b8d639f0b8f8efda2af5d8da62feb77160619b07a/570",
"extensions_unittests": "752503ca7f36d3cc241fb0d1ee97aa5a11d7e6d6818c7bbc0db7aab4b6bed40b/644",
"filesystem_service_unittests": "2001ac31cc7a7c561814f2c7dc9441d0072d328f75d06760605bb8506627063e/323",
"flatbuffers_unittests": "6483fdec8236d8d4c76af2836878c7d6363bd783a588899338d25eb408728523/408",
"fuzzing_unittests": "c12cf391528ddf9a0d98431fc1f75d1c24c208ee66b7bd3c63b7e45390e61aff/323",
"gcm_unit_tests": "0d03c2c2c8e6e3648eb225d441683b4515d76ac87b1bf22be320e8b22f4e940a/400",
"gfx_unittests": "30b0fc1df7b398bae510c77692338dd1a26bd581f49af54ca31ce0502e287211/399",
"gin_unittests": "0d412313b8d4f8023efb976b79fce1935170dab007eb1dd4cc4f79e604f9ee31/400",
"google_apis_unittests": "cb0180db604c5e8e8fd9aa100b456ad0eb634712aa954fe0180461157818a6ca/485",
"gpu_unittests": "b1f55e54ae53b25ddeaf03637a328b2fa94cf808929e061278aaf52cbf4d8f70/400",
"grit_python_unittests": "5cf013503ce6b924a40b0334b16bc2874ad454c4ad2f814432d5457b251b7f28/411",
"gwp_asan_unittests": "7f97fda508c0ccbc53aef139238d218e7054b193635a32244c2b6082b04f47ff/408",
"headless_browsertests": "a6c18798e8e43834f45ae157151c6b27d7088477b2ca6bc0993a431dc64000ca/653",
"headless_shell_wpt": "7fc144a4513a84aeb4e4646aca70dfae3058bd50cdffd6db7dc198728c36f6a3/566",
"headless_unittests": "f505d6d1a2c1dcce3adc64c1bd40fda8774948eeb2ff861911c8b14adad10ec5/323",
"interactive_ui_tests": "df01e902f4742d95c0f3550fd081c09d1f660ed10b84c040bbde0946da6a90d3/974",
"ipc_tests": "2faa1a33ae47677b0cadcd5777e1693123ce9fe5700b457e76d14fd404d6fbad/323",
"latency_unittests": "e85c6634df8622c131e0e0900fbc760233f01e138019c847fc4ae01d6958d1a8/323",
"leveldb_unittests": "feebc5928a2a015796d03777c0dcb0d5b834448818c21eab46eddc79c437086d/323",
"libjingle_xmpp_unittests": "21b23a767bc4d414c02bf39115d3223dc472cb5f216e19c268716e9cf9cdad06/323",
"liburlpattern_unittests": "6d4a708236a525d31df4727dcc8a744508d1d97758559aa3e8a08cefac83a6d5/323",
"media_unittests": "d1957dc636d038ee43799ebd99435be514f4de8f91ba167143de673aee2f2172/403",
"message_center_unittests": "7fa9535128f75b10b35e4e27cc880aee448efe398ab93cda1c7fc6c2003a7b8a/323",
"midi_unittests": "04ccfc95bbbb7c1fba224785773d05b9efd350a73c25d759f653b6ede6976039/323",
"mojo_unittests": "47c4e53a728ad8a1b5d408c7b6176593eda6ebdc91ab8769519fb964d91a9a0b/401",
"native_theme_unittests": "fd9729704cd9379c53abb23b2d9fa7b4c286b5ec70e61a9cd7b612268d6aaa81/323",
"net_unittests": "3290f227c558e8513511e1b69427f099951e9d2d628e6bea3267d55ef04e397f/486",
"openscreen_unittests": "8d37cb1d877401c63f41e12e2c3e2b62c60df66a25d41e7dad66456b638c8de6/408",
"pdf_unittests": "965d797ea124b8bf33c66009ee15353ce200ecba34d82139bc597d0e5a982e32/400",
"perfetto_unittests": "ff752043544cc839317c162eea9cfebfd6b311acd4393a902561705ff0dd138f/323",
"performance_test_suite": "be09bff9569ebe51c332c0a3117f00be5937f83e9011a4ecc172d59da0d31513/732",
"power_sampler_unittests": "1d8638f085165e30d67e44e625dbab6d2380d933834a381c583252986cdab7c3/323",
"printing_unittests": "8e98a634597625dbc63bbd87f9c8c3e4a3aeb6b11e904c0f207db5d7b7540585/482",
"remoting_unittests": "1a5a0cbe5ccbe151f8b1a276fba5582813b677a1f8a2d6fedbd249149f01afb1/400",
"sandbox_unittests": "f9ad79f7faa4551cd5fba89156098e2f2b22a3b85c4cadd14387d7c76d37a89c/323",
"services_unittests": "ba0356ddea4ecfafb72877e12e4bae82f3a561cd648e4b3b24aea6a0e330cb09/653",
"shell_dialogs_unittests": "859a3a1fad1d810410facd9d2d784539afc58c88dcb67db7cd12636891551ee6/323",
"skia_unittests": "56db0180669dc0d7456ed15baa3ef5545b6393edea22e33cc70783e2994dae6a/323",
"snapshot_unittests": "74f41ca9563303d8450301e98a62e7e2c3bd33a0692450c8d491917061ff56a3/323",
"sql_unittests": "b79d4c1598007a1cee8915466c237983b87c5405cdac13a6a04a79b881f02f17/400",
"storage_unittests": "f01199de5674e194993ce96db848ffd05929f451a6eb30b937263eb39cb241f8/481",
"sync_integration_tests": "e6567ee87c43b072f41de189be17e240c2f9d4ad22231644dd5c01f27d9a3785/646",
"telemetry_gpu_unittests": "bdcdcdaf63fb3219ee66f569369127f443c1837db3d6ecfadc6482aa7a101394/736",
"telemetry_unittests": "b6b27c498f6d59ee2b390199aa51f8bb0312127c1460eeacdf4c06cf43d5cf62/575",
"ui_base_unittests": "65765666558c24c1624063e06a27da28c0078d866f0098d4d7ccbea8998219ca/555",
"ui_touch_selection_unittests": "9acd04061dc3598a1c27713e4e7183dd0c442ac7ac73f61c8fcc5e0b51471b40/323",
"ui_unittests": "922d57dcf9ab7eef0c7df18162ba0f178fa715f955256adec1d836d7ea06c133/323",
"unit_tests": "17bc5df2997e519eee173b891e03157309a4c767c5fa0268e2cefa21b9ae2ed9/1061",
"updater_tests": "c9e1d3bc7fbbe6c3399ed3ff5890fbf3e745e158d63e8289274c89fbc6b5d161/566",
"url_unittests": "fa021875c5d2f6323281654170b74764a2bff5f95b080aed2e7a869a14da0c49/323",
"views_perftests": "9320415ff595d0117dc258ecfe74cbe3d42814dc5b148c0279c03a4662d1cef9/487",
"views_unittests": "498edb6b7b6a1e2025ad7fef6bec6341ecff5f58058f4ce864fbc5a7598f47d3/402",
"viz_unittests": "6ef070a8eb7cf8547cb104f805df25817cb34bf72c7b2c6f30d20b71cd4e9929/407",
"wtf_unittests": "8abf4b7b3be90e523b59896adad53625f564b9b63c726119daa10bee341ec971/323",
"xr_browser_tests": "3f516ffae74672ee3d9229c2cc9e6b80a33577de92f91157ba6edadd88529a10/645",
"zlib_unittests": "551ff625f713924bdd4fab1c1664e76ec57521bac23cda8df48d2864c340c513/408"
},
"swarming_command_lines_cwd": "out/38ad-Mac_Builder_(db",
"swarming_command_lines_digest": "07e6b3479b3c11c2c9a1d3696e884c2d2ab29c264c793cc2c8cd325ce244da36/94",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "0f36b6ec450075ef8b2d591672b05f44336d3d4b",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8868083299638026272"
},
{
"key": "scheduler_job_id",
"value": "chromium/mac15-tests-dbg"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Mac Builder (dbg)"
},
{
"key": "parent_buildnumber",
"value": "315610"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[15:22:25.415] Scheduled build:
{
"id": "8702653237736154993",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "mac15-tests-dbg"
},
"number": 2590,
"createdBy": "project:chromium",
"createTime": "2025-09-26T15:22:24.789554935Z",
"updateTime": "2025-09-26T15:22:24.789554935Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "0f36b6ec450075ef8b2d591672b05f44336d3d4b",
"ref": "refs/heads/main"
}
}
}
[15:22:25.415] Task URL: https://cr-buildbucket.appspot.com/build/8702653237736154993
[15:22:25.415] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:2:0) after 1m27s
[15:22:26.399] Received PubSub notification, asking Buildbucket for the build status
[15:22:26.423] Build status: SCHEDULED
[15:22:33.544] Received PubSub notification, asking Buildbucket for the build status
[15:22:33.565] Build status: STARTED
[15:23:52.487] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:2:0)
[15:23:52.510] Build status: STARTED
[15:23:52.510] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:5:0) after 7m50s
[15:31:42.575] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:5:0)
[15:31:42.597] Build status: STARTED
[15:31:42.597] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:6:0) after 7m18s
[15:39:00.701] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:6:0)
[15:39:00.719] Build status: STARTED
[15:39:00.719] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:7:0) after 5m3s
[15:44:03.754] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:7:0)
[15:44:03.771] Build status: STARTED
[15:44:03.771] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:8:0) after 3m5s
[15:47:08.794] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:8:0)
[15:47:08.811] Build status: STARTED
[15:47:08.811] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:9:0) after 8m32s
[15:55:40.831] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:9:0)
[15:55:40.852] Build status: STARTED
[15:55:40.852] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:10:0) after 3m57s
[15:59:37.971] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:10:0)
[15:59:37.989] Build status: STARTED
[15:59:37.989] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:11:0) after 2m4s
[16:01:42.008] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:11:0)
[16:01:42.068] Build status: STARTED
[16:01:42.068] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:12:0) after 7m47s
[16:09:29.076] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:12:0)
[16:09:29.132] Build status: STARTED
[16:09:29.132] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:13:0) after 6m27s
[16:15:56.148] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:13:0)
[16:15:56.170] Build status: STARTED
[16:15:56.170] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:14:0) after 3m50s
[16:19:46.190] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:14:0)
[16:19:46.205] Build status: STARTED
[16:19:46.205] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:15:0) after 5m36s
[16:25:22.220] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:15:0)
[16:25:22.235] Build status: STARTED
[16:25:22.235] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:16:0) after 1m47s
[16:27:09.263] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:16:0)
[16:27:09.280] Build status: STARTED
[16:27:09.280] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:17:0) after 4m53s
[16:32:02.327] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:17:0)
[16:32:02.399] Build status: STARTED
[16:32:02.399] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:18:0) after 9m55s
[16:41:57.435] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:18:0)
[16:41:57.480] Build status: STARTED
[16:41:57.481] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:19:0) after 6m17s
[16:48:14.521] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:19:0)
[16:48:14.536] Build status: STARTED
[16:48:14.536] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:20:0) after 1m49s
[16:50:03.619] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:20:0)
[16:50:03.634] Build status: STARTED
[16:50:03.634] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:21:0) after 3m8s
[16:53:11.753] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:21:0)
[16:53:11.778] Build status: STARTED
[16:53:11.778] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:22:0) after 9m7s
[17:02:18.812] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:22:0)
[17:02:18.828] Build status: STARTED
[17:02:18.828] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:23:0) after 6m24s
[17:08:42.848] Handling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:23:0)
[17:08:42.863] Build status: STARTED
[17:08:42.863] Scheduling timer "check-buildbucket-build-status" (chromium/mac15-tests-dbg:8868083299638026272:24:0) after 7m51s
[17:10:43.382] Received PubSub notification, asking Buildbucket for the build status
[17:10:43.400] Build:
{
"id": "8702653237736154993",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "mac15-tests-dbg"
},
"number": 2590,
"createdBy": "project:chromium",
"createTime": "2025-09-26T15:22:24.789554935Z",
"startTime": "2025-09-26T15:22:33.224856235Z",
"endTime": "2025-09-26T17:10:43.034326036Z",
"updateTime": "2025-09-26T17:10:43.034326036Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "0f36b6ec450075ef8b2d591672b05f44336d3d4b",
"ref": "refs/heads/main"
}
}
}
[17:10:43.400] Invocation finished in 1h48m19.807715289s with status FAILED