[03:26:07.531] New invocation is queued and will start shortly
[03:26:07.531] Triggered by user:angle-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[03:26:08.670] Starting the invocation (attempt 1)
[03:26:08.698] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/angle/angle/+/5ce747e1749411f12e03305a60bdca63d69113d0"
[03:26:08.699] Popped gitiles commit info from properties and tags
[03:26:08.699] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[03:26:08.699] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[03:26:08.699] Buildbucket request:
{
"requestId": "8831889576751790064",
"builder": {
"project": "angle",
"bucket": "ci",
"builder": "angle-android-arm64-google-pixel6-rel"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8831889576751790064",
"job": "angle/angle-android-arm64-google-pixel6-rel",
"triggers": [
{
"gitiles": {
"properties": {
"deps_revision_overrides": {
"angle": "5ce747e1749411f12e03305a60bdca63d69113d0"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8666461241737292097"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8666461241737292097"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8666461241737292097"
}
],
"parent_build_id": "8666461241737292097",
"parent_builder_group": "ci",
"parent_buildername": "angle-android-arm64-builder-rel",
"parent_got_angle_revision": "5ce747e1749411f12e03305a60bdca63d69113d0",
"parent_got_revision": "5ce747e1749411f12e03305a60bdca63d69113d0",
"revision": "5ce747e1749411f12e03305a60bdca63d69113d0",
"swarm_hashes": {
"angle_capture_tests": "c7a804eba4abe2f305567e1c5d4af5c3341a8f85ff960151297df2951152b10e/585",
"angle_deqp_egl_tests": "c3d7f395348531b591fed6c846a38a49232716574e317f3ba7c6d9e3bad6af32/585",
"angle_deqp_gles2_tests": "2ed46e2df77322c4b49cc8cfa0b2e33d1fcdf642caf72c6b23cf52deedffb656/585",
"angle_deqp_gles31_565_no_depth_no_stencil_tests": "d569515315eb2c27a721ac3c926b0bca27468a831ae955ece966b0a55a3f0e94/585",
"angle_deqp_gles31_multisample_tests": "1b3919227fa7ef6b65eb02861b84c0ce42c97b6d5531db993e382590cbe5e31b/585",
"angle_deqp_gles31_tests": "aa61bd40c1f533f7500325fb38c14e7e0fb941d1a1b12eda338dd0d25afe2380/585",
"angle_deqp_gles3_565_no_depth_no_stencil_tests": "71583db5aa42b03933395457736fde150e67d19b2c9d680949c3e917277d10de/585",
"angle_deqp_gles3_multisample_tests": "ab95db7ce8078ff37bb70b4b7613215724c713ddaaed7aa98d63acce65f899bf/585",
"angle_deqp_gles3_tests": "f6d7197dd1d3710fe7b1c60b617d4b21cd3eb91444460296d00f73656f2d4e7d/585",
"angle_deqp_khr_gles2_tests": "a5c8b22ce843043ae2426d6c8e0593658fcf6a6c3f504837f2cdcea57c52f13d/585",
"angle_deqp_khr_gles31_tests": "eea867ea45da3345502f92430bb0a900ae4484124f942665ff5a00c15145230d/585",
"angle_deqp_khr_gles32_tests": "0c61d8d938b30bd2df7ec9e66afbe36fb2bbf6097e97edc13839b22dc2b1357d/585",
"angle_deqp_khr_gles3_tests": "59fdd64b815d872e670fead9d7271c9dd87575a95b10df285783a2519d9120af/585",
"angle_deqp_khr_glesext_tests": "77297151305256868838aa13d9da7ad246bfa3ad3fcf2848c28a92bd17882619/585",
"angle_deqp_khr_noctx_gles2_tests": "e6c2976c9ac90a515fdd0355320be5a521ce6698d025a45f0e6aaa1746a0e9c3/585",
"angle_deqp_khr_noctx_gles32_tests": "4a0ab67a40b7740089c564692204af652d18512d2e0d6e3a76a88fb6733f2842/585",
"angle_deqp_khr_single_gles32_tests": "70c476b4619dae8e518c5f560d71a14cb3bc4bc4b9ebeb7aeb3319a49cd30933/585",
"angle_end2end_tests": "83517802267aaba4b978de31b99df3034b2086d2e08c6360a98fc1da69078d79/585",
"angle_gles1_conformance_tests": "2c5e8172cc8a65a42218488c4e59ab3a16f0e455ce06119cb1585df8a71c53ac/585",
"angle_perftests": "d5b752d79b2b80637313711cbedbe37f3ea4c09112282a1ebcfb63c8f35cf2b3/745",
"angle_restricted_trace_gold_tests": "64c082322155b7f1cff4ba38e118fd9053d7f693e82511613da35a7872b3eebe/746",
"angle_trace_perf_tests": "522d480b201a349f2a89ab2c077261892aa2e95bbdfc99e2c80ffecce8471e39/745",
"angle_unittests": "edc39036d3b61bb8a1035059d89a56fb2d6e9709e734b2c0372f9cf79eeb88c3/585"
},
"swarming_command_lines_cwd": "out/6da5-angle-android-a",
"swarming_command_lines_digest": "1742422d76183d48107f577ad98e20b3721e10cf0261b270a661231d5e24bea3/93",
"test_trigger_deps_digest": "677b69131341223ab8576aae83ff32dd7bb43074b2b70766a8bfe54acaaabdad/750"
},
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/angle/angle",
"revision": "5ce747e1749411f12e03305a60bdca63d69113d0",
"tags": [
"parent_buildername:angle-android-arm64-builder-rel",
"parent_buildnumber:1548",
"user_agent:recipe"
]
},
"id": "56ce4109-caa9-4f15-b4c2-0cc472cd4715",
"title": "angle-android-arm64-builder-rel/1548",
"url": "https://cr-buildbucket.appspot.com/build/8666461241737292097"
}
]
},
"deps_revision_overrides": {
"angle": "5ce747e1749411f12e03305a60bdca63d69113d0"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8666461241737292097"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8666461241737292097"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8666461241737292097"
}
],
"parent_build_id": "8666461241737292097",
"parent_builder_group": "ci",
"parent_buildername": "angle-android-arm64-builder-rel",
"parent_got_angle_revision": "5ce747e1749411f12e03305a60bdca63d69113d0",
"parent_got_revision": "5ce747e1749411f12e03305a60bdca63d69113d0",
"swarm_hashes": {
"angle_capture_tests": "c7a804eba4abe2f305567e1c5d4af5c3341a8f85ff960151297df2951152b10e/585",
"angle_deqp_egl_tests": "c3d7f395348531b591fed6c846a38a49232716574e317f3ba7c6d9e3bad6af32/585",
"angle_deqp_gles2_tests": "2ed46e2df77322c4b49cc8cfa0b2e33d1fcdf642caf72c6b23cf52deedffb656/585",
"angle_deqp_gles31_565_no_depth_no_stencil_tests": "d569515315eb2c27a721ac3c926b0bca27468a831ae955ece966b0a55a3f0e94/585",
"angle_deqp_gles31_multisample_tests": "1b3919227fa7ef6b65eb02861b84c0ce42c97b6d5531db993e382590cbe5e31b/585",
"angle_deqp_gles31_tests": "aa61bd40c1f533f7500325fb38c14e7e0fb941d1a1b12eda338dd0d25afe2380/585",
"angle_deqp_gles3_565_no_depth_no_stencil_tests": "71583db5aa42b03933395457736fde150e67d19b2c9d680949c3e917277d10de/585",
"angle_deqp_gles3_multisample_tests": "ab95db7ce8078ff37bb70b4b7613215724c713ddaaed7aa98d63acce65f899bf/585",
"angle_deqp_gles3_tests": "f6d7197dd1d3710fe7b1c60b617d4b21cd3eb91444460296d00f73656f2d4e7d/585",
"angle_deqp_khr_gles2_tests": "a5c8b22ce843043ae2426d6c8e0593658fcf6a6c3f504837f2cdcea57c52f13d/585",
"angle_deqp_khr_gles31_tests": "eea867ea45da3345502f92430bb0a900ae4484124f942665ff5a00c15145230d/585",
"angle_deqp_khr_gles32_tests": "0c61d8d938b30bd2df7ec9e66afbe36fb2bbf6097e97edc13839b22dc2b1357d/585",
"angle_deqp_khr_gles3_tests": "59fdd64b815d872e670fead9d7271c9dd87575a95b10df285783a2519d9120af/585",
"angle_deqp_khr_glesext_tests": "77297151305256868838aa13d9da7ad246bfa3ad3fcf2848c28a92bd17882619/585",
"angle_deqp_khr_noctx_gles2_tests": "e6c2976c9ac90a515fdd0355320be5a521ce6698d025a45f0e6aaa1746a0e9c3/585",
"angle_deqp_khr_noctx_gles32_tests": "4a0ab67a40b7740089c564692204af652d18512d2e0d6e3a76a88fb6733f2842/585",
"angle_deqp_khr_single_gles32_tests": "70c476b4619dae8e518c5f560d71a14cb3bc4bc4b9ebeb7aeb3319a49cd30933/585",
"angle_end2end_tests": "83517802267aaba4b978de31b99df3034b2086d2e08c6360a98fc1da69078d79/585",
"angle_gles1_conformance_tests": "2c5e8172cc8a65a42218488c4e59ab3a16f0e455ce06119cb1585df8a71c53ac/585",
"angle_perftests": "d5b752d79b2b80637313711cbedbe37f3ea4c09112282a1ebcfb63c8f35cf2b3/745",
"angle_restricted_trace_gold_tests": "64c082322155b7f1cff4ba38e118fd9053d7f693e82511613da35a7872b3eebe/746",
"angle_trace_perf_tests": "522d480b201a349f2a89ab2c077261892aa2e95bbdfc99e2c80ffecce8471e39/745",
"angle_unittests": "edc39036d3b61bb8a1035059d89a56fb2d6e9709e734b2c0372f9cf79eeb88c3/585"
},
"swarming_command_lines_cwd": "out/6da5-angle-android-a",
"swarming_command_lines_digest": "1742422d76183d48107f577ad98e20b3721e10cf0261b270a661231d5e24bea3/93",
"test_trigger_deps_digest": "677b69131341223ab8576aae83ff32dd7bb43074b2b70766a8bfe54acaaabdad/750"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "angle/angle",
"id": "5ce747e1749411f12e03305a60bdca63d69113d0",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8831889576751790064"
},
{
"key": "scheduler_job_id",
"value": "angle/angle-android-arm64-google-pixel6-rel"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "angle-android-arm64-builder-rel"
},
{
"key": "parent_buildnumber",
"value": "1548"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[03:26:09.523] Scheduled build:
{
"id": "8666459514698662097",
"builder": {
"project": "angle",
"bucket": "ci",
"builder": "angle-android-arm64-google-pixel6-rel"
},
"number": 1395,
"createdBy": "project:angle",
"createTime": "2026-10-31T03:26:08.865688167Z",
"updateTime": "2026-10-31T03:26:08.865688167Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "angle/angle",
"id": "5ce747e1749411f12e03305a60bdca63d69113d0",
"ref": "refs/heads/main"
}
}
}
[03:26:09.523] Task URL: https://cr-buildbucket.appspot.com/build/8666459514698662097
[03:26:09.523] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:2:0) after 4m29s
[03:26:10.329] Received PubSub notification, asking Buildbucket for the build status
[03:26:10.376] Build status: SCHEDULED
[03:30:38.540] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:2:0)
[03:30:38.562] Build status: SCHEDULED
[03:30:38.562] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:4:0) after 4m7s
[03:30:42.079] Received PubSub notification, asking Buildbucket for the build status
[03:30:42.108] Build status: STARTED
[03:34:45.581] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:4:0)
[03:34:45.603] Build status: STARTED
[03:34:45.603] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:6:0) after 5m5s
[03:39:50.636] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:6:0)
[03:39:50.695] Build status: STARTED
[03:39:50.695] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:7:0) after 6m38s
[03:46:28.711] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:7:0)
[03:46:28.732] Build status: STARTED
[03:46:28.732] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:8:0) after 8m39s
[03:55:07.747] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:8:0)
[03:55:07.765] Build status: STARTED
[03:55:07.765] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:9:0) after 2m10s
[03:57:17.797] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:9:0)
[03:57:17.816] Build status: STARTED
[03:57:17.816] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:10:0) after 1m3s
[03:58:20.829] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:10:0)
[03:58:20.910] Build status: STARTED
[03:58:20.910] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:11:0) after 9m16s
[04:07:37.006] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:11:0)
[04:07:37.042] Build status: STARTED
[04:07:37.042] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:12:0) after 9m22s
[04:16:59.136] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:12:0)
[04:16:59.177] Build status: STARTED
[04:16:59.177] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:13:0) after 7m40s
[04:24:39.196] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:13:0)
[04:24:39.214] Build status: STARTED
[04:24:39.214] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:14:0) after 2m52s
[04:27:31.234] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:14:0)
[04:27:31.251] Build status: STARTED
[04:27:31.251] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:15:0) after 3m6s
[04:30:37.303] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:15:0)
[04:30:37.339] Build status: STARTED
[04:30:37.339] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:16:0) after 2m22s
[04:32:59.358] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:16:0)
[04:32:59.381] Build status: STARTED
[04:32:59.382] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:17:0) after 7m51s
[04:40:50.413] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:17:0)
[04:40:50.496] Build status: STARTED
[04:40:50.496] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:18:0) after 5m40s
[04:46:30.513] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:18:0)
[04:46:30.534] Build status: STARTED
[04:46:30.534] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:19:0) after 9m5s
[04:55:35.551] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:19:0)
[04:55:35.568] Build status: STARTED
[04:55:35.568] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:20:0) after 4m41s
[05:00:16.649] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:20:0)
[05:00:16.680] Build status: STARTED
[05:00:16.680] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:21:0) after 5m15s
[05:05:31.673] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:21:0)
[05:05:31.777] Build status: STARTED
[05:05:31.777] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:22:0) after 4m36s
[05:10:07.810] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:22:0)
[05:10:07.847] Build status: STARTED
[05:10:07.847] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:23:0) after 7m9s
[05:17:16.869] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:23:0)
[05:17:16.884] Build status: STARTED
[05:17:16.884] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:24:0) after 6m1s
[05:23:17.958] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:24:0)
[05:23:17.977] Build status: STARTED
[05:23:17.977] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:25:0) after 9m13s
[05:32:30.996] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:25:0)
[05:32:31.132] Build status: STARTED
[05:32:31.132] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:26:0) after 6m12s
[05:38:43.151] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:26:0)
[05:38:43.177] Build status: STARTED
[05:38:43.177] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:27:0) after 5m21s
[05:44:04.194] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:27:0)
[05:44:04.261] Build status: STARTED
[05:44:04.261] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:28:0) after 3m28s
[05:47:32.391] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:28:0)
[05:47:32.458] Build status: STARTED
[05:47:32.458] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:29:0) after 7m12s
[05:54:44.530] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:29:0)
[05:54:44.558] Build status: STARTED
[05:54:44.558] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:30:0) after 2m6s
[05:56:50.625] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:30:0)
[05:56:50.663] Build status: STARTED
[05:56:50.663] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:31:0) after 8m10s
[06:05:00.785] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:31:0)
[06:05:00.811] Build status: STARTED
[06:05:00.811] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:32:0) after 3m21s
[06:08:21.948] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:32:0)
[06:08:21.981] Build status: STARTED
[06:08:21.981] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-rel:8831889576751790064:33:0) after 9m25s
[06:17:31.846] Received PubSub notification, asking Buildbucket for the build status
[06:17:31.895] Build:
{
"id": "8666459514698662097",
"builder": {
"project": "angle",
"bucket": "ci",
"builder": "angle-android-arm64-google-pixel6-rel"
},
"number": 1395,
"createdBy": "project:angle",
"createTime": "2026-10-31T03:26:08.865688167Z",
"startTime": "2026-10-31T03:30:41.703674988Z",
"endTime": "2026-10-31T06:17:30.942289070Z",
"updateTime": "2026-10-31T06:17:30.942289070Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "angle/angle",
"id": "5ce747e1749411f12e03305a60bdca63d69113d0",
"ref": "refs/heads/main"
}
}
}
[06:17:31.895] Invocation finished in 2h51m24.372445707s with status FAILED