Status: FAILED
Triggered by: angle-ci-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 58 minutes
Actions:

Definition (rev dc134c65447c1d0503bf7aae465a793917faad9d)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "angle-android-arm64-samsung-s24-rel"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "deps_revision_overrides": {
    "angle": "71959fdc46b449eb5b3de862d19abf01dac5328d"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8676659046188174145"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8676659046188174145"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8676659046188174145"
    }
  ],
  "parent_build_id": "8676659046188174145",
  "parent_builder_group": "ci",
  "parent_buildername": "angle-android-arm64-builder-rel",
  "parent_got_angle_revision": "71959fdc46b449eb5b3de862d19abf01dac5328d",
  "parent_got_revision": "71959fdc46b449eb5b3de862d19abf01dac5328d",
  "repository": "https://chromium.googlesource.com/angle/angle",
  "revision": "71959fdc46b449eb5b3de862d19abf01dac5328d",
  "swarm_hashes": {
    "angle_capture_tests": "a745357faf5933b268f7dde64719fa99a76f1a377af9507af86d178f20d56f42/486",
    "angle_deqp_egl_tests": "59c488e74cbec704812dc17d01260cd9823d7b534feb5b4daab1af7704877b1e/486",
    "angle_deqp_gles2_tests": "fceb316836353b4ba208955f3069f645a488794c1cc9d17c5c6f713ac68ba6d2/486",
    "angle_deqp_gles31_565_no_depth_no_stencil_tests": "aba3404f7b971929cf3b8c7d4b16ab7ccb6490b5705f67033c776a84ca4abafa/486",
    "angle_deqp_gles31_multisample_tests": "f69f0f769800372be5b4d169afd983f03c91c8b6f3004f6c21003a34ce56c1e4/486",
    "angle_deqp_gles31_tests": "b0221dfe4bef4e0514b7b087f780da8bd9a33b434af06fbab3d20201a978e30f/486",
    "angle_deqp_gles3_565_no_depth_no_stencil_tests": "4ee14b5086b5390d9bedef7e1712404bbc46226f1aa3c50df92afa83e05fd02c/486",
    "angle_deqp_gles3_multisample_tests": "35a577df74dcfac1db2ed205090122b017c3dab0eca396887f99ca434a0d9687/486",
    "angle_deqp_gles3_tests": "e0e2981155c2957dca048484f7cc6b7948475f14e21af8e7f34603b9478a6e55/486",
    "angle_deqp_khr_gles2_tests": "b9b600a21c58922e59fe00853ab8e359455861d59a6492e7426717b4770eade1/486",
    "angle_deqp_khr_gles31_tests": "42e9af0bb56e84dac7cbeac967762fd3d475e22bdcbe73ca1faf8629012e320d/486",
    "angle_deqp_khr_gles32_tests": "64741f6c182a368c931dafc3abd67bec138381c4599bb0305a8e9f4ec8b1588e/486",
    "angle_deqp_khr_gles3_tests": "22e08f1eadd16795c57a8e4f006b6ba9d6d62022d72b2c1f739114d0781b338c/486",
    "angle_deqp_khr_glesext_tests": "9e331dee44ff3cb47de5a66b68513cf23d0d624b114beaaf87536c77fe3ae9d4/486",
    "angle_deqp_khr_noctx_gles2_tests": "babf9ed5ce6478cffb046b50652a011c7e653628305be4e1d3d21c0b66c36277/486",
    "angle_deqp_khr_noctx_gles32_tests": "d5e4edc3a998612311b02cccf8bd02e0099bc5a18694dc3c52310f62a5832825/486",
    "angle_deqp_khr_single_gles32_tests": "7db78ebcd1b1492c558ac17c4cfe8effa440b2a8837b01d56f829340d99bf58c/486",
    "angle_end2end_tests": "8e5b259cc0eb3313ba6f939c730456cbbcc871084c0b02594e8291479e69638f/486",
    "angle_gles1_conformance_tests": "2ef1ceaa003359ed555c86a8c630035a58d8e1f5e02e62646612ed76c1b70715/486",
    "angle_perftests": "2bb17412dde722ee22113f5b910875845453811781ea12eb53ab26c2ceedd8b2/646",
    "angle_restricted_trace_gold_tests": "c16b1981739d53ce0cbddae105a2cfb7036ed8ec7f993a2e080ee9229b4f19a7/647",
    "angle_trace_perf_tests": "5a79cf61ec1bcf0a720e010e2c5ed4f28dee1413ac1319b36fca1a34b25b6b1d/646",
    "angle_unittests": "2d09ce40133c6aef8cdbe33c1f0783c8a6de3111832f294c6b5dfc452afda71b/486"
  },
  "swarming_command_lines_cwd": "out/6da5-angle-android-a",
  "swarming_command_lines_digest": "1742422d76183d48107f577ad98e20b3721e10cf0261b270a661231d5e24bea3/93",
  "test_trigger_deps_digest": "60dc335e3d11bf0c4965afe995b7fb6772f9d18b44c4448c3ea19cd942bf8246/651"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/angle/angle/+/71959fdc46b449eb5b3de862d19abf01dac5328d
  • gitiles_ref:refs/heads/main
  • parent_buildername:angle-android-arm64-builder-rel
  • parent_buildnumber:724
  • user_agent:recipe

Debug log (UTC)

[13:51:24.448] New invocation is queued and will start shortly
[13:51:24.448] Triggered by user:angle-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[13:51:25.848] Starting the invocation (attempt 1)
[13:51:25.872] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/angle/angle/+/71959fdc46b449eb5b3de862d19abf01dac5328d"
[13:51:25.872] Popped gitiles commit info from properties and tags
[13:51:25.872] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[13:51:25.873] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[13:51:25.874] Buildbucket request:
{
  "requestId": "8842087694613809840",
  "builder": {
    "project": "angle",
    "bucket": "ci",
    "builder": "angle-android-arm64-samsung-s24-rel"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8842087694613809840",
            "job": "angle/angle-android-arm64-samsung-s24-rel",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "deps_revision_overrides": {
                                          "angle": "71959fdc46b449eb5b3de862d19abf01dac5328d"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8676659046188174145"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8676659046188174145"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8676659046188174145"
                                              }
                                        ],
                                    "parent_build_id": "8676659046188174145",
                                    "parent_builder_group": "ci",
                                    "parent_buildername": "angle-android-arm64-builder-rel",
                                    "parent_got_angle_revision": "71959fdc46b449eb5b3de862d19abf01dac5328d",
                                    "parent_got_revision": "71959fdc46b449eb5b3de862d19abf01dac5328d",
                                    "revision": "71959fdc46b449eb5b3de862d19abf01dac5328d",
                                    "swarm_hashes": {
                                          "angle_capture_tests": "a745357faf5933b268f7dde64719fa99a76f1a377af9507af86d178f20d56f42/486",
                                          "angle_deqp_egl_tests": "59c488e74cbec704812dc17d01260cd9823d7b534feb5b4daab1af7704877b1e/486",
                                          "angle_deqp_gles2_tests": "fceb316836353b4ba208955f3069f645a488794c1cc9d17c5c6f713ac68ba6d2/486",
                                          "angle_deqp_gles31_565_no_depth_no_stencil_tests": "aba3404f7b971929cf3b8c7d4b16ab7ccb6490b5705f67033c776a84ca4abafa/486",
                                          "angle_deqp_gles31_multisample_tests": "f69f0f769800372be5b4d169afd983f03c91c8b6f3004f6c21003a34ce56c1e4/486",
                                          "angle_deqp_gles31_tests": "b0221dfe4bef4e0514b7b087f780da8bd9a33b434af06fbab3d20201a978e30f/486",
                                          "angle_deqp_gles3_565_no_depth_no_stencil_tests": "4ee14b5086b5390d9bedef7e1712404bbc46226f1aa3c50df92afa83e05fd02c/486",
                                          "angle_deqp_gles3_multisample_tests": "35a577df74dcfac1db2ed205090122b017c3dab0eca396887f99ca434a0d9687/486",
                                          "angle_deqp_gles3_tests": "e0e2981155c2957dca048484f7cc6b7948475f14e21af8e7f34603b9478a6e55/486",
                                          "angle_deqp_khr_gles2_tests": "b9b600a21c58922e59fe00853ab8e359455861d59a6492e7426717b4770eade1/486",
                                          "angle_deqp_khr_gles31_tests": "42e9af0bb56e84dac7cbeac967762fd3d475e22bdcbe73ca1faf8629012e320d/486",
                                          "angle_deqp_khr_gles32_tests": "64741f6c182a368c931dafc3abd67bec138381c4599bb0305a8e9f4ec8b1588e/486",
                                          "angle_deqp_khr_gles3_tests": "22e08f1eadd16795c57a8e4f006b6ba9d6d62022d72b2c1f739114d0781b338c/486",
                                          "angle_deqp_khr_glesext_tests": "9e331dee44ff3cb47de5a66b68513cf23d0d624b114beaaf87536c77fe3ae9d4/486",
                                          "angle_deqp_khr_noctx_gles2_tests": "babf9ed5ce6478cffb046b50652a011c7e653628305be4e1d3d21c0b66c36277/486",
                                          "angle_deqp_khr_noctx_gles32_tests": "d5e4edc3a998612311b02cccf8bd02e0099bc5a18694dc3c52310f62a5832825/486",
                                          "angle_deqp_khr_single_gles32_tests": "7db78ebcd1b1492c558ac17c4cfe8effa440b2a8837b01d56f829340d99bf58c/486",
                                          "angle_end2end_tests": "8e5b259cc0eb3313ba6f939c730456cbbcc871084c0b02594e8291479e69638f/486",
                                          "angle_gles1_conformance_tests": "2ef1ceaa003359ed555c86a8c630035a58d8e1f5e02e62646612ed76c1b70715/486",
                                          "angle_perftests": "2bb17412dde722ee22113f5b910875845453811781ea12eb53ab26c2ceedd8b2/646",
                                          "angle_restricted_trace_gold_tests": "c16b1981739d53ce0cbddae105a2cfb7036ed8ec7f993a2e080ee9229b4f19a7/647",
                                          "angle_trace_perf_tests": "5a79cf61ec1bcf0a720e010e2c5ed4f28dee1413ac1319b36fca1a34b25b6b1d/646",
                                          "angle_unittests": "2d09ce40133c6aef8cdbe33c1f0783c8a6de3111832f294c6b5dfc452afda71b/486"
                                        },
                                    "swarming_command_lines_cwd": "out/6da5-angle-android-a",
                                    "swarming_command_lines_digest": "1742422d76183d48107f577ad98e20b3721e10cf0261b270a661231d5e24bea3/93",
                                    "test_trigger_deps_digest": "60dc335e3d11bf0c4965afe995b7fb6772f9d18b44c4448c3ea19cd942bf8246/651"
                                  },
                              "ref": "refs/heads/main",
                              "repo": "https://chromium.googlesource.com/angle/angle",
                              "revision": "71959fdc46b449eb5b3de862d19abf01dac5328d",
                              "tags": [
                                    "parent_buildername:angle-android-arm64-builder-rel",
                                    "parent_buildnumber:724",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "24282ee2-6f4b-4ed5-8222-8da1052a0228",
                        "title": "angle-android-arm64-builder-rel/724",
                        "url": "https://cr-buildbucket.appspot.com/build/8676659046188174145"
                      }
                ]
          },
      "deps_revision_overrides": {
            "angle": "71959fdc46b449eb5b3de862d19abf01dac5328d"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8676659046188174145"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8676659046188174145"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8676659046188174145"
                }
          ],
      "parent_build_id": "8676659046188174145",
      "parent_builder_group": "ci",
      "parent_buildername": "angle-android-arm64-builder-rel",
      "parent_got_angle_revision": "71959fdc46b449eb5b3de862d19abf01dac5328d",
      "parent_got_revision": "71959fdc46b449eb5b3de862d19abf01dac5328d",
      "swarm_hashes": {
            "angle_capture_tests": "a745357faf5933b268f7dde64719fa99a76f1a377af9507af86d178f20d56f42/486",
            "angle_deqp_egl_tests": "59c488e74cbec704812dc17d01260cd9823d7b534feb5b4daab1af7704877b1e/486",
            "angle_deqp_gles2_tests": "fceb316836353b4ba208955f3069f645a488794c1cc9d17c5c6f713ac68ba6d2/486",
            "angle_deqp_gles31_565_no_depth_no_stencil_tests": "aba3404f7b971929cf3b8c7d4b16ab7ccb6490b5705f67033c776a84ca4abafa/486",
            "angle_deqp_gles31_multisample_tests": "f69f0f769800372be5b4d169afd983f03c91c8b6f3004f6c21003a34ce56c1e4/486",
            "angle_deqp_gles31_tests": "b0221dfe4bef4e0514b7b087f780da8bd9a33b434af06fbab3d20201a978e30f/486",
            "angle_deqp_gles3_565_no_depth_no_stencil_tests": "4ee14b5086b5390d9bedef7e1712404bbc46226f1aa3c50df92afa83e05fd02c/486",
            "angle_deqp_gles3_multisample_tests": "35a577df74dcfac1db2ed205090122b017c3dab0eca396887f99ca434a0d9687/486",
            "angle_deqp_gles3_tests": "e0e2981155c2957dca048484f7cc6b7948475f14e21af8e7f34603b9478a6e55/486",
            "angle_deqp_khr_gles2_tests": "b9b600a21c58922e59fe00853ab8e359455861d59a6492e7426717b4770eade1/486",
            "angle_deqp_khr_gles31_tests": "42e9af0bb56e84dac7cbeac967762fd3d475e22bdcbe73ca1faf8629012e320d/486",
            "angle_deqp_khr_gles32_tests": "64741f6c182a368c931dafc3abd67bec138381c4599bb0305a8e9f4ec8b1588e/486",
            "angle_deqp_khr_gles3_tests": "22e08f1eadd16795c57a8e4f006b6ba9d6d62022d72b2c1f739114d0781b338c/486",
            "angle_deqp_khr_glesext_tests": "9e331dee44ff3cb47de5a66b68513cf23d0d624b114beaaf87536c77fe3ae9d4/486",
            "angle_deqp_khr_noctx_gles2_tests": "babf9ed5ce6478cffb046b50652a011c7e653628305be4e1d3d21c0b66c36277/486",
            "angle_deqp_khr_noctx_gles32_tests": "d5e4edc3a998612311b02cccf8bd02e0099bc5a18694dc3c52310f62a5832825/486",
            "angle_deqp_khr_single_gles32_tests": "7db78ebcd1b1492c558ac17c4cfe8effa440b2a8837b01d56f829340d99bf58c/486",
            "angle_end2end_tests": "8e5b259cc0eb3313ba6f939c730456cbbcc871084c0b02594e8291479e69638f/486",
            "angle_gles1_conformance_tests": "2ef1ceaa003359ed555c86a8c630035a58d8e1f5e02e62646612ed76c1b70715/486",
            "angle_perftests": "2bb17412dde722ee22113f5b910875845453811781ea12eb53ab26c2ceedd8b2/646",
            "angle_restricted_trace_gold_tests": "c16b1981739d53ce0cbddae105a2cfb7036ed8ec7f993a2e080ee9229b4f19a7/647",
            "angle_trace_perf_tests": "5a79cf61ec1bcf0a720e010e2c5ed4f28dee1413ac1319b36fca1a34b25b6b1d/646",
            "angle_unittests": "2d09ce40133c6aef8cdbe33c1f0783c8a6de3111832f294c6b5dfc452afda71b/486"
          },
      "swarming_command_lines_cwd": "out/6da5-angle-android-a",
      "swarming_command_lines_digest": "1742422d76183d48107f577ad98e20b3721e10cf0261b270a661231d5e24bea3/93",
      "test_trigger_deps_digest": "60dc335e3d11bf0c4965afe995b7fb6772f9d18b44c4448c3ea19cd942bf8246/651"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "angle/angle",
    "id": "71959fdc46b449eb5b3de862d19abf01dac5328d",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8842087694613809840"
    },
    {
      "key": "scheduler_job_id",
      "value": "angle/angle-android-arm64-samsung-s24-rel"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "angle-android-arm64-builder-rel"
    },
    {
      "key": "parent_buildnumber",
      "value": "724"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[13:51:26.460] Scheduled build:
{
  "id": "8676657632342176849",
  "builder": {
    "project": "angle",
    "bucket": "ci",
    "builder": "angle-android-arm64-samsung-s24-rel"
  },
  "number": 629,
  "createdBy": "project:angle",
  "createTime": "2026-07-10T13:51:25.993134515Z",
  "updateTime": "2026-07-10T13:51:25.993134515Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "angle/angle",
      "id": "71959fdc46b449eb5b3de862d19abf01dac5328d",
      "ref": "refs/heads/main"
    }
  }
}
[13:51:26.460] Task URL: https://cr-buildbucket.appspot.com/build/8676657632342176849
[13:51:26.460] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:2:0) after 3m11s
[13:51:27.425] Received PubSub notification, asking Buildbucket for the build status
[13:51:27.439] Build status: SCHEDULED
[13:51:38.525] Received PubSub notification, asking Buildbucket for the build status
[13:51:38.572] Build status: STARTED
[13:54:37.470] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:2:0)
[13:54:37.493] Build status: STARTED
[13:54:37.493] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:5:0) after 6m59s
[14:01:36.546] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:5:0)
[14:01:36.580] Build status: STARTED
[14:01:36.580] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:6:0) after 2m43s
[14:04:19.563] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:6:0)
[14:04:19.590] Build status: STARTED
[14:04:19.590] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:7:0) after 1m16s
[14:05:35.597] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:7:0)
[14:05:35.615] Build status: STARTED
[14:05:35.615] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:8:0) after 6m0s
[14:11:35.633] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:8:0)
[14:11:35.653] Build status: STARTED
[14:11:35.653] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:9:0) after 2m16s
[14:13:52.068] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:9:0)
[14:13:52.090] Build status: STARTED
[14:13:52.090] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:10:0) after 5m48s
[14:19:40.141] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:10:0)
[14:19:40.170] Build status: STARTED
[14:19:40.170] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:11:0) after 5m47s
[14:25:27.200] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:11:0)
[14:25:27.220] Build status: STARTED
[14:25:27.220] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:12:0) after 9m52s
[14:35:19.235] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:12:0)
[14:35:19.265] Build status: STARTED
[14:35:19.265] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:13:0) after 8m1s
[14:43:20.416] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:13:0)
[14:43:20.448] Build status: STARTED
[14:43:20.448] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:14:0) after 1m44s
[14:45:04.439] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:14:0)
[14:45:04.520] Build status: STARTED
[14:45:04.520] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-samsung-s24-rel:8842087694613809840:15:0) after 6m47s
[14:50:07.854] Received PubSub notification, asking Buildbucket for the build status
[14:50:07.876] Build:
{
  "id": "8676657632342176849",
  "builder": {
    "project": "angle",
    "bucket": "ci",
    "builder": "angle-android-arm64-samsung-s24-rel"
  },
  "number": 629,
  "createdBy": "project:angle",
  "createTime": "2026-07-10T13:51:25.993134515Z",
  "startTime": "2026-07-10T13:51:38.350924227Z",
  "endTime": "2026-07-10T14:50:07.491981425Z",
  "updateTime": "2026-07-10T14:50:07.491981425Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "angle/angle",
      "id": "71959fdc46b449eb5b3de862d19abf01dac5328d",
      "ref": "refs/heads/main"
    }
  }
}
[14:50:07.876] Invocation finished in 58m43.432867232s with status FAILED