Status: FAILED
Triggered by: angle-ci-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 57 minutes
Actions:

Definition (rev dc134c65447c1d0503bf7aae465a793917faad9d)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "angle-android-arm64-google-pixel6-exp-rel"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "deps_revision_overrides": {
    "angle": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8667930972045820513"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8667930972045820513"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8667930972045820513"
    }
  ],
  "parent_build_id": "8667930972045820513",
  "parent_builder_group": "ci",
  "parent_buildername": "angle-android-arm64-builder-rel",
  "parent_got_angle_revision": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1",
  "parent_got_revision": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1",
  "repository": "https://chromium.googlesource.com/angle/angle",
  "revision": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1",
  "swarm_hashes": {
    "angle_capture_tests": "5add8540cef8cd1eace83f06e2cb15c6c150b75276758eb6fffb700afd875b8e/585",
    "angle_deqp_egl_tests": "9b33edd5d632f38d75a927bb63e5d5b753e1327dcf87ff5ee9d96c04a3a11572/585",
    "angle_deqp_gles2_tests": "3473fdd518e7f5311cc93fb186534fbca3e3a21e1cca7b3aeaa42cc97f2c61bc/585",
    "angle_deqp_gles31_565_no_depth_no_stencil_tests": "de0ac0a5c30866c669ca9c628b3cc8706bf63c9c660dbf0defb1fe5f208dcf70/585",
    "angle_deqp_gles31_multisample_tests": "8547d112a9c81c6ee063ac84c93f7f26c483ccff0f1dddf2d336b68d3b314b79/585",
    "angle_deqp_gles31_tests": "b6882858180459d4dfe4d155fc2e7e552b72c310da20b86050b486450fd0de3e/585",
    "angle_deqp_gles3_565_no_depth_no_stencil_tests": "4be1ac9c1a13120aacd701cf1e682ad80eb772630a47614e3e3a15b54c42ae43/585",
    "angle_deqp_gles3_multisample_tests": "cc3db1cf6f649191ff1c385c6b21f48af7c1516274d414f6cb2c91756c22a88a/585",
    "angle_deqp_gles3_tests": "2a81bae05203af21ed1a29302931380bc920ee63d292fc10c2e32c2152c4b9fc/585",
    "angle_deqp_khr_gles2_tests": "524f58a47312e0a2c2756463e91f88902ff037422472c69e02d3616ba944f9b2/585",
    "angle_deqp_khr_gles31_tests": "4fc9aa3e2b4aeb08894034aaf4153ba8f7aa4074ed3cb1beb9832c06dd02b4e0/585",
    "angle_deqp_khr_gles32_tests": "f4a6a9d9eccd180f685d9e1ec4dfa1eb05e769d714faf69cb17fc63a07d1b970/585",
    "angle_deqp_khr_gles3_tests": "7e203f16a82ec104acec2dd246853649b93673549cf14f951b849820b28246db/585",
    "angle_deqp_khr_glesext_tests": "281b7fbcb1cdf31a77a95bed48a50d87a3f4adc90743a60dfdc7e4b1e74b4290/585",
    "angle_deqp_khr_noctx_gles2_tests": "d6567e344f28187349fd09fd008a2348922e7beea0cb8cb8d6268de842a554a2/585",
    "angle_deqp_khr_noctx_gles32_tests": "2a65d01039aca40ee9840a93a6d65b5b0971e843108a3d6561cef36b28d187b4/585",
    "angle_deqp_khr_single_gles32_tests": "cdeef559d09a20060d159a704012ca80f90bf7f63685c621b536d066e49ee040/585",
    "angle_end2end_tests": "52c8768920fdb1a67b076c0689feb8f40d4d7fdfaff10ca85427f6f6f525c887/585",
    "angle_gles1_conformance_tests": "33eead3b13a27c66b55875b46eabe8e6b46d07c158b5531a84709386d56410b1/585",
    "angle_perftests": "a2117effe58bd76691a302fce77de08fc9d810fa9b8256566d6027060008c387/745",
    "angle_restricted_trace_gold_tests": "3510616cbd12e092b98633afb85e0fcd5144d3dbee932e788a77ab49e08ecbc8/746",
    "angle_trace_perf_tests": "07f3bdda69d2d811cea2473b66a480620ef3ffe68ff9dd2fb33f205409fd7580/745",
    "angle_unittests": "3036859214524a6a00b23b0a04a751f6f4b0b843ffd8aa29499086f4a08f031d/585"
  },
  "swarming_command_lines_cwd": "out/6da5-angle-android-a",
  "swarming_command_lines_digest": "1742422d76183d48107f577ad98e20b3721e10cf0261b270a661231d5e24bea3/93",
  "test_trigger_deps_digest": "408308804b44ca71fc271ae189da9d0187553bb3cfa770e3f9d42a365094f591/750"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/angle/angle/+/a33b086f57848be2a5b8b55f6cf95df36e1d50b1
  • gitiles_ref:refs/heads/main
  • parent_buildername:angle-android-arm64-builder-rel
  • parent_buildnumber:1399
  • user_agent:recipe

Debug log (UTC)

[00:23:30.381] New invocation is queued and will start shortly
[00:23:30.381] Triggered by user:angle-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[00:23:33.238] Starting the invocation (attempt 1)
[00:23:33.270] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/angle/angle/+/a33b086f57848be2a5b8b55f6cf95df36e1d50b1"
[00:23:33.270] Popped gitiles commit info from properties and tags
[00:23:33.270] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[00:23:33.270] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[00:23:33.271] Buildbucket request:
{
  "requestId": "8833350617617518592",
  "builder": {
    "project": "angle",
    "bucket": "ci",
    "builder": "angle-android-arm64-google-pixel6-exp-rel"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8833350617617518592",
            "job": "angle/angle-android-arm64-google-pixel6-exp-rel",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "deps_revision_overrides": {
                                          "angle": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8667930972045820513"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8667930972045820513"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8667930972045820513"
                                              }
                                        ],
                                    "parent_build_id": "8667930972045820513",
                                    "parent_builder_group": "ci",
                                    "parent_buildername": "angle-android-arm64-builder-rel",
                                    "parent_got_angle_revision": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1",
                                    "parent_got_revision": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1",
                                    "revision": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1",
                                    "swarm_hashes": {
                                          "angle_capture_tests": "5add8540cef8cd1eace83f06e2cb15c6c150b75276758eb6fffb700afd875b8e/585",
                                          "angle_deqp_egl_tests": "9b33edd5d632f38d75a927bb63e5d5b753e1327dcf87ff5ee9d96c04a3a11572/585",
                                          "angle_deqp_gles2_tests": "3473fdd518e7f5311cc93fb186534fbca3e3a21e1cca7b3aeaa42cc97f2c61bc/585",
                                          "angle_deqp_gles31_565_no_depth_no_stencil_tests": "de0ac0a5c30866c669ca9c628b3cc8706bf63c9c660dbf0defb1fe5f208dcf70/585",
                                          "angle_deqp_gles31_multisample_tests": "8547d112a9c81c6ee063ac84c93f7f26c483ccff0f1dddf2d336b68d3b314b79/585",
                                          "angle_deqp_gles31_tests": "b6882858180459d4dfe4d155fc2e7e552b72c310da20b86050b486450fd0de3e/585",
                                          "angle_deqp_gles3_565_no_depth_no_stencil_tests": "4be1ac9c1a13120aacd701cf1e682ad80eb772630a47614e3e3a15b54c42ae43/585",
                                          "angle_deqp_gles3_multisample_tests": "cc3db1cf6f649191ff1c385c6b21f48af7c1516274d414f6cb2c91756c22a88a/585",
                                          "angle_deqp_gles3_tests": "2a81bae05203af21ed1a29302931380bc920ee63d292fc10c2e32c2152c4b9fc/585",
                                          "angle_deqp_khr_gles2_tests": "524f58a47312e0a2c2756463e91f88902ff037422472c69e02d3616ba944f9b2/585",
                                          "angle_deqp_khr_gles31_tests": "4fc9aa3e2b4aeb08894034aaf4153ba8f7aa4074ed3cb1beb9832c06dd02b4e0/585",
                                          "angle_deqp_khr_gles32_tests": "f4a6a9d9eccd180f685d9e1ec4dfa1eb05e769d714faf69cb17fc63a07d1b970/585",
                                          "angle_deqp_khr_gles3_tests": "7e203f16a82ec104acec2dd246853649b93673549cf14f951b849820b28246db/585",
                                          "angle_deqp_khr_glesext_tests": "281b7fbcb1cdf31a77a95bed48a50d87a3f4adc90743a60dfdc7e4b1e74b4290/585",
                                          "angle_deqp_khr_noctx_gles2_tests": "d6567e344f28187349fd09fd008a2348922e7beea0cb8cb8d6268de842a554a2/585",
                                          "angle_deqp_khr_noctx_gles32_tests": "2a65d01039aca40ee9840a93a6d65b5b0971e843108a3d6561cef36b28d187b4/585",
                                          "angle_deqp_khr_single_gles32_tests": "cdeef559d09a20060d159a704012ca80f90bf7f63685c621b536d066e49ee040/585",
                                          "angle_end2end_tests": "52c8768920fdb1a67b076c0689feb8f40d4d7fdfaff10ca85427f6f6f525c887/585",
                                          "angle_gles1_conformance_tests": "33eead3b13a27c66b55875b46eabe8e6b46d07c158b5531a84709386d56410b1/585",
                                          "angle_perftests": "a2117effe58bd76691a302fce77de08fc9d810fa9b8256566d6027060008c387/745",
                                          "angle_restricted_trace_gold_tests": "3510616cbd12e092b98633afb85e0fcd5144d3dbee932e788a77ab49e08ecbc8/746",
                                          "angle_trace_perf_tests": "07f3bdda69d2d811cea2473b66a480620ef3ffe68ff9dd2fb33f205409fd7580/745",
                                          "angle_unittests": "3036859214524a6a00b23b0a04a751f6f4b0b843ffd8aa29499086f4a08f031d/585"
                                        },
                                    "swarming_command_lines_cwd": "out/6da5-angle-android-a",
                                    "swarming_command_lines_digest": "1742422d76183d48107f577ad98e20b3721e10cf0261b270a661231d5e24bea3/93",
                                    "test_trigger_deps_digest": "408308804b44ca71fc271ae189da9d0187553bb3cfa770e3f9d42a365094f591/750"
                                  },
                              "ref": "refs/heads/main",
                              "repo": "https://chromium.googlesource.com/angle/angle",
                              "revision": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1",
                              "tags": [
                                    "parent_buildername:angle-android-arm64-builder-rel",
                                    "parent_buildnumber:1399",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "3d4ab352-abf2-4be4-b25e-57a77d52f3b4",
                        "title": "angle-android-arm64-builder-rel/1399",
                        "url": "https://cr-buildbucket.appspot.com/build/8667930972045820513"
                      }
                ]
          },
      "deps_revision_overrides": {
            "angle": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8667930972045820513"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8667930972045820513"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8667930972045820513"
                }
          ],
      "parent_build_id": "8667930972045820513",
      "parent_builder_group": "ci",
      "parent_buildername": "angle-android-arm64-builder-rel",
      "parent_got_angle_revision": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1",
      "parent_got_revision": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1",
      "swarm_hashes": {
            "angle_capture_tests": "5add8540cef8cd1eace83f06e2cb15c6c150b75276758eb6fffb700afd875b8e/585",
            "angle_deqp_egl_tests": "9b33edd5d632f38d75a927bb63e5d5b753e1327dcf87ff5ee9d96c04a3a11572/585",
            "angle_deqp_gles2_tests": "3473fdd518e7f5311cc93fb186534fbca3e3a21e1cca7b3aeaa42cc97f2c61bc/585",
            "angle_deqp_gles31_565_no_depth_no_stencil_tests": "de0ac0a5c30866c669ca9c628b3cc8706bf63c9c660dbf0defb1fe5f208dcf70/585",
            "angle_deqp_gles31_multisample_tests": "8547d112a9c81c6ee063ac84c93f7f26c483ccff0f1dddf2d336b68d3b314b79/585",
            "angle_deqp_gles31_tests": "b6882858180459d4dfe4d155fc2e7e552b72c310da20b86050b486450fd0de3e/585",
            "angle_deqp_gles3_565_no_depth_no_stencil_tests": "4be1ac9c1a13120aacd701cf1e682ad80eb772630a47614e3e3a15b54c42ae43/585",
            "angle_deqp_gles3_multisample_tests": "cc3db1cf6f649191ff1c385c6b21f48af7c1516274d414f6cb2c91756c22a88a/585",
            "angle_deqp_gles3_tests": "2a81bae05203af21ed1a29302931380bc920ee63d292fc10c2e32c2152c4b9fc/585",
            "angle_deqp_khr_gles2_tests": "524f58a47312e0a2c2756463e91f88902ff037422472c69e02d3616ba944f9b2/585",
            "angle_deqp_khr_gles31_tests": "4fc9aa3e2b4aeb08894034aaf4153ba8f7aa4074ed3cb1beb9832c06dd02b4e0/585",
            "angle_deqp_khr_gles32_tests": "f4a6a9d9eccd180f685d9e1ec4dfa1eb05e769d714faf69cb17fc63a07d1b970/585",
            "angle_deqp_khr_gles3_tests": "7e203f16a82ec104acec2dd246853649b93673549cf14f951b849820b28246db/585",
            "angle_deqp_khr_glesext_tests": "281b7fbcb1cdf31a77a95bed48a50d87a3f4adc90743a60dfdc7e4b1e74b4290/585",
            "angle_deqp_khr_noctx_gles2_tests": "d6567e344f28187349fd09fd008a2348922e7beea0cb8cb8d6268de842a554a2/585",
            "angle_deqp_khr_noctx_gles32_tests": "2a65d01039aca40ee9840a93a6d65b5b0971e843108a3d6561cef36b28d187b4/585",
            "angle_deqp_khr_single_gles32_tests": "cdeef559d09a20060d159a704012ca80f90bf7f63685c621b536d066e49ee040/585",
            "angle_end2end_tests": "52c8768920fdb1a67b076c0689feb8f40d4d7fdfaff10ca85427f6f6f525c887/585",
            "angle_gles1_conformance_tests": "33eead3b13a27c66b55875b46eabe8e6b46d07c158b5531a84709386d56410b1/585",
            "angle_perftests": "a2117effe58bd76691a302fce77de08fc9d810fa9b8256566d6027060008c387/745",
            "angle_restricted_trace_gold_tests": "3510616cbd12e092b98633afb85e0fcd5144d3dbee932e788a77ab49e08ecbc8/746",
            "angle_trace_perf_tests": "07f3bdda69d2d811cea2473b66a480620ef3ffe68ff9dd2fb33f205409fd7580/745",
            "angle_unittests": "3036859214524a6a00b23b0a04a751f6f4b0b843ffd8aa29499086f4a08f031d/585"
          },
      "swarming_command_lines_cwd": "out/6da5-angle-android-a",
      "swarming_command_lines_digest": "1742422d76183d48107f577ad98e20b3721e10cf0261b270a661231d5e24bea3/93",
      "test_trigger_deps_digest": "408308804b44ca71fc271ae189da9d0187553bb3cfa770e3f9d42a365094f591/750"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "angle/angle",
    "id": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8833350617617518592"
    },
    {
      "key": "scheduler_job_id",
      "value": "angle/angle-android-arm64-google-pixel6-exp-rel"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "angle-android-arm64-builder-rel"
    },
    {
      "key": "parent_buildnumber",
      "value": "1399"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[00:23:33.935] Scheduled build:
{
  "id": "8667920553847060017",
  "builder": {
    "project": "angle",
    "bucket": "ci",
    "builder": "angle-android-arm64-google-pixel6-exp-rel"
  },
  "number": 1189,
  "createdBy": "project:angle",
  "createTime": "2026-10-15T00:23:33.354471410Z",
  "updateTime": "2026-10-15T00:23:33.354471410Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "angle/angle",
      "id": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1",
      "ref": "refs/heads/main"
    }
  }
}
[00:23:33.935] Task URL: https://cr-buildbucket.appspot.com/build/8667920553847060017
[00:23:33.935] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:2:0) after 2m29s
[00:23:34.848] Received PubSub notification, asking Buildbucket for the build status
[00:23:34.873] Build status: SCHEDULED
[00:23:52.945] Received PubSub notification, asking Buildbucket for the build status
[00:23:53.120] Build status: STARTED
[00:26:03.004] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:2:0)
[00:26:03.112] Build status: STARTED
[00:26:03.112] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:5:0) after 6m56s
[00:32:59.184] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:5:0)
[00:32:59.204] Build status: STARTED
[00:32:59.204] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:6:0) after 5m43s
[00:38:42.305] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:6:0)
[00:38:42.330] Build status: STARTED
[00:38:42.330] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:7:0) after 5m0s
[00:43:42.385] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:7:0)
[00:43:42.407] Build status: STARTED
[00:43:42.408] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:8:0) after 4m15s
[00:47:57.412] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:8:0)
[00:47:57.429] Build status: STARTED
[00:47:57.429] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:9:0) after 5m26s
[00:53:23.490] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:9:0)
[00:53:23.509] Build status: STARTED
[00:53:23.509] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:10:0) after 9m25s
[01:02:48.615] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:10:0)
[01:02:48.634] Build status: STARTED
[01:02:48.634] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:11:0) after 4m45s
[01:07:33.663] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:11:0)
[01:07:33.684] Build status: STARTED
[01:07:33.684] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:12:0) after 2m35s
[01:10:08.739] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:12:0)
[01:10:08.762] Build status: STARTED
[01:10:08.762] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:13:0) after 1m46s
[01:11:54.758] Handling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:13:0)
[01:11:54.785] Build status: STARTED
[01:11:54.785] Scheduling timer "check-buildbucket-build-status" (angle/angle-android-arm64-google-pixel6-exp-rel:8833350617617518592:14:0) after 9m38s
[01:21:09.863] Received PubSub notification, asking Buildbucket for the build status
[01:21:09.884] Build:
{
  "id": "8667920553847060017",
  "builder": {
    "project": "angle",
    "bucket": "ci",
    "builder": "angle-android-arm64-google-pixel6-exp-rel"
  },
  "number": 1189,
  "createdBy": "project:angle",
  "createTime": "2026-10-15T00:23:33.354471410Z",
  "startTime": "2026-10-15T00:23:49.902266649Z",
  "endTime": "2026-10-15T01:21:09.282794472Z",
  "updateTime": "2026-10-15T01:21:09.282794472Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "angle/angle",
      "id": "a33b086f57848be2a5b8b55f6cf95df36e1d50b1",
      "ref": "refs/heads/main"
    }
  }
}
[01:21:09.884] Invocation finished in 57m39.509971261s with status FAILED