Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 1 hour
Actions:

Definition (rev 1308c62b6ac00ddd1e8c3d9e8f882d4034bc9327)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Win10 x64 DEPS Release (Intel)"
>

Incoming triggers

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "8504baee8b7f193c5a919ba3859089cbadd2a63b",
        "project": "chromium/src",
        "ref": "refs/branch-heads/7827"
      }
    ]
  },
  "branch": "refs/branch-heads/7827",
  "deps_revision_overrides": {
    "src": "8504baee8b7f193c5a919ba3859089cbadd2a63b"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8678458509417406049"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8678458509417406049"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8678458509417406049"
    }
  ],
  "parent_build_id": "8678458509417406049",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Win10 x64 DEPS Builder",
  "parent_got_angle_revision": "355cc61af2aadd8f0494800325b2bf9908138108",
  "parent_got_dawn_revision": "5f4c5ef509c5ffa65822302341cf9b2ccad471f9",
  "parent_got_revision": "8504baee8b7f193c5a919ba3859089cbadd2a63b",
  "parent_got_revision_cp": "refs/branch-heads/7827@{#3533}",
  "parent_got_swiftshader_revision": "f9d5d49a3c599a315e3493dc1e9b5309cffb3305",
  "parent_got_v8_revision": "933ce636c562cd54d68e7f7c93ab5cdffd685fca",
  "parent_got_v8_revision_cp": "refs/branch-heads/14.9@{#69}",
  "parent_got_webrtc_revision": "28311abc149a9bb7e6761a3d54f362a8d77e6793",
  "parent_got_webrtc_revision_cp": "refs/branch-heads/7827@{#7}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "8504baee8b7f193c5a919ba3859089cbadd2a63b",
  "swarm_hashes": {
    "gl_tests": "c9f48870695b3d4ca3c9ae5e9b92baca2f9e211ce994f071bf5277ea40e093c9/323",
    "gl_unittests": "a90fb2aa1d7a5efb25ee4567ac0024f19a6865268aa7314f65754eb3eb3c4761/402",
    "telemetry_gpu_integration_test": "8e7929c30fb60332cebc17037cefc3167ccdfaee34b476bf36610943357cd848/815",
    "telemetry_gpu_unittests": "6fd7a38b78ffeb4979feb630142a539b4f0e29ffefc5e5f223981c24557feddc/736",
    "webgpu_blink_web_tests": "b186fcb6152264c812c63a96df7af81437099801e31266a8e64c9dc4700bc72a/486"
  },
  "swarming_command_lines_cwd": "out\\ea6c-Dawn_Win10_x64_",
  "swarming_command_lines_digest": "0f81065f11c77ea732bf7c88af88ac8791293bb7e35db22054436a6ce53daa18/93",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/8504baee8b7f193c5a919ba3859089cbadd2a63b
  • gitiles_ref:refs/branch-heads/7827
  • parent_buildername:Dawn Win10 x64 DEPS Builder
  • parent_buildnumber:2282
  • user_agent:recipe

Debug log (UTC)

[17:02:06.322] New invocation is queued and will start shortly
[17:02:06.322] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[17:02:07.374] Starting the invocation (attempt 1)
[17:02:07.411] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/8504baee8b7f193c5a919ba3859089cbadd2a63b"
[17:02:07.411] Popped gitiles commit info from properties and tags
[17:02:07.411] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[17:02:07.411] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[17:02:07.412] Buildbucket request:
{
  "requestId": "8843887636268484704",
  "builder": {
    "project": "chromium-m149",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 DEPS Release (Intel)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "8504baee8b7f193c5a919ba3859089cbadd2a63b",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/7827"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8843887636268484704",
            "job": "chromium-m149/Dawn Win10 x64 DEPS Release (Intel)",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "ad0bcbc9dea1e9e7131ae2313cf539c20ae39050",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7827"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "ad0bcbc9dea1e9e7131ae2313cf539c20ae39050"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8678461639849010945"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8678461639849010945"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8678461639849010945"
                                              }
                                        ],
                                    "parent_build_id": "8678461639849010945",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Win10 x64 DEPS Builder",
                                    "parent_got_angle_revision": "355cc61af2aadd8f0494800325b2bf9908138108",
                                    "parent_got_dawn_revision": "5f4c5ef509c5ffa65822302341cf9b2ccad471f9",
                                    "parent_got_revision": "ad0bcbc9dea1e9e7131ae2313cf539c20ae39050",
                                    "parent_got_revision_cp": "refs/branch-heads/7827@{#3532}",
                                    "parent_got_swiftshader_revision": "f9d5d49a3c599a315e3493dc1e9b5309cffb3305",
                                    "parent_got_v8_revision": "933ce636c562cd54d68e7f7c93ab5cdffd685fca",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/14.9@{#69}",
                                    "parent_got_webrtc_revision": "28311abc149a9bb7e6761a3d54f362a8d77e6793",
                                    "parent_got_webrtc_revision_cp": "refs/branch-heads/7827@{#7}",
                                    "revision": "ad0bcbc9dea1e9e7131ae2313cf539c20ae39050",
                                    "swarm_hashes": {
                                          "gl_tests": "c9f48870695b3d4ca3c9ae5e9b92baca2f9e211ce994f071bf5277ea40e093c9/323",
                                          "gl_unittests": "a90fb2aa1d7a5efb25ee4567ac0024f19a6865268aa7314f65754eb3eb3c4761/402",
                                          "telemetry_gpu_integration_test": "879dbeca74b8fd1d84f2180a42e8f120f8d1aa093956e110c4b95acf783ea370/815",
                                          "telemetry_gpu_unittests": "6fd7a38b78ffeb4979feb630142a539b4f0e29ffefc5e5f223981c24557feddc/736",
                                          "webgpu_blink_web_tests": "3bff787e57d461b4118b338208e5f1f087486de2f1895033b01957d8217e72f8/486"
                                        },
                                    "swarming_command_lines_cwd": "out\\ea6c-Dawn_Win10_x64_",
                                    "swarming_command_lines_digest": "0f81065f11c77ea732bf7c88af88ac8791293bb7e35db22054436a6ce53daa18/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7827",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "ad0bcbc9dea1e9e7131ae2313cf539c20ae39050",
                              "tags": [
                                    "parent_buildername:Dawn Win10 x64 DEPS Builder",
                                    "parent_buildnumber:2281",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "c27af007-a15a-4a17-af49-a098a04dd80e",
                        "title": "Dawn Win10 x64 DEPS Builder/2281",
                        "url": "https://cr-buildbucket.appspot.com/build/8678461639849010945"
                      },
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "8504baee8b7f193c5a919ba3859089cbadd2a63b",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7827"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "8504baee8b7f193c5a919ba3859089cbadd2a63b"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8678458509417406049"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8678458509417406049"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8678458509417406049"
                                              }
                                        ],
                                    "parent_build_id": "8678458509417406049",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Win10 x64 DEPS Builder",
                                    "parent_got_angle_revision": "355cc61af2aadd8f0494800325b2bf9908138108",
                                    "parent_got_dawn_revision": "5f4c5ef509c5ffa65822302341cf9b2ccad471f9",
                                    "parent_got_revision": "8504baee8b7f193c5a919ba3859089cbadd2a63b",
                                    "parent_got_revision_cp": "refs/branch-heads/7827@{#3533}",
                                    "parent_got_swiftshader_revision": "f9d5d49a3c599a315e3493dc1e9b5309cffb3305",
                                    "parent_got_v8_revision": "933ce636c562cd54d68e7f7c93ab5cdffd685fca",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/14.9@{#69}",
                                    "parent_got_webrtc_revision": "28311abc149a9bb7e6761a3d54f362a8d77e6793",
                                    "parent_got_webrtc_revision_cp": "refs/branch-heads/7827@{#7}",
                                    "revision": "8504baee8b7f193c5a919ba3859089cbadd2a63b",
                                    "swarm_hashes": {
                                          "gl_tests": "c9f48870695b3d4ca3c9ae5e9b92baca2f9e211ce994f071bf5277ea40e093c9/323",
                                          "gl_unittests": "a90fb2aa1d7a5efb25ee4567ac0024f19a6865268aa7314f65754eb3eb3c4761/402",
                                          "telemetry_gpu_integration_test": "8e7929c30fb60332cebc17037cefc3167ccdfaee34b476bf36610943357cd848/815",
                                          "telemetry_gpu_unittests": "6fd7a38b78ffeb4979feb630142a539b4f0e29ffefc5e5f223981c24557feddc/736",
                                          "webgpu_blink_web_tests": "b186fcb6152264c812c63a96df7af81437099801e31266a8e64c9dc4700bc72a/486"
                                        },
                                    "swarming_command_lines_cwd": "out\\ea6c-Dawn_Win10_x64_",
                                    "swarming_command_lines_digest": "0f81065f11c77ea732bf7c88af88ac8791293bb7e35db22054436a6ce53daa18/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7827",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "8504baee8b7f193c5a919ba3859089cbadd2a63b",
                              "tags": [
                                    "parent_buildername:Dawn Win10 x64 DEPS Builder",
                                    "parent_buildnumber:2282",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "0dc89762-7bc4-4a5f-bb4c-3540a26174d6",
                        "title": "Dawn Win10 x64 DEPS Builder/2282",
                        "url": "https://cr-buildbucket.appspot.com/build/8678458509417406049"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "8504baee8b7f193c5a919ba3859089cbadd2a63b"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8678458509417406049"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8678458509417406049"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8678458509417406049"
                }
          ],
      "parent_build_id": "8678458509417406049",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Win10 x64 DEPS Builder",
      "parent_got_angle_revision": "355cc61af2aadd8f0494800325b2bf9908138108",
      "parent_got_dawn_revision": "5f4c5ef509c5ffa65822302341cf9b2ccad471f9",
      "parent_got_revision": "8504baee8b7f193c5a919ba3859089cbadd2a63b",
      "parent_got_revision_cp": "refs/branch-heads/7827@{#3533}",
      "parent_got_swiftshader_revision": "f9d5d49a3c599a315e3493dc1e9b5309cffb3305",
      "parent_got_v8_revision": "933ce636c562cd54d68e7f7c93ab5cdffd685fca",
      "parent_got_v8_revision_cp": "refs/branch-heads/14.9@{#69}",
      "parent_got_webrtc_revision": "28311abc149a9bb7e6761a3d54f362a8d77e6793",
      "parent_got_webrtc_revision_cp": "refs/branch-heads/7827@{#7}",
      "swarm_hashes": {
            "gl_tests": "c9f48870695b3d4ca3c9ae5e9b92baca2f9e211ce994f071bf5277ea40e093c9/323",
            "gl_unittests": "a90fb2aa1d7a5efb25ee4567ac0024f19a6865268aa7314f65754eb3eb3c4761/402",
            "telemetry_gpu_integration_test": "8e7929c30fb60332cebc17037cefc3167ccdfaee34b476bf36610943357cd848/815",
            "telemetry_gpu_unittests": "6fd7a38b78ffeb4979feb630142a539b4f0e29ffefc5e5f223981c24557feddc/736",
            "webgpu_blink_web_tests": "b186fcb6152264c812c63a96df7af81437099801e31266a8e64c9dc4700bc72a/486"
          },
      "swarming_command_lines_cwd": "out\\ea6c-Dawn_Win10_x64_",
      "swarming_command_lines_digest": "0f81065f11c77ea732bf7c88af88ac8791293bb7e35db22054436a6ce53daa18/93",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "8504baee8b7f193c5a919ba3859089cbadd2a63b",
    "ref": "refs/branch-heads/7827"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8843887636268484704"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m149/Dawn Win10 x64 DEPS Release (Intel)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Win10 x64 DEPS Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "2282"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[17:02:07.937] Scheduled build:
{
  "id": "8678457574388589393",
  "builder": {
    "project": "chromium-m149",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 DEPS Release (Intel)"
  },
  "number": 774,
  "createdBy": "project:chromium-m149",
  "createTime": "2026-06-20T17:02:07.492705502Z",
  "updateTime": "2026-06-20T17:02:07.492705502Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "8504baee8b7f193c5a919ba3859089cbadd2a63b",
      "ref": "refs/branch-heads/7827"
    }
  }
}
[17:02:07.937] Task URL: https://cr-buildbucket.appspot.com/build/8678457574388589393
[17:02:07.937] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:2:0) after 8m8s
[17:02:09.351] Received PubSub notification, asking Buildbucket for the build status
[17:02:09.370] Build status: SCHEDULED
[17:02:27.778] Received PubSub notification, asking Buildbucket for the build status
[17:02:27.856] Build status: STARTED
[17:10:16.056] Handling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:2:0)
[17:10:16.090] Build status: STARTED
[17:10:16.090] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:5:0) after 2m36s
[17:12:52.107] Handling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:5:0)
[17:12:52.127] Build status: STARTED
[17:12:52.127] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:6:0) after 6m58s
[17:19:50.243] Handling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:6:0)
[17:19:50.350] Build status: STARTED
[17:19:50.350] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:7:0) after 4m21s
[17:24:11.540] Handling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:7:0)
[17:24:11.561] Build status: STARTED
[17:24:11.561] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:8:0) after 4m10s
[17:28:21.580] Handling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:8:0)
[17:28:21.599] Build status: STARTED
[17:28:21.599] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:9:0) after 4m44s
[17:33:05.583] Handling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:9:0)
[17:33:05.607] Build status: STARTED
[17:33:05.608] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:10:0) after 7m31s
[17:40:36.728] Handling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:10:0)
[17:40:36.746] Build status: STARTED
[17:40:36.746] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:11:0) after 5m34s
[17:46:10.822] Handling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:11:0)
[17:46:10.838] Build status: STARTED
[17:46:10.839] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:12:0) after 4m31s
[17:50:41.858] Handling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:12:0)
[17:50:41.878] Build status: STARTED
[17:50:41.878] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:13:0) after 6m6s
[17:56:47.896] Handling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:13:0)
[17:56:47.917] Build status: STARTED
[17:56:47.917] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:14:0) after 5m27s
[18:02:14.936] Handling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:14:0)
[18:02:15.102] Build status: STARTED
[18:02:15.102] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:15:0) after 5m30s
[18:07:45.172] Handling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:15:0)
[18:07:45.198] Build status: STARTED
[18:07:45.198] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:16:0) after 5m57s
[18:13:42.214] Handling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:16:0)
[18:13:42.232] Build status: STARTED
[18:13:42.232] Scheduling timer "check-buildbucket-build-status" (chromium-m149/Dawn Win10 x64 DEPS Release (Intel):8843887636268484704:17:0) after 7m26s
[18:17:28.907] Received PubSub notification, asking Buildbucket for the build status
[18:17:28.929] Build:
{
  "id": "8678457574388589393",
  "builder": {
    "project": "chromium-m149",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 DEPS Release (Intel)"
  },
  "number": 774,
  "createdBy": "project:chromium-m149",
  "createTime": "2026-06-20T17:02:07.492705502Z",
  "startTime": "2026-06-20T17:02:26.152324318Z",
  "endTime": "2026-06-20T18:17:27.737337537Z",
  "updateTime": "2026-06-20T18:17:27.737337537Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "8504baee8b7f193c5a919ba3859089cbadd2a63b",
      "ref": "refs/branch-heads/7827"
    }
  }
}
[18:17:28.929] Invocation finished in 1h15m22.613667738s with status FAILED