Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 1 hour
Actions:

Definition (rev 1ea5d64318a68cfeb1208ef2f27d1e815a3a23f6)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Win10 x64 DEPS Release (NVIDIA)"
>

Incoming triggers

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "42c5e37d9435bde1f46dfa22756908cb98c20c16",
        "project": "chromium/src",
        "ref": "refs/branch-heads/8078"
      }
    ]
  },
  "branch": "refs/branch-heads/8078",
  "deps_revision_overrides": {
    "src": "42c5e37d9435bde1f46dfa22756908cb98c20c16"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8669253274837953633"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8669253274837953633"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8669253274837953633"
    }
  ],
  "parent_build_id": "8669253274837953633",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Win10 x64 DEPS Builder",
  "parent_got_angle_revision": "dfeede58b6183e8964ae4b6d9612fe1d579403e3",
  "parent_got_dawn_revision": "0a2c7df818e285d6db0f085135139cda04cee8bb",
  "parent_got_revision": "42c5e37d9435bde1f46dfa22756908cb98c20c16",
  "parent_got_revision_cp": "refs/branch-heads/8078@{#29}",
  "parent_got_swiftshader_revision": "1e80438d2b93ef36a7c05f8d2b81233bac0e3d16",
  "parent_got_v8_revision": "b976f35da8356fcf4e5e176d1d810ba6a6f05718",
  "parent_got_v8_revision_cp": "refs/heads/15.6.75@{#1}",
  "parent_got_webrtc_revision": "3225248425fe2b3cafe389465b94ebc24968d9ba",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#48737}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "42c5e37d9435bde1f46dfa22756908cb98c20c16",
  "swarm_hashes": {
    "gl_tests": "27557a98e2e1fbe35cf9815a4dcaffa8c8b2c2c61b90f36757d0992a54d067e2/421",
    "gl_unittests": "c169851d1c9685057eafdb3c71aea84f0801f09b9b8caebba4938cecc7c6625a/500",
    "telemetry_gpu_integration_test": "9042273469bf5867bb7076a2545e372d56c56e1df7011ff8ec747b1b45df5ef0/913",
    "telemetry_gpu_unittests": "c5a972b45a5c5cdc646d4b4be7dd63e78be8deb506606c53a056feab9be5e6e3/834",
    "webgpu_blink_web_tests": "1691814a3ecb3c085c819450aa3b5051d39418f7a8ed007756da0fd36f2a1823/584"
  },
  "swarming_command_lines_cwd": "out\\ec9a-Dawn_Win10_x64_",
  "swarming_command_lines_digest": "0f81065f11c77ea732bf7c88af88ac8791293bb7e35db22054436a6ce53daa18/93"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/42c5e37d9435bde1f46dfa22756908cb98c20c16
  • gitiles_ref:refs/branch-heads/8078
  • parent_buildername:Dawn Win10 x64 DEPS Builder
  • parent_buildnumber:16
  • user_agent:recipe

Debug log (UTC)

[07:38:04.277] New invocation is queued and will start shortly
[07:38:04.277] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[07:38:05.371] Starting the invocation (attempt 1)
[07:38:05.469] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/42c5e37d9435bde1f46dfa22756908cb98c20c16"
[07:38:05.469] Popped gitiles commit info from properties and tags
[07:38:05.469] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[07:38:05.469] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[07:38:05.470] Buildbucket request:
{
  "requestId": "8834682231649784016",
  "builder": {
    "project": "chromium-m156",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 DEPS Release (NVIDIA)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "42c5e37d9435bde1f46dfa22756908cb98c20c16",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/8078"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8834682231649784016",
            "job": "chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA)",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "fb768ae143ce1c1e8de39b9581f3ab857461c21f",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/8078"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "fb768ae143ce1c1e8de39b9581f3ab857461c21f"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8669258042696460161"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8669258042696460161"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8669258042696460161"
                                              }
                                        ],
                                    "parent_build_id": "8669258042696460161",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Win10 x64 DEPS Builder",
                                    "parent_got_angle_revision": "dfeede58b6183e8964ae4b6d9612fe1d579403e3",
                                    "parent_got_dawn_revision": "0a2c7df818e285d6db0f085135139cda04cee8bb",
                                    "parent_got_revision": "fb768ae143ce1c1e8de39b9581f3ab857461c21f",
                                    "parent_got_revision_cp": "refs/branch-heads/8078@{#27}",
                                    "parent_got_swiftshader_revision": "1e80438d2b93ef36a7c05f8d2b81233bac0e3d16",
                                    "parent_got_v8_revision": "b976f35da8356fcf4e5e176d1d810ba6a6f05718",
                                    "parent_got_v8_revision_cp": "refs/heads/15.6.75@{#1}",
                                    "parent_got_webrtc_revision": "3225248425fe2b3cafe389465b94ebc24968d9ba",
                                    "parent_got_webrtc_revision_cp": "refs/heads/main@{#48737}",
                                    "revision": "fb768ae143ce1c1e8de39b9581f3ab857461c21f",
                                    "swarm_hashes": {
                                          "gl_tests": "27557a98e2e1fbe35cf9815a4dcaffa8c8b2c2c61b90f36757d0992a54d067e2/421",
                                          "gl_unittests": "c169851d1c9685057eafdb3c71aea84f0801f09b9b8caebba4938cecc7c6625a/500",
                                          "telemetry_gpu_integration_test": "fa389bc76ead05ef50705f675380f8c21b4d4309fd50e326fe1e0d32e9787e7a/913",
                                          "telemetry_gpu_unittests": "c5a972b45a5c5cdc646d4b4be7dd63e78be8deb506606c53a056feab9be5e6e3/834",
                                          "webgpu_blink_web_tests": "2a32cc83a64d262f1edd3ad8b9ebd440eb2f538507f774ace39c4dd575656d14/584"
                                        },
                                    "swarming_command_lines_cwd": "out\\ec9a-Dawn_Win10_x64_",
                                    "swarming_command_lines_digest": "0f81065f11c77ea732bf7c88af88ac8791293bb7e35db22054436a6ce53daa18/93"
                                  },
                              "ref": "refs/branch-heads/8078",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "fb768ae143ce1c1e8de39b9581f3ab857461c21f",
                              "tags": [
                                    "parent_buildername:Dawn Win10 x64 DEPS Builder",
                                    "parent_buildnumber:15",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "386d539f-c65e-4fa8-b473-a4f88147ba10",
                        "title": "Dawn Win10 x64 DEPS Builder/15",
                        "url": "https://cr-buildbucket.appspot.com/build/8669258042696460161"
                      },
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "42c5e37d9435bde1f46dfa22756908cb98c20c16",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/8078"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "42c5e37d9435bde1f46dfa22756908cb98c20c16"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8669253274837953633"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8669253274837953633"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8669253274837953633"
                                              }
                                        ],
                                    "parent_build_id": "8669253274837953633",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Win10 x64 DEPS Builder",
                                    "parent_got_angle_revision": "dfeede58b6183e8964ae4b6d9612fe1d579403e3",
                                    "parent_got_dawn_revision": "0a2c7df818e285d6db0f085135139cda04cee8bb",
                                    "parent_got_revision": "42c5e37d9435bde1f46dfa22756908cb98c20c16",
                                    "parent_got_revision_cp": "refs/branch-heads/8078@{#29}",
                                    "parent_got_swiftshader_revision": "1e80438d2b93ef36a7c05f8d2b81233bac0e3d16",
                                    "parent_got_v8_revision": "b976f35da8356fcf4e5e176d1d810ba6a6f05718",
                                    "parent_got_v8_revision_cp": "refs/heads/15.6.75@{#1}",
                                    "parent_got_webrtc_revision": "3225248425fe2b3cafe389465b94ebc24968d9ba",
                                    "parent_got_webrtc_revision_cp": "refs/heads/main@{#48737}",
                                    "revision": "42c5e37d9435bde1f46dfa22756908cb98c20c16",
                                    "swarm_hashes": {
                                          "gl_tests": "27557a98e2e1fbe35cf9815a4dcaffa8c8b2c2c61b90f36757d0992a54d067e2/421",
                                          "gl_unittests": "c169851d1c9685057eafdb3c71aea84f0801f09b9b8caebba4938cecc7c6625a/500",
                                          "telemetry_gpu_integration_test": "9042273469bf5867bb7076a2545e372d56c56e1df7011ff8ec747b1b45df5ef0/913",
                                          "telemetry_gpu_unittests": "c5a972b45a5c5cdc646d4b4be7dd63e78be8deb506606c53a056feab9be5e6e3/834",
                                          "webgpu_blink_web_tests": "1691814a3ecb3c085c819450aa3b5051d39418f7a8ed007756da0fd36f2a1823/584"
                                        },
                                    "swarming_command_lines_cwd": "out\\ec9a-Dawn_Win10_x64_",
                                    "swarming_command_lines_digest": "0f81065f11c77ea732bf7c88af88ac8791293bb7e35db22054436a6ce53daa18/93"
                                  },
                              "ref": "refs/branch-heads/8078",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "42c5e37d9435bde1f46dfa22756908cb98c20c16",
                              "tags": [
                                    "parent_buildername:Dawn Win10 x64 DEPS Builder",
                                    "parent_buildnumber:16",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "d13d497c-d389-4970-8d77-f26aef84540e",
                        "title": "Dawn Win10 x64 DEPS Builder/16",
                        "url": "https://cr-buildbucket.appspot.com/build/8669253274837953633"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "42c5e37d9435bde1f46dfa22756908cb98c20c16"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8669253274837953633"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8669253274837953633"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8669253274837953633"
                }
          ],
      "parent_build_id": "8669253274837953633",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Win10 x64 DEPS Builder",
      "parent_got_angle_revision": "dfeede58b6183e8964ae4b6d9612fe1d579403e3",
      "parent_got_dawn_revision": "0a2c7df818e285d6db0f085135139cda04cee8bb",
      "parent_got_revision": "42c5e37d9435bde1f46dfa22756908cb98c20c16",
      "parent_got_revision_cp": "refs/branch-heads/8078@{#29}",
      "parent_got_swiftshader_revision": "1e80438d2b93ef36a7c05f8d2b81233bac0e3d16",
      "parent_got_v8_revision": "b976f35da8356fcf4e5e176d1d810ba6a6f05718",
      "parent_got_v8_revision_cp": "refs/heads/15.6.75@{#1}",
      "parent_got_webrtc_revision": "3225248425fe2b3cafe389465b94ebc24968d9ba",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#48737}",
      "swarm_hashes": {
            "gl_tests": "27557a98e2e1fbe35cf9815a4dcaffa8c8b2c2c61b90f36757d0992a54d067e2/421",
            "gl_unittests": "c169851d1c9685057eafdb3c71aea84f0801f09b9b8caebba4938cecc7c6625a/500",
            "telemetry_gpu_integration_test": "9042273469bf5867bb7076a2545e372d56c56e1df7011ff8ec747b1b45df5ef0/913",
            "telemetry_gpu_unittests": "c5a972b45a5c5cdc646d4b4be7dd63e78be8deb506606c53a056feab9be5e6e3/834",
            "webgpu_blink_web_tests": "1691814a3ecb3c085c819450aa3b5051d39418f7a8ed007756da0fd36f2a1823/584"
          },
      "swarming_command_lines_cwd": "out\\ec9a-Dawn_Win10_x64_",
      "swarming_command_lines_digest": "0f81065f11c77ea732bf7c88af88ac8791293bb7e35db22054436a6ce53daa18/93"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "42c5e37d9435bde1f46dfa22756908cb98c20c16",
    "ref": "refs/branch-heads/8078"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8834682231649784016"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Win10 x64 DEPS Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "16"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[07:38:06.051] Scheduled build:
{
  "id": "8669252169602845297",
  "builder": {
    "project": "chromium-m156",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 DEPS Release (NVIDIA)"
  },
  "number": 8,
  "createdBy": "project:chromium-m156",
  "createTime": "2026-09-30T07:38:05.608641222Z",
  "updateTime": "2026-09-30T07:38:05.608641222Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "42c5e37d9435bde1f46dfa22756908cb98c20c16",
      "ref": "refs/branch-heads/8078"
    }
  }
}
[07:38:06.051] Task URL: https://cr-buildbucket.appspot.com/build/8669252169602845297
[07:38:06.051] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:2:0) after 9m53s
[07:38:07.052] Received PubSub notification, asking Buildbucket for the build status
[07:38:07.070] Build status: SCHEDULED
[07:38:19.242] Received PubSub notification, asking Buildbucket for the build status
[07:38:19.345] Build status: STARTED
[07:47:59.066] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:2:0)
[07:47:59.095] Build status: STARTED
[07:47:59.095] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:5:0) after 2m36s
[07:50:35.180] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:5:0)
[07:50:35.205] Build status: STARTED
[07:50:35.205] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:6:0) after 4m38s
[07:55:13.324] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:6:0)
[07:55:13.346] Build status: STARTED
[07:55:13.346] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:7:0) after 4m16s
[07:59:29.315] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:7:0)
[07:59:29.346] Build status: STARTED
[07:59:29.346] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:8:0) after 2m13s
[08:01:42.368] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:8:0)
[08:01:42.407] Build status: STARTED
[08:01:42.407] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:9:0) after 4m56s
[08:06:38.442] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:9:0)
[08:06:38.463] Build status: STARTED
[08:06:38.463] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:10:0) after 8m50s
[08:15:28.542] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:10:0)
[08:15:28.567] Build status: STARTED
[08:15:28.567] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:11:0) after 8m20s
[08:23:48.620] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:11:0)
[08:23:48.688] Build status: STARTED
[08:23:48.688] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:12:0) after 3m12s
[08:27:00.810] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:12:0)
[08:27:00.856] Build status: STARTED
[08:27:00.856] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:13:0) after 3m48s
[08:30:48.873] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:13:0)
[08:30:48.890] Build status: STARTED
[08:30:48.890] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:14:0) after 9m10s
[08:39:58.962] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:14:0)
[08:39:58.983] Build status: STARTED
[08:39:58.983] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:15:0) after 5m20s
[08:45:19.098] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:15:0)
[08:45:19.121] Build status: STARTED
[08:45:19.121] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:16:0) after 9m1s
[08:54:20.222] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:16:0)
[08:54:20.244] Build status: STARTED
[08:54:20.244] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:17:0) after 5m41s
[09:00:01.333] Handling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:17:0)
[09:00:01.358] Build status: STARTED
[09:00:01.358] Scheduling timer "check-buildbucket-build-status" (chromium-m156/Dawn Win10 x64 DEPS Release (NVIDIA):8834682231649784016:18:0) after 8m8s
[09:05:03.499] Received PubSub notification, asking Buildbucket for the build status
[09:05:03.521] Build:
{
  "id": "8669252169602845297",
  "builder": {
    "project": "chromium-m156",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 DEPS Release (NVIDIA)"
  },
  "number": 8,
  "createdBy": "project:chromium-m156",
  "createTime": "2026-09-30T07:38:05.608641222Z",
  "startTime": "2026-09-30T07:38:19.035338412Z",
  "endTime": "2026-09-30T09:05:02.882563696Z",
  "updateTime": "2026-09-30T09:05:02.882563696Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "42c5e37d9435bde1f46dfa22756908cb98c20c16",
      "ref": "refs/branch-heads/8078"
    }
  }
}
[09:05:03.521] Invocation finished in 1h26m59.249206828s with status FAILED