Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 1 hour
Actions:

Definition (rev 8c6a23373824b48b93461eca2a40ea83a3cf31fb)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Linux x64 DEPS Release (NVIDIA)"
>

Incoming triggers

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "a511f8fa92891ff56c616b163e50bd66095c5efd",
        "project": "chromium/src",
        "ref": "refs/branch-heads/7258"
      }
    ]
  },
  "branch": "refs/branch-heads/7258",
  "deps_revision_overrides": {
    "src": "a511f8fa92891ff56c616b163e50bd66095c5efd"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8709667155112213137"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8709667155112213137"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8709667155112213137"
    }
  ],
  "parent_build_id": "8709667155112213137",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Linux x64 DEPS Builder",
  "parent_got_angle_revision": "7ab02e1d49a649adaba62b8a7fdfabf8144b313f",
  "parent_got_dawn_revision": "15442662beda24122f371b96b37532329e50edc6",
  "parent_got_revision": "a511f8fa92891ff56c616b163e50bd66095c5efd",
  "parent_got_revision_cp": "refs/branch-heads/7258@{#1113}",
  "parent_got_swiftshader_revision": "ed01d9931de34d3a6fb4d615050db5080d9cea16",
  "parent_got_v8_revision": "d5e808d6d1f4f7b03c2a6d46be4b04ea0d10302c",
  "parent_got_v8_revision_cp": "refs/branch-heads/13.9@{#19}",
  "parent_got_webrtc_revision": "1ab982fa34894fa2f5d2e3b7b3f31244efdffaf9",
  "parent_got_webrtc_revision_cp": "refs/branch-heads/7258@{#1}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "a511f8fa92891ff56c616b163e50bd66095c5efd",
  "swarm_hashes": {
    "dawn_end2end_tests": "d36828db03c190e13d3e8ea99365391655b62e45fc480fdf29dafc5596092fd6/323",
    "dawn_perf_tests": "66ec7ccb4ef037b5b5aea1d1c6e7a52f816aa204361cab2ce712154b347681de/487",
    "gl_tests": "13c5961624f1c96ac6d9fe0c1e6d910cc1f46af5b6cff8b1c5ef0eec9a4bdb6b/323",
    "gl_unittests": "f827cf2b605192adfc2cd7384065957cc5bd77a48a81660a757c64dad0de6d38/323",
    "telemetry_gpu_integration_test": "3200965022afd0b2dca8b6bf224ea7f27a179f957aa2ad5e4e113724f137c5d9/815",
    "telemetry_gpu_unittests": "2f94115d641069e53f930c2086bac4bda538df975935c5adaa68f1f2f1aa6118/736",
    "webgpu_blink_web_tests": "7d9f6bc7b3ecf42f5dc52f0db28a264965aa043287294e8db0a7d45cac383d2b/566"
  },
  "swarming_command_lines_cwd": "out/7a4c-Dawn_Linux_x64_",
  "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/a511f8fa92891ff56c616b163e50bd66095c5efd
  • gitiles_ref:refs/branch-heads/7258
  • parent_buildername:Dawn Linux x64 DEPS Builder
  • parent_buildnumber:857
  • user_agent:recipe

Debug log (UTC)

[06:19:34.788] New invocation is queued and will start shortly
[06:19:34.788] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[06:19:35.874] Starting the invocation (attempt 1)
[06:19:35.898] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/a511f8fa92891ff56c616b163e50bd66095c5efd"
[06:19:35.898] Popped gitiles commit info from properties and tags
[06:19:35.899] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[06:19:35.899] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[06:19:35.899] Buildbucket request:
{
  "requestId": "8875093416928769120",
  "builder": {
    "project": "chromium-m139",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "a511f8fa92891ff56c616b163e50bd66095c5efd",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/7258"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8875093416928769120",
            "job": "chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA)",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "332616045a0eef8fb43c7b21414c5010abfab968",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7258"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "332616045a0eef8fb43c7b21414c5010abfab968"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8709668342243291249"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8709668342243291249"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8709668342243291249"
                                              }
                                        ],
                                    "parent_build_id": "8709668342243291249",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Linux x64 DEPS Builder",
                                    "parent_got_angle_revision": "7ab02e1d49a649adaba62b8a7fdfabf8144b313f",
                                    "parent_got_dawn_revision": "15442662beda24122f371b96b37532329e50edc6",
                                    "parent_got_revision": "332616045a0eef8fb43c7b21414c5010abfab968",
                                    "parent_got_revision_cp": "refs/branch-heads/7258@{#1111}",
                                    "parent_got_swiftshader_revision": "ed01d9931de34d3a6fb4d615050db5080d9cea16",
                                    "parent_got_v8_revision": "d5e808d6d1f4f7b03c2a6d46be4b04ea0d10302c",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.9@{#19}",
                                    "parent_got_webrtc_revision": "1ab982fa34894fa2f5d2e3b7b3f31244efdffaf9",
                                    "parent_got_webrtc_revision_cp": "refs/branch-heads/7258@{#1}",
                                    "revision": "332616045a0eef8fb43c7b21414c5010abfab968",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "d36828db03c190e13d3e8ea99365391655b62e45fc480fdf29dafc5596092fd6/323",
                                          "dawn_perf_tests": "66ec7ccb4ef037b5b5aea1d1c6e7a52f816aa204361cab2ce712154b347681de/487",
                                          "gl_tests": "13c5961624f1c96ac6d9fe0c1e6d910cc1f46af5b6cff8b1c5ef0eec9a4bdb6b/323",
                                          "gl_unittests": "f827cf2b605192adfc2cd7384065957cc5bd77a48a81660a757c64dad0de6d38/323",
                                          "telemetry_gpu_integration_test": "e8739d93646433ac71eb58460b5b77f5a99160de1c8e5a0782ab7d7f1a845558/815",
                                          "telemetry_gpu_unittests": "2f94115d641069e53f930c2086bac4bda538df975935c5adaa68f1f2f1aa6118/736",
                                          "webgpu_blink_web_tests": "33934add3277639311c779df5c0d901777209f18fd0afe8be72a98e9f90aba01/566"
                                        },
                                    "swarming_command_lines_cwd": "out/7a4c-Dawn_Linux_x64_",
                                    "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7258",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "332616045a0eef8fb43c7b21414c5010abfab968",
                              "tags": [
                                    "parent_buildername:Dawn Linux x64 DEPS Builder",
                                    "parent_buildnumber:856",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "87541373-3fee-4c4b-8fa8-fbab8b0541bd",
                        "title": "Dawn Linux x64 DEPS Builder/856",
                        "url": "https://cr-buildbucket.appspot.com/build/8709668342243291249"
                      },
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "a511f8fa92891ff56c616b163e50bd66095c5efd",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7258"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "a511f8fa92891ff56c616b163e50bd66095c5efd"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8709667155112213137"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8709667155112213137"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8709667155112213137"
                                              }
                                        ],
                                    "parent_build_id": "8709667155112213137",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Linux x64 DEPS Builder",
                                    "parent_got_angle_revision": "7ab02e1d49a649adaba62b8a7fdfabf8144b313f",
                                    "parent_got_dawn_revision": "15442662beda24122f371b96b37532329e50edc6",
                                    "parent_got_revision": "a511f8fa92891ff56c616b163e50bd66095c5efd",
                                    "parent_got_revision_cp": "refs/branch-heads/7258@{#1113}",
                                    "parent_got_swiftshader_revision": "ed01d9931de34d3a6fb4d615050db5080d9cea16",
                                    "parent_got_v8_revision": "d5e808d6d1f4f7b03c2a6d46be4b04ea0d10302c",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.9@{#19}",
                                    "parent_got_webrtc_revision": "1ab982fa34894fa2f5d2e3b7b3f31244efdffaf9",
                                    "parent_got_webrtc_revision_cp": "refs/branch-heads/7258@{#1}",
                                    "revision": "a511f8fa92891ff56c616b163e50bd66095c5efd",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "d36828db03c190e13d3e8ea99365391655b62e45fc480fdf29dafc5596092fd6/323",
                                          "dawn_perf_tests": "66ec7ccb4ef037b5b5aea1d1c6e7a52f816aa204361cab2ce712154b347681de/487",
                                          "gl_tests": "13c5961624f1c96ac6d9fe0c1e6d910cc1f46af5b6cff8b1c5ef0eec9a4bdb6b/323",
                                          "gl_unittests": "f827cf2b605192adfc2cd7384065957cc5bd77a48a81660a757c64dad0de6d38/323",
                                          "telemetry_gpu_integration_test": "3200965022afd0b2dca8b6bf224ea7f27a179f957aa2ad5e4e113724f137c5d9/815",
                                          "telemetry_gpu_unittests": "2f94115d641069e53f930c2086bac4bda538df975935c5adaa68f1f2f1aa6118/736",
                                          "webgpu_blink_web_tests": "7d9f6bc7b3ecf42f5dc52f0db28a264965aa043287294e8db0a7d45cac383d2b/566"
                                        },
                                    "swarming_command_lines_cwd": "out/7a4c-Dawn_Linux_x64_",
                                    "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7258",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "a511f8fa92891ff56c616b163e50bd66095c5efd",
                              "tags": [
                                    "parent_buildername:Dawn Linux x64 DEPS Builder",
                                    "parent_buildnumber:857",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "0c09157c-69e8-4b34-b7b9-0b2102f3009f",
                        "title": "Dawn Linux x64 DEPS Builder/857",
                        "url": "https://cr-buildbucket.appspot.com/build/8709667155112213137"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "a511f8fa92891ff56c616b163e50bd66095c5efd"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8709667155112213137"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8709667155112213137"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8709667155112213137"
                }
          ],
      "parent_build_id": "8709667155112213137",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Linux x64 DEPS Builder",
      "parent_got_angle_revision": "7ab02e1d49a649adaba62b8a7fdfabf8144b313f",
      "parent_got_dawn_revision": "15442662beda24122f371b96b37532329e50edc6",
      "parent_got_revision": "a511f8fa92891ff56c616b163e50bd66095c5efd",
      "parent_got_revision_cp": "refs/branch-heads/7258@{#1113}",
      "parent_got_swiftshader_revision": "ed01d9931de34d3a6fb4d615050db5080d9cea16",
      "parent_got_v8_revision": "d5e808d6d1f4f7b03c2a6d46be4b04ea0d10302c",
      "parent_got_v8_revision_cp": "refs/branch-heads/13.9@{#19}",
      "parent_got_webrtc_revision": "1ab982fa34894fa2f5d2e3b7b3f31244efdffaf9",
      "parent_got_webrtc_revision_cp": "refs/branch-heads/7258@{#1}",
      "swarm_hashes": {
            "dawn_end2end_tests": "d36828db03c190e13d3e8ea99365391655b62e45fc480fdf29dafc5596092fd6/323",
            "dawn_perf_tests": "66ec7ccb4ef037b5b5aea1d1c6e7a52f816aa204361cab2ce712154b347681de/487",
            "gl_tests": "13c5961624f1c96ac6d9fe0c1e6d910cc1f46af5b6cff8b1c5ef0eec9a4bdb6b/323",
            "gl_unittests": "f827cf2b605192adfc2cd7384065957cc5bd77a48a81660a757c64dad0de6d38/323",
            "telemetry_gpu_integration_test": "3200965022afd0b2dca8b6bf224ea7f27a179f957aa2ad5e4e113724f137c5d9/815",
            "telemetry_gpu_unittests": "2f94115d641069e53f930c2086bac4bda538df975935c5adaa68f1f2f1aa6118/736",
            "webgpu_blink_web_tests": "7d9f6bc7b3ecf42f5dc52f0db28a264965aa043287294e8db0a7d45cac383d2b/566"
          },
      "swarming_command_lines_cwd": "out/7a4c-Dawn_Linux_x64_",
      "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "a511f8fa92891ff56c616b163e50bd66095c5efd",
    "ref": "refs/branch-heads/7258"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8875093416928769120"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Linux x64 DEPS Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "857"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[06:19:36.385] Scheduled build:
{
  "id": "8709663355050202433",
  "builder": {
    "project": "chromium-m139",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "number": 359,
  "createdBy": "project:chromium-m139",
  "createTime": "2025-07-11T06:19:35.951895426Z",
  "updateTime": "2025-07-11T06:19:35.951895426Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "a511f8fa92891ff56c616b163e50bd66095c5efd",
      "ref": "refs/branch-heads/7258"
    }
  }
}
[06:19:36.385] Task URL: https://cr-buildbucket.appspot.com/build/8709663355050202433
[06:19:36.385] Scheduling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:2:0) after 1m16s
[06:20:15.676] Received PubSub notification, asking Buildbucket for the build status
[06:20:15.700] Build status: STARTED
[06:20:17.687] Received PubSub notification, asking Buildbucket for the build status
[06:20:17.711] Build status: STARTED
[06:20:52.373] Handling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:2:0)
[06:20:52.395] Build status: STARTED
[06:20:52.395] Scheduling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:5:0) after 7m32s
[06:28:24.413] Handling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:5:0)
[06:28:24.439] Build status: STARTED
[06:28:24.439] Scheduling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:6:0) after 7m47s
[06:36:11.512] Handling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:6:0)
[06:36:11.539] Build status: STARTED
[06:36:11.539] Scheduling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:7:0) after 1m53s
[06:38:04.605] Handling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:7:0)
[06:38:04.624] Build status: STARTED
[06:38:04.624] Scheduling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:8:0) after 8m11s
[06:46:15.702] Handling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:8:0)
[06:46:15.723] Build status: STARTED
[06:46:15.723] Scheduling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:9:0) after 2m57s
[06:49:12.740] Handling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:9:0)
[06:49:12.767] Build status: STARTED
[06:49:12.767] Scheduling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:10:0) after 8m17s
[06:57:29.886] Handling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:10:0)
[06:57:29.913] Build status: STARTED
[06:57:29.913] Scheduling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:11:0) after 9m14s
[07:06:44.001] Handling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:11:0)
[07:06:44.028] Build status: STARTED
[07:06:44.028] Scheduling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:12:0) after 7m36s
[07:14:20.052] Handling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:12:0)
[07:14:20.070] Build status: STARTED
[07:14:20.070] Scheduling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:13:0) after 5m26s
[07:19:46.093] Handling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:13:0)
[07:19:46.120] Build status: STARTED
[07:19:46.120] Scheduling timer "check-buildbucket-build-status" (chromium-m139/Dawn Linux x64 DEPS Release (NVIDIA):8875093416928769120:14:0) after 9m55s
[07:26:55.477] Received PubSub notification, asking Buildbucket for the build status
[07:26:55.499] Build:
{
  "id": "8709663355050202433",
  "builder": {
    "project": "chromium-m139",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "number": 359,
  "createdBy": "project:chromium-m139",
  "createTime": "2025-07-11T06:19:35.951895426Z",
  "startTime": "2025-07-11T06:19:45.077073835Z",
  "endTime": "2025-07-11T07:26:54.991535833Z",
  "updateTime": "2025-07-11T07:26:54.991535833Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "a511f8fa92891ff56c616b163e50bd66095c5efd",
      "ref": "refs/branch-heads/7258"
    }
  }
}
[07:26:55.499] Invocation finished in 1h7m20.72271442s with status FAILED