Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 1 hour
Actions:

Definition (rev e06c85a67c04bcbdc0ee7d29b76877746b7b8777)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Linux x64 DEPS Release (NVIDIA)"
>

Incoming triggers

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "76b9cd04107da9c6c26666ab21b7cae4fc34f409",
        "project": "chromium/src",
        "ref": "refs/branch-heads/7444"
      }
    ]
  },
  "branch": "refs/branch-heads/7444",
  "deps_revision_overrides": {
    "src": "76b9cd04107da9c6c26666ab21b7cae4fc34f409"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8701875213045026145"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8701875213045026145"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8701875213045026145"
    }
  ],
  "parent_build_id": "8701875213045026145",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Linux x64 DEPS Builder",
  "parent_got_angle_revision": "e5b338502c7c65bd38d4695bc557938ca4cc3aa7",
  "parent_got_dawn_revision": "cee9cb0d67e749bf42f5e90cb3b8a6f525dbb920",
  "parent_got_revision": "76b9cd04107da9c6c26666ab21b7cae4fc34f409",
  "parent_got_revision_cp": "refs/branch-heads/7444@{#327}",
  "parent_got_swiftshader_revision": "5f1c459a11bb4899301e89609cbf7547f9f31e20",
  "parent_got_v8_revision": "3a777f7fba0b3123738216fb46ab1a3a5cf13927",
  "parent_got_v8_revision_cp": "refs/branch-heads/14.2@{#3}",
  "parent_got_webrtc_revision": "f70dc714a073397356f6ed866481da73f90f0b96",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#45766}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "76b9cd04107da9c6c26666ab21b7cae4fc34f409",
  "swarm_hashes": {
    "dawn_end2end_tests": "b0c5dd6dfc3bfa081e6782c4bba0f7fe86f635e09c730c3dbd55944a8a6b3f40/323",
    "dawn_perf_tests": "d91b7568d2968ea66ef4ca819782a3124f5e4861830397deec2ca91f572e3cd7/487",
    "gl_tests": "4a0ac708505b0ec2006f44ffd9d37f843bbbe81707dca305911300e7a4781b02/323",
    "gl_unittests": "fd1a367fe598480336f0fab4ff68a920e4c5b18847cca87c5f1eb69424f7bb67/323",
    "telemetry_gpu_integration_test": "43447904986ec745b01e5781347b67676ae1860f24a14d314f091fdb61c2c99c/815",
    "telemetry_gpu_unittests": "b906c005d240397a20105694205a24d14c74e713c7ad3f11553bcb576bb54422/736",
    "webgpu_blink_web_tests": "345d1c407f9e20e51511985de1f8b6c322c9b544d7e144c79046e94384faa5c4/566"
  },
  "swarming_command_lines_cwd": "out/ee60-Dawn_Linux_x64_",
  "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/76b9cd04107da9c6c26666ab21b7cae4fc34f409
  • gitiles_ref:refs/branch-heads/7444
  • parent_buildername:Dawn Linux x64 DEPS Builder
  • parent_buildnumber:265
  • user_agent:recipe

Debug log (UTC)

[05:54:37.199] New invocation is queued and will start shortly
[05:54:37.199] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[05:54:38.455] Starting the invocation (attempt 1)
[05:54:38.489] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/76b9cd04107da9c6c26666ab21b7cae4fc34f409"
[05:54:38.489] Popped gitiles commit info from properties and tags
[05:54:38.489] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[05:54:38.489] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[05:54:38.489] Buildbucket request:
{
  "requestId": "8867303648181134112",
  "builder": {
    "project": "chromium-m142",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "76b9cd04107da9c6c26666ab21b7cae4fc34f409",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/7444"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8867303648181134112",
            "job": "chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA)",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "fd89887eda8aecd5f496756584d9bf13ffe08518",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7444"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "fd89887eda8aecd5f496756584d9bf13ffe08518"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8701877190698965537"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8701877190698965537"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8701877190698965537"
                                              }
                                        ],
                                    "parent_build_id": "8701877190698965537",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Linux x64 DEPS Builder",
                                    "parent_got_angle_revision": "e5b338502c7c65bd38d4695bc557938ca4cc3aa7",
                                    "parent_got_dawn_revision": "cee9cb0d67e749bf42f5e90cb3b8a6f525dbb920",
                                    "parent_got_revision": "fd89887eda8aecd5f496756584d9bf13ffe08518",
                                    "parent_got_revision_cp": "refs/branch-heads/7444@{#326}",
                                    "parent_got_swiftshader_revision": "5f1c459a11bb4899301e89609cbf7547f9f31e20",
                                    "parent_got_v8_revision": "3a777f7fba0b3123738216fb46ab1a3a5cf13927",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/14.2@{#3}",
                                    "parent_got_webrtc_revision": "f70dc714a073397356f6ed866481da73f90f0b96",
                                    "parent_got_webrtc_revision_cp": "refs/heads/main@{#45766}",
                                    "revision": "fd89887eda8aecd5f496756584d9bf13ffe08518",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "b0c5dd6dfc3bfa081e6782c4bba0f7fe86f635e09c730c3dbd55944a8a6b3f40/323",
                                          "dawn_perf_tests": "d91b7568d2968ea66ef4ca819782a3124f5e4861830397deec2ca91f572e3cd7/487",
                                          "gl_tests": "4a0ac708505b0ec2006f44ffd9d37f843bbbe81707dca305911300e7a4781b02/323",
                                          "gl_unittests": "fd1a367fe598480336f0fab4ff68a920e4c5b18847cca87c5f1eb69424f7bb67/323",
                                          "telemetry_gpu_integration_test": "0c62b31e84452cf1654f3d9522bf734e0269ba5408b8cd07fc7e3247dfd6640e/815",
                                          "telemetry_gpu_unittests": "b906c005d240397a20105694205a24d14c74e713c7ad3f11553bcb576bb54422/736",
                                          "webgpu_blink_web_tests": "c1acfba343cfd1658df5fa1d478457d57746a6eddc67b7899c4135536afe83e0/566"
                                        },
                                    "swarming_command_lines_cwd": "out/ee60-Dawn_Linux_x64_",
                                    "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7444",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "fd89887eda8aecd5f496756584d9bf13ffe08518",
                              "tags": [
                                    "parent_buildername:Dawn Linux x64 DEPS Builder",
                                    "parent_buildnumber:264",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "ea1f956d-76cb-4c94-a96b-29844becf1f9",
                        "title": "Dawn Linux x64 DEPS Builder/264",
                        "url": "https://cr-buildbucket.appspot.com/build/8701877190698965537"
                      },
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "76b9cd04107da9c6c26666ab21b7cae4fc34f409",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7444"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "76b9cd04107da9c6c26666ab21b7cae4fc34f409"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8701875213045026145"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8701875213045026145"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8701875213045026145"
                                              }
                                        ],
                                    "parent_build_id": "8701875213045026145",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Linux x64 DEPS Builder",
                                    "parent_got_angle_revision": "e5b338502c7c65bd38d4695bc557938ca4cc3aa7",
                                    "parent_got_dawn_revision": "cee9cb0d67e749bf42f5e90cb3b8a6f525dbb920",
                                    "parent_got_revision": "76b9cd04107da9c6c26666ab21b7cae4fc34f409",
                                    "parent_got_revision_cp": "refs/branch-heads/7444@{#327}",
                                    "parent_got_swiftshader_revision": "5f1c459a11bb4899301e89609cbf7547f9f31e20",
                                    "parent_got_v8_revision": "3a777f7fba0b3123738216fb46ab1a3a5cf13927",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/14.2@{#3}",
                                    "parent_got_webrtc_revision": "f70dc714a073397356f6ed866481da73f90f0b96",
                                    "parent_got_webrtc_revision_cp": "refs/heads/main@{#45766}",
                                    "revision": "76b9cd04107da9c6c26666ab21b7cae4fc34f409",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "b0c5dd6dfc3bfa081e6782c4bba0f7fe86f635e09c730c3dbd55944a8a6b3f40/323",
                                          "dawn_perf_tests": "d91b7568d2968ea66ef4ca819782a3124f5e4861830397deec2ca91f572e3cd7/487",
                                          "gl_tests": "4a0ac708505b0ec2006f44ffd9d37f843bbbe81707dca305911300e7a4781b02/323",
                                          "gl_unittests": "fd1a367fe598480336f0fab4ff68a920e4c5b18847cca87c5f1eb69424f7bb67/323",
                                          "telemetry_gpu_integration_test": "43447904986ec745b01e5781347b67676ae1860f24a14d314f091fdb61c2c99c/815",
                                          "telemetry_gpu_unittests": "b906c005d240397a20105694205a24d14c74e713c7ad3f11553bcb576bb54422/736",
                                          "webgpu_blink_web_tests": "345d1c407f9e20e51511985de1f8b6c322c9b544d7e144c79046e94384faa5c4/566"
                                        },
                                    "swarming_command_lines_cwd": "out/ee60-Dawn_Linux_x64_",
                                    "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7444",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "76b9cd04107da9c6c26666ab21b7cae4fc34f409",
                              "tags": [
                                    "parent_buildername:Dawn Linux x64 DEPS Builder",
                                    "parent_buildnumber:265",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "fdd79b5b-c663-4f30-acc1-6f4e0c437cc0",
                        "title": "Dawn Linux x64 DEPS Builder/265",
                        "url": "https://cr-buildbucket.appspot.com/build/8701875213045026145"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "76b9cd04107da9c6c26666ab21b7cae4fc34f409"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8701875213045026145"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8701875213045026145"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8701875213045026145"
                }
          ],
      "parent_build_id": "8701875213045026145",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Linux x64 DEPS Builder",
      "parent_got_angle_revision": "e5b338502c7c65bd38d4695bc557938ca4cc3aa7",
      "parent_got_dawn_revision": "cee9cb0d67e749bf42f5e90cb3b8a6f525dbb920",
      "parent_got_revision": "76b9cd04107da9c6c26666ab21b7cae4fc34f409",
      "parent_got_revision_cp": "refs/branch-heads/7444@{#327}",
      "parent_got_swiftshader_revision": "5f1c459a11bb4899301e89609cbf7547f9f31e20",
      "parent_got_v8_revision": "3a777f7fba0b3123738216fb46ab1a3a5cf13927",
      "parent_got_v8_revision_cp": "refs/branch-heads/14.2@{#3}",
      "parent_got_webrtc_revision": "f70dc714a073397356f6ed866481da73f90f0b96",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#45766}",
      "swarm_hashes": {
            "dawn_end2end_tests": "b0c5dd6dfc3bfa081e6782c4bba0f7fe86f635e09c730c3dbd55944a8a6b3f40/323",
            "dawn_perf_tests": "d91b7568d2968ea66ef4ca819782a3124f5e4861830397deec2ca91f572e3cd7/487",
            "gl_tests": "4a0ac708505b0ec2006f44ffd9d37f843bbbe81707dca305911300e7a4781b02/323",
            "gl_unittests": "fd1a367fe598480336f0fab4ff68a920e4c5b18847cca87c5f1eb69424f7bb67/323",
            "telemetry_gpu_integration_test": "43447904986ec745b01e5781347b67676ae1860f24a14d314f091fdb61c2c99c/815",
            "telemetry_gpu_unittests": "b906c005d240397a20105694205a24d14c74e713c7ad3f11553bcb576bb54422/736",
            "webgpu_blink_web_tests": "345d1c407f9e20e51511985de1f8b6c322c9b544d7e144c79046e94384faa5c4/566"
          },
      "swarming_command_lines_cwd": "out/ee60-Dawn_Linux_x64_",
      "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "76b9cd04107da9c6c26666ab21b7cae4fc34f409",
    "ref": "refs/branch-heads/7444"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8867303648181134112"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Linux x64 DEPS Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "265"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[05:54:39.007] Scheduled build:
{
  "id": "8701873586019045041",
  "builder": {
    "project": "chromium-m142",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "number": 88,
  "createdBy": "project:chromium-m142",
  "createTime": "2025-10-05T05:54:38.607153995Z",
  "updateTime": "2025-10-05T05:54:38.607153995Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "76b9cd04107da9c6c26666ab21b7cae4fc34f409",
      "ref": "refs/branch-heads/7444"
    }
  },
  "canary": true
}
[05:54:39.007] Task URL: https://cr-buildbucket.appspot.com/build/8701873586019045041
[05:54:39.007] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:2:0) after 7m38s
[05:54:40.022] Received PubSub notification, asking Buildbucket for the build status
[05:54:40.036] Build status: SCHEDULED
[05:55:39.301] Received PubSub notification, asking Buildbucket for the build status
[05:55:39.318] Build status: STARTED
[06:02:17.021] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:2:0)
[06:02:17.037] Build status: STARTED
[06:02:17.037] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:5:0) after 8m39s
[06:10:56.166] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:5:0)
[06:10:56.181] Build status: STARTED
[06:10:56.181] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:6:0) after 2m13s
[06:13:09.236] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:6:0)
[06:13:09.259] Build status: STARTED
[06:13:09.259] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:7:0) after 8m35s
[06:21:44.317] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:7:0)
[06:21:44.378] Build status: STARTED
[06:21:44.378] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:8:0) after 4m31s
[06:26:15.395] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:8:0)
[06:26:15.417] Build status: STARTED
[06:26:15.417] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:9:0) after 4m29s
[06:30:44.505] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:9:0)
[06:30:44.533] Build status: STARTED
[06:30:44.533] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:10:0) after 5m45s
[06:36:29.633] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:10:0)
[06:36:29.699] Build status: STARTED
[06:36:29.700] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:11:0) after 7m25s
[06:43:54.889] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:11:0)
[06:43:54.910] Build status: STARTED
[06:43:54.910] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:12:0) after 8m16s
[06:52:11.028] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:12:0)
[06:52:11.053] Build status: STARTED
[06:52:11.053] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:13:0) after 3m18s
[06:55:29.069] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:13:0)
[06:55:29.113] Build status: STARTED
[06:55:29.113] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8867303648181134112:14:0) after 5m40s
[06:57:51.559] Received PubSub notification, asking Buildbucket for the build status
[06:57:51.574] Build:
{
  "id": "8701873586019045041",
  "builder": {
    "project": "chromium-m142",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "number": 88,
  "createdBy": "project:chromium-m142",
  "createTime": "2025-10-05T05:54:38.607153995Z",
  "startTime": "2025-10-05T05:55:38.598618614Z",
  "endTime": "2025-10-05T06:57:50.936240747Z",
  "updateTime": "2025-10-05T06:57:50.936240747Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "76b9cd04107da9c6c26666ab21b7cae4fc34f409",
      "ref": "refs/branch-heads/7444"
    }
  },
  "canary": true
}
[06:57:51.574] Invocation finished in 1h3m14.411985273s with status FAILED