Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 45 minutes
Actions:

Definition (rev e06c85a67c04bcbdc0ee7d29b76877746b7b8777)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Win10 x86 DEPS Release (NVIDIA)"
>

Incoming triggers

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "333c861d352e0d53a9aca34efa424099373b977d",
        "project": "chromium/src",
        "ref": "refs/branch-heads/7444"
      }
    ]
  },
  "branch": "refs/branch-heads/7444",
  "deps_revision_overrides": {
    "src": "333c861d352e0d53a9aca34efa424099373b977d"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8702179513112630337"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8702179513112630337"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8702179513112630337"
    }
  ],
  "parent_build_id": "8702179513112630337",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Win10 x86 DEPS Builder",
  "parent_got_angle_revision": "e5b338502c7c65bd38d4695bc557938ca4cc3aa7",
  "parent_got_dawn_revision": "cee9cb0d67e749bf42f5e90cb3b8a6f525dbb920",
  "parent_got_revision": "333c861d352e0d53a9aca34efa424099373b977d",
  "parent_got_revision_cp": "refs/branch-heads/7444@{#79}",
  "parent_got_swiftshader_revision": "5f1c459a11bb4899301e89609cbf7547f9f31e20",
  "parent_got_v8_revision": "3a777f7fba0b3123738216fb46ab1a3a5cf13927",
  "parent_got_v8_revision_cp": "refs/branch-heads/14.2@{#3}",
  "parent_got_webrtc_revision": "f70dc714a073397356f6ed866481da73f90f0b96",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#45766}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "333c861d352e0d53a9aca34efa424099373b977d",
  "swarm_hashes": {
    "dawn_end2end_tests": "c32da3ea7a04c3b6cf9d3f5c5f221c6f02e84511c9281d8e4de7f21262584698/323",
    "dawn_perf_tests": "1e79133cb6077090448b4935bd0dac4795d3ef7fa0061938db976ded31eb9623/487",
    "gl_tests": "3b5a9399cb3bb96582aab7a8054d79ee4665c5a8ce4e479f62cd89e646f01b25/323",
    "gl_unittests": "cf1e153b28b1cd4f546d111cb62236091d6510b84922a94ecee165e2dd6e855a/402",
    "telemetry_gpu_integration_test": "a96f520beb3622968ec247e251c5a5c697e56d1d9e867d0f95f8bd5b2a0cb415/815",
    "telemetry_gpu_unittests": "38333da79992be6bdb5e4fdc402c8d9e13d569cc348a8db3ffcdf5a65be2924f/736",
    "webgpu_blink_web_tests": "d9a16d8c884b1c622afbab29731b7320d6528a9af67666fbb26dd276331031bd/566"
  },
  "swarming_command_lines_cwd": "out\\e62c-Dawn_Win10_x86_",
  "swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/333c861d352e0d53a9aca34efa424099373b977d
  • gitiles_ref:refs/branch-heads/7444
  • parent_buildername:Dawn Win10 x86 DEPS Builder
  • parent_buildnumber:53
  • user_agent:recipe

Debug log (UTC)

[21:18:36.813] New invocation is queued and will start shortly
[21:18:36.813] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[21:18:38.253] Starting the invocation (attempt 1)
[21:18:38.283] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/333c861d352e0d53a9aca34efa424099373b977d"
[21:18:38.283] Popped gitiles commit info from properties and tags
[21:18:38.283] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[21:18:38.284] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[21:18:38.284] Buildbucket request:
{
  "requestId": "8867607903370243312",
  "builder": {
    "project": "chromium-m142",
    "bucket": "ci",
    "builder": "Dawn Win10 x86 DEPS Release (NVIDIA)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "333c861d352e0d53a9aca34efa424099373b977d",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/7444"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8867607903370243312",
            "job": "chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA)",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "d9faf89f891fa7ebffe969b04c34df691da36eaa",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7444"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "d9faf89f891fa7ebffe969b04c34df691da36eaa"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8702180792638162833"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8702180792638162833"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8702180792638162833"
                                              }
                                        ],
                                    "parent_build_id": "8702180792638162833",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Win10 x86 DEPS Builder",
                                    "parent_got_angle_revision": "e5b338502c7c65bd38d4695bc557938ca4cc3aa7",
                                    "parent_got_dawn_revision": "cee9cb0d67e749bf42f5e90cb3b8a6f525dbb920",
                                    "parent_got_revision": "d9faf89f891fa7ebffe969b04c34df691da36eaa",
                                    "parent_got_revision_cp": "refs/branch-heads/7444@{#78}",
                                    "parent_got_swiftshader_revision": "5f1c459a11bb4899301e89609cbf7547f9f31e20",
                                    "parent_got_v8_revision": "3a777f7fba0b3123738216fb46ab1a3a5cf13927",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/14.2@{#3}",
                                    "parent_got_webrtc_revision": "f70dc714a073397356f6ed866481da73f90f0b96",
                                    "parent_got_webrtc_revision_cp": "refs/heads/main@{#45766}",
                                    "revision": "d9faf89f891fa7ebffe969b04c34df691da36eaa",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "c32da3ea7a04c3b6cf9d3f5c5f221c6f02e84511c9281d8e4de7f21262584698/323",
                                          "dawn_perf_tests": "1e79133cb6077090448b4935bd0dac4795d3ef7fa0061938db976ded31eb9623/487",
                                          "gl_tests": "3b5a9399cb3bb96582aab7a8054d79ee4665c5a8ce4e479f62cd89e646f01b25/323",
                                          "gl_unittests": "cf1e153b28b1cd4f546d111cb62236091d6510b84922a94ecee165e2dd6e855a/402",
                                          "telemetry_gpu_integration_test": "71443f4336c18da705c24d18c2c88aab53efc859c175cf91a24cb4f1b713122a/815",
                                          "telemetry_gpu_unittests": "38333da79992be6bdb5e4fdc402c8d9e13d569cc348a8db3ffcdf5a65be2924f/736",
                                          "webgpu_blink_web_tests": "17dba91fb7d6c8e7cc348f11868f67c915a89a0131c3d796d473f8bcac118fb1/566"
                                        },
                                    "swarming_command_lines_cwd": "out\\e62c-Dawn_Win10_x86_",
                                    "swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7444",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "d9faf89f891fa7ebffe969b04c34df691da36eaa",
                              "tags": [
                                    "parent_buildername:Dawn Win10 x86 DEPS Builder",
                                    "parent_buildnumber:52",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "2214bf36-ff53-4197-9abe-3c00092a5eca",
                        "title": "Dawn Win10 x86 DEPS Builder/52",
                        "url": "https://cr-buildbucket.appspot.com/build/8702180792638162833"
                      },
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "333c861d352e0d53a9aca34efa424099373b977d",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7444"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "333c861d352e0d53a9aca34efa424099373b977d"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8702179513112630337"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8702179513112630337"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8702179513112630337"
                                              }
                                        ],
                                    "parent_build_id": "8702179513112630337",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Win10 x86 DEPS Builder",
                                    "parent_got_angle_revision": "e5b338502c7c65bd38d4695bc557938ca4cc3aa7",
                                    "parent_got_dawn_revision": "cee9cb0d67e749bf42f5e90cb3b8a6f525dbb920",
                                    "parent_got_revision": "333c861d352e0d53a9aca34efa424099373b977d",
                                    "parent_got_revision_cp": "refs/branch-heads/7444@{#79}",
                                    "parent_got_swiftshader_revision": "5f1c459a11bb4899301e89609cbf7547f9f31e20",
                                    "parent_got_v8_revision": "3a777f7fba0b3123738216fb46ab1a3a5cf13927",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/14.2@{#3}",
                                    "parent_got_webrtc_revision": "f70dc714a073397356f6ed866481da73f90f0b96",
                                    "parent_got_webrtc_revision_cp": "refs/heads/main@{#45766}",
                                    "revision": "333c861d352e0d53a9aca34efa424099373b977d",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "c32da3ea7a04c3b6cf9d3f5c5f221c6f02e84511c9281d8e4de7f21262584698/323",
                                          "dawn_perf_tests": "1e79133cb6077090448b4935bd0dac4795d3ef7fa0061938db976ded31eb9623/487",
                                          "gl_tests": "3b5a9399cb3bb96582aab7a8054d79ee4665c5a8ce4e479f62cd89e646f01b25/323",
                                          "gl_unittests": "cf1e153b28b1cd4f546d111cb62236091d6510b84922a94ecee165e2dd6e855a/402",
                                          "telemetry_gpu_integration_test": "a96f520beb3622968ec247e251c5a5c697e56d1d9e867d0f95f8bd5b2a0cb415/815",
                                          "telemetry_gpu_unittests": "38333da79992be6bdb5e4fdc402c8d9e13d569cc348a8db3ffcdf5a65be2924f/736",
                                          "webgpu_blink_web_tests": "d9a16d8c884b1c622afbab29731b7320d6528a9af67666fbb26dd276331031bd/566"
                                        },
                                    "swarming_command_lines_cwd": "out\\e62c-Dawn_Win10_x86_",
                                    "swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7444",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "333c861d352e0d53a9aca34efa424099373b977d",
                              "tags": [
                                    "parent_buildername:Dawn Win10 x86 DEPS Builder",
                                    "parent_buildnumber:53",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "8a127d4f-fbd1-4c23-bf0d-01c526595735",
                        "title": "Dawn Win10 x86 DEPS Builder/53",
                        "url": "https://cr-buildbucket.appspot.com/build/8702179513112630337"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "333c861d352e0d53a9aca34efa424099373b977d"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8702179513112630337"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8702179513112630337"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8702179513112630337"
                }
          ],
      "parent_build_id": "8702179513112630337",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Win10 x86 DEPS Builder",
      "parent_got_angle_revision": "e5b338502c7c65bd38d4695bc557938ca4cc3aa7",
      "parent_got_dawn_revision": "cee9cb0d67e749bf42f5e90cb3b8a6f525dbb920",
      "parent_got_revision": "333c861d352e0d53a9aca34efa424099373b977d",
      "parent_got_revision_cp": "refs/branch-heads/7444@{#79}",
      "parent_got_swiftshader_revision": "5f1c459a11bb4899301e89609cbf7547f9f31e20",
      "parent_got_v8_revision": "3a777f7fba0b3123738216fb46ab1a3a5cf13927",
      "parent_got_v8_revision_cp": "refs/branch-heads/14.2@{#3}",
      "parent_got_webrtc_revision": "f70dc714a073397356f6ed866481da73f90f0b96",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#45766}",
      "swarm_hashes": {
            "dawn_end2end_tests": "c32da3ea7a04c3b6cf9d3f5c5f221c6f02e84511c9281d8e4de7f21262584698/323",
            "dawn_perf_tests": "1e79133cb6077090448b4935bd0dac4795d3ef7fa0061938db976ded31eb9623/487",
            "gl_tests": "3b5a9399cb3bb96582aab7a8054d79ee4665c5a8ce4e479f62cd89e646f01b25/323",
            "gl_unittests": "cf1e153b28b1cd4f546d111cb62236091d6510b84922a94ecee165e2dd6e855a/402",
            "telemetry_gpu_integration_test": "a96f520beb3622968ec247e251c5a5c697e56d1d9e867d0f95f8bd5b2a0cb415/815",
            "telemetry_gpu_unittests": "38333da79992be6bdb5e4fdc402c8d9e13d569cc348a8db3ffcdf5a65be2924f/736",
            "webgpu_blink_web_tests": "d9a16d8c884b1c622afbab29731b7320d6528a9af67666fbb26dd276331031bd/566"
          },
      "swarming_command_lines_cwd": "out\\e62c-Dawn_Win10_x86_",
      "swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "333c861d352e0d53a9aca34efa424099373b977d",
    "ref": "refs/branch-heads/7444"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8867607903370243312"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Win10 x86 DEPS Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "53"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[21:18:38.853] Scheduled build:
{
  "id": "8702177841048060337",
  "builder": {
    "project": "chromium-m142",
    "bucket": "ci",
    "builder": "Dawn Win10 x86 DEPS Release (NVIDIA)"
  },
  "number": 29,
  "createdBy": "project:chromium-m142",
  "createTime": "2025-10-01T21:18:38.400888796Z",
  "updateTime": "2025-10-01T21:18:38.400888796Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "333c861d352e0d53a9aca34efa424099373b977d",
      "ref": "refs/branch-heads/7444"
    }
  },
  "canary": true
}
[21:18:38.853] Task URL: https://cr-buildbucket.appspot.com/build/8702177841048060337
[21:18:38.853] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:2:0) after 4m26s
[21:18:39.831] Received PubSub notification, asking Buildbucket for the build status
[21:18:39.847] Build status: SCHEDULED
[21:19:52.601] Received PubSub notification, asking Buildbucket for the build status
[21:19:52.628] Build status: STARTED
[21:23:04.868] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:2:0)
[21:23:04.889] Build status: STARTED
[21:23:04.889] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:5:0) after 5m15s
[21:28:19.981] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:5:0)
[21:28:20.015] Build status: STARTED
[21:28:20.015] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:6:0) after 2m1s
[21:30:21.080] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:6:0)
[21:30:21.104] Build status: STARTED
[21:30:21.104] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:7:0) after 5m30s
[21:35:51.271] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:7:0)
[21:35:51.292] Build status: STARTED
[21:35:51.292] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:8:0) after 4m27s
[21:40:18.354] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:8:0)
[21:40:18.388] Build status: STARTED
[21:40:18.388] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:9:0) after 5m18s
[21:45:36.408] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:9:0)
[21:45:36.433] Build status: STARTED
[21:45:36.433] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:10:0) after 5m2s
[21:50:38.535] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:10:0)
[21:50:38.551] Build status: STARTED
[21:50:38.551] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:11:0) after 7m20s
[21:57:58.567] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:11:0)
[21:57:58.587] Build status: STARTED
[21:57:58.587] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:12:0) after 1m33s
[21:59:31.698] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:12:0)
[21:59:31.718] Build status: STARTED
[21:59:31.718] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:13:0) after 2m55s
[22:02:26.746] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:13:0)
[22:02:26.767] Build status: STARTED
[22:02:26.767] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Win10 x86 DEPS Release (NVIDIA):8867607903370243312:14:0) after 3m22s
[22:04:04.490] Received PubSub notification, asking Buildbucket for the build status
[22:04:04.509] Build:
{
  "id": "8702177841048060337",
  "builder": {
    "project": "chromium-m142",
    "bucket": "ci",
    "builder": "Dawn Win10 x86 DEPS Release (NVIDIA)"
  },
  "number": 29,
  "createdBy": "project:chromium-m142",
  "createTime": "2025-10-01T21:18:38.400888796Z",
  "startTime": "2025-10-01T21:19:52.417212170Z",
  "endTime": "2025-10-01T22:04:04.083884232Z",
  "updateTime": "2025-10-01T22:04:04.083884232Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "333c861d352e0d53a9aca34efa424099373b977d",
      "ref": "refs/branch-heads/7444"
    }
  },
  "canary": true
}
[22:04:04.509] Invocation finished in 45m27.707260131s with status FAILED