Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 58 minutes
Actions:

Definition (rev 88792a3ee1d8f828380e93e1e6b2012886013f02)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Win10 x64 DEPS Release (NVIDIA)"
>

Incoming triggers

  • Dawn Win10 x64 DEPS Builder/1700 (1 month ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Win10 x64 DEPS Builder/1701 (1 month ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab",
        "project": "chromium/src",
        "ref": "refs/branch-heads/6778"
      }
    ]
  },
  "branch": "refs/branch-heads/6778",
  "deps_revision_overrides": {
    "src": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8728583016201376753"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8728583016201376753"
    }
  ],
  "parent_build_id": "8728583016201376753",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Win10 x64 DEPS Builder",
  "parent_got_angle_revision": "ac6cda4cbd716102ded6a965f79573b41581898d",
  "parent_got_dawn_revision": "740d2502dbbd719a76c5a8d3fb4dac1b5363f42e",
  "parent_got_revision": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab",
  "parent_got_revision_cp": "refs/branch-heads/6778@{#3003}",
  "parent_got_swiftshader_revision": "7a9a492a38b7c701f7c96a15a76046aed8f8c0c3",
  "parent_got_v8_revision": "77c955a3d2aa07c0256d85a7bc72d0c1cb6a79ba",
  "parent_got_v8_revision_cp": "refs/branch-heads/13.1@{#35}",
  "parent_got_webrtc_revision": "79aff54b0fa9238ce3518dd9eaf9610cd6f22e82",
  "parent_got_webrtc_revision_cp": "refs/branch-heads/6778@{#4}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab",
  "swarm_hashes": {
    "dawn_end2end_tests": "5bef9a5b3a2b3014f790966251790d78bcff78055eaae5e2f265c4593975fa89/323",
    "dawn_perf_tests": "8688d8d738d5c0c15435565a1cae596d599a483727ff6e9826a975936fbffdaa/487",
    "gl_tests": "3f261b211ac379774a5d0385feaf312283801cdf159824487dd179a6c7e41650/323",
    "gl_unittests": "398e809d8e7309bc9582130f4c62dc210ff75ca02c616783b3c8aa3c92f2ec7c/402",
    "telemetry_gpu_integration_test": "0d6d276367515be6d89baff8ea85f5834e57d78a1dce8a69ca19bf96093d8e98/815",
    "telemetry_gpu_unittests": "d2d742f332c0b2a0632ec9c3062c8fe94510317a8b990bcb978b776101955068/736",
    "webgpu_blink_web_tests": "bfcbaac7c0e0db9082851dc4760dab63f13b0a281cb137352abd9f681c9fc8d5/566"
  },
  "swarming_command_lines_cwd": "out\\Release_x64",
  "swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/e4c59cdb67890c8daad386039eff6a7c5cdf06ab
  • gitiles_ref:refs/branch-heads/6778
  • parent_buildername:Dawn Win10 x64 DEPS Builder
  • parent_buildnumber:1701
  • user_agent:recipe

Debug log (UTC)

[11:22:14.937] New invocation is queued and will start shortly
[11:22:14.937] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[11:22:15.989] Starting the invocation (attempt 1)
[11:22:16.019] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/e4c59cdb67890c8daad386039eff6a7c5cdf06ab"
[11:22:16.019] Popped gitiles commit info from properties and tags
[11:22:16.019] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[11:22:16.020] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[11:22:16.021] Buildbucket request:
{
  "requestId": "8894009140603547392",
  "builder": {
    "project": "chromium-m131",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 DEPS Release (NVIDIA)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/6778"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8894009140603547392",
            "job": "chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA)",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "e973dc09da4039260eb05a49e753bc5cbac604ac",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/6778"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "e973dc09da4039260eb05a49e753bc5cbac604ac"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8728584714671321345"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8728584714671321345"
                                              }
                                        ],
                                    "parent_build_id": "8728584714671321345",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Win10 x64 DEPS Builder",
                                    "parent_got_angle_revision": "ac6cda4cbd716102ded6a965f79573b41581898d",
                                    "parent_got_dawn_revision": "740d2502dbbd719a76c5a8d3fb4dac1b5363f42e",
                                    "parent_got_revision": "e973dc09da4039260eb05a49e753bc5cbac604ac",
                                    "parent_got_revision_cp": "refs/branch-heads/6778@{#3002}",
                                    "parent_got_swiftshader_revision": "7a9a492a38b7c701f7c96a15a76046aed8f8c0c3",
                                    "parent_got_v8_revision": "77c955a3d2aa07c0256d85a7bc72d0c1cb6a79ba",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.1@{#35}",
                                    "parent_got_webrtc_revision": "79aff54b0fa9238ce3518dd9eaf9610cd6f22e82",
                                    "parent_got_webrtc_revision_cp": "refs/branch-heads/6778@{#4}",
                                    "revision": "e973dc09da4039260eb05a49e753bc5cbac604ac",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "6e5cb14b8bfee6acd1c2b61664e65664c5e3f55b7ad5e82f584b8ddf745200ed/323",
                                          "dawn_perf_tests": "bf9b7941d791ff51ab5bc82060d4ff4774500d49af31c83d71fd5e87259cb9dd/487",
                                          "gl_tests": "528574498321bea7f47216eda1cd5097dbf510234a58ea4429ba6457350efa27/323",
                                          "gl_unittests": "6556cb02b17f6a08cd89164507db6c6cc5a9bc8971efd0868f1bba1352e1afb3/402",
                                          "telemetry_gpu_integration_test": "b4d6530691256c689ca695442aa4db4f1ccbc75d387ce66bd1add384ebc2e939/815",
                                          "telemetry_gpu_unittests": "e7a79a31c1748e4fb97092082d20d3132918b02c8b2024193466c02de1757434/736",
                                          "webgpu_blink_web_tests": "e426af898636189687f8141496b04c293c794bc0c40376828a73cfa8ca9195eb/566"
                                        },
                                    "swarming_command_lines_cwd": "out\\Release_x64",
                                    "swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93"
                                  },
                              "ref": "refs/branch-heads/6778",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "e973dc09da4039260eb05a49e753bc5cbac604ac",
                              "tags": [
                                    "parent_buildername:Dawn Win10 x64 DEPS Builder",
                                    "parent_buildnumber:1700",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "2fcf6abe-f78d-46a9-ba06-fbfac5355f9f",
                        "title": "Dawn Win10 x64 DEPS Builder/1700"
                      },
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/6778"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8728583016201376753"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8728583016201376753"
                                              }
                                        ],
                                    "parent_build_id": "8728583016201376753",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Win10 x64 DEPS Builder",
                                    "parent_got_angle_revision": "ac6cda4cbd716102ded6a965f79573b41581898d",
                                    "parent_got_dawn_revision": "740d2502dbbd719a76c5a8d3fb4dac1b5363f42e",
                                    "parent_got_revision": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab",
                                    "parent_got_revision_cp": "refs/branch-heads/6778@{#3003}",
                                    "parent_got_swiftshader_revision": "7a9a492a38b7c701f7c96a15a76046aed8f8c0c3",
                                    "parent_got_v8_revision": "77c955a3d2aa07c0256d85a7bc72d0c1cb6a79ba",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.1@{#35}",
                                    "parent_got_webrtc_revision": "79aff54b0fa9238ce3518dd9eaf9610cd6f22e82",
                                    "parent_got_webrtc_revision_cp": "refs/branch-heads/6778@{#4}",
                                    "revision": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "5bef9a5b3a2b3014f790966251790d78bcff78055eaae5e2f265c4593975fa89/323",
                                          "dawn_perf_tests": "8688d8d738d5c0c15435565a1cae596d599a483727ff6e9826a975936fbffdaa/487",
                                          "gl_tests": "3f261b211ac379774a5d0385feaf312283801cdf159824487dd179a6c7e41650/323",
                                          "gl_unittests": "398e809d8e7309bc9582130f4c62dc210ff75ca02c616783b3c8aa3c92f2ec7c/402",
                                          "telemetry_gpu_integration_test": "0d6d276367515be6d89baff8ea85f5834e57d78a1dce8a69ca19bf96093d8e98/815",
                                          "telemetry_gpu_unittests": "d2d742f332c0b2a0632ec9c3062c8fe94510317a8b990bcb978b776101955068/736",
                                          "webgpu_blink_web_tests": "bfcbaac7c0e0db9082851dc4760dab63f13b0a281cb137352abd9f681c9fc8d5/566"
                                        },
                                    "swarming_command_lines_cwd": "out\\Release_x64",
                                    "swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93"
                                  },
                              "ref": "refs/branch-heads/6778",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab",
                              "tags": [
                                    "parent_buildername:Dawn Win10 x64 DEPS Builder",
                                    "parent_buildnumber:1701",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "cff345a2-9c8c-468a-8f4b-88e624604e21",
                        "title": "Dawn Win10 x64 DEPS Builder/1701"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8728583016201376753"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8728583016201376753"
                }
          ],
      "parent_build_id": "8728583016201376753",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Win10 x64 DEPS Builder",
      "parent_got_angle_revision": "ac6cda4cbd716102ded6a965f79573b41581898d",
      "parent_got_dawn_revision": "740d2502dbbd719a76c5a8d3fb4dac1b5363f42e",
      "parent_got_revision": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab",
      "parent_got_revision_cp": "refs/branch-heads/6778@{#3003}",
      "parent_got_swiftshader_revision": "7a9a492a38b7c701f7c96a15a76046aed8f8c0c3",
      "parent_got_v8_revision": "77c955a3d2aa07c0256d85a7bc72d0c1cb6a79ba",
      "parent_got_v8_revision_cp": "refs/branch-heads/13.1@{#35}",
      "parent_got_webrtc_revision": "79aff54b0fa9238ce3518dd9eaf9610cd6f22e82",
      "parent_got_webrtc_revision_cp": "refs/branch-heads/6778@{#4}",
      "swarm_hashes": {
            "dawn_end2end_tests": "5bef9a5b3a2b3014f790966251790d78bcff78055eaae5e2f265c4593975fa89/323",
            "dawn_perf_tests": "8688d8d738d5c0c15435565a1cae596d599a483727ff6e9826a975936fbffdaa/487",
            "gl_tests": "3f261b211ac379774a5d0385feaf312283801cdf159824487dd179a6c7e41650/323",
            "gl_unittests": "398e809d8e7309bc9582130f4c62dc210ff75ca02c616783b3c8aa3c92f2ec7c/402",
            "telemetry_gpu_integration_test": "0d6d276367515be6d89baff8ea85f5834e57d78a1dce8a69ca19bf96093d8e98/815",
            "telemetry_gpu_unittests": "d2d742f332c0b2a0632ec9c3062c8fe94510317a8b990bcb978b776101955068/736",
            "webgpu_blink_web_tests": "bfcbaac7c0e0db9082851dc4760dab63f13b0a281cb137352abd9f681c9fc8d5/566"
          },
      "swarming_command_lines_cwd": "out\\Release_x64",
      "swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab",
    "ref": "refs/branch-heads/6778"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8894009140603547392"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Win10 x64 DEPS Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "1701"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[11:22:16.672] Scheduled build:
{
  "id": "8728579078731839713",
  "builder": {
    "project": "chromium-m131",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 DEPS Release (NVIDIA)"
  },
  "number": 1066,
  "createdBy": "project:chromium-m131",
  "createTime": "2024-12-14T11:22:16.100011940Z",
  "updateTime": "2024-12-14T11:22:16.100011940Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab",
      "ref": "refs/branch-heads/6778"
    }
  }
}
[11:22:16.672] Task URL: https://cr-buildbucket.appspot.com/build/8728579078731839713
[11:22:16.672] Scheduling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:2:0) after 2m54s
[11:22:17.639] Received PubSub notification, asking Buildbucket for the build status
[11:22:17.664] Build status: SCHEDULED
[11:22:31.236] Received PubSub notification, asking Buildbucket for the build status
[11:22:31.282] Build status: STARTED
[11:25:10.732] Handling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:2:0)
[11:25:10.796] Build status: STARTED
[11:25:10.796] Scheduling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:5:0) after 4m48s
[11:29:58.840] Handling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:5:0)
[11:29:58.859] Build status: STARTED
[11:29:58.859] Scheduling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:6:0) after 5m55s
[11:35:54.018] Handling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:6:0)
[11:35:54.042] Build status: STARTED
[11:35:54.042] Scheduling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:7:0) after 3m27s
[11:39:21.376] Handling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:7:0)
[11:39:21.418] Build status: STARTED
[11:39:21.418] Scheduling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:8:0) after 1m53s
[11:41:14.436] Handling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:8:0)
[11:41:14.460] Build status: STARTED
[11:41:14.460] Scheduling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:9:0) after 8m14s
[11:49:28.778] Handling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:9:0)
[11:49:28.821] Build status: STARTED
[11:49:28.821] Scheduling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:10:0) after 2m38s
[11:52:06.925] Handling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:10:0)
[11:52:06.946] Build status: STARTED
[11:52:06.946] Scheduling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:11:0) after 5m54s
[11:58:01.042] Handling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:11:0)
[11:58:01.063] Build status: STARTED
[11:58:01.063] Scheduling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:12:0) after 6m50s
[12:04:51.126] Handling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:12:0)
[12:04:51.148] Build status: STARTED
[12:04:51.148] Scheduling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:13:0) after 2m30s
[12:07:21.169] Handling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:13:0)
[12:07:21.188] Build status: STARTED
[12:07:21.188] Scheduling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:14:0) after 1m48s
[12:09:09.475] Handling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:14:0)
[12:09:09.497] Build status: STARTED
[12:09:09.497] Scheduling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:15:0) after 6m13s
[12:15:22.512] Handling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:15:0)
[12:15:22.531] Build status: STARTED
[12:15:22.531] Scheduling timer "check-buildbucket-build-status" (chromium-m131/Dawn Win10 x64 DEPS Release (NVIDIA):8894009140603547392:16:0) after 6m5s
[12:20:30.673] Received PubSub notification, asking Buildbucket for the build status
[12:20:30.697] Build:
{
  "id": "8728579078731839713",
  "builder": {
    "project": "chromium-m131",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 DEPS Release (NVIDIA)"
  },
  "number": 1066,
  "createdBy": "project:chromium-m131",
  "createTime": "2024-12-14T11:22:16.100011940Z",
  "startTime": "2024-12-14T11:22:31.016897634Z",
  "endTime": "2024-12-14T12:20:30.254660145Z",
  "updateTime": "2024-12-14T12:20:30.254660145Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "e4c59cdb67890c8daad386039eff6a7c5cdf06ab",
      "ref": "refs/branch-heads/6778"
    }
  }
}
[12:20:30.697] Invocation finished in 58m15.766735046s with status FAILED