Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 2 hours
Actions:

Definition (rev 1f1b4513521524d405025696b8777ff82db0bd9d)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Win10 x64 ASAN Release (NVIDIA)"
>

Incoming triggers

  • Dawn Win10 x64 ASAN Builder/8843 (11 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Win10 x64 ASAN Builder/8844 (11 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "9e3e20cd4875442fc1aaab7b65604a53cab1f1a3",
        "project": "chromium/src",
        "ref": "refs/heads/main"
      }
    ]
  },
  "branch": "refs/heads/main",
  "deps_revision_overrides": {
    "src": "9e3e20cd4875442fc1aaab7b65604a53cab1f1a3",
    "src/third_party/dawn": "9bbe2114a1f103fe1dde36b23db2238c485fd57e"
  },
  "parent_build_id": "8743471525522657521",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Win10 x64 ASAN Builder",
  "parent_got_angle_revision": "2efce9babc183666620dc57f8506aa9acb523186",
  "parent_got_dawn_revision": "9bbe2114a1f103fe1dde36b23db2238c485fd57e",
  "parent_got_revision": "9e3e20cd4875442fc1aaab7b65604a53cab1f1a3",
  "parent_got_revision_cp": "refs/heads/main@{#1322529}",
  "parent_got_swiftshader_revision": "a0ec371d8331d787c61eccc89fb411019330314e",
  "parent_got_v8_revision": "22a564262998b1843c87139add4e9a372934ec03",
  "parent_got_v8_revision_cp": "refs/heads/12.8.204@{#1}",
  "parent_got_webrtc_revision": "fbf754b581529219c1abdd56a66ab1ad79fda11c",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#42576}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "9e3e20cd4875442fc1aaab7b65604a53cab1f1a3",
  "swarm_hashes": {
    "dawn_end2end_tests": "0a2ea2e1149bfaba1bd62f5d7028ddfa22f37e1864753cc6c01738547d8cce84/487",
    "dawn_perf_tests": "3c2fe0069b926863d5226515da719810ea8e0655dd38a16f50bd0e591f4cccb8/488",
    "gl_tests": "16e1c6b3d283773447c9811747c5e43560c99a028567a4bb85441c82991cadb4/487",
    "gl_unittests": "85272f0a9ab6fc73aa2f833c687cfc70f5d3a1884625cc08ed4979399704fe53/487",
    "telemetry_gpu_integration_test": "bba6cf11a2313749d5b8d8da3c7b89080c0de14069c8fd788cd25784f8c4b2f3/814",
    "telemetry_gpu_unittests": "b31aa027c3960131fca3e07d1f7cc056998cf2f7a1401838383e97831e98fd6e/735",
    "webgpu_blink_web_tests": "eb7cd060802ae2e8fd1d6f96c43809e48303822444cea442a70c92121c2becdc/566"
  },
  "swarming_command_lines_cwd": "out\\Release_x64",
  "swarming_command_lines_digest": "241fcb002f7cd4a5f15c614bfa0f9cb4f8170a72e39dc8fb0659328aadd8d894/93"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/9e3e20cd4875442fc1aaab7b65604a53cab1f1a3
  • gitiles_ref:refs/heads/main
  • parent_buildername:Dawn Win10 x64 ASAN Builder
  • parent_buildnumber:8844
  • user_agent:recipe

Debug log (UTC)

[04:17:15.630] New invocation is queued and will start shortly
[04:17:15.630] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[04:17:16.710] Starting the invocation (attempt 1)
[04:17:16.752] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/9e3e20cd4875442fc1aaab7b65604a53cab1f1a3"
[04:17:16.752] Popped gitiles commit info from properties and tags
[04:17:16.752] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[04:17:16.752] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[04:17:16.752] Buildbucket request:
{
  "requestId": "8908893781060484880",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 ASAN Release (NVIDIA)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "9e3e20cd4875442fc1aaab7b65604a53cab1f1a3",
                        "project": "chromium/src",
                        "ref": "refs/heads/main"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "9e3e20cd4875442fc1aaab7b65604a53cab1f1a3",
            "src/third_party/dawn": "9bbe2114a1f103fe1dde36b23db2238c485fd57e"
          },
      "parent_build_id": "8743471525522657521",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Win10 x64 ASAN Builder",
      "parent_got_angle_revision": "2efce9babc183666620dc57f8506aa9acb523186",
      "parent_got_dawn_revision": "9bbe2114a1f103fe1dde36b23db2238c485fd57e",
      "parent_got_revision": "9e3e20cd4875442fc1aaab7b65604a53cab1f1a3",
      "parent_got_revision_cp": "refs/heads/main@{#1322529}",
      "parent_got_swiftshader_revision": "a0ec371d8331d787c61eccc89fb411019330314e",
      "parent_got_v8_revision": "22a564262998b1843c87139add4e9a372934ec03",
      "parent_got_v8_revision_cp": "refs/heads/12.8.204@{#1}",
      "parent_got_webrtc_revision": "fbf754b581529219c1abdd56a66ab1ad79fda11c",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#42576}",
      "swarm_hashes": {
            "dawn_end2end_tests": "0a2ea2e1149bfaba1bd62f5d7028ddfa22f37e1864753cc6c01738547d8cce84/487",
            "dawn_perf_tests": "3c2fe0069b926863d5226515da719810ea8e0655dd38a16f50bd0e591f4cccb8/488",
            "gl_tests": "16e1c6b3d283773447c9811747c5e43560c99a028567a4bb85441c82991cadb4/487",
            "gl_unittests": "85272f0a9ab6fc73aa2f833c687cfc70f5d3a1884625cc08ed4979399704fe53/487",
            "telemetry_gpu_integration_test": "bba6cf11a2313749d5b8d8da3c7b89080c0de14069c8fd788cd25784f8c4b2f3/814",
            "telemetry_gpu_unittests": "b31aa027c3960131fca3e07d1f7cc056998cf2f7a1401838383e97831e98fd6e/735",
            "webgpu_blink_web_tests": "eb7cd060802ae2e8fd1d6f96c43809e48303822444cea442a70c92121c2becdc/566"
          },
      "swarming_command_lines_cwd": "out\\Release_x64",
      "swarming_command_lines_digest": "241fcb002f7cd4a5f15c614bfa0f9cb4f8170a72e39dc8fb0659328aadd8d894/93"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "9e3e20cd4875442fc1aaab7b65604a53cab1f1a3",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8908893781060484880"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/Dawn Win10 x64 ASAN Release (NVIDIA)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Win10 x64 ASAN Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "8844"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[04:17:17.595] Scheduled build:
{
  "id": "8743463719155141409",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 ASAN Release (NVIDIA)"
  },
  "number": 3951,
  "createdBy": "project:chromium",
  "createTime": "2024-07-03T04:17:16.819734559Z",
  "updateTime": "2024-07-03T04:17:16.819734559Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "9e3e20cd4875442fc1aaab7b65604a53cab1f1a3",
      "ref": "refs/heads/main"
    }
  }
}
[04:17:17.595] Task URL: https://cr-buildbucket.appspot.com/build/8743463719155141409
[04:17:17.595] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:2:0) after 5m45s
[04:17:18.501] Received PubSub notification, asking Buildbucket for the build status
[04:17:18.533] Build status: SCHEDULED
[04:18:27.465] Received PubSub notification, asking Buildbucket for the build status
[04:18:27.486] Build status: STARTED
[04:23:03.189] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:2:0)
[04:23:03.265] Build status: STARTED
[04:23:03.265] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:5:0) after 1m37s
[04:24:40.463] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:5:0)
[04:24:40.480] Build status: STARTED
[04:24:40.480] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:6:0) after 2m56s
[04:27:36.942] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:6:0)
[04:27:36.960] Build status: STARTED
[04:27:36.960] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:7:0) after 1m33s
[04:29:10.260] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:7:0)
[04:29:10.424] Build status: STARTED
[04:29:10.424] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:8:0) after 3m45s
[04:32:55.754] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:8:0)
[04:32:55.778] Build status: STARTED
[04:32:55.778] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:9:0) after 7m21s
[04:40:16.844] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:9:0)
[04:40:16.895] Build status: STARTED
[04:40:16.895] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:10:0) after 5m7s
[04:45:24.256] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:10:0)
[04:45:24.281] Build status: STARTED
[04:45:24.281] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:11:0) after 2m49s
[04:48:13.445] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:11:0)
[04:48:13.475] Build status: STARTED
[04:48:13.475] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:12:0) after 4m36s
[04:52:49.594] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:12:0)
[04:52:49.625] Build status: STARTED
[04:52:49.625] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:13:0) after 3m4s
[04:55:53.933] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:13:0)
[04:55:53.955] Build status: STARTED
[04:55:53.955] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:14:0) after 6m28s
[05:02:22.119] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:14:0)
[05:02:22.150] Build status: STARTED
[05:02:22.150] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:15:0) after 1m45s
[05:04:07.699] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:15:0)
[05:04:07.722] Build status: STARTED
[05:04:07.722] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:16:0) after 2m45s
[05:06:52.768] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:16:0)
[05:06:52.785] Build status: STARTED
[05:06:52.785] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:17:0) after 7m44s
[05:14:36.917] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:17:0)
[05:14:36.938] Build status: STARTED
[05:14:36.938] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:18:0) after 1m20s
[05:15:57.101] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:18:0)
[05:15:57.179] Build status: STARTED
[05:15:57.179] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:19:0) after 2m28s
[05:18:25.182] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:19:0)
[05:18:25.252] Build status: STARTED
[05:18:25.252] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:20:0) after 2m47s
[05:21:12.423] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:20:0)
[05:21:12.452] Build status: STARTED
[05:21:12.452] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:21:0) after 2m15s
[05:23:27.554] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:21:0)
[05:23:27.587] Build status: STARTED
[05:23:27.587] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:22:0) after 7m11s
[05:30:38.934] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:22:0)
[05:30:38.953] Build status: STARTED
[05:30:38.953] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:23:0) after 3m4s
[05:33:43.879] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:23:0)
[05:33:43.911] Build status: STARTED
[05:33:43.911] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:24:0) after 7m36s
[05:41:19.930] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:24:0)
[05:41:19.948] Build status: STARTED
[05:41:19.948] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:25:0) after 9m35s
[05:50:55.160] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:25:0)
[05:50:55.185] Build status: STARTED
[05:50:55.185] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:26:0) after 6m51s
[05:57:46.498] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:26:0)
[05:57:46.516] Build status: STARTED
[05:57:46.516] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:27:0) after 3m9s
[06:00:55.509] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:27:0)
[06:00:55.526] Build status: STARTED
[06:00:55.526] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:28:0) after 2m54s
[06:03:49.843] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:28:0)
[06:03:49.860] Build status: STARTED
[06:03:49.860] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:29:0) after 2m58s
[06:06:48.206] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:29:0)
[06:06:48.222] Build status: STARTED
[06:06:48.222] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:30:0) after 8m0s
[06:14:48.241] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:30:0)
[06:14:48.259] Build status: STARTED
[06:14:48.259] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:31:0) after 8m17s
[06:23:05.704] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:31:0)
[06:23:05.725] Build status: STARTED
[06:23:05.725] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:32:0) after 9m56s
[06:33:01.819] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:32:0)
[06:33:01.840] Build status: STARTED
[06:33:01.840] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:33:0) after 8m37s
[06:41:38.885] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:33:0)
[06:41:38.945] Build status: STARTED
[06:41:38.945] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:34:0) after 1m16s
[06:42:55.086] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:34:0)
[06:42:55.105] Build status: STARTED
[06:42:55.105] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:35:0) after 8m58s
[06:51:53.299] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:35:0)
[06:51:53.322] Build status: STARTED
[06:51:53.322] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:36:0) after 1m32s
[06:53:25.783] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:36:0)
[06:53:25.806] Build status: STARTED
[06:53:25.806] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:37:0) after 4m34s
[06:57:59.824] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:37:0)
[06:57:59.905] Build status: STARTED
[06:57:59.905] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8908893781060484880:38:0) after 9m38s
[07:06:37.797] Received PubSub notification, asking Buildbucket for the build status
[07:06:37.818] Build:
{
  "id": "8743463719155141409",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 ASAN Release (NVIDIA)"
  },
  "number": 3951,
  "createdBy": "project:chromium",
  "createTime": "2024-07-03T04:17:16.819734559Z",
  "startTime": "2024-07-03T04:18:27.127071679Z",
  "endTime": "2024-07-03T07:06:33.022823642Z",
  "updateTime": "2024-07-03T07:06:33.022823642Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "9e3e20cd4875442fc1aaab7b65604a53cab1f1a3",
      "ref": "refs/heads/main"
    }
  }
}
[07:06:37.818] Invocation finished in 2h49m22.198886292s with status FAILED