Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 2 hours
Actions:

Definition (rev 1f1b4513521524d405025696b8777ff82db0bd9d)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Win10 x64 ASAN Release (Intel)"
>

Incoming triggers

  • Dawn Win10 x64 ASAN Builder/296 (2 years ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Win10 x64 ASAN Builder/297 (2 years ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Win10 x64 ASAN Builder/298 (2 years ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "bef35c27336173917ae56d757c4f6546ef18f6e4",
        "project": "chromium/src",
        "ref": "refs/heads/main"
      }
    ]
  },
  "branch": "refs/heads/main",
  "deps_revision_overrides": {
    "src": "bef35c27336173917ae56d757c4f6546ef18f6e4",
    "src/third_party/dawn": "015fbe706cc4b865994106353aaf15e2b43fdfb7"
  },
  "parent_build_id": "8773242173545243713",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Win10 x64 ASAN Builder",
  "parent_got_angle_revision": "0581a81c8f3ee9409c5564352246cb4ddf18e0a1",
  "parent_got_dawn_revision": "015fbe706cc4b865994106353aaf15e2b43fdfb7",
  "parent_got_nacl_revision": "f9cb132c419b9e019853103e88c518026b9f5083",
  "parent_got_revision": "bef35c27336173917ae56d757c4f6546ef18f6e4",
  "parent_got_revision_cp": "refs/heads/main@{#1181397}",
  "parent_got_swiftshader_revision": "729e92f8ae07d7b695bdcf346318dec4d11d899e",
  "parent_got_v8_revision": "fe608691065e23ae83720392bf61a59e8166bee6",
  "parent_got_v8_revision_cp": "refs/heads/11.7.439@{#1}",
  "parent_got_webrtc_revision": "82e5f91a2bdf955aa870142008fbdc9ac12f6acd",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#40524}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "bef35c27336173917ae56d757c4f6546ef18f6e4",
  "swarm_hashes": {
    "dawn_end2end_tests": "f2fb49033378e36da15660ec24463fead686d61f7d51d72d09c8d9e9e7cd0ab8/486",
    "dawn_perf_tests": "7a8f007d0c80e9ae4b20b734abb15afe5bd31e87469b88b15770fa4e09249549/488",
    "gl_tests": "b721ce3ee88766034b6445e1f6b78fbfd30d9b82f8cfe96195093d5c30010e9d/486",
    "gl_unittests": "f765e75b65e439e8c04a9898f2e8ca34e0e68e84ffbe038e6cc423a7db669f50/487",
    "telemetry_gpu_integration_test": "621400ddb535c8a8ff88c2ac0b58c436947601222fd065cfacc6de84644479f7/734",
    "telemetry_gpu_unittests": "fb9740da5877e9d5a5e1fe23d9d9179617fd5977c855f848ce2700c0c5e57677/655",
    "webgpu_blink_web_tests": "03813942e7cbc76c92ea481d75aac3f16e444e59bfcb2c8bd2f570f684e7366c/566"
  },
  "swarming_command_lines_cwd": "out\\Release_x64",
  "swarming_command_lines_digest": "eca0b9676d87a6ce8af3ba81b10e6d5bd10fc75c351264a541e8d91d0e717a42/93",
  "swarming_inverted_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/bef35c27336173917ae56d757c4f6546ef18f6e4
  • gitiles_ref:refs/heads/main
  • parent_buildername:Dawn Win10 x64 ASAN Builder
  • parent_buildnumber:298
  • user_agent:recipe

Debug log (UTC)

[13:19:48.224] New invocation is queued and will start shortly
[13:19:48.224] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[13:19:49.476] Starting the invocation (attempt 1)
[13:19:49.505] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/bef35c27336173917ae56d757c4f6546ef18f6e4"
[13:19:49.505] Popped gitiles commit info from properties and tags
[13:19:49.505] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[13:19:49.505] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[13:19:49.505] Buildbucket request:
{
  "requestId": "8938666049134791248",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 ASAN Release (Intel)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "bef35c27336173917ae56d757c4f6546ef18f6e4",
                        "project": "chromium/src",
                        "ref": "refs/heads/main"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "bef35c27336173917ae56d757c4f6546ef18f6e4",
            "src/third_party/dawn": "015fbe706cc4b865994106353aaf15e2b43fdfb7"
          },
      "parent_build_id": "8773242173545243713",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Win10 x64 ASAN Builder",
      "parent_got_angle_revision": "0581a81c8f3ee9409c5564352246cb4ddf18e0a1",
      "parent_got_dawn_revision": "015fbe706cc4b865994106353aaf15e2b43fdfb7",
      "parent_got_nacl_revision": "f9cb132c419b9e019853103e88c518026b9f5083",
      "parent_got_revision": "bef35c27336173917ae56d757c4f6546ef18f6e4",
      "parent_got_revision_cp": "refs/heads/main@{#1181397}",
      "parent_got_swiftshader_revision": "729e92f8ae07d7b695bdcf346318dec4d11d899e",
      "parent_got_v8_revision": "fe608691065e23ae83720392bf61a59e8166bee6",
      "parent_got_v8_revision_cp": "refs/heads/11.7.439@{#1}",
      "parent_got_webrtc_revision": "82e5f91a2bdf955aa870142008fbdc9ac12f6acd",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#40524}",
      "swarm_hashes": {
            "dawn_end2end_tests": "f2fb49033378e36da15660ec24463fead686d61f7d51d72d09c8d9e9e7cd0ab8/486",
            "dawn_perf_tests": "7a8f007d0c80e9ae4b20b734abb15afe5bd31e87469b88b15770fa4e09249549/488",
            "gl_tests": "b721ce3ee88766034b6445e1f6b78fbfd30d9b82f8cfe96195093d5c30010e9d/486",
            "gl_unittests": "f765e75b65e439e8c04a9898f2e8ca34e0e68e84ffbe038e6cc423a7db669f50/487",
            "telemetry_gpu_integration_test": "621400ddb535c8a8ff88c2ac0b58c436947601222fd065cfacc6de84644479f7/734",
            "telemetry_gpu_unittests": "fb9740da5877e9d5a5e1fe23d9d9179617fd5977c855f848ce2700c0c5e57677/655",
            "webgpu_blink_web_tests": "03813942e7cbc76c92ea481d75aac3f16e444e59bfcb2c8bd2f570f684e7366c/566"
          },
      "swarming_command_lines_cwd": "out\\Release_x64",
      "swarming_command_lines_digest": "eca0b9676d87a6ce8af3ba81b10e6d5bd10fc75c351264a541e8d91d0e717a42/93",
      "swarming_inverted_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "bef35c27336173917ae56d757c4f6546ef18f6e4",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8938666049134791248"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/Dawn Win10 x64 ASAN Release (Intel)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Win10 x64 ASAN Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "298"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[13:19:51.481] Scheduled build:
{
  "id": "8773235987068527921",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 ASAN Release (Intel)"
  },
  "number": 140,
  "createdBy": "project:chromium",
  "createTime": "2023-08-09T13:19:49.569055879Z",
  "updateTime": "2023-08-09T13:19:49.569055879Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "bef35c27336173917ae56d757c4f6546ef18f6e4",
      "ref": "refs/heads/main"
    }
  }
}
[13:19:51.481] Task URL: https://cr-buildbucket.appspot.com/build/8773235987068527921
[13:19:51.481] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:2:0) after 3m58s
[13:19:51.764] Received PubSub notification, asking Buildbucket for the build status
[13:19:51.790] Build status: SCHEDULED
[13:22:15.851] Received PubSub notification, asking Buildbucket for the build status
[13:22:15.993] Build status: STARTED
[13:23:49.608] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:2:0)
[13:23:49.650] Build status: STARTED
[13:23:49.650] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:5:0) after 1m1s
[13:24:50.673] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:5:0)
[13:24:50.694] Build status: STARTED
[13:24:50.694] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:6:0) after 2m38s
[13:27:28.723] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:6:0)
[13:27:28.748] Build status: STARTED
[13:27:28.748] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:7:0) after 3m53s
[13:31:21.745] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:7:0)
[13:31:21.763] Build status: STARTED
[13:31:21.763] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:8:0) after 4m53s
[13:36:15.131] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:8:0)
[13:36:15.150] Build status: STARTED
[13:36:15.150] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:9:0) after 2m56s
[13:39:11.301] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:9:0)
[13:39:11.318] Build status: STARTED
[13:39:11.318] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:10:0) after 1m18s
[13:40:29.345] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:10:0)
[13:40:29.471] Build status: STARTED
[13:40:29.471] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:11:0) after 1m45s
[13:42:14.556] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:11:0)
[13:42:14.575] Build status: STARTED
[13:42:14.575] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:12:0) after 6m10s
[13:48:24.705] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:12:0)
[13:48:24.724] Build status: STARTED
[13:48:24.724] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:13:0) after 6m37s
[13:55:01.761] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:13:0)
[13:55:01.786] Build status: STARTED
[13:55:01.786] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:14:0) after 7m16s
[14:02:17.830] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:14:0)
[14:02:17.854] Build status: STARTED
[14:02:17.854] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:15:0) after 7m49s
[14:10:06.925] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:15:0)
[14:10:06.954] Build status: STARTED
[14:10:06.954] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:16:0) after 3m18s
[14:13:25.012] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:16:0)
[14:13:25.041] Build status: STARTED
[14:13:25.041] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:17:0) after 4m20s
[14:17:45.270] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:17:0)
[14:17:45.287] Build status: STARTED
[14:17:45.287] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:18:0) after 2m40s
[14:20:25.304] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:18:0)
[14:20:25.323] Build status: STARTED
[14:20:25.323] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:19:0) after 9m33s
[14:29:58.391] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:19:0)
[14:29:58.410] Build status: STARTED
[14:29:58.410] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:20:0) after 2m27s
[14:32:25.441] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:20:0)
[14:32:25.460] Build status: STARTED
[14:32:25.460] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:21:0) after 4m55s
[14:37:20.853] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:21:0)
[14:37:20.953] Build status: STARTED
[14:37:20.953] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:22:0) after 6m48s
[14:44:09.158] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:22:0)
[14:44:09.177] Build status: STARTED
[14:44:09.177] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:23:0) after 8m14s
[14:52:23.411] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:23:0)
[14:52:23.438] Build status: STARTED
[14:52:23.438] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:24:0) after 5m28s
[14:57:51.456] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:24:0)
[14:57:51.531] Build status: STARTED
[14:57:51.531] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:25:0) after 5m40s
[15:03:31.857] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:25:0)
[15:03:31.881] Build status: STARTED
[15:03:31.881] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:26:0) after 7m53s
[15:11:25.086] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:26:0)
[15:11:25.173] Build status: STARTED
[15:11:25.173] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:27:0) after 6m6s
[15:17:31.256] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:27:0)
[15:17:31.284] Build status: STARTED
[15:17:31.284] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:28:0) after 5m12s
[15:22:43.300] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:28:0)
[15:22:43.320] Build status: STARTED
[15:22:43.320] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:29:0) after 6m25s
[15:29:08.350] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:29:0)
[15:29:08.373] Build status: STARTED
[15:29:08.373] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:30:0) after 3m14s
[15:32:22.368] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:30:0)
[15:32:22.400] Build status: STARTED
[15:32:22.400] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:31:0) after 8m58s
[15:41:20.521] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:31:0)
[15:41:20.545] Build status: STARTED
[15:41:20.545] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:32:0) after 5m53s
[15:47:13.619] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:32:0)
[15:47:13.640] Build status: STARTED
[15:47:13.640] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:33:0) after 1m12s
[15:48:26.011] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:33:0)
[15:48:26.028] Build status: STARTED
[15:48:26.028] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:34:0) after 2m52s
[15:51:18.442] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:34:0)
[15:51:18.465] Build status: STARTED
[15:51:18.465] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:35:0) after 7m24s
[15:58:42.493] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:35:0)
[15:58:42.514] Build status: STARTED
[15:58:42.514] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:36:0) after 3m53s
[16:02:35.572] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:36:0)
[16:02:35.597] Build status: STARTED
[16:02:35.597] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8938666049134791248:37:0) after 4m34s
[16:04:04.520] Received PubSub notification, asking Buildbucket for the build status
[16:04:04.544] Build:
{
  "id": "8773235987068527921",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 ASAN Release (Intel)"
  },
  "number": 140,
  "createdBy": "project:chromium",
  "createTime": "2023-08-09T13:19:49.569055879Z",
  "startTime": "2023-08-09T13:22:15.187299Z",
  "endTime": "2023-08-09T16:04:03.942903735Z",
  "updateTime": "2023-08-09T16:04:03.942903735Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "bef35c27336173917ae56d757c4f6546ef18f6e4",
      "ref": "refs/heads/main"
    }
  }
}
[16:04:04.544] Invocation finished in 2h44m16.329583511s with status FAILED