Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 2 hours
Actions:

Definition (rev cb5dce3b5608486899cb3dda521a0bf219feee99)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Win10 x64 Release (Intel UHD 770)"
>

Incoming triggers

  • Dawn Win10 x64 Builder/134275 (1 week ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Win10 x64 Builder/134276 (1 week ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "905b7f207425242704cf24e7917a8b02d237f836",
        "project": "chromium/src",
        "ref": "refs/heads/main"
      }
    ]
  },
  "branch": "refs/heads/main",
  "deps_revision_overrides": {
    "src": "905b7f207425242704cf24e7917a8b02d237f836",
    "src/third_party/dawn": "a8eb33756b228cf587ed065c0144e306def8f902"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8732007564205689329"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8732007564205689329"
    }
  ],
  "parent_build_id": "8732007564205689329",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Win10 x64 Builder",
  "parent_got_angle_revision": "d399841a9226bbe2c6ad4e0c55a9d50b25ec1728",
  "parent_got_dawn_revision": "a8eb33756b228cf587ed065c0144e306def8f902",
  "parent_got_revision": "905b7f207425242704cf24e7917a8b02d237f836",
  "parent_got_revision_cp": "refs/heads/main@{#1379000}",
  "parent_got_swiftshader_revision": "76855a9baecc97fa144ce70d7ae43a9f878e14c8",
  "parent_got_v8_revision": "9f63a260f0c63c901ac182e851113fa6374e41c7",
  "parent_got_v8_revision_cp": "refs/heads/13.2.133@{#1}",
  "parent_got_webrtc_revision": "8702008d4f5029d91b929100ee5c7ad9413306ef",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#43361}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "905b7f207425242704cf24e7917a8b02d237f836",
  "swarm_hashes": {
    "dawn_end2end_tests": "56d6c96031fde2a44b9a185504a5fea2d66accf5bc266c64963f91579562b3d1/323",
    "dawn_perf_tests": "bb15928a5c316f005d5dc3f25cb3b6b267f8fdc346630c4251276678dfaca597/487",
    "gl_tests": "61e8b9a78318bd994c84b2babacbd0adc211a96865ed9c44f6b827881f172350/323",
    "gl_unittests": "92d273b12dd0ed923f0774209e200538cd7248c795e33fbce498a1817396bfaa/402",
    "telemetry_gpu_integration_test": "6a03ab3dbc438e8d1b672878c2377ca75dfdf9f783370bff2a907a5e595395f3/815",
    "telemetry_gpu_unittests": "48ccb27f5f406942d54b68bd771c06c3e9012d56402a6916d4b4272bec043cc8/736",
    "webgpu_blink_web_tests": "31633763ccf7ce54d6a1a4d7696ff4c275ce35a1f8a3cb29c215f8ad9bac75bf/566"
  },
  "swarming_command_lines_cwd": "out\\9f0d-Dawn_Win10_x64_",
  "swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/905b7f207425242704cf24e7917a8b02d237f836
  • gitiles_ref:refs/heads/main
  • parent_buildername:Dawn Win10 x64 Builder
  • parent_buildnumber:134276
  • user_agent:recipe

Debug log (UTC)

[17:06:00.699] New invocation is queued and will start shortly
[17:06:00.699] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[17:06:01.817] Starting the invocation (attempt 1)
[17:06:01.849] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/905b7f207425242704cf24e7917a8b02d237f836"
[17:06:01.849] Popped gitiles commit info from properties and tags
[17:06:01.849] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[17:06:01.849] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[17:06:01.850] Buildbucket request:
{
  "requestId": "8897430197655273952",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 Release (Intel UHD 770)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "905b7f207425242704cf24e7917a8b02d237f836",
                        "project": "chromium/src",
                        "ref": "refs/heads/main"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "905b7f207425242704cf24e7917a8b02d237f836",
            "src/third_party/dawn": "a8eb33756b228cf587ed065c0144e306def8f902"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8732007564205689329"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8732007564205689329"
                }
          ],
      "parent_build_id": "8732007564205689329",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Win10 x64 Builder",
      "parent_got_angle_revision": "d399841a9226bbe2c6ad4e0c55a9d50b25ec1728",
      "parent_got_dawn_revision": "a8eb33756b228cf587ed065c0144e306def8f902",
      "parent_got_revision": "905b7f207425242704cf24e7917a8b02d237f836",
      "parent_got_revision_cp": "refs/heads/main@{#1379000}",
      "parent_got_swiftshader_revision": "76855a9baecc97fa144ce70d7ae43a9f878e14c8",
      "parent_got_v8_revision": "9f63a260f0c63c901ac182e851113fa6374e41c7",
      "parent_got_v8_revision_cp": "refs/heads/13.2.133@{#1}",
      "parent_got_webrtc_revision": "8702008d4f5029d91b929100ee5c7ad9413306ef",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#43361}",
      "swarm_hashes": {
            "dawn_end2end_tests": "56d6c96031fde2a44b9a185504a5fea2d66accf5bc266c64963f91579562b3d1/323",
            "dawn_perf_tests": "bb15928a5c316f005d5dc3f25cb3b6b267f8fdc346630c4251276678dfaca597/487",
            "gl_tests": "61e8b9a78318bd994c84b2babacbd0adc211a96865ed9c44f6b827881f172350/323",
            "gl_unittests": "92d273b12dd0ed923f0774209e200538cd7248c795e33fbce498a1817396bfaa/402",
            "telemetry_gpu_integration_test": "6a03ab3dbc438e8d1b672878c2377ca75dfdf9f783370bff2a907a5e595395f3/815",
            "telemetry_gpu_unittests": "48ccb27f5f406942d54b68bd771c06c3e9012d56402a6916d4b4272bec043cc8/736",
            "webgpu_blink_web_tests": "31633763ccf7ce54d6a1a4d7696ff4c275ce35a1f8a3cb29c215f8ad9bac75bf/566"
          },
      "swarming_command_lines_cwd": "out\\9f0d-Dawn_Win10_x64_",
      "swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "905b7f207425242704cf24e7917a8b02d237f836",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897430197655273952"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/Dawn Win10 x64 Release (Intel UHD 770)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Win10 x64 Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "134276"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[17:06:02.505] Scheduled build:
{
  "id": "8732000135719933873",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 Release (Intel UHD 770)"
  },
  "number": 2125,
  "createdBy": "project:chromium",
  "createTime": "2024-11-06T17:06:01.916049861Z",
  "updateTime": "2024-11-06T17:06:01.916049861Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "905b7f207425242704cf24e7917a8b02d237f836",
      "ref": "refs/heads/main"
    }
  }
}
[17:06:02.505] Task URL: https://cr-buildbucket.appspot.com/build/8732000135719933873
[17:06:02.505] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:2:0) after 8m51s
[17:06:03.554] Received PubSub notification, asking Buildbucket for the build status
[17:06:03.572] Build status: SCHEDULED
[17:07:29.928] Received PubSub notification, asking Buildbucket for the build status
[17:07:29.956] Build status: STARTED
[17:14:53.522] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:2:0)
[17:14:53.544] Build status: STARTED
[17:14:53.544] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:5:0) after 1m30s
[17:16:23.561] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:5:0)
[17:16:23.587] Build status: STARTED
[17:16:23.587] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:6:0) after 6m49s
[17:23:12.660] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:6:0)
[17:23:12.703] Build status: STARTED
[17:23:12.703] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:7:0) after 4m5s
[17:27:17.723] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:7:0)
[17:27:17.739] Build status: STARTED
[17:27:17.739] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:8:0) after 5m19s
[17:32:36.803] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:8:0)
[17:32:36.913] Build status: STARTED
[17:32:36.913] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:9:0) after 2m46s
[17:35:22.939] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:9:0)
[17:35:22.957] Build status: STARTED
[17:35:22.957] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:10:0) after 3m5s
[17:38:27.977] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:10:0)
[17:38:28.025] Build status: STARTED
[17:38:28.025] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:11:0) after 1m53s
[17:40:21.038] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:11:0)
[17:40:21.060] Build status: STARTED
[17:40:21.060] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:12:0) after 6m31s
[17:46:52.080] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:12:0)
[17:46:52.109] Build status: STARTED
[17:46:52.109] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:13:0) after 3m50s
[17:50:42.164] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:13:0)
[17:50:42.186] Build status: STARTED
[17:50:42.186] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:14:0) after 7m46s
[17:58:28.256] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:14:0)
[17:58:28.276] Build status: STARTED
[17:58:28.276] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:15:0) after 2m59s
[18:01:27.351] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:15:0)
[18:01:27.378] Build status: STARTED
[18:01:27.378] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:16:0) after 8m4s
[18:09:31.398] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:16:0)
[18:09:31.424] Build status: STARTED
[18:09:31.424] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:17:0) after 4m6s
[18:13:37.443] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:17:0)
[18:13:37.464] Build status: STARTED
[18:13:37.464] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:18:0) after 1m13s
[18:14:50.603] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:18:0)
[18:14:50.620] Build status: STARTED
[18:14:50.620] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:19:0) after 7m58s
[18:22:48.684] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:19:0)
[18:22:48.704] Build status: STARTED
[18:22:48.704] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:20:0) after 8m48s
[18:31:36.723] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:20:0)
[18:31:36.748] Build status: STARTED
[18:31:36.748] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:21:0) after 7m53s
[18:39:29.769] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:21:0)
[18:39:29.791] Build status: STARTED
[18:39:29.791] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:22:0) after 7m53s
[18:47:22.807] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:22:0)
[18:47:22.828] Build status: STARTED
[18:47:22.828] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:23:0) after 1m43s
[18:49:05.815] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:23:0)
[18:49:05.845] Build status: STARTED
[18:49:05.845] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:24:0) after 7m43s
[18:56:48.863] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:24:0)
[18:56:48.892] Build status: STARTED
[18:56:48.892] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:25:0) after 3m23s
[19:00:11.912] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:25:0)
[19:00:11.940] Build status: STARTED
[19:00:11.940] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:26:0) after 4m18s
[19:04:29.938] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:26:0)
[19:04:29.961] Build status: STARTED
[19:04:29.961] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:27:0) after 3m33s
[19:08:02.978] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:27:0)
[19:08:02.996] Build status: STARTED
[19:08:02.997] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:28:0) after 7m59s
[19:16:02.058] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:28:0)
[19:16:02.079] Build status: STARTED
[19:16:02.079] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:29:0) after 9m38s
[19:25:40.146] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:29:0)
[19:25:40.173] Build status: STARTED
[19:25:40.173] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:30:0) after 4m54s
[19:30:34.194] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:30:0)
[19:30:34.217] Build status: STARTED
[19:30:34.218] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:31:0) after 7m59s
[19:38:33.241] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:31:0)
[19:38:33.266] Build status: STARTED
[19:38:33.266] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:32:0) after 2m15s
[19:40:48.387] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:32:0)
[19:40:48.408] Build status: STARTED
[19:40:48.408] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:33:0) after 1m49s
[19:42:37.526] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:33:0)
[19:42:37.545] Build status: STARTED
[19:42:37.545] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:34:0) after 4m38s
[19:47:15.606] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:34:0)
[19:47:15.625] Build status: STARTED
[19:47:15.625] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:35:0) after 6m56s
[19:54:11.628] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:35:0)
[19:54:11.644] Build status: STARTED
[19:54:11.644] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:36:0) after 1m53s
[19:56:04.670] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:36:0)
[19:56:04.688] Build status: STARTED
[19:56:04.688] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:37:0) after 5m10s
[20:01:14.705] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:37:0)
[20:01:14.729] Build status: STARTED
[20:01:14.729] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (Intel UHD 770):8897430197655273952:38:0) after 8m22s
[20:01:23.810] Received PubSub notification, asking Buildbucket for the build status
[20:01:23.828] Build:
{
  "id": "8732000135719933873",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 Release (Intel UHD 770)"
  },
  "number": 2125,
  "createdBy": "project:chromium",
  "createTime": "2024-11-06T17:06:01.916049861Z",
  "startTime": "2024-11-06T17:07:29.713204768Z",
  "endTime": "2024-11-06T20:01:17.903337682Z",
  "updateTime": "2024-11-06T20:01:17.903337682Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "905b7f207425242704cf24e7917a8b02d237f836",
      "ref": "refs/heads/main"
    }
  }
}
[20:01:23.828] Invocation finished in 2h55m23.142214139s with status FAILED