LUCI Scheduler :: chromium :: Dawn Win10 x64 Release (NVIDIA)

Status: FAILED

Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com

Duration: 2 hours

Actions:

Definition (rev 36bc28844ffd5e030af6e41e2e0e4ddc5c92b699)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Win10 x64 Release (NVIDIA)"
>

Incoming triggers

Dawn Win10 x64 Builder/145434 (2 days ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
Dawn Win10 x64 Builder/145435 (2 days ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
Dawn Win10 x64 Builder/145436 (2 days ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
Dawn Win10 x64 Builder/145437 (2 days ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "fcc1d44337b43a81c2e5df685f814466e6ce07b4",
        "project": "chromium/src",
        "ref": "refs/heads/main"
      }
    ]
  },
  "branch": "refs/heads/main",
  "deps_revision_overrides": {
    "src": "fcc1d44337b43a81c2e5df685f814466e6ce07b4",
    "src/third_party/dawn": "bec13072f0cf66a8ba39de67806d61c5ce236e42"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8700816581408128721"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8700816581408128721"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8700816581408128721"
    }
  ],
  "parent_build_id": "8700816581408128721",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Win10 x64 Builder",
  "parent_got_angle_revision": "7f5958c7e1790088efbe31e82695a005f176e7f0",
  "parent_got_dawn_revision": "bec13072f0cf66a8ba39de67806d61c5ce236e42",
  "parent_got_revision": "fcc1d44337b43a81c2e5df685f814466e6ce07b4",
  "parent_got_revision_cp": "refs/heads/main@{#1531090}",
  "parent_got_swiftshader_revision": "3d536c0fc62b1cdea0f78c3c38d79be559855b88",
  "parent_got_v8_revision": "830cbc691fd164add58264ddf56d1a473a1e04f4",
  "parent_got_v8_revision_cp": "refs/heads/14.3.96@{#1}",
  "parent_got_webrtc_revision": "a10265058d96e2e17ef796ac74c0efb9a32f236d",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#45956}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "fcc1d44337b43a81c2e5df685f814466e6ce07b4",
  "swarm_hashes": {
    "dawn_end2end_tests": "c70246db8b53f2e24cc5aa6f46dc0d40589f0fcec6ae6c138ad917f33a9e71a5/323",
    "dawn_perf_tests": "ee1690f74ef4ee09310318e31466641b8cfa88d100279145c975ef0669d121b5/487",
    "gl_tests": "8bfca0b49e73ebbb9e0ea95251c2fc38f434c735e8f01dd49286fee0c090f927/323",
    "gl_unittests": "f8ee6fcf077f9ed6f652ffe5a7accd2cf4851c9f0e0ed16a1985da4fbf709489/402",
    "telemetry_gpu_integration_test": "0891ede1dba84171fe1cb626cc66af0e61a0f53921ca9aaac58bfc019bc33ec5/815",
    "telemetry_gpu_unittests": "7b4bf2bdb8c5510dab2a5cc9880711b19d5b20454492cbf3518d064212b347b2/736",
    "webgpu_blink_web_tests": "c90594f0287a2141e34224c806be1eb7882b96a7305dc4da8701bc742fc00af0/566"
  },
  "swarming_command_lines_cwd": "out\\9f0d-Dawn_Win10_x64_",
  "swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/fcc1d44337b43a81c2e5df685f814466e6ce07b4
gitiles_ref:refs/heads/main
parent_buildername:Dawn Win10 x64 Builder
parent_buildnumber:145437
user_agent:recipe

Debug log (UTC)

[22:39:06.613] New invocation is queued and will start shortly
[22:39:06.613] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[22:39:07.815] Starting the invocation (attempt 1)
[22:39:07.841] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/fcc1d44337b43a81c2e5df685f814466e6ce07b4"
[22:39:07.841] Popped gitiles commit info from properties and tags
[22:39:07.841] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[22:39:07.842] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[22:39:07.842] Buildbucket request:
{
  "requestId": "8866243884463074448",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 Release (NVIDIA)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "fcc1d44337b43a81c2e5df685f814466e6ce07b4",
                        "project": "chromium/src",
                        "ref": "refs/heads/main"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "fcc1d44337b43a81c2e5df685f814466e6ce07b4",
            "src/third_party/dawn": "bec13072f0cf66a8ba39de67806d61c5ce236e42"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8700816581408128721"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8700816581408128721"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8700816581408128721"
                }
          ],
      "parent_build_id": "8700816581408128721",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Win10 x64 Builder",
      "parent_got_angle_revision": "7f5958c7e1790088efbe31e82695a005f176e7f0",
      "parent_got_dawn_revision": "bec13072f0cf66a8ba39de67806d61c5ce236e42",
      "parent_got_revision": "fcc1d44337b43a81c2e5df685f814466e6ce07b4",
      "parent_got_revision_cp": "refs/heads/main@{#1531090}",
      "parent_got_swiftshader_revision": "3d536c0fc62b1cdea0f78c3c38d79be559855b88",
      "parent_got_v8_revision": "830cbc691fd164add58264ddf56d1a473a1e04f4",
      "parent_got_v8_revision_cp": "refs/heads/14.3.96@{#1}",
      "parent_got_webrtc_revision": "a10265058d96e2e17ef796ac74c0efb9a32f236d",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#45956}",
      "swarm_hashes": {
            "dawn_end2end_tests": "c70246db8b53f2e24cc5aa6f46dc0d40589f0fcec6ae6c138ad917f33a9e71a5/323",
            "dawn_perf_tests": "ee1690f74ef4ee09310318e31466641b8cfa88d100279145c975ef0669d121b5/487",
            "gl_tests": "8bfca0b49e73ebbb9e0ea95251c2fc38f434c735e8f01dd49286fee0c090f927/323",
            "gl_unittests": "f8ee6fcf077f9ed6f652ffe5a7accd2cf4851c9f0e0ed16a1985da4fbf709489/402",
            "telemetry_gpu_integration_test": "0891ede1dba84171fe1cb626cc66af0e61a0f53921ca9aaac58bfc019bc33ec5/815",
            "telemetry_gpu_unittests": "7b4bf2bdb8c5510dab2a5cc9880711b19d5b20454492cbf3518d064212b347b2/736",
            "webgpu_blink_web_tests": "c90594f0287a2141e34224c806be1eb7882b96a7305dc4da8701bc742fc00af0/566"
          },
      "swarming_command_lines_cwd": "out\\9f0d-Dawn_Win10_x64_",
      "swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "fcc1d44337b43a81c2e5df685f814466e6ce07b4",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8866243884463074448"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/Dawn Win10 x64 Release (NVIDIA)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Win10 x64 Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "145437"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[22:39:08.408] Scheduled build:
{
  "id": "8700813822448810481",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 Release (NVIDIA)"
  },
  "number": 88424,
  "createdBy": "project:chromium",
  "createTime": "2025-10-16T22:39:07.906506511Z",
  "updateTime": "2025-10-16T22:39:07.906506511Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "fcc1d44337b43a81c2e5df685f814466e6ce07b4",
      "ref": "refs/heads/main"
    }
  }
}
[22:39:08.408] Task URL: https://cr-buildbucket.appspot.com/build/8700813822448810481
[22:39:08.408] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:2:0) after 5m37s
[22:39:09.414] Received PubSub notification, asking Buildbucket for the build status
[22:39:09.435] Build status: SCHEDULED
[22:39:16.813] Received PubSub notification, asking Buildbucket for the build status
[22:39:16.840] Build status: STARTED
[22:44:45.435] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:2:0)
[22:44:45.455] Build status: STARTED
[22:44:45.455] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:5:0) after 7m2s
[22:51:47.518] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:5:0)
[22:51:47.537] Build status: STARTED
[22:51:47.537] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:6:0) after 9m10s
[23:00:57.703] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:6:0)
[23:00:57.717] Build status: STARTED
[23:00:57.717] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:7:0) after 2m26s
[23:03:23.731] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:7:0)
[23:03:23.749] Build status: STARTED
[23:03:23.749] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:8:0) after 2m23s
[23:05:46.766] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:8:0)
[23:05:46.783] Build status: STARTED
[23:05:46.783] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:9:0) after 6m35s
[23:12:21.850] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:9:0)
[23:12:21.881] Build status: STARTED
[23:12:21.881] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:10:0) after 8m54s
[23:21:15.956] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:10:0)
[23:21:15.973] Build status: STARTED
[23:21:15.973] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:11:0) after 2m53s
[23:24:08.979] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:11:0)
[23:24:08.998] Build status: STARTED
[23:24:08.998] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:12:0) after 9m38s
[23:33:47.017] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:12:0)
[23:33:47.050] Build status: STARTED
[23:33:47.050] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:13:0) after 9m34s
[23:43:21.088] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:13:0)
[23:43:21.113] Build status: STARTED
[23:43:21.113] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:14:0) after 8m47s
[23:52:08.129] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:14:0)
[23:52:08.148] Build status: STARTED
[23:52:08.148] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:15:0) after 4m22s
[23:56:30.214] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:15:0)
[23:56:30.255] Build status: STARTED
[23:56:30.255] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:16:0) after 3m3s
[23:59:33.273] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:16:0)
[23:59:33.304] Build status: STARTED
[23:59:33.304] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:17:0) after 3m19s
[00:02:52.321] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:17:0)
[00:02:52.338] Build status: STARTED
[00:02:52.338] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:18:0) after 2m14s
[00:05:06.329] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:18:0)
[00:05:06.345] Build status: STARTED
[00:05:06.345] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:19:0) after 6m57s
[00:12:03.502] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:19:0)
[00:12:03.520] Build status: STARTED
[00:12:03.520] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:20:0) after 4m11s
[00:16:14.533] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:20:0)
[00:16:14.571] Build status: STARTED
[00:16:14.571] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:21:0) after 8m52s
[00:25:06.589] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:21:0)
[00:25:06.657] Build status: STARTED
[00:25:06.657] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:22:0) after 3m57s
[00:29:03.707] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:22:0)
[00:29:03.724] Build status: STARTED
[00:29:03.724] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:23:0) after 5m50s
[00:34:53.765] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:23:0)
[00:34:53.830] Build status: STARTED
[00:34:53.830] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:24:0) after 5m46s
[00:40:39.848] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:24:0)
[00:40:39.864] Build status: STARTED
[00:40:39.864] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:25:0) after 5m13s
[00:45:52.962] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:25:0)
[00:45:52.978] Build status: STARTED
[00:45:52.978] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:26:0) after 1m24s
[00:47:17.043] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:26:0)
[00:47:17.058] Build status: STARTED
[00:47:17.058] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:27:0) after 8m7s
[00:55:24.083] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:27:0)
[00:55:24.163] Build status: STARTED
[00:55:24.164] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:28:0) after 7m38s
[01:03:02.172] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:28:0)
[01:03:02.209] Build status: STARTED
[01:03:02.209] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:29:0) after 3m59s
[01:07:01.224] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:29:0)
[01:07:01.249] Build status: STARTED
[01:07:01.249] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:30:0) after 5m31s
[01:12:32.265] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:30:0)
[01:12:32.289] Build status: STARTED
[01:12:32.289] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:31:0) after 4m46s
[01:17:18.355] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:31:0)
[01:17:18.370] Build status: STARTED
[01:17:18.370] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:32:0) after 1m16s
[01:18:34.507] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:32:0)
[01:18:34.534] Build status: STARTED
[01:18:34.534] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:33:0) after 5m12s
[01:23:46.578] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:33:0)
[01:23:46.597] Build status: STARTED
[01:23:46.597] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:34:0) after 6m4s
[01:29:50.618] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:34:0)
[01:29:50.635] Build status: STARTED
[01:29:50.635] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:35:0) after 1m24s
[01:31:14.749] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:35:0)
[01:31:14.804] Build status: STARTED
[01:31:14.804] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Release (NVIDIA):8866243884463074448:36:0) after 7m8s
[01:32:34.618] Received PubSub notification, asking Buildbucket for the build status
[01:32:34.642] Build:
{
  "id": "8700813822448810481",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 Release (NVIDIA)"
  },
  "number": 88424,
  "createdBy": "project:chromium",
  "createTime": "2025-10-16T22:39:07.906506511Z",
  "startTime": "2025-10-16T22:39:16.648977685Z",
  "endTime": "2025-10-17T01:32:34.032610232Z",
  "updateTime": "2025-10-17T01:32:34.032610232Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "fcc1d44337b43a81c2e5df685f814466e6ce07b4",
      "ref": "refs/heads/main"
    }
  }
}
[01:32:34.642] Invocation finished in 2h53m28.041141743s with status FAILED