Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 3 hours
Actions:

Definition (rev f48072dfb60d8fe758af47cc9af397252e2280e7)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Win10 x64 Experimental Release (NVIDIA)"
>

Incoming triggers

  • Dawn Win10 x64 Builder/124165 (7 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Win10 x64 Builder/124166 (7 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Win10 x64 Builder/124167 (7 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Win10 x64 Builder/124168 (7 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Win10 x64 Builder/124169 (7 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "b907e47bd88ee4d1a8869392e84173ad8a96a94a",
        "project": "chromium/src",
        "ref": "refs/heads/main"
      }
    ]
  },
  "branch": "refs/heads/main",
  "deps_revision_overrides": {
    "src": "b907e47bd88ee4d1a8869392e84173ad8a96a94a",
    "src/third_party/dawn": "3d6882ad8522501fadc0515b4669ece6773520af"
  },
  "parent_build_id": "8756069501556860161",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Win10 x64 Builder",
  "parent_got_angle_revision": "2cae27c296a8314f728132839b52f22441e52fa5",
  "parent_got_dawn_revision": "3d6882ad8522501fadc0515b4669ece6773520af",
  "parent_got_revision": "b907e47bd88ee4d1a8869392e84173ad8a96a94a",
  "parent_got_revision_cp": "refs/heads/main@{#1260821}",
  "parent_got_swiftshader_revision": "eb75201a4e0354a36d315dd01077092ec9aa2356",
  "parent_got_v8_revision": "08294ff5efd6e8902f1988d405fa9501e496ee8d",
  "parent_got_v8_revision_cp": "refs/heads/12.3.200@{#1}",
  "parent_got_webrtc_revision": "23c32da48ae3f5f146dbd60b8b7fe4416f3750b4",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#41735}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "b907e47bd88ee4d1a8869392e84173ad8a96a94a",
  "swarm_hashes": {
    "dawn_end2end_tests": "435015407614bfe4d747d863d41d5de1e1eb7d5da20862108c84b2ab08e00809/322",
    "dawn_perf_tests": "49567c53734b4a8620e9e17d794f4ef1b00f70e28c65ed9bb1f72c29ac6c1561/486",
    "gl_tests": "84351fbdd2cf39da728d01ede52dc9ac7b38c3459bec3b80a65e1e887010388b/322",
    "gl_unittests": "b10f66d3149f7ee7c90568f55a3c000c546c8dff434638614f4bbcda325b123e/401",
    "telemetry_gpu_integration_test": "18c34df44019d80eff9867d9be7f22137126a39ed43f4e36e324b7904b8c697f/814",
    "telemetry_gpu_unittests": "f53d6c9ab04ddc8daeb2c1574a3279d86e784317b6e2c0befa9c92e83e9c3345/735",
    "webgpu_blink_web_tests": "079e94eea89f879ddd9a149b6804e7ad9fb79ebe9260a72de5ab2e22c8dd1238/565"
  },
  "swarming_command_lines_cwd": "out\\Release_x64",
  "swarming_command_lines_digest": "40420e5889a0e7d57b0c3d6e43ef878629908374a8dee055acb98f5dece44ba4/93"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/b907e47bd88ee4d1a8869392e84173ad8a96a94a
  • gitiles_ref:refs/heads/main
  • parent_buildername:Dawn Win10 x64 Builder
  • parent_buildnumber:124169
  • user_agent:recipe

Debug log (UTC)

[01:32:59.195] New invocation is queued and will start shortly
[01:32:59.195] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[01:33:00.356] Starting the invocation (attempt 1)
[01:33:00.392] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/b907e47bd88ee4d1a8869392e84173ad8a96a94a"
[01:33:00.392] Popped gitiles commit info from properties and tags
[01:33:00.392] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[01:33:00.392] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[01:33:00.392] Buildbucket request:
{
  "requestId": "8921497094606844144",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 Experimental Release (NVIDIA)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "b907e47bd88ee4d1a8869392e84173ad8a96a94a",
                        "project": "chromium/src",
                        "ref": "refs/heads/main"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "b907e47bd88ee4d1a8869392e84173ad8a96a94a",
            "src/third_party/dawn": "3d6882ad8522501fadc0515b4669ece6773520af"
          },
      "parent_build_id": "8756069501556860161",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Win10 x64 Builder",
      "parent_got_angle_revision": "2cae27c296a8314f728132839b52f22441e52fa5",
      "parent_got_dawn_revision": "3d6882ad8522501fadc0515b4669ece6773520af",
      "parent_got_revision": "b907e47bd88ee4d1a8869392e84173ad8a96a94a",
      "parent_got_revision_cp": "refs/heads/main@{#1260821}",
      "parent_got_swiftshader_revision": "eb75201a4e0354a36d315dd01077092ec9aa2356",
      "parent_got_v8_revision": "08294ff5efd6e8902f1988d405fa9501e496ee8d",
      "parent_got_v8_revision_cp": "refs/heads/12.3.200@{#1}",
      "parent_got_webrtc_revision": "23c32da48ae3f5f146dbd60b8b7fe4416f3750b4",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#41735}",
      "swarm_hashes": {
            "dawn_end2end_tests": "435015407614bfe4d747d863d41d5de1e1eb7d5da20862108c84b2ab08e00809/322",
            "dawn_perf_tests": "49567c53734b4a8620e9e17d794f4ef1b00f70e28c65ed9bb1f72c29ac6c1561/486",
            "gl_tests": "84351fbdd2cf39da728d01ede52dc9ac7b38c3459bec3b80a65e1e887010388b/322",
            "gl_unittests": "b10f66d3149f7ee7c90568f55a3c000c546c8dff434638614f4bbcda325b123e/401",
            "telemetry_gpu_integration_test": "18c34df44019d80eff9867d9be7f22137126a39ed43f4e36e324b7904b8c697f/814",
            "telemetry_gpu_unittests": "f53d6c9ab04ddc8daeb2c1574a3279d86e784317b6e2c0befa9c92e83e9c3345/735",
            "webgpu_blink_web_tests": "079e94eea89f879ddd9a149b6804e7ad9fb79ebe9260a72de5ab2e22c8dd1238/565"
          },
      "swarming_command_lines_cwd": "out\\Release_x64",
      "swarming_command_lines_digest": "40420e5889a0e7d57b0c3d6e43ef878629908374a8dee055acb98f5dece44ba4/93"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "b907e47bd88ee4d1a8869392e84173ad8a96a94a",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8921497094606844144"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/Dawn Win10 x64 Experimental Release (NVIDIA)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Win10 x64 Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "124169"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[01:33:01.052] Scheduled build:
{
  "id": "8756067032586473649",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 Experimental Release (NVIDIA)"
  },
  "number": 96,
  "createdBy": "project:chromium",
  "createTime": "2024-02-15T01:33:00.498688994Z",
  "updateTime": "2024-02-15T01:33:00.498688994Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "b907e47bd88ee4d1a8869392e84173ad8a96a94a",
      "ref": "refs/heads/main"
    }
  }
}
[01:33:01.052] Task URL: https://cr-buildbucket.appspot.com/build/8756067032586473649
[01:33:01.052] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:2:0) after 1m37s
[01:33:02.006] Received PubSub notification, asking Buildbucket for the build status
[01:33:02.107] Build status: STARTED
[01:33:03.201] Received PubSub notification, asking Buildbucket for the build status
[01:33:03.231] Build status: STARTED
[01:34:38.050] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:2:0)
[01:34:38.085] Build status: STARTED
[01:34:38.085] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:5:0) after 6m22s
[01:41:00.102] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:5:0)
[01:41:00.177] Build status: STARTED
[01:41:00.177] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:6:0) after 8m16s
[01:49:16.154] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:6:0)
[01:49:16.192] Build status: STARTED
[01:49:16.192] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:7:0) after 5m11s
[01:54:27.217] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:7:0)
[01:54:27.242] Build status: STARTED
[01:54:27.242] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:8:0) after 9m9s
[02:03:36.296] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:8:0)
[02:03:36.347] Build status: STARTED
[02:03:36.347] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:9:0) after 1m53s
[02:05:29.351] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:9:0)
[02:05:29.397] Build status: STARTED
[02:05:29.397] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:10:0) after 4m7s
[02:09:36.416] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:10:0)
[02:09:36.479] Build status: STARTED
[02:09:36.479] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:11:0) after 1m7s
[02:10:43.521] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:11:0)
[02:10:43.548] Build status: STARTED
[02:10:43.548] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:12:0) after 3m50s
[02:14:33.624] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:12:0)
[02:14:33.647] Build status: STARTED
[02:14:33.647] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:13:0) after 6m8s
[02:20:41.669] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:13:0)
[02:20:41.687] Build status: STARTED
[02:20:41.687] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:14:0) after 8m59s
[02:29:40.665] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:14:0)
[02:29:40.691] Build status: STARTED
[02:29:40.692] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:15:0) after 3m49s
[02:33:29.709] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:15:0)
[02:33:29.727] Build status: STARTED
[02:33:29.727] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:16:0) after 1m40s
[02:35:09.753] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:16:0)
[02:35:09.907] Build status: STARTED
[02:35:09.907] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:17:0) after 5m17s
[02:40:26.938] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:17:0)
[02:40:26.987] Build status: STARTED
[02:40:26.987] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:18:0) after 3m42s
[02:44:09.318] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:18:0)
[02:44:09.339] Build status: STARTED
[02:44:09.339] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:19:0) after 9m58s
[02:54:07.508] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:19:0)
[02:54:07.667] Build status: STARTED
[02:54:07.667] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:20:0) after 9m47s
[03:03:54.684] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:20:0)
[03:03:54.709] Build status: STARTED
[03:03:54.709] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:21:0) after 2m49s
[03:06:43.737] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:21:0)
[03:06:43.760] Build status: STARTED
[03:06:43.760] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:22:0) after 6m6s
[03:12:49.953] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:22:0)
[03:12:49.971] Build status: STARTED
[03:12:49.971] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:23:0) after 2m45s
[03:15:35.535] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:23:0)
[03:15:35.578] Build status: STARTED
[03:15:35.578] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:24:0) after 8m44s
[03:24:19.564] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:24:0)
[03:24:19.581] Build status: STARTED
[03:24:19.581] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:25:0) after 4m7s
[03:28:26.704] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:25:0)
[03:28:26.724] Build status: STARTED
[03:28:26.724] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:26:0) after 7m28s
[03:35:54.842] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:26:0)
[03:35:54.859] Build status: STARTED
[03:35:54.859] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:27:0) after 7m55s
[03:43:49.857] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:27:0)
[03:43:49.874] Build status: STARTED
[03:43:49.874] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:28:0) after 9m34s
[03:53:23.972] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:28:0)
[03:53:24.028] Build status: STARTED
[03:53:24.028] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:29:0) after 4m21s
[03:57:45.327] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:29:0)
[03:57:45.347] Build status: STARTED
[03:57:45.347] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:30:0) after 3m20s
[04:01:05.464] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:30:0)
[04:01:05.550] Build status: STARTED
[04:01:05.550] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:31:0) after 2m24s
[04:03:29.583] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:31:0)
[04:03:29.600] Build status: STARTED
[04:03:29.600] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:32:0) after 4m0s
[04:07:29.710] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:32:0)
[04:07:29.729] Build status: STARTED
[04:07:29.729] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:33:0) after 8m55s
[04:16:24.842] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:33:0)
[04:16:24.868] Build status: STARTED
[04:16:24.868] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:34:0) after 9m56s
[04:26:20.888] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:34:0)
[04:26:20.915] Build status: STARTED
[04:26:20.915] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:35:0) after 6m45s
[04:33:05.937] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:35:0)
[04:33:05.961] Build status: STARTED
[04:33:05.961] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:36:0) after 2m10s
[04:35:16.009] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:36:0)
[04:35:16.059] Build status: STARTED
[04:35:16.059] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 Experimental Release (NVIDIA):8921497094606844144:37:0) after 9m26s
[04:35:46.388] Received PubSub notification, asking Buildbucket for the build status
[04:35:46.408] Build:
{
  "id": "8756067032586473649",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 Experimental Release (NVIDIA)"
  },
  "number": 96,
  "createdBy": "project:chromium",
  "createTime": "2024-02-15T01:33:00.498688994Z",
  "startTime": "2024-02-15T01:33:01.416417Z",
  "endTime": "2024-02-15T04:35:45.746291Z",
  "updateTime": "2024-02-15T04:35:45.746291Z",
  "status": "INFRA_FAILURE",
  "statusDetails": {
    "timeout": {

    }
  },
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "b907e47bd88ee4d1a8869392e84173ad8a96a94a",
      "ref": "refs/heads/main"
    }
  }
}
[04:35:46.408] Invocation finished in 3h2m47.220504065s with status FAILED