Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 2 hours
Actions:

Definition (rev 1f1b4513521524d405025696b8777ff82db0bd9d)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Win10 x64 ASAN Release (Intel)"
>

Incoming triggers

  • Dawn Win10 x64 ASAN Builder/5181 (2 years ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Win10 x64 ASAN Builder/5182 (2 years ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Win10 x64 ASAN Builder/5183 (2 years ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "0871bdb2cbe5e168e208d6032fcbb39d18bef23e",
        "project": "chromium/src",
        "ref": "refs/heads/main"
      }
    ]
  },
  "branch": "refs/heads/main",
  "deps_revision_overrides": {
    "src": "0871bdb2cbe5e168e208d6032fcbb39d18bef23e",
    "src/third_party/dawn": "da33625f315343cddce1108f3fffc40b6b7b4733"
  },
  "parent_build_id": "8756715501901853265",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Win10 x64 ASAN Builder",
  "parent_got_angle_revision": "f431641a948660f5e1709aa7cd89b16ccf93f1de",
  "parent_got_dawn_revision": "da33625f315343cddce1108f3fffc40b6b7b4733",
  "parent_got_revision": "0871bdb2cbe5e168e208d6032fcbb39d18bef23e",
  "parent_got_revision_cp": "refs/heads/main@{#1257589}",
  "parent_got_swiftshader_revision": "eb75201a4e0354a36d315dd01077092ec9aa2356",
  "parent_got_v8_revision": "93c5bbc582e38f3a543ae0e5a28e593f2e7cc6dd",
  "parent_got_v8_revision_cp": "refs/heads/12.3.140@{#1}",
  "parent_got_webrtc_revision": "aaa123debbd106669fc849425b92607edbd5d26f",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#41688}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "0871bdb2cbe5e168e208d6032fcbb39d18bef23e",
  "swarm_hashes": {
    "dawn_end2end_tests": "62233f18355d428c7b55124185e4de6f72a4ce421d46758e082bb90666548ffe/486",
    "dawn_perf_tests": "f73fca3e3a257c4b9baf991ad366f59c88c08d42f0c6c09e1bdbed8c1bb62cf1/488",
    "gl_tests": "3ad736ba4ad2a5ffc773da881e0cc9c0eaa802ed5341c7a104586a36418a0b5e/486",
    "gl_unittests": "e3e9acb02be1d108bbc8ae591ac7696a641bdaef942bedd041977af68262a56f/487",
    "telemetry_gpu_integration_test": "7508365090c752076e965ecc7551c092a182a959185bde72eb6de94e99c2a196/814",
    "telemetry_gpu_unittests": "960c54378eb17b0986161d3ed39013b10e21bbc77162cb5df783726b39a97da4/735",
    "webgpu_blink_web_tests": "4bbaaefe64f3bbc1110f4f5767c26f6efea2c6d60f7bad5cdc64917e89874997/566"
  },
  "swarming_command_lines_cwd": "out\\Release_x64",
  "swarming_command_lines_digest": "eca0b9676d87a6ce8af3ba81b10e6d5bd10fc75c351264a541e8d91d0e717a42/93"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/0871bdb2cbe5e168e208d6032fcbb39d18bef23e
  • gitiles_ref:refs/heads/main
  • parent_buildername:Dawn Win10 x64 ASAN Builder
  • parent_buildnumber:5183
  • user_agent:recipe

Debug log (UTC)

[23:19:18.136] New invocation is queued and will start shortly
[23:19:18.136] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[23:19:19.198] Starting the invocation (attempt 1)
[23:19:19.222] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/0871bdb2cbe5e168e208d6032fcbb39d18bef23e"
[23:19:19.222] Popped gitiles commit info from properties and tags
[23:19:19.222] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[23:19:19.222] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[23:19:19.223] Buildbucket request:
{
  "requestId": "8922139684060281520",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 ASAN Release (Intel)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "0871bdb2cbe5e168e208d6032fcbb39d18bef23e",
                        "project": "chromium/src",
                        "ref": "refs/heads/main"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "0871bdb2cbe5e168e208d6032fcbb39d18bef23e",
            "src/third_party/dawn": "da33625f315343cddce1108f3fffc40b6b7b4733"
          },
      "parent_build_id": "8756715501901853265",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Win10 x64 ASAN Builder",
      "parent_got_angle_revision": "f431641a948660f5e1709aa7cd89b16ccf93f1de",
      "parent_got_dawn_revision": "da33625f315343cddce1108f3fffc40b6b7b4733",
      "parent_got_revision": "0871bdb2cbe5e168e208d6032fcbb39d18bef23e",
      "parent_got_revision_cp": "refs/heads/main@{#1257589}",
      "parent_got_swiftshader_revision": "eb75201a4e0354a36d315dd01077092ec9aa2356",
      "parent_got_v8_revision": "93c5bbc582e38f3a543ae0e5a28e593f2e7cc6dd",
      "parent_got_v8_revision_cp": "refs/heads/12.3.140@{#1}",
      "parent_got_webrtc_revision": "aaa123debbd106669fc849425b92607edbd5d26f",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#41688}",
      "swarm_hashes": {
            "dawn_end2end_tests": "62233f18355d428c7b55124185e4de6f72a4ce421d46758e082bb90666548ffe/486",
            "dawn_perf_tests": "f73fca3e3a257c4b9baf991ad366f59c88c08d42f0c6c09e1bdbed8c1bb62cf1/488",
            "gl_tests": "3ad736ba4ad2a5ffc773da881e0cc9c0eaa802ed5341c7a104586a36418a0b5e/486",
            "gl_unittests": "e3e9acb02be1d108bbc8ae591ac7696a641bdaef942bedd041977af68262a56f/487",
            "telemetry_gpu_integration_test": "7508365090c752076e965ecc7551c092a182a959185bde72eb6de94e99c2a196/814",
            "telemetry_gpu_unittests": "960c54378eb17b0986161d3ed39013b10e21bbc77162cb5df783726b39a97da4/735",
            "webgpu_blink_web_tests": "4bbaaefe64f3bbc1110f4f5767c26f6efea2c6d60f7bad5cdc64917e89874997/566"
          },
      "swarming_command_lines_cwd": "out\\Release_x64",
      "swarming_command_lines_digest": "eca0b9676d87a6ce8af3ba81b10e6d5bd10fc75c351264a541e8d91d0e717a42/93"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "0871bdb2cbe5e168e208d6032fcbb39d18bef23e",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8922139684060281520"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/Dawn Win10 x64 ASAN Release (Intel)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Win10 x64 ASAN Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "5183"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[23:19:19.811] Scheduled build:
{
  "id": "8756709622214853233",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 ASAN Release (Intel)"
  },
  "number": 2090,
  "createdBy": "project:chromium",
  "createTime": "2024-02-07T23:19:19.274738147Z",
  "updateTime": "2024-02-07T23:19:19.274738147Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "0871bdb2cbe5e168e208d6032fcbb39d18bef23e",
      "ref": "refs/heads/main"
    }
  }
}
[23:19:19.811] Task URL: https://cr-buildbucket.appspot.com/build/8756709622214853233
[23:19:19.811] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:2:0) after 6m6s
[23:19:20.814] Received PubSub notification, asking Buildbucket for the build status
[23:19:20.841] Build status: SCHEDULED
[23:20:31.408] Received PubSub notification, asking Buildbucket for the build status
[23:20:31.428] Build status: STARTED
[23:25:25.829] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:2:0)
[23:25:25.858] Build status: STARTED
[23:25:25.858] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:5:0) after 7m56s
[23:33:21.888] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:5:0)
[23:33:21.911] Build status: STARTED
[23:33:21.911] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:6:0) after 2m47s
[23:36:10.000] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:6:0)
[23:36:10.018] Build status: STARTED
[23:36:10.018] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:7:0) after 9m12s
[23:45:22.040] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:7:0)
[23:45:22.086] Build status: STARTED
[23:45:22.086] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:8:0) after 7m48s
[23:53:10.237] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:8:0)
[23:53:10.256] Build status: STARTED
[23:53:10.256] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:9:0) after 2m10s
[23:55:20.374] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:9:0)
[23:55:20.396] Build status: STARTED
[23:55:20.396] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:10:0) after 8m6s
[00:03:26.413] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:10:0)
[00:03:26.434] Build status: STARTED
[00:03:26.434] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:11:0) after 4m46s
[00:08:12.460] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:11:0)
[00:08:12.620] Build status: STARTED
[00:08:12.620] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:12:0) after 3m6s
[00:11:18.824] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:12:0)
[00:11:18.841] Build status: STARTED
[00:11:18.841] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:13:0) after 9m15s
[00:20:33.894] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:13:0)
[00:20:33.924] Build status: STARTED
[00:20:33.924] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:14:0) after 1m3s
[00:21:36.958] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:14:0)
[00:21:36.984] Build status: STARTED
[00:21:36.984] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:15:0) after 1m26s
[00:23:02.988] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:15:0)
[00:23:03.070] Build status: STARTED
[00:23:03.070] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:16:0) after 6m12s
[00:29:15.183] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:16:0)
[00:29:15.266] Build status: STARTED
[00:29:15.266] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:17:0) after 1m37s
[00:30:52.292] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:17:0)
[00:30:52.361] Build status: STARTED
[00:30:52.361] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:18:0) after 4m45s
[00:35:37.384] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:18:0)
[00:35:37.424] Build status: STARTED
[00:35:37.424] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:19:0) after 3m57s
[00:39:34.519] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:19:0)
[00:39:34.552] Build status: STARTED
[00:39:34.552] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:20:0) after 1m23s
[00:40:57.575] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:20:0)
[00:40:57.597] Build status: STARTED
[00:40:57.597] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:21:0) after 6m24s
[00:47:21.613] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:21:0)
[00:47:21.636] Build status: STARTED
[00:47:21.636] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:22:0) after 5m16s
[00:52:37.707] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:22:0)
[00:52:37.749] Build status: STARTED
[00:52:37.749] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:23:0) after 4m1s
[00:56:38.763] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:23:0)
[00:56:38.784] Build status: STARTED
[00:56:38.784] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:24:0) after 2m54s
[00:59:32.806] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:24:0)
[00:59:32.824] Build status: STARTED
[00:59:32.825] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:25:0) after 2m46s
[01:02:18.845] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:25:0)
[01:02:18.865] Build status: STARTED
[01:02:18.865] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:26:0) after 5m53s
[01:08:11.865] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:26:0)
[01:08:11.907] Build status: STARTED
[01:08:11.907] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:27:0) after 8m59s
[01:17:10.927] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:27:0)
[01:17:11.015] Build status: STARTED
[01:17:11.015] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:28:0) after 9m55s
[01:27:06.043] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:28:0)
[01:27:06.068] Build status: STARTED
[01:27:06.068] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:29:0) after 5m47s
[01:32:53.085] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:29:0)
[01:32:53.102] Build status: STARTED
[01:32:53.102] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:30:0) after 7m21s
[01:40:14.126] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:30:0)
[01:40:14.142] Build status: STARTED
[01:40:14.142] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:31:0) after 2m5s
[01:42:19.185] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:31:0)
[01:42:19.215] Build status: STARTED
[01:42:19.215] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:32:0) after 1m14s
[01:43:33.287] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:32:0)
[01:43:33.305] Build status: STARTED
[01:43:33.305] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:33:0) after 5m53s
[01:49:26.281] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:33:0)
[01:49:26.301] Build status: STARTED
[01:49:26.301] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:34:0) after 2m22s
[01:51:48.333] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:34:0)
[01:51:48.355] Build status: STARTED
[01:51:48.356] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:35:0) after 3m32s
[01:55:20.377] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:35:0)
[01:55:20.403] Build status: STARTED
[01:55:20.403] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:36:0) after 6m59s
[02:02:19.540] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:36:0)
[02:02:19.593] Build status: STARTED
[02:02:19.593] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:37:0) after 7m34s
[02:09:53.616] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:37:0)
[02:09:53.641] Build status: STARTED
[02:09:53.641] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:38:0) after 2m4s
[02:11:57.670] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:38:0)
[02:11:57.748] Build status: STARTED
[02:11:57.748] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8922139684060281520:39:0) after 8m13s
[02:13:12.351] Received PubSub notification, asking Buildbucket for the build status
[02:13:12.398] Build:
{
  "id": "8756709622214853233",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 ASAN Release (Intel)"
  },
  "number": 2090,
  "createdBy": "project:chromium",
  "createTime": "2024-02-07T23:19:19.274738147Z",
  "startTime": "2024-02-07T23:20:30.933642066Z",
  "endTime": "2024-02-08T02:13:10.709189Z",
  "updateTime": "2024-02-08T02:13:10.709189Z",
  "status": "INFRA_FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "0871bdb2cbe5e168e208d6032fcbb39d18bef23e",
      "ref": "refs/heads/main"
    }
  }
}
[02:13:12.398] Invocation finished in 2h53m54.268456479s with status FAILED