Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 2 hours
Actions:

Definition (rev 1f1b4513521524d405025696b8777ff82db0bd9d)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Win10 x64 ASAN Release (Intel)"
>

Incoming triggers

  • Dawn Win10 x64 ASAN Builder/7560 (1 year ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Win10 x64 ASAN Builder/7561 (1 year ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Win10 x64 ASAN Builder/7562 (1 year ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "b27803efe6f7f9d322148ac276d83aecc547b003",
        "project": "chromium/src",
        "ref": "refs/heads/main"
      }
    ]
  },
  "branch": "refs/heads/main",
  "deps_revision_overrides": {
    "src": "b27803efe6f7f9d322148ac276d83aecc547b003",
    "src/third_party/dawn": "66e1c18ee075a04577b13d49d09a9d0e31af8cdb"
  },
  "parent_build_id": "8749074470547774193",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Win10 x64 ASAN Builder",
  "parent_got_angle_revision": "caebfea1f9dd447eb3dc5883a8780cd66549aeeb",
  "parent_got_dawn_revision": "66e1c18ee075a04577b13d49d09a9d0e31af8cdb",
  "parent_got_revision": "b27803efe6f7f9d322148ac276d83aecc547b003",
  "parent_got_revision_cp": "refs/heads/main@{#1295280}",
  "parent_got_swiftshader_revision": "76f7f8cfea80d26fa41d51f52cfbade49ec1f838",
  "parent_got_v8_revision": "b30ac7c708ab74ea4924e0cf5a196f6f3c74e6fd",
  "parent_got_v8_revision_cp": "refs/heads/12.6.145@{#1}",
  "parent_got_webrtc_revision": "57b09eca64df9315e603ac1a43934d8fbda64f46",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#42204}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "b27803efe6f7f9d322148ac276d83aecc547b003",
  "swarm_hashes": {
    "dawn_end2end_tests": "ca6d5fa80ca1a1397f2a348ca93b0bcd5818154cbf6184013dfcd5d069570f18/486",
    "dawn_perf_tests": "b3507f6752a63fa5333019dc1b6a7be5345d9ebf10481acc417add61fb4ca46e/488",
    "gl_tests": "bc458b94c097c691074ae5b1390e911343a18b8c0589ca92bbbb23eb7d120979/486",
    "gl_unittests": "d9238b9740bd5ea371169c598cf224d6e46461559724b64ebd3aaef2872bb428/487",
    "telemetry_gpu_integration_test": "fa8a98484b299e9ded4fcdd587765e34b16f7168b80b26e584b42857c23b9f09/814",
    "telemetry_gpu_unittests": "d5d0d432b73d44f19a647ee5fb2588706b203245fb2b5afa480e53c9cc2fd791/735",
    "webgpu_blink_web_tests": "85741cd4c903a9481bc1e3eeaf2d1b66f69c3d0588bbbb3c4a58e6a78101c334/566"
  },
  "swarming_command_lines_cwd": "out\\Release_x64",
  "swarming_command_lines_digest": "eca0b9676d87a6ce8af3ba81b10e6d5bd10fc75c351264a541e8d91d0e717a42/93"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/b27803efe6f7f9d322148ac276d83aecc547b003
  • gitiles_ref:refs/heads/main
  • parent_buildername:Dawn Win10 x64 ASAN Builder
  • parent_buildnumber:7562
  • user_agent:recipe

Debug log (UTC)

[07:02:46.335] New invocation is queued and will start shortly
[07:02:46.335] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[07:02:47.611] Starting the invocation (attempt 1)
[07:02:47.642] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/b27803efe6f7f9d322148ac276d83aecc547b003"
[07:02:47.642] Popped gitiles commit info from properties and tags
[07:02:47.642] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[07:02:47.644] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[07:02:47.644] Buildbucket request:
{
  "requestId": "8914500379881341152",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 ASAN Release (Intel)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "b27803efe6f7f9d322148ac276d83aecc547b003",
                        "project": "chromium/src",
                        "ref": "refs/heads/main"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "b27803efe6f7f9d322148ac276d83aecc547b003",
            "src/third_party/dawn": "66e1c18ee075a04577b13d49d09a9d0e31af8cdb"
          },
      "parent_build_id": "8749074470547774193",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Win10 x64 ASAN Builder",
      "parent_got_angle_revision": "caebfea1f9dd447eb3dc5883a8780cd66549aeeb",
      "parent_got_dawn_revision": "66e1c18ee075a04577b13d49d09a9d0e31af8cdb",
      "parent_got_revision": "b27803efe6f7f9d322148ac276d83aecc547b003",
      "parent_got_revision_cp": "refs/heads/main@{#1295280}",
      "parent_got_swiftshader_revision": "76f7f8cfea80d26fa41d51f52cfbade49ec1f838",
      "parent_got_v8_revision": "b30ac7c708ab74ea4924e0cf5a196f6f3c74e6fd",
      "parent_got_v8_revision_cp": "refs/heads/12.6.145@{#1}",
      "parent_got_webrtc_revision": "57b09eca64df9315e603ac1a43934d8fbda64f46",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#42204}",
      "swarm_hashes": {
            "dawn_end2end_tests": "ca6d5fa80ca1a1397f2a348ca93b0bcd5818154cbf6184013dfcd5d069570f18/486",
            "dawn_perf_tests": "b3507f6752a63fa5333019dc1b6a7be5345d9ebf10481acc417add61fb4ca46e/488",
            "gl_tests": "bc458b94c097c691074ae5b1390e911343a18b8c0589ca92bbbb23eb7d120979/486",
            "gl_unittests": "d9238b9740bd5ea371169c598cf224d6e46461559724b64ebd3aaef2872bb428/487",
            "telemetry_gpu_integration_test": "fa8a98484b299e9ded4fcdd587765e34b16f7168b80b26e584b42857c23b9f09/814",
            "telemetry_gpu_unittests": "d5d0d432b73d44f19a647ee5fb2588706b203245fb2b5afa480e53c9cc2fd791/735",
            "webgpu_blink_web_tests": "85741cd4c903a9481bc1e3eeaf2d1b66f69c3d0588bbbb3c4a58e6a78101c334/566"
          },
      "swarming_command_lines_cwd": "out\\Release_x64",
      "swarming_command_lines_digest": "eca0b9676d87a6ce8af3ba81b10e6d5bd10fc75c351264a541e8d91d0e717a42/93"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "b27803efe6f7f9d322148ac276d83aecc547b003",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8914500379881341152"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/Dawn Win10 x64 ASAN Release (Intel)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Win10 x64 ASAN Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "7562"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[07:02:48.258] Scheduled build:
{
  "id": "8749070317778178209",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 ASAN Release (Intel)"
  },
  "number": 2969,
  "createdBy": "project:chromium",
  "createTime": "2024-05-02T07:02:47.710762110Z",
  "updateTime": "2024-05-02T07:02:47.710762110Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "b27803efe6f7f9d322148ac276d83aecc547b003",
      "ref": "refs/heads/main"
    }
  }
}
[07:02:48.258] Task URL: https://cr-buildbucket.appspot.com/build/8749070317778178209
[07:02:48.258] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:2:0) after 4m45s
[07:02:49.179] Received PubSub notification, asking Buildbucket for the build status
[07:02:49.205] Build status: SCHEDULED
[07:04:00.331] Received PubSub notification, asking Buildbucket for the build status
[07:04:00.352] Build status: STARTED
[07:07:33.374] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:2:0)
[07:07:33.433] Build status: STARTED
[07:07:33.433] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:5:0) after 2m14s
[07:09:47.584] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:5:0)
[07:09:47.606] Build status: STARTED
[07:09:47.606] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:6:0) after 8m22s
[07:18:09.627] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:6:0)
[07:18:09.652] Build status: STARTED
[07:18:09.652] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:7:0) after 8m54s
[07:27:03.881] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:7:0)
[07:27:03.918] Build status: STARTED
[07:27:03.918] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:8:0) after 6m1s
[07:33:04.938] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:8:0)
[07:33:04.960] Build status: STARTED
[07:33:04.960] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:9:0) after 8m0s
[07:41:04.987] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:9:0)
[07:41:05.032] Build status: STARTED
[07:41:05.032] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:10:0) after 2m30s
[07:43:35.152] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:10:0)
[07:43:35.169] Build status: STARTED
[07:43:35.169] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:11:0) after 5m3s
[07:48:38.199] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:11:0)
[07:48:38.218] Build status: STARTED
[07:48:38.218] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:12:0) after 4m4s
[07:52:42.254] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:12:0)
[07:52:42.277] Build status: STARTED
[07:52:42.277] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:13:0) after 3m35s
[07:56:17.294] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:13:0)
[07:56:17.358] Build status: STARTED
[07:56:17.358] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:14:0) after 4m56s
[08:01:13.531] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:14:0)
[08:01:13.565] Build status: STARTED
[08:01:13.565] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:15:0) after 6m11s
[08:07:24.601] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:15:0)
[08:07:24.672] Build status: STARTED
[08:07:24.672] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:16:0) after 3m2s
[08:10:27.077] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:16:0)
[08:10:27.108] Build status: STARTED
[08:10:27.108] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:17:0) after 1m58s
[08:12:25.293] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:17:0)
[08:12:25.310] Build status: STARTED
[08:12:25.310] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:18:0) after 9m15s
[08:21:40.429] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:18:0)
[08:21:40.463] Build status: STARTED
[08:21:40.463] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:19:0) after 1m37s
[08:23:17.861] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:19:0)
[08:23:17.893] Build status: STARTED
[08:23:17.893] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:20:0) after 1m40s
[08:24:57.915] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:20:0)
[08:24:57.934] Build status: STARTED
[08:24:57.934] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:21:0) after 4m19s
[08:29:17.086] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:21:0)
[08:29:17.103] Build status: STARTED
[08:29:17.103] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:22:0) after 7m19s
[08:36:36.254] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:22:0)
[08:36:36.296] Build status: STARTED
[08:36:36.296] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:23:0) after 3m55s
[08:40:31.607] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:23:0)
[08:40:31.626] Build status: STARTED
[08:40:31.626] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:24:0) after 3m21s
[08:43:52.989] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:24:0)
[08:43:53.120] Build status: STARTED
[08:43:53.120] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:25:0) after 9m55s
[08:53:48.215] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:25:0)
[08:53:48.270] Build status: STARTED
[08:53:48.270] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:26:0) after 5m21s
[08:59:09.293] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:26:0)
[08:59:09.310] Build status: STARTED
[08:59:09.310] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:27:0) after 8m2s
[09:07:11.607] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:27:0)
[09:07:11.643] Build status: STARTED
[09:07:11.644] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:28:0) after 6m23s
[09:13:34.684] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:28:0)
[09:13:34.704] Build status: STARTED
[09:13:34.704] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:29:0) after 6m54s
[09:20:28.821] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:29:0)
[09:20:28.839] Build status: STARTED
[09:20:28.840] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:30:0) after 6m54s
[09:27:23.076] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:30:0)
[09:27:23.112] Build status: STARTED
[09:27:23.112] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:31:0) after 1m19s
[09:28:42.323] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:31:0)
[09:28:42.344] Build status: STARTED
[09:28:42.344] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:32:0) after 7m6s
[09:35:48.491] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:32:0)
[09:35:48.559] Build status: STARTED
[09:35:48.559] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:33:0) after 1m47s
[09:37:35.629] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:33:0)
[09:37:35.647] Build status: STARTED
[09:37:35.647] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:34:0) after 6m1s
[09:43:36.838] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:34:0)
[09:43:36.863] Build status: STARTED
[09:43:36.863] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:35:0) after 6m15s
[09:49:51.892] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:35:0)
[09:49:51.934] Build status: STARTED
[09:49:51.934] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:36:0) after 2m18s
[09:52:09.959] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:36:0)
[09:52:09.987] Build status: STARTED
[09:52:09.987] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:37:0) after 1m42s
[09:53:52.008] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:37:0)
[09:53:52.026] Build status: STARTED
[09:53:52.026] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:38:0) after 4m44s
[09:58:36.045] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:38:0)
[09:58:36.063] Build status: STARTED
[09:58:36.063] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (Intel):8914500379881341152:39:0) after 6m5s
[10:00:07.830] Received PubSub notification, asking Buildbucket for the build status
[10:00:07.851] Build:
{
  "id": "8749070317778178209",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Win10 x64 ASAN Release (Intel)"
  },
  "number": 2969,
  "createdBy": "project:chromium",
  "createTime": "2024-05-02T07:02:47.710762110Z",
  "startTime": "2024-05-02T07:04:00.093866941Z",
  "endTime": "2024-05-02T10:00:07.074686219Z",
  "updateTime": "2024-05-02T10:00:07.074686219Z",
  "status": "INFRA_FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "b27803efe6f7f9d322148ac276d83aecc547b003",
      "ref": "refs/heads/main"
    }
  }
}
[10:00:07.851] Invocation finished in 2h57m21.52998774s with status FAILED