Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 54 minutes
Actions:

Definition (rev dbc5bbb4d5c973373f7e6afa20eb56bb1648ac04)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Linux x64 DEPS Release (NVIDIA)"
>

Incoming triggers

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "9172228cce29568102eb1df412495d0b2354bc69",
        "project": "chromium/src",
        "ref": "refs/branch-heads/7103"
      }
    ]
  },
  "branch": "refs/branch-heads/7103",
  "deps_revision_overrides": {
    "src": "9172228cce29568102eb1df412495d0b2354bc69"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8712583990928172929"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8712583990928172929"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8712583990928172929"
    }
  ],
  "parent_build_id": "8712583990928172929",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Linux x64 DEPS Builder",
  "parent_got_angle_revision": "fa40b7c586fd2da9fd7e5c4d893ecb1334553b9e",
  "parent_got_dawn_revision": "1cffe7ec763900d104e4df62bc96d93f572157cb",
  "parent_got_revision": "9172228cce29568102eb1df412495d0b2354bc69",
  "parent_got_revision_cp": "refs/branch-heads/7103@{#3306}",
  "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
  "parent_got_v8_revision": "5b6eec8d9bf009d2263d784714e6bfa1777dbc7d",
  "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#31}",
  "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "9172228cce29568102eb1df412495d0b2354bc69",
  "swarm_hashes": {
    "dawn_end2end_tests": "19eec0e9a8d73c7a7e0be34b3cee0ab47a9bce01ef7ea46becd497f9dd1f7940/323",
    "dawn_perf_tests": "b671bea58e0bfca6e3889dd2bbc3793d592c3a91c3ed3088d6f5e2f2499fb0fb/487",
    "gl_tests": "c95d4ed6493938e8e1c86e279450436859ccba0d5c1f6715aa9569c202af7acc/323",
    "gl_unittests": "22b4bea10b0300a791747de9ffb12e0d07b1a0d2781a7b78829bcec12db853bd/323",
    "telemetry_gpu_integration_test": "b6070bb6313fe56537e7c542532a8048a04fb1f3c90c0f4ee70e5552c2c3ea38/815",
    "telemetry_gpu_unittests": "4879fc7297a6cbd3a0db263f9bcabc2d7f91f0a18b926a05842943dd23a85ee6/736",
    "webgpu_blink_web_tests": "b780f10988352c629a14b4e13670ecc4ae7bc35e64cca646326e6f3a98ee7930/566"
  },
  "swarming_command_lines_cwd": "out/1d86-Dawn_Linux_x64_",
  "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/9172228cce29568102eb1df412495d0b2354bc69
  • gitiles_ref:refs/branch-heads/7103
  • parent_buildername:Dawn Linux x64 DEPS Builder
  • parent_buildnumber:2722
  • user_agent:recipe

Debug log (UTC)

[01:12:42.586] New invocation is queued and will start shortly
[01:12:42.586] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[01:12:43.726] Starting the invocation (attempt 1)
[01:12:43.753] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/9172228cce29568102eb1df412495d0b2354bc69"
[01:12:43.753] Popped gitiles commit info from properties and tags
[01:12:43.754] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[01:12:43.754] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[01:12:43.754] Buildbucket request:
{
  "requestId": "8878011826447236640",
  "builder": {
    "project": "chromium-m136",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "9172228cce29568102eb1df412495d0b2354bc69",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/7103"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8878011826447236640",
            "job": "chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA)",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "73f914391ffabb5d649a35d0edc2065dd9236c61",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7103"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "73f914391ffabb5d649a35d0edc2065dd9236c61"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8712585405313090417"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8712585405313090417"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8712585405313090417"
                                              }
                                        ],
                                    "parent_build_id": "8712585405313090417",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Linux x64 DEPS Builder",
                                    "parent_got_angle_revision": "fa40b7c586fd2da9fd7e5c4d893ecb1334553b9e",
                                    "parent_got_dawn_revision": "1cffe7ec763900d104e4df62bc96d93f572157cb",
                                    "parent_got_revision": "73f914391ffabb5d649a35d0edc2065dd9236c61",
                                    "parent_got_revision_cp": "refs/branch-heads/7103@{#3305}",
                                    "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
                                    "parent_got_v8_revision": "5b6eec8d9bf009d2263d784714e6bfa1777dbc7d",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#31}",
                                    "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
                                    "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
                                    "revision": "73f914391ffabb5d649a35d0edc2065dd9236c61",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "19eec0e9a8d73c7a7e0be34b3cee0ab47a9bce01ef7ea46becd497f9dd1f7940/323",
                                          "dawn_perf_tests": "b671bea58e0bfca6e3889dd2bbc3793d592c3a91c3ed3088d6f5e2f2499fb0fb/487",
                                          "gl_tests": "c95d4ed6493938e8e1c86e279450436859ccba0d5c1f6715aa9569c202af7acc/323",
                                          "gl_unittests": "22b4bea10b0300a791747de9ffb12e0d07b1a0d2781a7b78829bcec12db853bd/323",
                                          "telemetry_gpu_integration_test": "71802537f679561d0250dcbf148278f04bd3b37de787a4af963fe107c40d5247/815",
                                          "telemetry_gpu_unittests": "4879fc7297a6cbd3a0db263f9bcabc2d7f91f0a18b926a05842943dd23a85ee6/736",
                                          "webgpu_blink_web_tests": "70f8eb5e6d0be9523b4b14c3acb6bf0802f4d5bf2306322dfce9163da2480814/566"
                                        },
                                    "swarming_command_lines_cwd": "out/1d86-Dawn_Linux_x64_",
                                    "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7103",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "73f914391ffabb5d649a35d0edc2065dd9236c61",
                              "tags": [
                                    "parent_buildername:Dawn Linux x64 DEPS Builder",
                                    "parent_buildnumber:2721",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "4a73b025-680e-4cf3-9fe2-ed04ce5efd4a",
                        "title": "Dawn Linux x64 DEPS Builder/2721",
                        "url": "https://cr-buildbucket.appspot.com/build/8712585405313090417"
                      },
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "9172228cce29568102eb1df412495d0b2354bc69",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7103"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "9172228cce29568102eb1df412495d0b2354bc69"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8712583990928172929"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8712583990928172929"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8712583990928172929"
                                              }
                                        ],
                                    "parent_build_id": "8712583990928172929",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Linux x64 DEPS Builder",
                                    "parent_got_angle_revision": "fa40b7c586fd2da9fd7e5c4d893ecb1334553b9e",
                                    "parent_got_dawn_revision": "1cffe7ec763900d104e4df62bc96d93f572157cb",
                                    "parent_got_revision": "9172228cce29568102eb1df412495d0b2354bc69",
                                    "parent_got_revision_cp": "refs/branch-heads/7103@{#3306}",
                                    "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
                                    "parent_got_v8_revision": "5b6eec8d9bf009d2263d784714e6bfa1777dbc7d",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#31}",
                                    "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
                                    "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
                                    "revision": "9172228cce29568102eb1df412495d0b2354bc69",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "19eec0e9a8d73c7a7e0be34b3cee0ab47a9bce01ef7ea46becd497f9dd1f7940/323",
                                          "dawn_perf_tests": "b671bea58e0bfca6e3889dd2bbc3793d592c3a91c3ed3088d6f5e2f2499fb0fb/487",
                                          "gl_tests": "c95d4ed6493938e8e1c86e279450436859ccba0d5c1f6715aa9569c202af7acc/323",
                                          "gl_unittests": "22b4bea10b0300a791747de9ffb12e0d07b1a0d2781a7b78829bcec12db853bd/323",
                                          "telemetry_gpu_integration_test": "b6070bb6313fe56537e7c542532a8048a04fb1f3c90c0f4ee70e5552c2c3ea38/815",
                                          "telemetry_gpu_unittests": "4879fc7297a6cbd3a0db263f9bcabc2d7f91f0a18b926a05842943dd23a85ee6/736",
                                          "webgpu_blink_web_tests": "b780f10988352c629a14b4e13670ecc4ae7bc35e64cca646326e6f3a98ee7930/566"
                                        },
                                    "swarming_command_lines_cwd": "out/1d86-Dawn_Linux_x64_",
                                    "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7103",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "9172228cce29568102eb1df412495d0b2354bc69",
                              "tags": [
                                    "parent_buildername:Dawn Linux x64 DEPS Builder",
                                    "parent_buildnumber:2722",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "c6accc68-f54e-4657-b1d1-eb5ac2e9d9f4",
                        "title": "Dawn Linux x64 DEPS Builder/2722",
                        "url": "https://cr-buildbucket.appspot.com/build/8712583990928172929"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "9172228cce29568102eb1df412495d0b2354bc69"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8712583990928172929"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8712583990928172929"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8712583990928172929"
                }
          ],
      "parent_build_id": "8712583990928172929",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Linux x64 DEPS Builder",
      "parent_got_angle_revision": "fa40b7c586fd2da9fd7e5c4d893ecb1334553b9e",
      "parent_got_dawn_revision": "1cffe7ec763900d104e4df62bc96d93f572157cb",
      "parent_got_revision": "9172228cce29568102eb1df412495d0b2354bc69",
      "parent_got_revision_cp": "refs/branch-heads/7103@{#3306}",
      "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
      "parent_got_v8_revision": "5b6eec8d9bf009d2263d784714e6bfa1777dbc7d",
      "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#31}",
      "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
      "swarm_hashes": {
            "dawn_end2end_tests": "19eec0e9a8d73c7a7e0be34b3cee0ab47a9bce01ef7ea46becd497f9dd1f7940/323",
            "dawn_perf_tests": "b671bea58e0bfca6e3889dd2bbc3793d592c3a91c3ed3088d6f5e2f2499fb0fb/487",
            "gl_tests": "c95d4ed6493938e8e1c86e279450436859ccba0d5c1f6715aa9569c202af7acc/323",
            "gl_unittests": "22b4bea10b0300a791747de9ffb12e0d07b1a0d2781a7b78829bcec12db853bd/323",
            "telemetry_gpu_integration_test": "b6070bb6313fe56537e7c542532a8048a04fb1f3c90c0f4ee70e5552c2c3ea38/815",
            "telemetry_gpu_unittests": "4879fc7297a6cbd3a0db263f9bcabc2d7f91f0a18b926a05842943dd23a85ee6/736",
            "webgpu_blink_web_tests": "b780f10988352c629a14b4e13670ecc4ae7bc35e64cca646326e6f3a98ee7930/566"
          },
      "swarming_command_lines_cwd": "out/1d86-Dawn_Linux_x64_",
      "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "9172228cce29568102eb1df412495d0b2354bc69",
    "ref": "refs/branch-heads/7103"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8878011826447236640"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Linux x64 DEPS Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "2722"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[01:12:44.282] Scheduled build:
{
  "id": "8712581764489384065",
  "builder": {
    "project": "chromium-m136",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "number": 1399,
  "createdBy": "project:chromium-m136",
  "createTime": "2025-06-09T01:12:43.825671845Z",
  "updateTime": "2025-06-09T01:12:43.825671845Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "9172228cce29568102eb1df412495d0b2354bc69",
      "ref": "refs/branch-heads/7103"
    }
  }
}
[01:12:44.283] Task URL: https://cr-buildbucket.appspot.com/build/8712581764489384065
[01:12:44.283] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:2:0) after 9m52s
[01:12:44.802] Received PubSub notification, asking Buildbucket for the build status
[01:12:44.827] Build status: SCHEDULED
[01:12:52.843] Received PubSub notification, asking Buildbucket for the build status
[01:12:52.864] Build status: STARTED
[01:22:36.425] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:2:0)
[01:22:36.449] Build status: STARTED
[01:22:36.449] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:5:0) after 1m26s
[01:24:02.642] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:5:0)
[01:24:02.669] Build status: STARTED
[01:24:02.669] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:6:0) after 5m42s
[01:29:44.695] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:6:0)
[01:29:44.724] Build status: STARTED
[01:29:44.724] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:7:0) after 7m56s
[01:37:40.735] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:7:0)
[01:37:40.760] Build status: STARTED
[01:37:40.760] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:8:0) after 3m26s
[01:41:06.786] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:8:0)
[01:41:06.810] Build status: STARTED
[01:41:06.810] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:9:0) after 9m31s
[01:50:37.947] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:9:0)
[01:50:37.972] Build status: STARTED
[01:50:37.972] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:10:0) after 6m23s
[01:57:00.992] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:10:0)
[01:57:01.010] Build status: STARTED
[01:57:01.010] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:11:0) after 2m19s
[01:59:20.010] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:11:0)
[01:59:20.049] Build status: STARTED
[01:59:20.049] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:12:0) after 1m10s
[02:00:30.347] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:12:0)
[02:00:30.423] Build status: STARTED
[02:00:30.423] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:13:0) after 5m51s
[02:06:21.616] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:13:0)
[02:06:21.641] Build status: STARTED
[02:06:21.641] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8878011826447236640:14:0) after 9m2s
[02:07:28.508] Received PubSub notification, asking Buildbucket for the build status
[02:07:28.534] Build:
{
  "id": "8712581764489384065",
  "builder": {
    "project": "chromium-m136",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "number": 1399,
  "createdBy": "project:chromium-m136",
  "createTime": "2025-06-09T01:12:43.825671845Z",
  "startTime": "2025-06-09T01:12:52.542106538Z",
  "endTime": "2025-06-09T02:07:27.849956861Z",
  "updateTime": "2025-06-09T02:07:27.849956861Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "9172228cce29568102eb1df412495d0b2354bc69",
      "ref": "refs/branch-heads/7103"
    }
  }
}
[02:07:28.534] Invocation finished in 54m45.960284041s with status FAILED