Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 1 hour
Actions:

Definition (rev d38987cedf2afbb1d8d389bb18434e45348ab684)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Mac x64 DEPS Release (AMD)"
>

Incoming triggers

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "7d5e2efb6ecffca96bdbfba53070f18a66871303",
        "project": "chromium/src",
        "ref": "refs/branch-heads/7499"
      }
    ]
  },
  "branch": "refs/branch-heads/7499",
  "deps_revision_overrides": {
    "src": "7d5e2efb6ecffca96bdbfba53070f18a66871303"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8696789998706655025"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8696789998706655025"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8696789998706655025"
    }
  ],
  "parent_build_id": "8696789998706655025",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Mac x64 DEPS Builder",
  "parent_got_angle_revision": "9b75aecb1fa769e357d1dad2614228b6a04ddee6",
  "parent_got_dawn_revision": "e1ce227ebf75378c5f60a9d531579982bcdd93ee",
  "parent_got_revision": "7d5e2efb6ecffca96bdbfba53070f18a66871303",
  "parent_got_revision_cp": "refs/branch-heads/7499@{#2916}",
  "parent_got_swiftshader_revision": "3d536c0fc62b1cdea0f78c3c38d79be559855b88",
  "parent_got_v8_revision": "beee9f5cafde91bbd086077a11db16cb9768e62a",
  "parent_got_v8_revision_cp": "refs/branch-heads/14.3@{#31}",
  "parent_got_webrtc_revision": "1788a81407183acc98163a4e1507c5c63fb175cc",
  "parent_got_webrtc_revision_cp": "refs/branch-heads/7499@{#2}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "7d5e2efb6ecffca96bdbfba53070f18a66871303",
  "swarm_hashes": {
    "dawn_end2end_tests": "ce260e4def75c436983cbfad670037e06cbaf09d9a0df1133b78bc82f6f4d4bb/323",
    "dawn_perf_tests": "9356fbc0a051bbf268120d20a36d4598278cedf39ecf7d4d9aaf86f884c26116/487",
    "gl_tests": "cf40547918acbe023866a8d8c8f84dc06603a019fde7f0aecf56915990776a3e/323",
    "gl_unittests": "aaf8a9a6a02b1be52b24b8431234920c0163c49b3c936a309ee655781a20a2ec/323",
    "telemetry_gpu_integration_test": "9e679239268f4d34879c271343af8a214b7b2a738c7849dbe788acab364c042b/815",
    "telemetry_gpu_unittests": "a2d82009c88bef38d41d9966d37daccb62932cac26053eee849ff36039049041/736",
    "webgpu_blink_web_tests": "3514564dc9875ff1834071a177cd372503f8a744f5625055be279ccca10fd042/566"
  },
  "swarming_command_lines_cwd": "out/aeae-Dawn_Mac_x64_DE",
  "swarming_command_lines_digest": "60de9dd905cce771078a83e00991e4c89813c7ce0fd35ffdc98650f989deaf14/93",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/7d5e2efb6ecffca96bdbfba53070f18a66871303
  • gitiles_ref:refs/branch-heads/7499
  • parent_buildername:Dawn Mac x64 DEPS Builder
  • parent_buildnumber:2273
  • user_agent:recipe

Debug log (UTC)

[08:46:45.248] New invocation is queued and will start shortly
[08:46:45.248] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[08:46:46.351] Starting the invocation (attempt 1)
[08:46:46.374] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/7d5e2efb6ecffca96bdbfba53070f18a66871303"
[08:46:46.374] Popped gitiles commit info from properties and tags
[08:46:46.374] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[08:46:46.374] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[08:46:46.375] Buildbucket request:
{
  "requestId": "8862219388292580288",
  "builder": {
    "project": "chromium-m143",
    "bucket": "ci",
    "builder": "Dawn Mac x64 DEPS Release (AMD)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "7d5e2efb6ecffca96bdbfba53070f18a66871303",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/7499"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8862219388292580288",
            "job": "chromium-m143/Dawn Mac x64 DEPS Release (AMD)",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "92817c9b0b7e5c15833bf514e1357735f1559211",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7499"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "92817c9b0b7e5c15833bf514e1357735f1559211"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8696792563560280353"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8696792563560280353"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8696792563560280353"
                                              }
                                        ],
                                    "parent_build_id": "8696792563560280353",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Mac x64 DEPS Builder",
                                    "parent_got_angle_revision": "9b75aecb1fa769e357d1dad2614228b6a04ddee6",
                                    "parent_got_dawn_revision": "e1ce227ebf75378c5f60a9d531579982bcdd93ee",
                                    "parent_got_revision": "92817c9b0b7e5c15833bf514e1357735f1559211",
                                    "parent_got_revision_cp": "refs/branch-heads/7499@{#2915}",
                                    "parent_got_swiftshader_revision": "3d536c0fc62b1cdea0f78c3c38d79be559855b88",
                                    "parent_got_v8_revision": "beee9f5cafde91bbd086077a11db16cb9768e62a",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/14.3@{#31}",
                                    "parent_got_webrtc_revision": "1788a81407183acc98163a4e1507c5c63fb175cc",
                                    "parent_got_webrtc_revision_cp": "refs/branch-heads/7499@{#2}",
                                    "revision": "92817c9b0b7e5c15833bf514e1357735f1559211",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "ce260e4def75c436983cbfad670037e06cbaf09d9a0df1133b78bc82f6f4d4bb/323",
                                          "dawn_perf_tests": "9356fbc0a051bbf268120d20a36d4598278cedf39ecf7d4d9aaf86f884c26116/487",
                                          "gl_tests": "cf40547918acbe023866a8d8c8f84dc06603a019fde7f0aecf56915990776a3e/323",
                                          "gl_unittests": "aaf8a9a6a02b1be52b24b8431234920c0163c49b3c936a309ee655781a20a2ec/323",
                                          "telemetry_gpu_integration_test": "559605083c27c25e4aacc71c7acd98fe4d2eedb1f41121aca622ae5b59f75044/815",
                                          "telemetry_gpu_unittests": "a2d82009c88bef38d41d9966d37daccb62932cac26053eee849ff36039049041/736",
                                          "webgpu_blink_web_tests": "2409c2c285ca042b51769fa128465e12ce00aeb34d5807dab860343e7509a2e6/566"
                                        },
                                    "swarming_command_lines_cwd": "out/aeae-Dawn_Mac_x64_DE",
                                    "swarming_command_lines_digest": "60de9dd905cce771078a83e00991e4c89813c7ce0fd35ffdc98650f989deaf14/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7499",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "92817c9b0b7e5c15833bf514e1357735f1559211",
                              "tags": [
                                    "parent_buildername:Dawn Mac x64 DEPS Builder",
                                    "parent_buildnumber:2272",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "a7e9b174-7dbe-4d2e-974d-e30579d2aaf1",
                        "title": "Dawn Mac x64 DEPS Builder/2272",
                        "url": "https://cr-buildbucket.appspot.com/build/8696792563560280353"
                      },
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "7d5e2efb6ecffca96bdbfba53070f18a66871303",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7499"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "7d5e2efb6ecffca96bdbfba53070f18a66871303"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8696789998706655025"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8696789998706655025"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8696789998706655025"
                                              }
                                        ],
                                    "parent_build_id": "8696789998706655025",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Mac x64 DEPS Builder",
                                    "parent_got_angle_revision": "9b75aecb1fa769e357d1dad2614228b6a04ddee6",
                                    "parent_got_dawn_revision": "e1ce227ebf75378c5f60a9d531579982bcdd93ee",
                                    "parent_got_revision": "7d5e2efb6ecffca96bdbfba53070f18a66871303",
                                    "parent_got_revision_cp": "refs/branch-heads/7499@{#2916}",
                                    "parent_got_swiftshader_revision": "3d536c0fc62b1cdea0f78c3c38d79be559855b88",
                                    "parent_got_v8_revision": "beee9f5cafde91bbd086077a11db16cb9768e62a",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/14.3@{#31}",
                                    "parent_got_webrtc_revision": "1788a81407183acc98163a4e1507c5c63fb175cc",
                                    "parent_got_webrtc_revision_cp": "refs/branch-heads/7499@{#2}",
                                    "revision": "7d5e2efb6ecffca96bdbfba53070f18a66871303",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "ce260e4def75c436983cbfad670037e06cbaf09d9a0df1133b78bc82f6f4d4bb/323",
                                          "dawn_perf_tests": "9356fbc0a051bbf268120d20a36d4598278cedf39ecf7d4d9aaf86f884c26116/487",
                                          "gl_tests": "cf40547918acbe023866a8d8c8f84dc06603a019fde7f0aecf56915990776a3e/323",
                                          "gl_unittests": "aaf8a9a6a02b1be52b24b8431234920c0163c49b3c936a309ee655781a20a2ec/323",
                                          "telemetry_gpu_integration_test": "9e679239268f4d34879c271343af8a214b7b2a738c7849dbe788acab364c042b/815",
                                          "telemetry_gpu_unittests": "a2d82009c88bef38d41d9966d37daccb62932cac26053eee849ff36039049041/736",
                                          "webgpu_blink_web_tests": "3514564dc9875ff1834071a177cd372503f8a744f5625055be279ccca10fd042/566"
                                        },
                                    "swarming_command_lines_cwd": "out/aeae-Dawn_Mac_x64_DE",
                                    "swarming_command_lines_digest": "60de9dd905cce771078a83e00991e4c89813c7ce0fd35ffdc98650f989deaf14/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7499",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "7d5e2efb6ecffca96bdbfba53070f18a66871303",
                              "tags": [
                                    "parent_buildername:Dawn Mac x64 DEPS Builder",
                                    "parent_buildnumber:2273",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "29b0b481-690c-487e-a54d-36c49c11539b",
                        "title": "Dawn Mac x64 DEPS Builder/2273",
                        "url": "https://cr-buildbucket.appspot.com/build/8696789998706655025"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "7d5e2efb6ecffca96bdbfba53070f18a66871303"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8696789998706655025"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8696789998706655025"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8696789998706655025"
                }
          ],
      "parent_build_id": "8696789998706655025",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Mac x64 DEPS Builder",
      "parent_got_angle_revision": "9b75aecb1fa769e357d1dad2614228b6a04ddee6",
      "parent_got_dawn_revision": "e1ce227ebf75378c5f60a9d531579982bcdd93ee",
      "parent_got_revision": "7d5e2efb6ecffca96bdbfba53070f18a66871303",
      "parent_got_revision_cp": "refs/branch-heads/7499@{#2916}",
      "parent_got_swiftshader_revision": "3d536c0fc62b1cdea0f78c3c38d79be559855b88",
      "parent_got_v8_revision": "beee9f5cafde91bbd086077a11db16cb9768e62a",
      "parent_got_v8_revision_cp": "refs/branch-heads/14.3@{#31}",
      "parent_got_webrtc_revision": "1788a81407183acc98163a4e1507c5c63fb175cc",
      "parent_got_webrtc_revision_cp": "refs/branch-heads/7499@{#2}",
      "swarm_hashes": {
            "dawn_end2end_tests": "ce260e4def75c436983cbfad670037e06cbaf09d9a0df1133b78bc82f6f4d4bb/323",
            "dawn_perf_tests": "9356fbc0a051bbf268120d20a36d4598278cedf39ecf7d4d9aaf86f884c26116/487",
            "gl_tests": "cf40547918acbe023866a8d8c8f84dc06603a019fde7f0aecf56915990776a3e/323",
            "gl_unittests": "aaf8a9a6a02b1be52b24b8431234920c0163c49b3c936a309ee655781a20a2ec/323",
            "telemetry_gpu_integration_test": "9e679239268f4d34879c271343af8a214b7b2a738c7849dbe788acab364c042b/815",
            "telemetry_gpu_unittests": "a2d82009c88bef38d41d9966d37daccb62932cac26053eee849ff36039049041/736",
            "webgpu_blink_web_tests": "3514564dc9875ff1834071a177cd372503f8a744f5625055be279ccca10fd042/566"
          },
      "swarming_command_lines_cwd": "out/aeae-Dawn_Mac_x64_DE",
      "swarming_command_lines_digest": "60de9dd905cce771078a83e00991e4c89813c7ce0fd35ffdc98650f989deaf14/93",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "7d5e2efb6ecffca96bdbfba53070f18a66871303",
    "ref": "refs/branch-heads/7499"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8862219388292580288"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m143/Dawn Mac x64 DEPS Release (AMD)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Mac x64 DEPS Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "2273"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[08:46:46.874] Scheduled build:
{
  "id": "8696789326400381169",
  "builder": {
    "project": "chromium-m143",
    "bucket": "ci",
    "builder": "Dawn Mac x64 DEPS Release (AMD)"
  },
  "number": 572,
  "createdBy": "project:chromium-m143",
  "createTime": "2025-11-30T08:46:46.423200912Z",
  "updateTime": "2025-11-30T08:46:46.423200912Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "7d5e2efb6ecffca96bdbfba53070f18a66871303",
      "ref": "refs/branch-heads/7499"
    }
  }
}
[08:46:46.874] Task URL: https://cr-buildbucket.appspot.com/build/8696789326400381169
[08:46:46.874] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:2:0) after 6m9s
[08:47:07.183] Received PubSub notification, asking Buildbucket for the build status
[08:47:07.208] Build status: SCHEDULED
[08:49:47.961] Received PubSub notification, asking Buildbucket for the build status
[08:49:47.983] Build status: STARTED
[08:52:55.879] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:2:0)
[08:52:55.895] Build status: STARTED
[08:52:55.895] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:5:0) after 7m5s
[09:00:01.008] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:5:0)
[09:00:01.026] Build status: STARTED
[09:00:01.026] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:6:0) after 7m50s
[09:07:51.560] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:6:0)
[09:07:51.577] Build status: STARTED
[09:07:51.577] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:7:0) after 7m15s
[09:15:06.628] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:7:0)
[09:15:06.653] Build status: STARTED
[09:15:06.653] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:8:0) after 8m36s
[09:23:42.674] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:8:0)
[09:23:42.770] Build status: STARTED
[09:23:42.770] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:9:0) after 8m54s
[09:32:36.886] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:9:0)
[09:32:37.490] Build status: STARTED
[09:32:37.490] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:10:0) after 2m39s
[09:35:16.507] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:10:0)
[09:35:16.531] Build status: STARTED
[09:35:16.531] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:11:0) after 1m8s
[09:36:24.797] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:11:0)
[09:36:24.811] Build status: STARTED
[09:36:24.811] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:12:0) after 7m10s
[09:43:34.877] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:12:0)
[09:43:34.931] Build status: STARTED
[09:43:34.932] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:13:0) after 9m35s
[09:53:09.946] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:13:0)
[09:53:09.969] Build status: STARTED
[09:53:09.969] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:14:0) after 6m39s
[09:59:49.066] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:14:0)
[09:59:49.082] Build status: STARTED
[09:59:49.082] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (AMD):8862219388292580288:15:0) after 9m29s
[09:59:55.776] Received PubSub notification, asking Buildbucket for the build status
[09:59:55.792] Build:
{
  "id": "8696789326400381169",
  "builder": {
    "project": "chromium-m143",
    "bucket": "ci",
    "builder": "Dawn Mac x64 DEPS Release (AMD)"
  },
  "number": 572,
  "createdBy": "project:chromium-m143",
  "createTime": "2025-11-30T08:46:46.423200912Z",
  "startTime": "2025-11-30T08:49:47.799190285Z",
  "endTime": "2025-11-30T09:59:53.378261866Z",
  "updateTime": "2025-11-30T09:59:53.378261866Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "7d5e2efb6ecffca96bdbfba53070f18a66871303",
      "ref": "refs/branch-heads/7499"
    }
  }
}
[09:59:55.792] Invocation finished in 1h13m10.556817634s with status FAILED