Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 53 minutes
Actions:

Definition (rev dbc5bbb4d5c973373f7e6afa20eb56bb1648ac04)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Linux x64 DEPS Release (NVIDIA)"
>

Incoming triggers

  • Dawn Linux x64 DEPS Builder/588 (2 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Linux x64 DEPS Builder/589 (2 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "d2cfcce686c9c80f41780036c5a49895dfde7d59",
        "project": "chromium/src",
        "ref": "refs/branch-heads/7103"
      }
    ]
  },
  "branch": "refs/branch-heads/7103",
  "deps_revision_overrides": {
    "src": "d2cfcce686c9c80f41780036c5a49895dfde7d59"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8717668081761141601"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8717668081761141601"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8717668081761141601"
    }
  ],
  "parent_build_id": "8717668081761141601",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Linux x64 DEPS Builder",
  "parent_got_angle_revision": "ecc378cc61109732d174d6542c41fd523c331b13",
  "parent_got_dawn_revision": "1cffe7ec763900d104e4df62bc96d93f572157cb",
  "parent_got_revision": "d2cfcce686c9c80f41780036c5a49895dfde7d59",
  "parent_got_revision_cp": "refs/branch-heads/7103@{#810}",
  "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
  "parent_got_v8_revision": "1dbf680799488c0699f6e3c5bdfa3a35e7f28a2d",
  "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#9}",
  "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "d2cfcce686c9c80f41780036c5a49895dfde7d59",
  "swarm_hashes": {
    "dawn_end2end_tests": "61b610a9f1dd31da0c2b7a2c082527fea24c5d4a4357dd8c60c9316a1e2a3fba/323",
    "dawn_perf_tests": "6fe965370a678c690d11e834bc4e03503c009f45881dbc60d81ae12122ed878f/487",
    "gl_tests": "cf1e343e8405db9570d885475e9bececf9586ff19b7713f95dfcdbfd99c1df52/323",
    "gl_unittests": "ead8ece03a4271431a46a598ac4fc9c12fcd424fd32564a9a3bc06449447f112/323",
    "telemetry_gpu_integration_test": "f9c0835299beb0b16ec509b598788c4621edf43cd0300964cd602ecccf569fdf/815",
    "telemetry_gpu_unittests": "74ad77517f2f4e8d8f30dded402a9bf4f08eaea426f613255ed5b8cc5fdb4164/736",
    "webgpu_blink_web_tests": "e569917e80da41e9a009933dec42518705f3fb2ced686eca9c96713b380c2354/566"
  },
  "swarming_command_lines_cwd": "out/1d86-Dawn_Linux_x64_",
  "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/d2cfcce686c9c80f41780036c5a49895dfde7d59
  • gitiles_ref:refs/branch-heads/7103
  • parent_buildername:Dawn Linux x64 DEPS Builder
  • parent_buildnumber:589
  • user_agent:recipe

Debug log (UTC)

[22:13:17.013] New invocation is queued and will start shortly
[22:13:17.013] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[22:13:18.301] Starting the invocation (attempt 1)
[22:13:18.326] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/d2cfcce686c9c80f41780036c5a49895dfde7d59"
[22:13:18.326] Popped gitiles commit info from properties and tags
[22:13:18.326] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[22:13:18.326] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[22:13:18.327] Buildbucket request:
{
  "requestId": "8883096545083012512",
  "builder": {
    "project": "chromium-m136",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "d2cfcce686c9c80f41780036c5a49895dfde7d59",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/7103"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8883096545083012512",
            "job": "chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA)",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "e70942200703d91a83789d04e7a952c845c5e029",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7103"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "e70942200703d91a83789d04e7a952c845c5e029"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8717670032106481809"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8717670032106481809"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8717670032106481809"
                                              }
                                        ],
                                    "parent_build_id": "8717670032106481809",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Linux x64 DEPS Builder",
                                    "parent_got_angle_revision": "ecc378cc61109732d174d6542c41fd523c331b13",
                                    "parent_got_dawn_revision": "1cffe7ec763900d104e4df62bc96d93f572157cb",
                                    "parent_got_revision": "e70942200703d91a83789d04e7a952c845c5e029",
                                    "parent_got_revision_cp": "refs/branch-heads/7103@{#809}",
                                    "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
                                    "parent_got_v8_revision": "1dbf680799488c0699f6e3c5bdfa3a35e7f28a2d",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#9}",
                                    "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
                                    "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
                                    "revision": "e70942200703d91a83789d04e7a952c845c5e029",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "61b610a9f1dd31da0c2b7a2c082527fea24c5d4a4357dd8c60c9316a1e2a3fba/323",
                                          "dawn_perf_tests": "6fe965370a678c690d11e834bc4e03503c009f45881dbc60d81ae12122ed878f/487",
                                          "gl_tests": "cf1e343e8405db9570d885475e9bececf9586ff19b7713f95dfcdbfd99c1df52/323",
                                          "gl_unittests": "ead8ece03a4271431a46a598ac4fc9c12fcd424fd32564a9a3bc06449447f112/323",
                                          "telemetry_gpu_integration_test": "eb616d93fd60a4575a39b65f7f992ef048d90f16da81d3e9d93659e31563a618/815",
                                          "telemetry_gpu_unittests": "74ad77517f2f4e8d8f30dded402a9bf4f08eaea426f613255ed5b8cc5fdb4164/736",
                                          "webgpu_blink_web_tests": "766edbc47c8168d9b49c16b7aba790e1537762dfa8f33284ea072d80f5e75c30/566"
                                        },
                                    "swarming_command_lines_cwd": "out/1d86-Dawn_Linux_x64_",
                                    "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7103",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "e70942200703d91a83789d04e7a952c845c5e029",
                              "tags": [
                                    "parent_buildername:Dawn Linux x64 DEPS Builder",
                                    "parent_buildnumber:588",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "7d4ba165-eb3e-455b-9fe3-9aefb739355c",
                        "title": "Dawn Linux x64 DEPS Builder/588"
                      },
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "d2cfcce686c9c80f41780036c5a49895dfde7d59",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7103"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "d2cfcce686c9c80f41780036c5a49895dfde7d59"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8717668081761141601"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8717668081761141601"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8717668081761141601"
                                              }
                                        ],
                                    "parent_build_id": "8717668081761141601",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Linux x64 DEPS Builder",
                                    "parent_got_angle_revision": "ecc378cc61109732d174d6542c41fd523c331b13",
                                    "parent_got_dawn_revision": "1cffe7ec763900d104e4df62bc96d93f572157cb",
                                    "parent_got_revision": "d2cfcce686c9c80f41780036c5a49895dfde7d59",
                                    "parent_got_revision_cp": "refs/branch-heads/7103@{#810}",
                                    "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
                                    "parent_got_v8_revision": "1dbf680799488c0699f6e3c5bdfa3a35e7f28a2d",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#9}",
                                    "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
                                    "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
                                    "revision": "d2cfcce686c9c80f41780036c5a49895dfde7d59",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "61b610a9f1dd31da0c2b7a2c082527fea24c5d4a4357dd8c60c9316a1e2a3fba/323",
                                          "dawn_perf_tests": "6fe965370a678c690d11e834bc4e03503c009f45881dbc60d81ae12122ed878f/487",
                                          "gl_tests": "cf1e343e8405db9570d885475e9bececf9586ff19b7713f95dfcdbfd99c1df52/323",
                                          "gl_unittests": "ead8ece03a4271431a46a598ac4fc9c12fcd424fd32564a9a3bc06449447f112/323",
                                          "telemetry_gpu_integration_test": "f9c0835299beb0b16ec509b598788c4621edf43cd0300964cd602ecccf569fdf/815",
                                          "telemetry_gpu_unittests": "74ad77517f2f4e8d8f30dded402a9bf4f08eaea426f613255ed5b8cc5fdb4164/736",
                                          "webgpu_blink_web_tests": "e569917e80da41e9a009933dec42518705f3fb2ced686eca9c96713b380c2354/566"
                                        },
                                    "swarming_command_lines_cwd": "out/1d86-Dawn_Linux_x64_",
                                    "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7103",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "d2cfcce686c9c80f41780036c5a49895dfde7d59",
                              "tags": [
                                    "parent_buildername:Dawn Linux x64 DEPS Builder",
                                    "parent_buildnumber:589",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "9f0ea9ab-83b4-4c1a-8c7f-8011c5a0d758",
                        "title": "Dawn Linux x64 DEPS Builder/589"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "d2cfcce686c9c80f41780036c5a49895dfde7d59"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8717668081761141601"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8717668081761141601"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8717668081761141601"
                }
          ],
      "parent_build_id": "8717668081761141601",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Linux x64 DEPS Builder",
      "parent_got_angle_revision": "ecc378cc61109732d174d6542c41fd523c331b13",
      "parent_got_dawn_revision": "1cffe7ec763900d104e4df62bc96d93f572157cb",
      "parent_got_revision": "d2cfcce686c9c80f41780036c5a49895dfde7d59",
      "parent_got_revision_cp": "refs/branch-heads/7103@{#810}",
      "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
      "parent_got_v8_revision": "1dbf680799488c0699f6e3c5bdfa3a35e7f28a2d",
      "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#9}",
      "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
      "swarm_hashes": {
            "dawn_end2end_tests": "61b610a9f1dd31da0c2b7a2c082527fea24c5d4a4357dd8c60c9316a1e2a3fba/323",
            "dawn_perf_tests": "6fe965370a678c690d11e834bc4e03503c009f45881dbc60d81ae12122ed878f/487",
            "gl_tests": "cf1e343e8405db9570d885475e9bececf9586ff19b7713f95dfcdbfd99c1df52/323",
            "gl_unittests": "ead8ece03a4271431a46a598ac4fc9c12fcd424fd32564a9a3bc06449447f112/323",
            "telemetry_gpu_integration_test": "f9c0835299beb0b16ec509b598788c4621edf43cd0300964cd602ecccf569fdf/815",
            "telemetry_gpu_unittests": "74ad77517f2f4e8d8f30dded402a9bf4f08eaea426f613255ed5b8cc5fdb4164/736",
            "webgpu_blink_web_tests": "e569917e80da41e9a009933dec42518705f3fb2ced686eca9c96713b380c2354/566"
          },
      "swarming_command_lines_cwd": "out/1d86-Dawn_Linux_x64_",
      "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "d2cfcce686c9c80f41780036c5a49895dfde7d59",
    "ref": "refs/branch-heads/7103"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8883096545083012512"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Linux x64 DEPS Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "589"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[22:13:18.813] Scheduled build:
{
  "id": "8717666482982125121",
  "builder": {
    "project": "chromium-m136",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "number": 292,
  "createdBy": "project:chromium-m136",
  "createTime": "2025-04-13T22:13:18.392201422Z",
  "updateTime": "2025-04-13T22:13:18.392201422Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "d2cfcce686c9c80f41780036c5a49895dfde7d59",
      "ref": "refs/branch-heads/7103"
    }
  }
}
[22:13:18.813] Task URL: https://cr-buildbucket.appspot.com/build/8717666482982125121
[22:13:18.813] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:2:0) after 4m16s
[22:13:24.517] Received PubSub notification, asking Buildbucket for the build status
[22:13:24.713] Build status: SCHEDULED
[22:13:48.071] Received PubSub notification, asking Buildbucket for the build status
[22:13:48.096] Build status: STARTED
[22:17:34.850] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:2:0)
[22:17:34.877] Build status: STARTED
[22:17:34.877] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:5:0) after 5m44s
[22:23:18.973] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:5:0)
[22:23:18.996] Build status: STARTED
[22:23:18.996] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:6:0) after 6m54s
[22:30:13.011] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:6:0)
[22:30:13.034] Build status: STARTED
[22:30:13.034] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:7:0) after 4m11s
[22:34:24.049] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:7:0)
[22:34:24.086] Build status: STARTED
[22:34:24.086] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:8:0) after 6m43s
[22:41:07.098] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:8:0)
[22:41:07.133] Build status: STARTED
[22:41:07.133] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:9:0) after 3m25s
[22:44:32.593] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:9:0)
[22:44:32.628] Build status: STARTED
[22:44:32.628] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:10:0) after 3m51s
[22:48:23.745] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:10:0)
[22:48:23.773] Build status: STARTED
[22:48:23.773] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:11:0) after 2m26s
[22:50:49.871] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:11:0)
[22:50:49.903] Build status: STARTED
[22:50:49.903] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:12:0) after 9m17s
[23:00:07.139] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:12:0)
[23:00:07.213] Build status: STARTED
[23:00:07.214] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:13:0) after 6m31s
[23:06:38.238] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:13:0)
[23:06:38.258] Build status: STARTED
[23:06:38.259] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883096545083012512:14:0) after 6m40s
[23:06:40.056] Received PubSub notification, asking Buildbucket for the build status
[23:06:40.079] Build:
{
  "id": "8717666482982125121",
  "builder": {
    "project": "chromium-m136",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "number": 292,
  "createdBy": "project:chromium-m136",
  "createTime": "2025-04-13T22:13:18.392201422Z",
  "startTime": "2025-04-13T22:13:47.832088468Z",
  "endTime": "2025-04-13T23:06:39.388406385Z",
  "updateTime": "2025-04-13T23:06:39.388406385Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "d2cfcce686c9c80f41780036c5a49895dfde7d59",
      "ref": "refs/branch-heads/7103"
    }
  }
}
[23:06:40.079] Invocation finished in 53m23.073535293s with status FAILED