Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 53 minutes
Actions:

Definition (rev dbc5bbb4d5c973373f7e6afa20eb56bb1648ac04)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Linux x64 DEPS Release (NVIDIA)"
>

Incoming triggers

  • Dawn Linux x64 DEPS Builder/259 (2 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Linux x64 DEPS Builder/260 (2 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "42c4089c9a5a344d1a247d317e649406e7fffcb4",
        "project": "chromium/src",
        "ref": "refs/branch-heads/7103"
      }
    ]
  },
  "branch": "refs/branch-heads/7103",
  "deps_revision_overrides": {
    "src": "42c4089c9a5a344d1a247d317e649406e7fffcb4"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8718260115792271393"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8718260115792271393"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8718260115792271393"
    }
  ],
  "parent_build_id": "8718260115792271393",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Linux x64 DEPS Builder",
  "parent_got_angle_revision": "ecc378cc61109732d174d6542c41fd523c331b13",
  "parent_got_dawn_revision": "cb024ff08d896770493c74e61d0510746f96f6df",
  "parent_got_revision": "42c4089c9a5a344d1a247d317e649406e7fffcb4",
  "parent_got_revision_cp": "refs/branch-heads/7103@{#334}",
  "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
  "parent_got_v8_revision": "ce92cefac8e62faf9fbac8388753efa33dabb6cb",
  "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#3}",
  "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "42c4089c9a5a344d1a247d317e649406e7fffcb4",
  "swarm_hashes": {
    "dawn_end2end_tests": "5f17f74471f58ba77a418f78189e19a2700b3c4f499cf0d2d418a520cdf78004/323",
    "dawn_perf_tests": "4a509d18e33951369b181927bb664db97cb5fa033fa580e408cecb6c35370f4e/487",
    "gl_tests": "03d824db0303a5a30e5afc1318f95cee09ea49e0903620cc52ce924e546a033e/323",
    "gl_unittests": "72cb369dde4e8325f6a86a0ec630944cc671d22695cff14516d3215210fba4fc/323",
    "telemetry_gpu_integration_test": "702ec07ff600ebd194f85e6e5152c640e4e22804b9c0f45c141ccf492a6d172c/815",
    "telemetry_gpu_unittests": "9d2fdddd5bcf3c9c653bc1cef1e27b83635d62ad89f8ffe9d459e30ac7a6282f/736",
    "webgpu_blink_web_tests": "80045c3631b9b28b83ee708d0314c9b7657ef95a50871aa0ecc93a65a88aa833/566"
  },
  "swarming_command_lines_cwd": "out/1d86-Dawn_Linux_x64_",
  "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/42c4089c9a5a344d1a247d317e649406e7fffcb4
  • gitiles_ref:refs/branch-heads/7103
  • parent_buildername:Dawn Linux x64 DEPS Builder
  • parent_buildnumber:260
  • user_agent:recipe

Debug log (UTC)

[09:23:53.269] New invocation is queued and will start shortly
[09:23:53.269] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[09:23:54.628] Starting the invocation (attempt 1)
[09:23:54.655] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/42c4089c9a5a344d1a247d317e649406e7fffcb4"
[09:23:54.656] Popped gitiles commit info from properties and tags
[09:23:54.656] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[09:23:54.657] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[09:23:54.657] Buildbucket request:
{
  "requestId": "8883688533082567264",
  "builder": {
    "project": "chromium-m136",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "42c4089c9a5a344d1a247d317e649406e7fffcb4",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/7103"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8883688533082567264",
            "job": "chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA)",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "ea29a59191c08c57a3867a8aabf437abc259f0ac",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7103"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "ea29a59191c08c57a3867a8aabf437abc259f0ac"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8718261425027873905"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8718261425027873905"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8718261425027873905"
                                              }
                                        ],
                                    "parent_build_id": "8718261425027873905",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Linux x64 DEPS Builder",
                                    "parent_got_angle_revision": "ecc378cc61109732d174d6542c41fd523c331b13",
                                    "parent_got_dawn_revision": "cb024ff08d896770493c74e61d0510746f96f6df",
                                    "parent_got_revision": "ea29a59191c08c57a3867a8aabf437abc259f0ac",
                                    "parent_got_revision_cp": "refs/branch-heads/7103@{#331}",
                                    "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
                                    "parent_got_v8_revision": "ce92cefac8e62faf9fbac8388753efa33dabb6cb",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#3}",
                                    "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
                                    "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
                                    "revision": "ea29a59191c08c57a3867a8aabf437abc259f0ac",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "5f17f74471f58ba77a418f78189e19a2700b3c4f499cf0d2d418a520cdf78004/323",
                                          "dawn_perf_tests": "4a509d18e33951369b181927bb664db97cb5fa033fa580e408cecb6c35370f4e/487",
                                          "gl_tests": "03d824db0303a5a30e5afc1318f95cee09ea49e0903620cc52ce924e546a033e/323",
                                          "gl_unittests": "72cb369dde4e8325f6a86a0ec630944cc671d22695cff14516d3215210fba4fc/323",
                                          "telemetry_gpu_integration_test": "89b040e5714c8d22054b7dd730a426002881e27b752905a3cd83cb8478605fdd/815",
                                          "telemetry_gpu_unittests": "9d2fdddd5bcf3c9c653bc1cef1e27b83635d62ad89f8ffe9d459e30ac7a6282f/736",
                                          "webgpu_blink_web_tests": "2f1571bbae259411efc55aa7080c31a34d69907e18deb715603bd20903e28345/566"
                                        },
                                    "swarming_command_lines_cwd": "out/1d86-Dawn_Linux_x64_",
                                    "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7103",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "ea29a59191c08c57a3867a8aabf437abc259f0ac",
                              "tags": [
                                    "parent_buildername:Dawn Linux x64 DEPS Builder",
                                    "parent_buildnumber:259",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "a822e5d2-29c7-41d3-b885-c5d798cd9734",
                        "title": "Dawn Linux x64 DEPS Builder/259"
                      },
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "42c4089c9a5a344d1a247d317e649406e7fffcb4",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7103"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "42c4089c9a5a344d1a247d317e649406e7fffcb4"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8718260115792271393"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8718260115792271393"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8718260115792271393"
                                              }
                                        ],
                                    "parent_build_id": "8718260115792271393",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Linux x64 DEPS Builder",
                                    "parent_got_angle_revision": "ecc378cc61109732d174d6542c41fd523c331b13",
                                    "parent_got_dawn_revision": "cb024ff08d896770493c74e61d0510746f96f6df",
                                    "parent_got_revision": "42c4089c9a5a344d1a247d317e649406e7fffcb4",
                                    "parent_got_revision_cp": "refs/branch-heads/7103@{#334}",
                                    "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
                                    "parent_got_v8_revision": "ce92cefac8e62faf9fbac8388753efa33dabb6cb",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#3}",
                                    "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
                                    "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
                                    "revision": "42c4089c9a5a344d1a247d317e649406e7fffcb4",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "5f17f74471f58ba77a418f78189e19a2700b3c4f499cf0d2d418a520cdf78004/323",
                                          "dawn_perf_tests": "4a509d18e33951369b181927bb664db97cb5fa033fa580e408cecb6c35370f4e/487",
                                          "gl_tests": "03d824db0303a5a30e5afc1318f95cee09ea49e0903620cc52ce924e546a033e/323",
                                          "gl_unittests": "72cb369dde4e8325f6a86a0ec630944cc671d22695cff14516d3215210fba4fc/323",
                                          "telemetry_gpu_integration_test": "702ec07ff600ebd194f85e6e5152c640e4e22804b9c0f45c141ccf492a6d172c/815",
                                          "telemetry_gpu_unittests": "9d2fdddd5bcf3c9c653bc1cef1e27b83635d62ad89f8ffe9d459e30ac7a6282f/736",
                                          "webgpu_blink_web_tests": "80045c3631b9b28b83ee708d0314c9b7657ef95a50871aa0ecc93a65a88aa833/566"
                                        },
                                    "swarming_command_lines_cwd": "out/1d86-Dawn_Linux_x64_",
                                    "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7103",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "42c4089c9a5a344d1a247d317e649406e7fffcb4",
                              "tags": [
                                    "parent_buildername:Dawn Linux x64 DEPS Builder",
                                    "parent_buildnumber:260",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "9061aff7-4804-4289-bd29-df41a019a158",
                        "title": "Dawn Linux x64 DEPS Builder/260"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "42c4089c9a5a344d1a247d317e649406e7fffcb4"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8718260115792271393"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8718260115792271393"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8718260115792271393"
                }
          ],
      "parent_build_id": "8718260115792271393",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Linux x64 DEPS Builder",
      "parent_got_angle_revision": "ecc378cc61109732d174d6542c41fd523c331b13",
      "parent_got_dawn_revision": "cb024ff08d896770493c74e61d0510746f96f6df",
      "parent_got_revision": "42c4089c9a5a344d1a247d317e649406e7fffcb4",
      "parent_got_revision_cp": "refs/branch-heads/7103@{#334}",
      "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
      "parent_got_v8_revision": "ce92cefac8e62faf9fbac8388753efa33dabb6cb",
      "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#3}",
      "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
      "swarm_hashes": {
            "dawn_end2end_tests": "5f17f74471f58ba77a418f78189e19a2700b3c4f499cf0d2d418a520cdf78004/323",
            "dawn_perf_tests": "4a509d18e33951369b181927bb664db97cb5fa033fa580e408cecb6c35370f4e/487",
            "gl_tests": "03d824db0303a5a30e5afc1318f95cee09ea49e0903620cc52ce924e546a033e/323",
            "gl_unittests": "72cb369dde4e8325f6a86a0ec630944cc671d22695cff14516d3215210fba4fc/323",
            "telemetry_gpu_integration_test": "702ec07ff600ebd194f85e6e5152c640e4e22804b9c0f45c141ccf492a6d172c/815",
            "telemetry_gpu_unittests": "9d2fdddd5bcf3c9c653bc1cef1e27b83635d62ad89f8ffe9d459e30ac7a6282f/736",
            "webgpu_blink_web_tests": "80045c3631b9b28b83ee708d0314c9b7657ef95a50871aa0ecc93a65a88aa833/566"
          },
      "swarming_command_lines_cwd": "out/1d86-Dawn_Linux_x64_",
      "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "42c4089c9a5a344d1a247d317e649406e7fffcb4",
    "ref": "refs/branch-heads/7103"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8883688533082567264"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Linux x64 DEPS Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "260"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[09:23:55.126] Scheduled build:
{
  "id": "8718258470892752273",
  "builder": {
    "project": "chromium-m136",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "number": 131,
  "createdBy": "project:chromium-m136",
  "createTime": "2025-04-07T09:23:54.726028380Z",
  "updateTime": "2025-04-07T09:23:54.726028380Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "42c4089c9a5a344d1a247d317e649406e7fffcb4",
      "ref": "refs/branch-heads/7103"
    }
  },
  "canary": true
}
[09:23:55.126] Task URL: https://cr-buildbucket.appspot.com/build/8718258470892752273
[09:23:55.126] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:2:0) after 7m34s
[09:23:56.116] Received PubSub notification, asking Buildbucket for the build status
[09:23:56.141] Build status: SCHEDULED
[09:24:03.074] Received PubSub notification, asking Buildbucket for the build status
[09:24:03.093] Build status: STARTED
[09:31:29.149] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:2:0)
[09:31:29.172] Build status: STARTED
[09:31:29.172] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:5:0) after 7m53s
[09:39:22.305] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:5:0)
[09:39:22.348] Build status: STARTED
[09:39:22.348] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:6:0) after 6m16s
[09:45:38.374] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:6:0)
[09:45:38.400] Build status: STARTED
[09:45:38.400] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:7:0) after 4m19s
[09:49:57.526] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:7:0)
[09:49:57.550] Build status: STARTED
[09:49:57.550] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:8:0) after 6m58s
[09:56:55.574] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:8:0)
[09:56:55.604] Build status: STARTED
[09:56:55.605] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:9:0) after 2m29s
[09:59:24.731] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:9:0)
[09:59:24.751] Build status: STARTED
[09:59:24.751] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:10:0) after 2m18s
[10:01:42.798] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:10:0)
[10:01:42.826] Build status: STARTED
[10:01:42.826] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:11:0) after 3m17s
[10:04:59.855] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:11:0)
[10:04:59.882] Build status: STARTED
[10:04:59.882] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:12:0) after 7m0s
[10:12:00.021] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:12:0)
[10:12:00.119] Build status: STARTED
[10:12:00.119] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:13:0) after 2m12s
[10:14:12.144] Handling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:13:0)
[10:14:12.180] Build status: STARTED
[10:14:12.180] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Dawn Linux x64 DEPS Release (NVIDIA):8883688533082567264:14:0) after 7m57s
[10:17:17.639] Received PubSub notification, asking Buildbucket for the build status
[10:17:17.733] Build:
{
  "id": "8718258470892752273",
  "builder": {
    "project": "chromium-m136",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "number": 131,
  "createdBy": "project:chromium-m136",
  "createTime": "2025-04-07T09:23:54.726028380Z",
  "startTime": "2025-04-07T09:24:02.809144042Z",
  "endTime": "2025-04-07T10:17:17.239183027Z",
  "updateTime": "2025-04-07T10:17:17.239183027Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "42c4089c9a5a344d1a247d317e649406e7fffcb4",
      "ref": "refs/branch-heads/7103"
    }
  },
  "canary": true
}
[10:17:17.733] Invocation finished in 53m24.478836821s with status FAILED