Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 1 hour
Actions:

Definition (rev 027538db86d56496591602437654bb2f6274d0bf)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Linux x64 DEPS Release (NVIDIA)"
>

Incoming triggers

  • Dawn Linux x64 DEPS Builder/1238 (1 week ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • Dawn Linux x64 DEPS Builder/1239 (1 week ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "861f128bf98c83e95232d017b585b91c757d9168",
        "project": "chromium/src",
        "ref": "refs/branch-heads/7049"
      }
    ]
  },
  "branch": "refs/branch-heads/7049",
  "deps_revision_overrides": {
    "src": "861f128bf98c83e95232d017b585b91c757d9168"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8719031684089144753"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8719031684089144753"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8719031684089144753"
    }
  ],
  "parent_build_id": "8719031684089144753",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Linux x64 DEPS Builder",
  "parent_got_angle_revision": "079266db445215380befce453b1ab3bbdfeaf73d",
  "parent_got_dawn_revision": "bdc68b25b620d7302a955e2c38c548ebfe74ef31",
  "parent_got_revision": "861f128bf98c83e95232d017b585b91c757d9168",
  "parent_got_revision_cp": "refs/branch-heads/7049@{#1549}",
  "parent_got_swiftshader_revision": "c12c13839d689f411911326b1f72e96eb525686c",
  "parent_got_v8_revision": "e2591684c45463aa1e46ebefc3fd35deee63f37c",
  "parent_got_v8_revision_cp": "refs/branch-heads/13.5@{#19}",
  "parent_got_webrtc_revision": "04413d62f754a7b1a3a2d8c3df23bcde040112b2",
  "parent_got_webrtc_revision_cp": "refs/branch-heads/7049@{#2}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "861f128bf98c83e95232d017b585b91c757d9168",
  "swarm_hashes": {
    "dawn_end2end_tests": "f1fb36fd7c3a5e8c5dcdd5429be50465dc4ee41f9199f7fe609a8bff8bd257eb/323",
    "dawn_perf_tests": "58526942527291a1533e23affb246d8392eff2da22f8bef69af1e137c20dff46/487",
    "gl_tests": "7170ca0cb7f7839aa9ac0b9445e0969ea1fbd78d91c81d6cbc32a0e058387c58/323",
    "gl_unittests": "9eaf5fda2735dbd8b49dbd5dc5cf1daa5515095864e6af019bbad76d04361200/323",
    "telemetry_gpu_integration_test": "a66b0ae18f68ed83be6bdc630142a70beb08028754c1851c036083cb80235a28/815",
    "telemetry_gpu_unittests": "013788890b1b44def4e2306cf40e5e3c40cccf5025273af8997a108a66b2b5a0/736",
    "webgpu_blink_web_tests": "8e2d1ddd9c7209b4fa6e201e6be639c933bea0edc2daf9a8ad6784570978ec05/566"
  },
  "swarming_command_lines_cwd": "out/0836-Dawn_Linux_x64_",
  "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/861f128bf98c83e95232d017b585b91c757d9168
  • gitiles_ref:refs/branch-heads/7049
  • parent_buildername:Dawn Linux x64 DEPS Builder
  • parent_buildnumber:1239
  • user_agent:recipe

Debug log (UTC)

[20:56:42.874] New invocation is queued and will start shortly
[20:56:42.874] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[20:56:44.145] Starting the invocation (attempt 1)
[20:56:44.169] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/861f128bf98c83e95232d017b585b91c757d9168"
[20:56:44.169] Popped gitiles commit info from properties and tags
[20:56:44.169] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[20:56:44.169] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[20:56:44.169] Buildbucket request:
{
  "requestId": "8884460316880950480",
  "builder": {
    "project": "chromium-m135",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "861f128bf98c83e95232d017b585b91c757d9168",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/7049"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8884460316880950480",
            "job": "chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA)",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "0767ab8ed6481bc2c99aaa5da740cc3fae901633",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7049"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "0767ab8ed6481bc2c99aaa5da740cc3fae901633"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8719034869795074705"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8719034869795074705"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8719034869795074705"
                                              }
                                        ],
                                    "parent_build_id": "8719034869795074705",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Linux x64 DEPS Builder",
                                    "parent_got_angle_revision": "079266db445215380befce453b1ab3bbdfeaf73d",
                                    "parent_got_dawn_revision": "bdc68b25b620d7302a955e2c38c548ebfe74ef31",
                                    "parent_got_revision": "0767ab8ed6481bc2c99aaa5da740cc3fae901633",
                                    "parent_got_revision_cp": "refs/branch-heads/7049@{#1548}",
                                    "parent_got_swiftshader_revision": "c12c13839d689f411911326b1f72e96eb525686c",
                                    "parent_got_v8_revision": "e2591684c45463aa1e46ebefc3fd35deee63f37c",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.5@{#19}",
                                    "parent_got_webrtc_revision": "04413d62f754a7b1a3a2d8c3df23bcde040112b2",
                                    "parent_got_webrtc_revision_cp": "refs/branch-heads/7049@{#2}",
                                    "revision": "0767ab8ed6481bc2c99aaa5da740cc3fae901633",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "f1fb36fd7c3a5e8c5dcdd5429be50465dc4ee41f9199f7fe609a8bff8bd257eb/323",
                                          "dawn_perf_tests": "58526942527291a1533e23affb246d8392eff2da22f8bef69af1e137c20dff46/487",
                                          "gl_tests": "7170ca0cb7f7839aa9ac0b9445e0969ea1fbd78d91c81d6cbc32a0e058387c58/323",
                                          "gl_unittests": "9eaf5fda2735dbd8b49dbd5dc5cf1daa5515095864e6af019bbad76d04361200/323",
                                          "telemetry_gpu_integration_test": "e6ae5c79bfcb0053baec3e7fbd9335420e6ed4b8adf6808ad5637babfe95cc92/815",
                                          "telemetry_gpu_unittests": "013788890b1b44def4e2306cf40e5e3c40cccf5025273af8997a108a66b2b5a0/736",
                                          "webgpu_blink_web_tests": "6d3d25a7d02bbe74f8cb937f0364480d23391233d4cef8d6bfb059a19bdb9201/566"
                                        },
                                    "swarming_command_lines_cwd": "out/0836-Dawn_Linux_x64_",
                                    "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7049",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "0767ab8ed6481bc2c99aaa5da740cc3fae901633",
                              "tags": [
                                    "parent_buildername:Dawn Linux x64 DEPS Builder",
                                    "parent_buildnumber:1238",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "fc86dab9-a073-4057-afe4-aa481aaa743a",
                        "title": "Dawn Linux x64 DEPS Builder/1238"
                      },
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "861f128bf98c83e95232d017b585b91c757d9168",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7049"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "861f128bf98c83e95232d017b585b91c757d9168"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8719031684089144753"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8719031684089144753"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8719031684089144753"
                                              }
                                        ],
                                    "parent_build_id": "8719031684089144753",
                                    "parent_builder_group": "chromium.dawn",
                                    "parent_buildername": "Dawn Linux x64 DEPS Builder",
                                    "parent_got_angle_revision": "079266db445215380befce453b1ab3bbdfeaf73d",
                                    "parent_got_dawn_revision": "bdc68b25b620d7302a955e2c38c548ebfe74ef31",
                                    "parent_got_revision": "861f128bf98c83e95232d017b585b91c757d9168",
                                    "parent_got_revision_cp": "refs/branch-heads/7049@{#1549}",
                                    "parent_got_swiftshader_revision": "c12c13839d689f411911326b1f72e96eb525686c",
                                    "parent_got_v8_revision": "e2591684c45463aa1e46ebefc3fd35deee63f37c",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.5@{#19}",
                                    "parent_got_webrtc_revision": "04413d62f754a7b1a3a2d8c3df23bcde040112b2",
                                    "parent_got_webrtc_revision_cp": "refs/branch-heads/7049@{#2}",
                                    "revision": "861f128bf98c83e95232d017b585b91c757d9168",
                                    "swarm_hashes": {
                                          "dawn_end2end_tests": "f1fb36fd7c3a5e8c5dcdd5429be50465dc4ee41f9199f7fe609a8bff8bd257eb/323",
                                          "dawn_perf_tests": "58526942527291a1533e23affb246d8392eff2da22f8bef69af1e137c20dff46/487",
                                          "gl_tests": "7170ca0cb7f7839aa9ac0b9445e0969ea1fbd78d91c81d6cbc32a0e058387c58/323",
                                          "gl_unittests": "9eaf5fda2735dbd8b49dbd5dc5cf1daa5515095864e6af019bbad76d04361200/323",
                                          "telemetry_gpu_integration_test": "a66b0ae18f68ed83be6bdc630142a70beb08028754c1851c036083cb80235a28/815",
                                          "telemetry_gpu_unittests": "013788890b1b44def4e2306cf40e5e3c40cccf5025273af8997a108a66b2b5a0/736",
                                          "webgpu_blink_web_tests": "8e2d1ddd9c7209b4fa6e201e6be639c933bea0edc2daf9a8ad6784570978ec05/566"
                                        },
                                    "swarming_command_lines_cwd": "out/0836-Dawn_Linux_x64_",
                                    "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7049",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "861f128bf98c83e95232d017b585b91c757d9168",
                              "tags": [
                                    "parent_buildername:Dawn Linux x64 DEPS Builder",
                                    "parent_buildnumber:1239",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "ca22a3f6-f7f2-473f-9244-80070815d0d6",
                        "title": "Dawn Linux x64 DEPS Builder/1239"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "861f128bf98c83e95232d017b585b91c757d9168"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8719031684089144753"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8719031684089144753"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8719031684089144753"
                }
          ],
      "parent_build_id": "8719031684089144753",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Linux x64 DEPS Builder",
      "parent_got_angle_revision": "079266db445215380befce453b1ab3bbdfeaf73d",
      "parent_got_dawn_revision": "bdc68b25b620d7302a955e2c38c548ebfe74ef31",
      "parent_got_revision": "861f128bf98c83e95232d017b585b91c757d9168",
      "parent_got_revision_cp": "refs/branch-heads/7049@{#1549}",
      "parent_got_swiftshader_revision": "c12c13839d689f411911326b1f72e96eb525686c",
      "parent_got_v8_revision": "e2591684c45463aa1e46ebefc3fd35deee63f37c",
      "parent_got_v8_revision_cp": "refs/branch-heads/13.5@{#19}",
      "parent_got_webrtc_revision": "04413d62f754a7b1a3a2d8c3df23bcde040112b2",
      "parent_got_webrtc_revision_cp": "refs/branch-heads/7049@{#2}",
      "swarm_hashes": {
            "dawn_end2end_tests": "f1fb36fd7c3a5e8c5dcdd5429be50465dc4ee41f9199f7fe609a8bff8bd257eb/323",
            "dawn_perf_tests": "58526942527291a1533e23affb246d8392eff2da22f8bef69af1e137c20dff46/487",
            "gl_tests": "7170ca0cb7f7839aa9ac0b9445e0969ea1fbd78d91c81d6cbc32a0e058387c58/323",
            "gl_unittests": "9eaf5fda2735dbd8b49dbd5dc5cf1daa5515095864e6af019bbad76d04361200/323",
            "telemetry_gpu_integration_test": "a66b0ae18f68ed83be6bdc630142a70beb08028754c1851c036083cb80235a28/815",
            "telemetry_gpu_unittests": "013788890b1b44def4e2306cf40e5e3c40cccf5025273af8997a108a66b2b5a0/736",
            "webgpu_blink_web_tests": "8e2d1ddd9c7209b4fa6e201e6be639c933bea0edc2daf9a8ad6784570978ec05/566"
          },
      "swarming_command_lines_cwd": "out/0836-Dawn_Linux_x64_",
      "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "861f128bf98c83e95232d017b585b91c757d9168",
    "ref": "refs/branch-heads/7049"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8884460316880950480"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Linux x64 DEPS Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "1239"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[20:56:44.709] Scheduled build:
{
  "id": "8719030254775415217",
  "builder": {
    "project": "chromium-m135",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "number": 611,
  "createdBy": "project:chromium-m135",
  "createTime": "2025-03-29T20:56:44.259643537Z",
  "updateTime": "2025-03-29T20:56:44.259643537Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "861f128bf98c83e95232d017b585b91c757d9168",
      "ref": "refs/branch-heads/7049"
    }
  },
  "canary": true
}
[20:56:44.709] Task URL: https://cr-buildbucket.appspot.com/build/8719030254775415217
[20:56:44.709] Scheduling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:2:0) after 7m21s
[20:57:51.820] Received PubSub notification, asking Buildbucket for the build status
[20:57:51.850] Build status: STARTED
[20:58:14.715] Received PubSub notification, asking Buildbucket for the build status
[20:58:14.751] Build status: STARTED
[21:04:05.722] Handling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:2:0)
[21:04:05.751] Build status: STARTED
[21:04:05.751] Scheduling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:5:0) after 8m6s
[21:12:11.868] Handling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:5:0)
[21:12:11.916] Build status: STARTED
[21:12:11.916] Scheduling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:6:0) after 2m50s
[21:15:01.948] Handling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:6:0)
[21:15:02.000] Build status: STARTED
[21:15:02.000] Scheduling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:7:0) after 6m44s
[21:21:46.155] Handling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:7:0)
[21:21:46.181] Build status: STARTED
[21:21:46.181] Scheduling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:8:0) after 8m44s
[21:30:30.364] Handling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:8:0)
[21:30:30.386] Build status: STARTED
[21:30:30.386] Scheduling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:9:0) after 6m23s
[21:36:53.415] Handling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:9:0)
[21:36:53.451] Build status: STARTED
[21:36:53.451] Scheduling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:10:0) after 5m1s
[21:41:54.497] Handling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:10:0)
[21:41:54.541] Build status: STARTED
[21:41:54.541] Scheduling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:11:0) after 3m58s
[21:45:52.776] Handling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:11:0)
[21:45:52.805] Build status: STARTED
[21:45:52.805] Scheduling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:12:0) after 4m41s
[21:50:33.980] Handling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:12:0)
[21:50:34.000] Build status: STARTED
[21:50:34.000] Scheduling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:13:0) after 7m34s
[21:58:08.128] Handling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:13:0)
[21:58:08.286] Build status: STARTED
[21:58:08.286] Scheduling timer "check-buildbucket-build-status" (chromium-m135/Dawn Linux x64 DEPS Release (NVIDIA):8884460316880950480:14:0) after 9m7s
[21:58:44.690] Received PubSub notification, asking Buildbucket for the build status
[21:58:44.706] Build:
{
  "id": "8719030254775415217",
  "builder": {
    "project": "chromium-m135",
    "bucket": "ci",
    "builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
  },
  "number": 611,
  "createdBy": "project:chromium-m135",
  "createTime": "2025-03-29T20:56:44.259643537Z",
  "startTime": "2025-03-29T20:57:14.535699161Z",
  "endTime": "2025-03-29T21:58:44.319012073Z",
  "updateTime": "2025-03-29T21:58:44.319012073Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "861f128bf98c83e95232d017b585b91c757d9168",
      "ref": "refs/branch-heads/7049"
    }
  },
  "canary": true
}
[21:58:44.706] Invocation finished in 1h2m1.838087891s with status FAILED