Status: FAILED
Triggered by: v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 1 hour
Actions:

Definition (rev c922df7f3ebbf09d5a6f144d12699a27ed6f4f07)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "V8 Linux - arm64 - sim - debug"
>

Incoming triggers

Incoming properties

{
  "newest_gitiles_revision": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
  "oldest_gitiles_revision": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
  "parent_build": "https://cr-buildbucket.appspot.com/build/8704943343832357313",
  "parent_buildername": "V8 Linux - arm64 - sim - debug builder",
  "parent_gn_args": [
    "is_component_build = true",
    "is_debug = true",
    "target_cpu = \"x64\"",
    "use_remoteexec = true",
    "v8_enable_backtrace = true",
    "v8_enable_google_benchmark = true",
    "v8_enable_slow_dchecks = true",
    "v8_target_cpu = \"arm64\""
  ],
  "parent_got_revision": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
  "parent_got_revision_cp": "refs/heads/main@{#102149}",
  "parent_test_spec": {
    "swarming_dimensions": {
      "os": "Ubuntu-22.04"
    },
    "swarming_task_attrs": {
      "hard_timeout": 3600
    },
    "tests": [
      [
        "mozilla",
        2,
        null,
        "",
        [],
        {},
        {}
      ],
      [
        "test262",
        2,
        "default",
        "default",
        [],
        {},
        {}
      ],
      [
        "v8testing",
        12,
        null,
        "",
        [],
        {},
        {}
      ],
      [
        "v8testing",
        14,
        "extra",
        "extra",
        [],
        {},
        {}
      ],
      [
        "v8testing",
        3,
        "default",
        "all_features",
        [
          "--extra-flags",
          "--sim-arm64-optional-features=\"all\""
        ],
        {},
        {}
      ]
    ]
  },
  "revision": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
  "swarm_hashes": {
    "bot_default": "be6247bc37792ea823d2e8870b60ebe89a3558adc89a9cd3fcd2a7e206406d44/320",
    "mozilla": "915cf971735dedd7b7595c3d7f4fbcc9bacc6b19690d692a5a797a8535373d49/319",
    "test262": "80af5ca31326dda00634def0f94880c954e9d07f695a9dab9e283b12bd1cc2b4/319"
  }
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/cd6944c05d268ef7d734cf33af86bc94c6172c2f
  • gitiles_ref:refs/heads/main
  • parent_buildername:V8 Linux - arm64 - sim - debug builder
  • parent_buildnumber:23073
  • user_agent:recipe

Debug log (UTC)

[09:01:45.157] New invocation is queued and will start shortly
[09:01:45.157] Triggered by user:v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[09:01:46.256] Starting the invocation (attempt 1)
[09:01:46.284] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/cd6944c05d268ef7d734cf33af86bc94c6172c2f"
[09:01:46.284] Reconstructed gitiles commit from tags
[09:01:46.284] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[09:01:46.284] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[09:01:46.285] Buildbucket request:
{
  "requestId": "8870372171643159264",
  "builder": {
    "project": "v8",
    "bucket": "ci",
    "builder": "V8 Linux - arm64 - sim - debug"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8870372171643159264",
            "job": "v8/ci-V8 Linux - arm64 - sim - debug",
            "triggers": [
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "0e16715a8c7c23d1fe4885829b5d35fedca63c41",
                                    "oldest_gitiles_revision": "0e16715a8c7c23d1fe4885829b5d35fedca63c41",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8704945668863120705",
                                    "parent_buildername": "V8 Linux - arm64 - sim - debug builder",
                                    "parent_gn_args": [
                                          "is_component_build = true",
                                          "is_debug = true",
                                          "target_cpu = \"x64\"",
                                          "use_remoteexec = true",
                                          "v8_enable_backtrace = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_enable_slow_dchecks = true",
                                          "v8_target_cpu = \"arm64\""
                                        ],
                                    "parent_got_revision": "0e16715a8c7c23d1fe4885829b5d35fedca63c41",
                                    "parent_got_revision_cp": "refs/heads/main@{#102147}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-22.04"
                                              },
                                          "swarming_task_attrs": {
                                                "hard_timeout": 3600
                                              },
                                          "tests": [
                                                [
                                                      "mozilla",
                                                      2,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "test262",
                                                      2,
                                                      "default",
                                                      "default",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "v8testing",
                                                      12,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "v8testing",
                                                      14,
                                                      "extra",
                                                      "extra",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "v8testing",
                                                      3,
                                                      "default",
                                                      "all_features",
                                                      [
                                                            "--extra-flags",
                                                            "--sim-arm64-optional-features=\"all\""
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "0e16715a8c7c23d1fe4885829b5d35fedca63c41",
                                    "swarm_hashes": {
                                          "bot_default": "30e57761d5ef682ea4e9f91ede386275f373278e85d5a4b1e67d466230838f5e/320",
                                          "mozilla": "c525204577ac31602263d65a07131e535a06687e210f7d70edab35c8c87a298a/319",
                                          "test262": "737bdbc279928e0ad8a947d5265b92b932c1c745efe060c411c227ac54aed910/319"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/0e16715a8c7c23d1fe4885829b5d35fedca63c41",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Linux - arm64 - sim - debug builder",
                                    "parent_buildnumber:23071",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "897ce06d-0d53-40b1-ad26-328cd864c9c3",
                        "title": "V8 Linux - arm64 - sim - debug builder/23071",
                        "url": "https://cr-buildbucket.appspot.com/build/8704945668863120705"
                      },
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "e92e18f765c80e98974f3f75a7c2979fd96457e4",
                                    "oldest_gitiles_revision": "e92e18f765c80e98974f3f75a7c2979fd96457e4",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8704944478060471185",
                                    "parent_buildername": "V8 Linux - arm64 - sim - debug builder",
                                    "parent_gn_args": [
                                          "is_component_build = true",
                                          "is_debug = true",
                                          "target_cpu = \"x64\"",
                                          "use_remoteexec = true",
                                          "v8_enable_backtrace = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_enable_slow_dchecks = true",
                                          "v8_target_cpu = \"arm64\""
                                        ],
                                    "parent_got_revision": "e92e18f765c80e98974f3f75a7c2979fd96457e4",
                                    "parent_got_revision_cp": "refs/heads/main@{#102148}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-22.04"
                                              },
                                          "swarming_task_attrs": {
                                                "hard_timeout": 3600
                                              },
                                          "tests": [
                                                [
                                                      "mozilla",
                                                      2,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "test262",
                                                      2,
                                                      "default",
                                                      "default",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "v8testing",
                                                      12,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "v8testing",
                                                      14,
                                                      "extra",
                                                      "extra",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "v8testing",
                                                      3,
                                                      "default",
                                                      "all_features",
                                                      [
                                                            "--extra-flags",
                                                            "--sim-arm64-optional-features=\"all\""
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "e92e18f765c80e98974f3f75a7c2979fd96457e4",
                                    "swarm_hashes": {
                                          "bot_default": "30e57761d5ef682ea4e9f91ede386275f373278e85d5a4b1e67d466230838f5e/320",
                                          "mozilla": "c525204577ac31602263d65a07131e535a06687e210f7d70edab35c8c87a298a/319",
                                          "test262": "737bdbc279928e0ad8a947d5265b92b932c1c745efe060c411c227ac54aed910/319"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/e92e18f765c80e98974f3f75a7c2979fd96457e4",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Linux - arm64 - sim - debug builder",
                                    "parent_buildnumber:23072",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "c24bac4f-92a4-49b1-a8be-4d5e8180d719",
                        "title": "V8 Linux - arm64 - sim - debug builder/23072",
                        "url": "https://cr-buildbucket.appspot.com/build/8704944478060471185"
                      },
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
                                    "oldest_gitiles_revision": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8704943343832357313",
                                    "parent_buildername": "V8 Linux - arm64 - sim - debug builder",
                                    "parent_gn_args": [
                                          "is_component_build = true",
                                          "is_debug = true",
                                          "target_cpu = \"x64\"",
                                          "use_remoteexec = true",
                                          "v8_enable_backtrace = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_enable_slow_dchecks = true",
                                          "v8_target_cpu = \"arm64\""
                                        ],
                                    "parent_got_revision": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
                                    "parent_got_revision_cp": "refs/heads/main@{#102149}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-22.04"
                                              },
                                          "swarming_task_attrs": {
                                                "hard_timeout": 3600
                                              },
                                          "tests": [
                                                [
                                                      "mozilla",
                                                      2,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "test262",
                                                      2,
                                                      "default",
                                                      "default",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "v8testing",
                                                      12,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "v8testing",
                                                      14,
                                                      "extra",
                                                      "extra",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "v8testing",
                                                      3,
                                                      "default",
                                                      "all_features",
                                                      [
                                                            "--extra-flags",
                                                            "--sim-arm64-optional-features=\"all\""
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
                                    "swarm_hashes": {
                                          "bot_default": "be6247bc37792ea823d2e8870b60ebe89a3558adc89a9cd3fcd2a7e206406d44/320",
                                          "mozilla": "915cf971735dedd7b7595c3d7f4fbcc9bacc6b19690d692a5a797a8535373d49/319",
                                          "test262": "80af5ca31326dda00634def0f94880c954e9d07f695a9dab9e283b12bd1cc2b4/319"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/cd6944c05d268ef7d734cf33af86bc94c6172c2f",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Linux - arm64 - sim - debug builder",
                                    "parent_buildnumber:23073",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "2afe3712-c85c-423e-a55d-e9b6b4e5f336",
                        "title": "V8 Linux - arm64 - sim - debug builder/23073",
                        "url": "https://cr-buildbucket.appspot.com/build/8704943343832357313"
                      }
                ]
          },
      "newest_gitiles_revision": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
      "oldest_gitiles_revision": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
      "parent_build": "https://cr-buildbucket.appspot.com/build/8704943343832357313",
      "parent_buildername": "V8 Linux - arm64 - sim - debug builder",
      "parent_gn_args": [
            "is_component_build = true",
            "is_debug = true",
            "target_cpu = \"x64\"",
            "use_remoteexec = true",
            "v8_enable_backtrace = true",
            "v8_enable_google_benchmark = true",
            "v8_enable_slow_dchecks = true",
            "v8_target_cpu = \"arm64\""
          ],
      "parent_got_revision": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
      "parent_got_revision_cp": "refs/heads/main@{#102149}",
      "parent_test_spec": {
            "swarming_dimensions": {
                  "os": "Ubuntu-22.04"
                },
            "swarming_task_attrs": {
                  "hard_timeout": 3600
                },
            "tests": [
                  [
                        "mozilla",
                        2,
                        null,
                        "",
                        [
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "test262",
                        2,
                        "default",
                        "default",
                        [
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "v8testing",
                        12,
                        null,
                        "",
                        [
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "v8testing",
                        14,
                        "extra",
                        "extra",
                        [
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "v8testing",
                        3,
                        "default",
                        "all_features",
                        [
                              "--extra-flags",
                              "--sim-arm64-optional-features=\"all\""
                            ],
                        {
                            },
                        {
                            }
                      ]
                ]
          },
      "swarm_hashes": {
            "bot_default": "be6247bc37792ea823d2e8870b60ebe89a3558adc89a9cd3fcd2a7e206406d44/320",
            "mozilla": "915cf971735dedd7b7595c3d7f4fbcc9bacc6b19690d692a5a797a8535373d49/319",
            "test262": "80af5ca31326dda00634def0f94880c954e9d07f695a9dab9e283b12bd1cc2b4/319"
          }
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "v8/v8",
    "id": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8870372171643159264"
    },
    {
      "key": "scheduler_job_id",
      "value": "v8/ci-V8 Linux - arm64 - sim - debug"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "V8 Linux - arm64 - sim - debug builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "23073"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[09:01:46.875] Scheduled build:
{
  "id": "8704942109742556657",
  "builder": {
    "project": "v8",
    "bucket": "ci",
    "builder": "V8 Linux - arm64 - sim - debug"
  },
  "number": 41007,
  "createdBy": "project:v8",
  "createTime": "2025-09-01T09:01:46.343558689Z",
  "updateTime": "2025-09-01T09:01:46.343558689Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "v8/v8",
      "id": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[09:01:46.875] Task URL: https://cr-buildbucket.appspot.com/build/8704942109742556657
[09:01:46.875] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:2:0) after 6m41s
[09:01:47.129] Received PubSub notification, asking Buildbucket for the build status
[09:01:47.151] Build status: SCHEDULED
[09:01:56.580] Received PubSub notification, asking Buildbucket for the build status
[09:01:56.597] Build status: STARTED
[09:08:27.937] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:2:0)
[09:08:27.959] Build status: STARTED
[09:08:27.959] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:5:0) after 4m35s
[09:13:03.088] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:5:0)
[09:13:03.114] Build status: STARTED
[09:13:03.114] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:6:0) after 8m32s
[09:21:35.183] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:6:0)
[09:21:35.204] Build status: STARTED
[09:21:35.204] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:7:0) after 1m52s
[09:23:27.292] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:7:0)
[09:23:27.401] Build status: STARTED
[09:23:27.401] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:8:0) after 3m23s
[09:26:50.417] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:8:0)
[09:26:50.462] Build status: STARTED
[09:26:50.462] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:9:0) after 1m17s
[09:28:07.480] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:9:0)
[09:28:07.532] Build status: STARTED
[09:28:07.532] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:10:0) after 7m36s
[09:35:43.602] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:10:0)
[09:35:43.625] Build status: STARTED
[09:35:43.625] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:11:0) after 9m33s
[09:45:16.717] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:11:0)
[09:45:16.733] Build status: STARTED
[09:45:16.733] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:12:0) after 7m1s
[09:52:17.801] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:12:0)
[09:52:17.817] Build status: STARTED
[09:52:17.817] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:13:0) after 6m22s
[09:58:39.879] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:13:0)
[09:58:39.899] Build status: STARTED
[09:58:39.899] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:14:0) after 2m35s
[10:01:14.941] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:14:0)
[10:01:14.959] Build status: STARTED
[10:01:14.959] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:15:0) after 1m48s
[10:03:02.939] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:15:0)
[10:03:03.006] Build status: STARTED
[10:03:03.006] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - arm64 - sim - debug:8870372171643159264:16:0) after 2m59s
[10:03:09.709] Received PubSub notification, asking Buildbucket for the build status
[10:03:09.727] Build:
{
  "id": "8704942109742556657",
  "builder": {
    "project": "v8",
    "bucket": "ci",
    "builder": "V8 Linux - arm64 - sim - debug"
  },
  "number": 41007,
  "createdBy": "project:v8",
  "createTime": "2025-09-01T09:01:46.343558689Z",
  "startTime": "2025-09-01T09:01:56.153252681Z",
  "endTime": "2025-09-01T10:03:07.944103800Z",
  "updateTime": "2025-09-01T10:03:07.944103800Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "v8/v8",
      "id": "cd6944c05d268ef7d734cf33af86bc94c6172c2f",
      "ref": "refs/heads/main"
    }
  },
  "canary": true
}
[10:03:09.728] Invocation finished in 1h1m24.58031686s with status FAILED