Status: FAILED
Triggered by: v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 36 minutes
Actions:

Definition (rev c922df7f3ebbf09d5a6f144d12699a27ed6f4f07)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "V8 NumFuzz - debug"
>

Incoming triggers

  • V8 Clusterfuzz Linux64 - debug builder/39432 (2 years ago by v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com)
  • V8 Clusterfuzz Linux64 - debug builder/39433 (2 years ago by v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "newest_gitiles_revision": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
  "oldest_gitiles_revision": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
  "parent_build": "https://cr-buildbucket.appspot.com/build/8759195585438110161",
  "parent_buildername": "V8 Clusterfuzz Linux64 - debug builder",
  "parent_gn_args": [
    "is_component_build = true",
    "is_debug = true",
    "target_cpu = \"x64\"",
    "use_remoteexec = true",
    "v8_enable_backtrace = true",
    "v8_enable_google_benchmark = true",
    "v8_enable_slow_dchecks = true"
  ],
  "parent_got_revision": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
  "parent_got_revision_cp": "refs/heads/main@{#91781}",
  "parent_test_spec": {
    "swarming_dimensions": {
      "os": "Ubuntu-22.04"
    },
    "swarming_task_attrs": {
      "expiration": 13800,
      "hard_timeout": 4200,
      "priority": 35
    },
    "tests": [
      [
        "d8testing_random_gc",
        1,
        null,
        "",
        [],
        {},
        {}
      ],
      [
        "numfuzz",
        2,
        null,
        "marking",
        [
          "--total-timeout-sec=2100",
          "--stress-marking=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "delay",
        [
          "--total-timeout-sec=2100",
          "--stress-delay-tasks=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "interrupt",
        [
          "--total-timeout-sec=2100",
          "--stress-interrupt-budget=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "threads",
        [
          "--total-timeout-sec=2100",
          "--stress-thread-pool-size=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "stack",
        [
          "--total-timeout-sec=2100",
          "--stress-stack-size=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        3,
        null,
        "combined",
        [
          "--total-timeout-sec=2100",
          "--stress-delay-tasks=4",
          "--stress-deopt=2",
          "--stress-compaction=2",
          "--stress-gc=4",
          "--stress-marking=4",
          "--stress-scavenge=4",
          "--stress-thread-pool-size=2",
          "--stress-stack-size=1",
          "--stress-interrupt-budget=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "scavenge",
        [
          "--total-timeout-sec=2100",
          "--stress-scavenge=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        2,
        null,
        "deopt",
        [
          "--total-timeout-sec=2100",
          "--stress-deopt=1"
        ],
        {},
        {}
      ]
    ]
  },
  "revision": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
  "swarm_hashes": {
    "d8_default": "6c6519cd5432add87fa6736d5e2d4bed6ae98d889d0b03b5688e4e425fb1dee7/320",
    "run-num-fuzzer": "5fec191cef0d982c4496d1d99f658f5ba95afb2a18b563d3195da5a25f7c3a49/320"
  }
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/6f98fbe86a0d11e6c902e2ee50f609db046daf71
  • gitiles_ref:refs/heads/main
  • parent_buildername:V8 Clusterfuzz Linux64 - debug builder
  • parent_buildnumber:39433
  • user_agent:recipe

Debug log (UTC)

[13:02:31.033] New invocation is queued and will start shortly
[13:02:31.033] Triggered by user:v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[13:02:32.204] Starting the invocation (attempt 1)
[13:02:32.237] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/6f98fbe86a0d11e6c902e2ee50f609db046daf71"
[13:02:32.237] Reconstructed gitiles commit from tags
[13:02:32.237] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[13:02:32.241] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[13:02:32.242] Buildbucket request:
{
  "requestId": "8924624606914059296",
  "builder": {
    "project": "v8",
    "bucket": "ci",
    "builder": "V8 NumFuzz - debug"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8924624606914059296",
            "job": "v8/V8 NumFuzz - debug",
            "triggers": [
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "8c5bbf8540927b85e6e98f440efe41925035bb65",
                                    "oldest_gitiles_revision": "8c5bbf8540927b85e6e98f440efe41925035bb65",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8759196239595812065",
                                    "parent_buildername": "V8 Clusterfuzz Linux64 - debug builder",
                                    "parent_gn_args": [
                                          "is_component_build = true",
                                          "is_debug = true",
                                          "target_cpu = \"x64\"",
                                          "use_remoteexec = true",
                                          "v8_enable_backtrace = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_enable_slow_dchecks = true"
                                        ],
                                    "parent_got_revision": "8c5bbf8540927b85e6e98f440efe41925035bb65",
                                    "parent_got_revision_cp": "refs/heads/main@{#91780}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-22.04"
                                              },
                                          "swarming_task_attrs": {
                                                "expiration": 13800,
                                                "hard_timeout": 4200,
                                                "priority": 35
                                              },
                                          "tests": [
                                                [
                                                      "d8testing_random_gc",
                                                      1,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      2,
                                                      null,
                                                      "marking",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-marking=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "delay",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-delay-tasks=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "interrupt",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "threads",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-thread-pool-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "stack",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-stack-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      3,
                                                      null,
                                                      "combined",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-delay-tasks=4",
                                                            "--stress-deopt=2",
                                                            "--stress-compaction=2",
                                                            "--stress-gc=4",
                                                            "--stress-marking=4",
                                                            "--stress-scavenge=4",
                                                            "--stress-thread-pool-size=2",
                                                            "--stress-stack-size=1",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "scavenge",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-scavenge=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      2,
                                                      null,
                                                      "deopt",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-deopt=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "8c5bbf8540927b85e6e98f440efe41925035bb65",
                                    "swarm_hashes": {
                                          "d8_default": "270e7b862b9e847a38e6ac73c6b529c18c94618d9467ad4c2615d90aff1d07d2/320",
                                          "run-num-fuzzer": "ee8e6c85153028b4dcc47b18ce71af874ddd6f1c450e28029a19be8119f9e011/320"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/8c5bbf8540927b85e6e98f440efe41925035bb65",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Clusterfuzz Linux64 - debug builder",
                                    "parent_buildnumber:39432",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "1c167a50-2ada-4bb4-bd79-fa149fd9b3c7",
                        "title": "V8 Clusterfuzz Linux64 - debug builder/39432"
                      },
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
                                    "oldest_gitiles_revision": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8759195585438110161",
                                    "parent_buildername": "V8 Clusterfuzz Linux64 - debug builder",
                                    "parent_gn_args": [
                                          "is_component_build = true",
                                          "is_debug = true",
                                          "target_cpu = \"x64\"",
                                          "use_remoteexec = true",
                                          "v8_enable_backtrace = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_enable_slow_dchecks = true"
                                        ],
                                    "parent_got_revision": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
                                    "parent_got_revision_cp": "refs/heads/main@{#91781}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-22.04"
                                              },
                                          "swarming_task_attrs": {
                                                "expiration": 13800,
                                                "hard_timeout": 4200,
                                                "priority": 35
                                              },
                                          "tests": [
                                                [
                                                      "d8testing_random_gc",
                                                      1,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      2,
                                                      null,
                                                      "marking",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-marking=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "delay",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-delay-tasks=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "interrupt",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "threads",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-thread-pool-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "stack",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-stack-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      3,
                                                      null,
                                                      "combined",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-delay-tasks=4",
                                                            "--stress-deopt=2",
                                                            "--stress-compaction=2",
                                                            "--stress-gc=4",
                                                            "--stress-marking=4",
                                                            "--stress-scavenge=4",
                                                            "--stress-thread-pool-size=2",
                                                            "--stress-stack-size=1",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "scavenge",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-scavenge=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      2,
                                                      null,
                                                      "deopt",
                                                      [
                                                            "--total-timeout-sec=2100",
                                                            "--stress-deopt=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
                                    "swarm_hashes": {
                                          "d8_default": "6c6519cd5432add87fa6736d5e2d4bed6ae98d889d0b03b5688e4e425fb1dee7/320",
                                          "run-num-fuzzer": "5fec191cef0d982c4496d1d99f658f5ba95afb2a18b563d3195da5a25f7c3a49/320"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/6f98fbe86a0d11e6c902e2ee50f609db046daf71",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Clusterfuzz Linux64 - debug builder",
                                    "parent_buildnumber:39433",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "62eeb9d6-53c4-4edb-9965-43b85876eca2",
                        "title": "V8 Clusterfuzz Linux64 - debug builder/39433"
                      }
                ]
          },
      "newest_gitiles_revision": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
      "oldest_gitiles_revision": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
      "parent_build": "https://cr-buildbucket.appspot.com/build/8759195585438110161",
      "parent_buildername": "V8 Clusterfuzz Linux64 - debug builder",
      "parent_gn_args": [
            "is_component_build = true",
            "is_debug = true",
            "target_cpu = \"x64\"",
            "use_remoteexec = true",
            "v8_enable_backtrace = true",
            "v8_enable_google_benchmark = true",
            "v8_enable_slow_dchecks = true"
          ],
      "parent_got_revision": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
      "parent_got_revision_cp": "refs/heads/main@{#91781}",
      "parent_test_spec": {
            "swarming_dimensions": {
                  "os": "Ubuntu-22.04"
                },
            "swarming_task_attrs": {
                  "expiration": 13800,
                  "hard_timeout": 4200,
                  "priority": 35
                },
            "tests": [
                  [
                        "d8testing_random_gc",
                        1,
                        null,
                        "",
                        [
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        2,
                        null,
                        "marking",
                        [
                              "--total-timeout-sec=2100",
                              "--stress-marking=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "delay",
                        [
                              "--total-timeout-sec=2100",
                              "--stress-delay-tasks=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "interrupt",
                        [
                              "--total-timeout-sec=2100",
                              "--stress-interrupt-budget=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "threads",
                        [
                              "--total-timeout-sec=2100",
                              "--stress-thread-pool-size=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "stack",
                        [
                              "--total-timeout-sec=2100",
                              "--stress-stack-size=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        3,
                        null,
                        "combined",
                        [
                              "--total-timeout-sec=2100",
                              "--stress-delay-tasks=4",
                              "--stress-deopt=2",
                              "--stress-compaction=2",
                              "--stress-gc=4",
                              "--stress-marking=4",
                              "--stress-scavenge=4",
                              "--stress-thread-pool-size=2",
                              "--stress-stack-size=1",
                              "--stress-interrupt-budget=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "scavenge",
                        [
                              "--total-timeout-sec=2100",
                              "--stress-scavenge=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        2,
                        null,
                        "deopt",
                        [
                              "--total-timeout-sec=2100",
                              "--stress-deopt=1"
                            ],
                        {
                            },
                        {
                            }
                      ]
                ]
          },
      "swarm_hashes": {
            "d8_default": "6c6519cd5432add87fa6736d5e2d4bed6ae98d889d0b03b5688e4e425fb1dee7/320",
            "run-num-fuzzer": "5fec191cef0d982c4496d1d99f658f5ba95afb2a18b563d3195da5a25f7c3a49/320"
          }
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "v8/v8",
    "id": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8924624606914059296"
    },
    {
      "key": "scheduler_job_id",
      "value": "v8/V8 NumFuzz - debug"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "V8 Clusterfuzz Linux64 - debug builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "39433"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[13:02:32.878] Scheduled build:
{
  "id": "8759194544896830705",
  "builder": {
    "project": "v8",
    "bucket": "ci",
    "builder": "V8 NumFuzz - debug"
  },
  "number": 24118,
  "createdBy": "project:v8",
  "createTime": "2024-01-11T13:02:32.334742489Z",
  "updateTime": "2024-01-11T13:02:32.334742489Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "v8/v8",
      "id": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
      "ref": "refs/heads/main"
    }
  }
}
[13:02:32.878] Task URL: https://cr-buildbucket.appspot.com/build/8759194544896830705
[13:02:32.878] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - debug:8924624606914059296:2:0) after 7m38s
[13:02:33.821] Received PubSub notification, asking Buildbucket for the build status
[13:02:33.881] Build status: SCHEDULED
[13:02:37.590] Received PubSub notification, asking Buildbucket for the build status
[13:02:37.667] Build status: STARTED
[13:10:10.929] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - debug:8924624606914059296:2:0)
[13:10:11.041] Build status: STARTED
[13:10:11.041] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - debug:8924624606914059296:5:0) after 2m56s
[13:13:07.175] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - debug:8924624606914059296:5:0)
[13:13:07.196] Build status: STARTED
[13:13:07.196] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - debug:8924624606914059296:6:0) after 6m9s
[13:19:16.320] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - debug:8924624606914059296:6:0)
[13:19:16.412] Build status: STARTED
[13:19:16.412] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - debug:8924624606914059296:7:0) after 4m8s
[13:23:24.423] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - debug:8924624606914059296:7:0)
[13:23:24.443] Build status: STARTED
[13:23:24.443] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - debug:8924624606914059296:8:0) after 7m27s
[13:30:51.464] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - debug:8924624606914059296:8:0)
[13:30:51.525] Build status: STARTED
[13:30:51.525] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - debug:8924624606914059296:9:0) after 3m53s
[13:34:44.558] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - debug:8924624606914059296:9:0)
[13:34:44.721] Build status: STARTED
[13:34:44.721] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - debug:8924624606914059296:10:0) after 9m23s
[13:39:09.283] Received PubSub notification, asking Buildbucket for the build status
[13:39:09.306] Build:
{
  "id": "8759194544896830705",
  "builder": {
    "project": "v8",
    "bucket": "ci",
    "builder": "V8 NumFuzz - debug"
  },
  "number": 24118,
  "createdBy": "project:v8",
  "createTime": "2024-01-11T13:02:32.334742489Z",
  "startTime": "2024-01-11T13:02:36.852893Z",
  "endTime": "2024-01-11T13:39:08.523662Z",
  "updateTime": "2024-01-11T13:39:08.523662Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "v8/v8",
      "id": "6f98fbe86a0d11e6c902e2ee50f609db046daf71",
      "ref": "refs/heads/main"
    }
  }
}
[13:39:09.306] Invocation finished in 36m38.279827929s with status FAILED