Status: FAILED
Triggered by: v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 1 hour
Actions:

Definition (rev c922df7f3ebbf09d5a6f144d12699a27ed6f4f07)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "V8 Linux - mips64el - sim"
>

Incoming triggers

  • V8 Linux - mips64el - sim - builder/48533 (2 years ago by v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com)
  • V8 Linux - mips64el - sim - builder/48534 (2 years ago by v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com)
  • V8 Linux - mips64el - sim - builder/48535 (2 years ago by v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com)
  • V8 Linux - mips64el - sim - builder/48536 (2 years ago by v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "newest_gitiles_revision": "90db103e4b63c4229bf9d07b5504776429444bfd",
  "oldest_gitiles_revision": "90db103e4b63c4229bf9d07b5504776429444bfd",
  "parent_build": "https://cr-buildbucket.appspot.com/build/8785367490328403249",
  "parent_buildername": "V8 Linux - mips64el - sim - builder",
  "parent_gn_args": [
    "dcheck_always_on = false",
    "is_component_build = false",
    "is_debug = false",
    "target_cpu = \"x64\"",
    "use_goma = false",
    "use_remoteexec = true",
    "v8_enable_google_benchmark = true",
    "v8_target_cpu = \"mips64el\""
  ],
  "parent_got_revision": "90db103e4b63c4229bf9d07b5504776429444bfd",
  "parent_got_revision_cp": "refs/heads/main@{#86747}",
  "parent_test_spec": {
    "swarming_dimensions": {
      "os": "Ubuntu-18.04"
    },
    "swarming_task_attrs": {
      "expiration": 14400,
      "hard_timeout": 3600,
      "priority": 35
    },
    "tests": [
      [
        "test262",
        2,
        "default",
        "default",
        [],
        {},
        {}
      ],
      [
        "v8testing",
        4,
        null,
        "",
        [],
        {},
        {}
      ]
    ]
  },
  "revision": "90db103e4b63c4229bf9d07b5504776429444bfd",
  "swarm_hashes": {
    "bot_default": "26033680cdf606b61c649a47d2e435467a9ff7afe01f80a793d60b19f75f7363/320",
    "test262": "7c174e6283f53db262404c4d1c42500ee83d82dc7e6cb72c1d9043e8c2ebf44c/404"
  }
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/90db103e4b63c4229bf9d07b5504776429444bfd
  • gitiles_ref:refs/heads/main
  • parent_buildername:V8 Linux - mips64el - sim - builder
  • parent_buildnumber:48536
  • user_agent:recipe

Debug log (UTC)

[16:02:28.571] New invocation is queued and will start shortly
[16:02:28.571] Triggered by user:v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[16:02:29.725] Starting the invocation (attempt 1)
[16:02:29.805] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/90db103e4b63c4229bf9d07b5504776429444bfd"
[16:02:29.805] Reconstructed gitiles commit from tags
[16:02:29.805] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[16:02:29.805] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[16:02:29.806] Buildbucket request:
{
  "requestId": "8950795808170810752",
  "builder": {
    "project": "v8",
    "bucket": "ci",
    "builder": "V8 Linux - mips64el - sim"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8950795808170810752",
            "job": "v8/ci-V8 Linux - mips64el - sim",
            "triggers": [
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "d7a31ed3b9b927d9ce509b09b429800e58445120",
                                    "oldest_gitiles_revision": "d7a31ed3b9b927d9ce509b09b429800e58445120",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8785369838698741825",
                                    "parent_buildername": "V8 Linux - mips64el - sim - builder",
                                    "parent_gn_args": [
                                          "dcheck_always_on = false",
                                          "is_component_build = false",
                                          "is_debug = false",
                                          "target_cpu = \"x64\"",
                                          "use_goma = false",
                                          "use_remoteexec = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_target_cpu = \"mips64el\""
                                        ],
                                    "parent_got_revision": "d7a31ed3b9b927d9ce509b09b429800e58445120",
                                    "parent_got_revision_cp": "refs/heads/main@{#86744}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-18.04"
                                              },
                                          "swarming_task_attrs": {
                                                "expiration": 14400,
                                                "hard_timeout": 3600,
                                                "priority": 35
                                              },
                                          "tests": [
                                                [
                                                      "test262",
                                                      2,
                                                      "default",
                                                      "default",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "v8testing",
                                                      4,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "d7a31ed3b9b927d9ce509b09b429800e58445120",
                                    "swarm_hashes": {
                                          "bot_default": "a77e3fd6eb696a5aeaf820d359488ffc686b957ae82b8a235345f6bdaef5430f/320",
                                          "test262": "d98067e91503fff3112cf71d04fa8542f586aac2579276a7301bf3355d491663/404"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/d7a31ed3b9b927d9ce509b09b429800e58445120",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Linux - mips64el - sim - builder",
                                    "parent_buildnumber:48533",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "13dd74e0-f38d-41c8-9ef9-13de144df686",
                        "title": "V8 Linux - mips64el - sim - builder/48533"
                      },
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "6c195bb1b1d166fddabe8ee1458bc423468af72e",
                                    "oldest_gitiles_revision": "6c195bb1b1d166fddabe8ee1458bc423468af72e",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8785368572733930721",
                                    "parent_buildername": "V8 Linux - mips64el - sim - builder",
                                    "parent_gn_args": [
                                          "dcheck_always_on = false",
                                          "is_component_build = false",
                                          "is_debug = false",
                                          "target_cpu = \"x64\"",
                                          "use_goma = false",
                                          "use_remoteexec = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_target_cpu = \"mips64el\""
                                        ],
                                    "parent_got_revision": "6c195bb1b1d166fddabe8ee1458bc423468af72e",
                                    "parent_got_revision_cp": "refs/heads/main@{#86745}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-18.04"
                                              },
                                          "swarming_task_attrs": {
                                                "expiration": 14400,
                                                "hard_timeout": 3600,
                                                "priority": 35
                                              },
                                          "tests": [
                                                [
                                                      "test262",
                                                      2,
                                                      "default",
                                                      "default",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "v8testing",
                                                      4,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "6c195bb1b1d166fddabe8ee1458bc423468af72e",
                                    "swarm_hashes": {
                                          "bot_default": "24466f15e750325de7a91fedc2d830b3a5e0a931c15a4df2cfbda9521326ed96/320",
                                          "test262": "893a895003883344d63937ba7a8a684e363552ab15b0e90dffe5a45debf42cbf/404"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/6c195bb1b1d166fddabe8ee1458bc423468af72e",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Linux - mips64el - sim - builder",
                                    "parent_buildnumber:48534",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "7b20012b-051a-4652-bbca-da7dee2a987a",
                        "title": "V8 Linux - mips64el - sim - builder/48534"
                      },
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "ffc737c8ab5e22037823ed5403a1c5dd24177312",
                                    "oldest_gitiles_revision": "ffc737c8ab5e22037823ed5403a1c5dd24177312",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8785368154174851697",
                                    "parent_buildername": "V8 Linux - mips64el - sim - builder",
                                    "parent_gn_args": [
                                          "dcheck_always_on = false",
                                          "is_component_build = false",
                                          "is_debug = false",
                                          "target_cpu = \"x64\"",
                                          "use_goma = false",
                                          "use_remoteexec = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_target_cpu = \"mips64el\""
                                        ],
                                    "parent_got_revision": "ffc737c8ab5e22037823ed5403a1c5dd24177312",
                                    "parent_got_revision_cp": "refs/heads/main@{#86746}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-18.04"
                                              },
                                          "swarming_task_attrs": {
                                                "expiration": 14400,
                                                "hard_timeout": 3600,
                                                "priority": 35
                                              },
                                          "tests": [
                                                [
                                                      "test262",
                                                      2,
                                                      "default",
                                                      "default",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "v8testing",
                                                      4,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "ffc737c8ab5e22037823ed5403a1c5dd24177312",
                                    "swarm_hashes": {
                                          "bot_default": "82bbc61e7f1041ee604c87268fe09c47cde1df1fc8a22e91460242fec56055f9/320",
                                          "test262": "0bf228b4bfee3cceed1074c87024c5563f121b76759cfa145df63dc843e14a95/404"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/ffc737c8ab5e22037823ed5403a1c5dd24177312",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Linux - mips64el - sim - builder",
                                    "parent_buildnumber:48535",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "666f9644-12cb-4a30-a288-74f831d66cbb",
                        "title": "V8 Linux - mips64el - sim - builder/48535"
                      },
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "90db103e4b63c4229bf9d07b5504776429444bfd",
                                    "oldest_gitiles_revision": "90db103e4b63c4229bf9d07b5504776429444bfd",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8785367490328403249",
                                    "parent_buildername": "V8 Linux - mips64el - sim - builder",
                                    "parent_gn_args": [
                                          "dcheck_always_on = false",
                                          "is_component_build = false",
                                          "is_debug = false",
                                          "target_cpu = \"x64\"",
                                          "use_goma = false",
                                          "use_remoteexec = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_target_cpu = \"mips64el\""
                                        ],
                                    "parent_got_revision": "90db103e4b63c4229bf9d07b5504776429444bfd",
                                    "parent_got_revision_cp": "refs/heads/main@{#86747}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-18.04"
                                              },
                                          "swarming_task_attrs": {
                                                "expiration": 14400,
                                                "hard_timeout": 3600,
                                                "priority": 35
                                              },
                                          "tests": [
                                                [
                                                      "test262",
                                                      2,
                                                      "default",
                                                      "default",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "v8testing",
                                                      4,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "90db103e4b63c4229bf9d07b5504776429444bfd",
                                    "swarm_hashes": {
                                          "bot_default": "26033680cdf606b61c649a47d2e435467a9ff7afe01f80a793d60b19f75f7363/320",
                                          "test262": "7c174e6283f53db262404c4d1c42500ee83d82dc7e6cb72c1d9043e8c2ebf44c/404"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/90db103e4b63c4229bf9d07b5504776429444bfd",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Linux - mips64el - sim - builder",
                                    "parent_buildnumber:48536",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "1824391f-f1f0-465c-b2b1-3994b18c6fe4",
                        "title": "V8 Linux - mips64el - sim - builder/48536"
                      }
                ]
          },
      "newest_gitiles_revision": "90db103e4b63c4229bf9d07b5504776429444bfd",
      "oldest_gitiles_revision": "90db103e4b63c4229bf9d07b5504776429444bfd",
      "parent_build": "https://cr-buildbucket.appspot.com/build/8785367490328403249",
      "parent_buildername": "V8 Linux - mips64el - sim - builder",
      "parent_gn_args": [
            "dcheck_always_on = false",
            "is_component_build = false",
            "is_debug = false",
            "target_cpu = \"x64\"",
            "use_goma = false",
            "use_remoteexec = true",
            "v8_enable_google_benchmark = true",
            "v8_target_cpu = \"mips64el\""
          ],
      "parent_got_revision": "90db103e4b63c4229bf9d07b5504776429444bfd",
      "parent_got_revision_cp": "refs/heads/main@{#86747}",
      "parent_test_spec": {
            "swarming_dimensions": {
                  "os": "Ubuntu-18.04"
                },
            "swarming_task_attrs": {
                  "expiration": 14400,
                  "hard_timeout": 3600,
                  "priority": 35
                },
            "tests": [
                  [
                        "test262",
                        2,
                        "default",
                        "default",
                        [
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "v8testing",
                        4,
                        null,
                        "",
                        [
                            ],
                        {
                            },
                        {
                            }
                      ]
                ]
          },
      "swarm_hashes": {
            "bot_default": "26033680cdf606b61c649a47d2e435467a9ff7afe01f80a793d60b19f75f7363/320",
            "test262": "7c174e6283f53db262404c4d1c42500ee83d82dc7e6cb72c1d9043e8c2ebf44c/404"
          }
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "v8/v8",
    "id": "90db103e4b63c4229bf9d07b5504776429444bfd",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8950795808170810752"
    },
    {
      "key": "scheduler_job_id",
      "value": "v8/ci-V8 Linux - mips64el - sim"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "V8 Linux - mips64el - sim - builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "48536"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[16:02:30.531] Scheduled build:
{
  "id": "8785365745946104353",
  "builder": {
    "project": "v8",
    "bucket": "ci",
    "builder": "V8 Linux - mips64el - sim"
  },
  "number": 19901,
  "createdBy": "project:v8",
  "createTime": "2023-03-28T16:02:30.063110457Z",
  "updateTime": "2023-03-28T16:02:30.063110457Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "v8/v8",
      "id": "90db103e4b63c4229bf9d07b5504776429444bfd",
      "ref": "refs/heads/main"
    }
  }
}
[16:02:30.531] Task URL: https://cr-buildbucket.appspot.com/build/8785365745946104353
[16:02:30.531] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:2:0) after 8m50s
[16:02:33.412] Received PubSub notification, asking Buildbucket for the build status
[16:02:33.442] Build status: STARTED
[16:11:20.555] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:2:0)
[16:11:20.592] Build status: STARTED
[16:11:20.592] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:4:0) after 4m56s
[16:16:16.615] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:4:0)
[16:16:16.645] Build status: STARTED
[16:16:16.646] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:5:0) after 9m7s
[16:25:23.667] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:5:0)
[16:25:23.695] Build status: STARTED
[16:25:23.695] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:6:0) after 1m30s
[16:26:53.771] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:6:0)
[16:26:53.794] Build status: STARTED
[16:26:53.795] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:7:0) after 5m25s
[16:32:18.825] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:7:0)
[16:32:18.901] Build status: STARTED
[16:32:18.901] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:8:0) after 8m32s
[16:40:51.086] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:8:0)
[16:40:51.115] Build status: STARTED
[16:40:51.115] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:9:0) after 5m37s
[16:46:28.138] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:9:0)
[16:46:28.171] Build status: STARTED
[16:46:28.171] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:10:0) after 7m23s
[16:53:51.192] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:10:0)
[16:53:51.223] Build status: STARTED
[16:53:51.223] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:11:0) after 6m38s
[17:00:29.244] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:11:0)
[17:00:29.267] Build status: STARTED
[17:00:29.267] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:12:0) after 6m48s
[17:07:17.512] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:12:0)
[17:07:17.534] Build status: STARTED
[17:07:17.534] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:13:0) after 7m34s
[17:14:51.791] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:13:0)
[17:14:51.832] Build status: STARTED
[17:14:51.832] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:14:0) after 6m7s
[17:20:59.082] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:14:0)
[17:20:59.111] Build status: STARTED
[17:20:59.111] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:15:0) after 5m14s
[17:26:13.181] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:15:0)
[17:26:13.216] Build status: STARTED
[17:26:13.216] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:16:0) after 8m1s
[17:34:14.947] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:16:0)
[17:34:14.973] Build status: STARTED
[17:34:14.973] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:17:0) after 7m14s
[17:41:29.227] Handling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:17:0)
[17:41:29.269] Build status: STARTED
[17:41:29.269] Scheduling timer "check-buildbucket-build-status" (v8/ci-V8 Linux - mips64el - sim:8950795808170810752:18:0) after 8m57s
[17:49:55.792] Received PubSub notification, asking Buildbucket for the build status
[17:49:55.833] Build:
{
  "id": "8785365745946104353",
  "builder": {
    "project": "v8",
    "bucket": "ci",
    "builder": "V8 Linux - mips64el - sim"
  },
  "number": 19901,
  "createdBy": "project:v8",
  "createTime": "2023-03-28T16:02:30.063110457Z",
  "startTime": "2023-03-28T16:02:31.940017Z",
  "endTime": "2023-03-28T17:49:55.403449088Z",
  "updateTime": "2023-03-28T17:49:55.403449088Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "v8/v8",
      "id": "90db103e4b63c4229bf9d07b5504776429444bfd",
      "ref": "refs/heads/main"
    }
  }
}
[17:49:55.833] Invocation finished in 1h47m27.27565791s with status FAILED