1. All projects
  2. fuchsia
  3. core.x64-full_determinism-e5a0ec7f
  4. 8915596325506378128
Status: FAILED
Triggered by: -
Duration: 3 hours
Actions:

Definition (rev a6f4092eb6d344c81211966c5995265ee62c6dba) 

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "global.ci"
  builder: "core.x64-full_determinism"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://fuchsia.googlesource.com/integration",
  "revision": "1abd05df7844fcceddbdc9865db3f73b6ab02051"
}

Incoming tags

  • buildset:commit/gitiles/fuchsia.googlesource.com/integration/+/1abd05df7844fcceddbdc9865db3f73b6ab02051
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[04:43:11.140] New invocation is queued and will start shortly
[04:43:12.206] Starting the invocation (attempt 1)
[04:43:12.232] Popped buildset tag "buildset:commit/gitiles/fuchsia.googlesource.com/integration/+/1abd05df7844fcceddbdc9865db3f73b6ab02051"
[04:43:12.232] Popped gitiles commit info from properties and tags
[04:43:12.232] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[04:43:12.233] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[04:43:12.233] Buildbucket request:
{
  "requestId": "8915596325506378128",
  "builder": {
    "project": "fuchsia",
    "bucket": "global.ci",
    "builder": "core.x64-full_determinism"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8915596325506378128",
            "job": "fuchsia/core.x64-full_determinism-e5a0ec7f",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://fuchsia.googlesource.com/integration",
                              "revision": "f4a96bd16c41b657f4713c865a3bf5af1e0486f7"
                            },
                        "id": "https://fuchsia.googlesource.com/integration/+/refs/heads/main@f4a96bd16c41b657f4713c865a3bf5af1e0486f7",
                        "title": "f4a96bd16c41b657f4713c865a3bf5af1e0486f7",
                        "url": "https://fuchsia.googlesource.com/integration/+/f4a96bd16c41b657f4713c865a3bf5af1e0486f7"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://fuchsia.googlesource.com/integration",
                              "revision": "029e70bd07fcdda9814d876e15f1ddcfd201c872"
                            },
                        "id": "https://fuchsia.googlesource.com/integration/+/refs/heads/main@029e70bd07fcdda9814d876e15f1ddcfd201c872",
                        "title": "029e70bd07fcdda9814d876e15f1ddcfd201c872",
                        "url": "https://fuchsia.googlesource.com/integration/+/029e70bd07fcdda9814d876e15f1ddcfd201c872"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://fuchsia.googlesource.com/integration",
                              "revision": "99a26b2fb02986c463a7fba79d734dd0b857aabb"
                            },
                        "id": "https://fuchsia.googlesource.com/integration/+/refs/heads/main@99a26b2fb02986c463a7fba79d734dd0b857aabb",
                        "title": "99a26b2fb02986c463a7fba79d734dd0b857aabb",
                        "url": "https://fuchsia.googlesource.com/integration/+/99a26b2fb02986c463a7fba79d734dd0b857aabb"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://fuchsia.googlesource.com/integration",
                              "revision": "43b97b7ef2497ecb4211fb4524a4d03c4e951c7a"
                            },
                        "id": "https://fuchsia.googlesource.com/integration/+/refs/heads/main@43b97b7ef2497ecb4211fb4524a4d03c4e951c7a",
                        "title": "43b97b7ef2497ecb4211fb4524a4d03c4e951c7a",
                        "url": "https://fuchsia.googlesource.com/integration/+/43b97b7ef2497ecb4211fb4524a4d03c4e951c7a"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://fuchsia.googlesource.com/integration",
                              "revision": "39bc623ca66239fd39501f9e61b7fe616d6cff84"
                            },
                        "id": "https://fuchsia.googlesource.com/integration/+/refs/heads/main@39bc623ca66239fd39501f9e61b7fe616d6cff84",
                        "title": "39bc623ca66239fd39501f9e61b7fe616d6cff84",
                        "url": "https://fuchsia.googlesource.com/integration/+/39bc623ca66239fd39501f9e61b7fe616d6cff84"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://fuchsia.googlesource.com/integration",
                              "revision": "457467864c3f1b94009605028ea4a45f225b850f"
                            },
                        "id": "https://fuchsia.googlesource.com/integration/+/refs/heads/main@457467864c3f1b94009605028ea4a45f225b850f",
                        "title": "457467864c3f1b94009605028ea4a45f225b850f",
                        "url": "https://fuchsia.googlesource.com/integration/+/457467864c3f1b94009605028ea4a45f225b850f"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://fuchsia.googlesource.com/integration",
                              "revision": "e9e9c994be93ab6d1cc6798e36353fa4b5ded6ce"
                            },
                        "id": "https://fuchsia.googlesource.com/integration/+/refs/heads/main@e9e9c994be93ab6d1cc6798e36353fa4b5ded6ce",
                        "title": "e9e9c994be93ab6d1cc6798e36353fa4b5ded6ce",
                        "url": "https://fuchsia.googlesource.com/integration/+/e9e9c994be93ab6d1cc6798e36353fa4b5ded6ce"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://fuchsia.googlesource.com/integration",
                              "revision": "1f9fa148d26164489313be81ccbaddb43277620e"
                            },
                        "id": "https://fuchsia.googlesource.com/integration/+/refs/heads/main@1f9fa148d26164489313be81ccbaddb43277620e",
                        "title": "1f9fa148d26164489313be81ccbaddb43277620e",
                        "url": "https://fuchsia.googlesource.com/integration/+/1f9fa148d26164489313be81ccbaddb43277620e"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://fuchsia.googlesource.com/integration",
                              "revision": "db93fed9118ef95da1ab75ec71ec80c16435f2a0"
                            },
                        "id": "https://fuchsia.googlesource.com/integration/+/refs/heads/main@db93fed9118ef95da1ab75ec71ec80c16435f2a0",
                        "title": "db93fed9118ef95da1ab75ec71ec80c16435f2a0",
                        "url": "https://fuchsia.googlesource.com/integration/+/db93fed9118ef95da1ab75ec71ec80c16435f2a0"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://fuchsia.googlesource.com/integration",
                              "revision": "1abd05df7844fcceddbdc9865db3f73b6ab02051"
                            },
                        "id": "https://fuchsia.googlesource.com/integration/+/refs/heads/main@1abd05df7844fcceddbdc9865db3f73b6ab02051",
                        "title": "1abd05df7844fcceddbdc9865db3f73b6ab02051",
                        "url": "https://fuchsia.googlesource.com/integration/+/1abd05df7844fcceddbdc9865db3f73b6ab02051"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "fuchsia.googlesource.com",
    "project": "integration",
    "id": "1abd05df7844fcceddbdc9865db3f73b6ab02051",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8915596325506378128"
    },
    {
      "key": "scheduler_job_id",
      "value": "fuchsia/core.x64-full_determinism-e5a0ec7f"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[04:43:12.834] Scheduled build:
{
  "id": "8750166263578859489",
  "builder": {
    "project": "fuchsia",
    "bucket": "global.ci",
    "builder": "core.x64-full_determinism"
  },
  "createdBy": "project:fuchsia",
  "createTime": "2024-04-20T04:43:12.348009752Z",
  "updateTime": "2024-04-20T04:43:12.348009752Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "fuchsia.googlesource.com",
      "project": "integration",
      "id": "1abd05df7844fcceddbdc9865db3f73b6ab02051",
      "ref": "refs/heads/main"
    }
  }
}
[04:43:12.834] Task URL: https://cr-buildbucket.appspot.com/build/8750166263578859489
[04:43:12.834] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:2:0) after 6m43s
[04:43:37.861] Received PubSub notification, asking Buildbucket for the build status
[04:43:37.893] Build status: STARTED
[04:43:40.766] Received PubSub notification, asking Buildbucket for the build status
[04:43:40.785] Build status: STARTED
[04:49:55.851] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:2:0)
[04:49:55.885] Build status: STARTED
[04:49:55.885] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:5:0) after 7m13s
[04:57:08.904] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:5:0)
[04:57:09.243] Build status: STARTED
[04:57:09.243] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:6:0) after 5m2s
[05:02:11.283] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:6:0)
[05:02:11.327] Build status: STARTED
[05:02:11.327] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:7:0) after 6m43s
[05:08:54.438] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:7:0)
[05:08:54.532] Build status: STARTED
[05:08:54.532] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:8:0) after 9m59s
[05:18:53.687] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:8:0)
[05:18:53.705] Build status: STARTED
[05:18:53.706] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:9:0) after 9m53s
[05:28:46.723] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:9:0)
[05:28:46.741] Build status: STARTED
[05:28:46.741] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:10:0) after 6m47s
[05:35:33.758] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:10:0)
[05:35:33.792] Build status: STARTED
[05:35:33.792] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:11:0) after 5m38s
[05:41:11.946] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:11:0)
[05:41:11.974] Build status: STARTED
[05:41:11.974] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:12:0) after 6m33s
[05:47:45.005] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:12:0)
[05:47:45.027] Build status: STARTED
[05:47:45.027] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:13:0) after 3m44s
[05:51:29.090] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:13:0)
[05:51:29.115] Build status: STARTED
[05:51:29.115] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:14:0) after 5m37s
[05:57:06.143] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:14:0)
[05:57:06.164] Build status: STARTED
[05:57:06.164] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:15:0) after 9m17s
[06:06:23.227] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:15:0)
[06:06:23.286] Build status: STARTED
[06:06:23.286] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:16:0) after 7m14s
[06:13:37.622] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:16:0)
[06:13:37.648] Build status: STARTED
[06:13:37.648] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:17:0) after 4m56s
[06:18:33.700] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:17:0)
[06:18:33.830] Build status: STARTED
[06:18:33.830] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:18:0) after 8m26s
[06:26:59.987] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:18:0)
[06:27:00.017] Build status: STARTED
[06:27:00.017] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:19:0) after 9m36s
[06:36:36.125] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:19:0)
[06:36:36.192] Build status: STARTED
[06:36:36.192] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:20:0) after 2m50s
[06:39:26.253] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:20:0)
[06:39:26.274] Build status: STARTED
[06:39:26.274] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:21:0) after 5m20s
[06:44:46.339] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:21:0)
[06:44:46.356] Build status: STARTED
[06:44:46.356] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:22:0) after 5m40s
[06:50:26.411] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:22:0)
[06:50:26.430] Build status: STARTED
[06:50:26.430] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:23:0) after 5m46s
[06:56:12.584] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:23:0)
[06:56:12.610] Build status: STARTED
[06:56:12.610] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:24:0) after 4m32s
[07:00:44.701] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:24:0)
[07:00:44.728] Build status: STARTED
[07:00:44.728] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:25:0) after 7m22s
[07:08:06.847] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:25:0)
[07:08:06.867] Build status: STARTED
[07:08:06.867] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:26:0) after 3m53s
[07:12:00.247] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:26:0)
[07:12:00.285] Build status: STARTED
[07:12:00.285] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:27:0) after 1m33s
[07:13:33.304] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:27:0)
[07:13:33.327] Build status: STARTED
[07:13:33.327] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:28:0) after 4m31s
[07:18:04.468] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:28:0)
[07:18:04.491] Build status: STARTED
[07:18:04.491] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:29:0) after 3m26s
[07:21:30.510] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:29:0)
[07:21:30.540] Build status: STARTED
[07:21:30.540] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:30:0) after 3m30s
[07:25:00.562] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:30:0)
[07:25:00.609] Build status: STARTED
[07:25:00.609] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:31:0) after 5m3s
[07:30:03.641] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:31:0)
[07:30:03.671] Build status: STARTED
[07:30:03.671] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:32:0) after 1m13s
[07:31:16.726] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:32:0)
[07:31:16.748] Build status: STARTED
[07:31:16.748] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:33:0) after 1m25s
[07:32:41.784] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:33:0)
[07:32:41.894] Build status: STARTED
[07:32:41.894] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:34:0) after 1m23s
[07:34:05.145] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:34:0)
[07:34:05.162] Build status: STARTED
[07:34:05.162] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:35:0) after 6m55s
[07:41:00.180] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:35:0)
[07:41:00.207] Build status: STARTED
[07:41:00.207] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:36:0) after 8m30s
[07:49:30.378] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:36:0)
[07:49:30.419] Build status: STARTED
[07:49:30.419] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:37:0) after 4m10s
[07:53:40.609] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:37:0)
[07:53:40.638] Build status: STARTED
[07:53:40.638] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:38:0) after 2m32s
[07:56:12.647] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:38:0)
[07:56:12.672] Build status: STARTED
[07:56:12.672] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:39:0) after 8m9s
[08:04:21.755] Handling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:39:0)
[08:04:21.797] Build status: STARTED
[08:04:21.797] Scheduling timer "check-buildbucket-build-status" (fuchsia/core.x64-full_determinism-e5a0ec7f:8915596325506378128:40:0) after 6m41s
[08:06:32.499] Received PubSub notification, asking Buildbucket for the build status
[08:06:32.521] Build:
{
  "id": "8750166263578859489",
  "builder": {
    "project": "fuchsia",
    "bucket": "global.ci",
    "builder": "core.x64-full_determinism"
  },
  "createdBy": "project:fuchsia",
  "createTime": "2024-04-20T04:43:12.348009752Z",
  "startTime": "2024-04-20T04:43:20.274097256Z",
  "endTime": "2024-04-20T08:06:31.964360487Z",
  "updateTime": "2024-04-20T08:06:31.964360487Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "fuchsia.googlesource.com",
      "project": "integration",
      "id": "1abd05df7844fcceddbdc9865db3f73b6ab02051",
      "ref": "refs/heads/main"
    }
  }
}
[08:06:32.521] Invocation finished in 3h23m21.395251289s with status FAILED