Status: FAILED
Triggered by: -
Duration: 5 hours
Actions:

Definition (rev 36bc28844ffd5e030af6e41e2e0e4ddc5c92b699)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "ToTLinux (dbg)"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "6e6d9e6536514cac003cf92130ad951b21d8c60f"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/6e6d9e6536514cac003cf92130ad951b21d8c60f
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[09:03:10.370] New invocation is queued and will start shortly
[09:03:11.559] Starting the invocation (attempt 1)
[09:03:11.589] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/6e6d9e6536514cac003cf92130ad951b21d8c60f"
[09:03:11.589] Popped gitiles commit info from properties and tags
[09:03:11.589] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[09:03:11.590] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[09:03:11.590] Buildbucket request:
{
  "requestId": "8910778329312094960",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "ToTLinux (dbg)"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "6e6d9e6536514cac003cf92130ad951b21d8c60f",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8910778329312094960"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/ToTLinux (dbg)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[09:03:12.646] Scheduled build:
{
  "id": "8745348267252957249",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "ToTLinux (dbg)"
  },
  "number": 28292,
  "createdBy": "project:chromium",
  "createTime": "2024-06-12T09:03:11.700947439Z",
  "updateTime": "2024-06-12T09:03:11.700947439Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "6e6d9e6536514cac003cf92130ad951b21d8c60f",
      "ref": "refs/heads/main"
    }
  }
}
[09:03:12.646] Task URL: https://cr-buildbucket.appspot.com/build/8745348267252957249
[09:03:12.646] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:2:0) after 5m0s
[09:03:13.511] Received PubSub notification, asking Buildbucket for the build status
[09:03:13.546] Build status: SCHEDULED
[09:04:41.302] Received PubSub notification, asking Buildbucket for the build status
[09:04:41.322] Build status: STARTED
[09:08:12.858] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:2:0)
[09:08:12.879] Build status: STARTED
[09:08:12.879] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:5:0) after 2m47s
[09:11:00.223] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:5:0)
[09:11:00.300] Build status: STARTED
[09:11:00.300] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:6:0) after 6m4s
[09:17:04.558] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:6:0)
[09:17:04.578] Build status: STARTED
[09:17:04.578] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:7:0) after 2m10s
[09:19:15.029] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:7:0)
[09:19:15.047] Build status: STARTED
[09:19:15.047] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:8:0) after 5m8s
[09:24:23.082] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:8:0)
[09:24:23.105] Build status: STARTED
[09:24:23.105] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:9:0) after 6m43s
[09:31:06.177] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:9:0)
[09:31:06.240] Build status: STARTED
[09:31:06.240] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:10:0) after 1m50s
[09:32:56.313] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:10:0)
[09:32:56.339] Build status: STARTED
[09:32:56.339] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:11:0) after 5m13s
[09:38:09.356] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:11:0)
[09:38:09.378] Build status: STARTED
[09:38:09.378] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:12:0) after 5m5s
[09:43:14.419] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:12:0)
[09:43:14.441] Build status: STARTED
[09:43:14.441] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:13:0) after 4m17s
[09:47:31.759] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:13:0)
[09:47:31.776] Build status: STARTED
[09:47:31.776] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:14:0) after 2m32s
[09:50:04.004] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:14:0)
[09:50:04.020] Build status: STARTED
[09:50:04.020] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:15:0) after 6m57s
[09:57:01.159] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:15:0)
[09:57:01.180] Build status: STARTED
[09:57:01.180] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:16:0) after 5m55s
[10:02:56.209] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:16:0)
[10:02:56.235] Build status: STARTED
[10:02:56.235] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:17:0) after 5m56s
[10:08:52.630] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:17:0)
[10:08:52.753] Build status: STARTED
[10:08:52.753] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:18:0) after 8m31s
[10:17:23.882] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:18:0)
[10:17:23.900] Build status: STARTED
[10:17:23.900] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:19:0) after 6m38s
[10:24:01.921] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:19:0)
[10:24:01.997] Build status: STARTED
[10:24:01.997] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:20:0) after 2m52s
[10:26:54.149] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:20:0)
[10:26:54.168] Build status: STARTED
[10:26:54.168] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:21:0) after 8m57s
[10:35:51.694] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:21:0)
[10:35:51.802] Build status: STARTED
[10:35:51.802] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:22:0) after 7m18s
[10:43:09.933] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:22:0)
[10:43:09.950] Build status: STARTED
[10:43:09.950] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:23:0) after 7m29s
[10:50:38.968] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:23:0)
[10:50:38.988] Build status: STARTED
[10:50:38.989] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:24:0) after 8m48s
[10:59:27.089] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:24:0)
[10:59:27.112] Build status: STARTED
[10:59:27.112] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:25:0) after 3m42s
[11:03:09.303] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:25:0)
[11:03:09.321] Build status: STARTED
[11:03:09.321] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:26:0) after 1m15s
[11:04:24.517] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:26:0)
[11:04:24.547] Build status: STARTED
[11:04:24.547] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:27:0) after 9m55s
[11:14:19.564] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:27:0)
[11:14:19.583] Build status: STARTED
[11:14:19.583] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:28:0) after 8m29s
[11:22:48.780] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:28:0)
[11:22:48.820] Build status: STARTED
[11:22:48.820] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:29:0) after 9m0s
[11:31:48.816] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:29:0)
[11:31:48.892] Build status: STARTED
[11:31:48.892] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:30:0) after 7m46s
[11:39:35.010] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:30:0)
[11:39:35.128] Build status: STARTED
[11:39:35.128] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:31:0) after 5m37s
[11:45:12.236] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:31:0)
[11:45:12.262] Build status: STARTED
[11:45:12.262] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:32:0) after 2m37s
[11:47:49.971] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:32:0)
[11:47:49.988] Build status: STARTED
[11:47:49.988] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:33:0) after 7m55s
[11:55:44.965] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:33:0)
[11:55:44.982] Build status: STARTED
[11:55:44.982] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:34:0) after 2m13s
[11:57:57.967] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:34:0)
[11:57:57.989] Build status: STARTED
[11:57:57.989] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:35:0) after 1m20s
[11:59:17.992] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:35:0)
[11:59:18.009] Build status: STARTED
[11:59:18.009] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:36:0) after 3m13s
[12:02:31.030] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:36:0)
[12:02:31.122] Build status: STARTED
[12:02:31.122] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:37:0) after 1m17s
[12:03:48.172] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:37:0)
[12:03:48.253] Build status: STARTED
[12:03:48.253] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:38:0) after 9m1s
[12:12:49.273] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:38:0)
[12:12:49.353] Build status: STARTED
[12:12:49.353] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:39:0) after 3m48s
[12:16:37.505] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:39:0)
[12:16:37.524] Build status: STARTED
[12:16:37.524] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:40:0) after 8m6s
[12:24:43.599] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:40:0)
[12:24:43.627] Build status: STARTED
[12:24:43.627] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:41:0) after 3m24s
[12:28:07.651] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:41:0)
[12:28:07.668] Build status: STARTED
[12:28:07.668] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:42:0) after 7m44s
[12:35:51.691] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:42:0)
[12:35:51.707] Build status: STARTED
[12:35:51.707] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:43:0) after 1m20s
[12:37:11.825] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:43:0)
[12:37:11.844] Build status: STARTED
[12:37:11.845] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:44:0) after 3m26s
[12:40:37.884] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:44:0)
[12:40:38.024] Build status: STARTED
[12:40:38.024] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:45:0) after 3m45s
[12:44:23.216] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:45:0)
[12:44:23.271] Build status: STARTED
[12:44:23.271] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:46:0) after 1m40s
[12:46:03.305] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:46:0)
[12:46:03.328] Build status: STARTED
[12:46:03.328] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:47:0) after 5m52s
[12:51:55.349] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:47:0)
[12:51:55.373] Build status: STARTED
[12:51:55.373] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:48:0) after 4m31s
[12:56:26.414] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:48:0)
[12:56:26.437] Build status: STARTED
[12:56:26.437] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:49:0) after 1m57s
[12:58:23.476] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:49:0)
[12:58:23.495] Build status: STARTED
[12:58:23.495] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:50:0) after 9m11s
[13:07:34.596] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:50:0)
[13:07:34.616] Build status: STARTED
[13:07:34.616] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:51:0) after 4m34s
[13:12:08.665] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:51:0)
[13:12:08.684] Build status: STARTED
[13:12:08.684] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:52:0) after 8m12s
[13:20:20.707] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:52:0)
[13:20:20.749] Build status: STARTED
[13:20:20.749] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:53:0) after 8m28s
[13:28:48.767] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:53:0)
[13:28:48.782] Build status: STARTED
[13:28:48.782] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:54:0) after 3m46s
[13:32:34.800] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:54:0)
[13:32:34.820] Build status: STARTED
[13:32:34.820] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:55:0) after 1m39s
[13:34:13.819] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:55:0)
[13:34:13.859] Build status: STARTED
[13:34:13.859] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:56:0) after 9m48s
[13:44:01.970] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:56:0)
[13:44:02.001] Build status: STARTED
[13:44:02.001] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:57:0) after 4m31s
[13:48:33.027] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:57:0)
[13:48:33.050] Build status: STARTED
[13:48:33.050] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:58:0) after 2m28s
[13:51:01.067] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:58:0)
[13:51:01.087] Build status: STARTED
[13:51:01.087] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:59:0) after 9m28s
[14:00:29.114] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:59:0)
[14:00:29.140] Build status: STARTED
[14:00:29.140] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:60:0) after 8m44s
[14:09:13.164] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:60:0)
[14:09:13.235] Build status: STARTED
[14:09:13.235] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:61:0) after 4m14s
[14:13:27.274] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:61:0)
[14:13:27.292] Build status: STARTED
[14:13:27.292] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:62:0) after 5m27s
[14:18:54.388] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:62:0)
[14:18:54.439] Build status: STARTED
[14:18:54.439] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:63:0) after 4m24s
[14:23:18.516] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:63:0)
[14:23:18.573] Build status: STARTED
[14:23:18.573] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:64:0) after 2m39s
[14:25:57.600] Handling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:64:0)
[14:25:57.624] Build status: STARTED
[14:25:57.624] Scheduling timer "check-buildbucket-build-status" (chromium/ToTLinux (dbg):8910778329312094960:65:0) after 6m39s
[14:26:49.824] Received PubSub notification, asking Buildbucket for the build status
[14:26:49.850] Build:
{
  "id": "8745348267252957249",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "ToTLinux (dbg)"
  },
  "number": 28292,
  "createdBy": "project:chromium",
  "createTime": "2024-06-12T09:03:11.700947439Z",
  "startTime": "2024-06-12T09:04:39.424741638Z",
  "endTime": "2024-06-12T14:26:49.527878730Z",
  "updateTime": "2024-06-12T14:26:49.527878730Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "6e6d9e6536514cac003cf92130ad951b21d8c60f",
      "ref": "refs/heads/main"
    }
  }
}
[14:26:49.850] Invocation finished in 5h23m39.497729779s with status FAILED