1. All projects
  2. chromium
  3. win-build-perf-ci-siso
  4. 8897847399823647936
Status: FAILED
Triggered by: -
Duration: 5 hours
Actions:

Definition (rev db0b978b2708bf07e4f95065a32a0db817a9e4bf) 

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "build"
  builder: "win-build-perf-ci-siso"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "65acb19403bd05c5c2f1e9463bafea2e3584de21"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/65acb19403bd05c5c2f1e9463bafea2e3584de21
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[02:34:45.709] New invocation is queued and will start shortly
[02:34:46.859] Starting the invocation (attempt 1)
[02:34:46.959] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/65acb19403bd05c5c2f1e9463bafea2e3584de21"
[02:34:46.959] Popped gitiles commit info from properties and tags
[02:34:46.959] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[02:34:46.959] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[02:34:46.960] Buildbucket request:
{
  "requestId": "8897847399823647936",
  "builder": {
    "project": "chromium",
    "bucket": "build",
    "builder": "win-build-perf-ci-siso"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "65acb19403bd05c5c2f1e9463bafea2e3584de21",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897847399823647936"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/win-build-perf-ci-siso"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[02:34:47.495] Scheduled build:
{
  "id": "8732417337743251537",
  "builder": {
    "project": "chromium",
    "bucket": "build",
    "builder": "win-build-perf-ci-siso"
  },
  "number": 84,
  "createdBy": "project:chromium",
  "createTime": "2024-11-02T02:34:47.061570150Z",
  "updateTime": "2024-11-02T02:34:47.061570150Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "65acb19403bd05c5c2f1e9463bafea2e3584de21",
      "ref": "refs/heads/main"
    }
  }
}
[02:34:47.495] Task URL: https://cr-buildbucket.appspot.com/build/8732417337743251537
[02:34:47.495] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:2:0) after 2m7s
[02:34:47.704] Received PubSub notification, asking Buildbucket for the build status
[02:34:47.720] Build status: SCHEDULED
[02:34:57.573] Received PubSub notification, asking Buildbucket for the build status
[02:34:57.722] Build status: STARTED
[02:36:54.530] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:2:0)
[02:36:54.547] Build status: STARTED
[02:36:54.547] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:5:0) after 5m20s
[02:42:14.671] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:5:0)
[02:42:14.761] Build status: STARTED
[02:42:14.761] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:6:0) after 4m54s
[02:47:08.961] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:6:0)
[02:47:08.981] Build status: STARTED
[02:47:08.981] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:7:0) after 8m17s
[02:55:26.048] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:7:0)
[02:55:26.077] Build status: STARTED
[02:55:26.077] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:8:0) after 3m7s
[02:58:33.195] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:8:0)
[02:58:33.223] Build status: STARTED
[02:58:33.223] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:9:0) after 7m15s
[03:05:48.330] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:9:0)
[03:05:48.348] Build status: STARTED
[03:05:48.348] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:10:0) after 5m32s
[03:11:20.363] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:10:0)
[03:11:20.380] Build status: STARTED
[03:11:20.380] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:11:0) after 8m14s
[03:19:34.870] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:11:0)
[03:19:34.891] Build status: STARTED
[03:19:34.891] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:12:0) after 1m36s
[03:21:10.986] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:12:0)
[03:21:11.025] Build status: STARTED
[03:21:11.025] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:13:0) after 5m14s
[03:26:25.040] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:13:0)
[03:26:25.055] Build status: STARTED
[03:26:25.055] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:14:0) after 5m10s
[03:31:35.158] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:14:0)
[03:31:35.191] Build status: STARTED
[03:31:35.191] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:15:0) after 6m44s
[03:38:19.293] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:15:0)
[03:38:19.312] Build status: STARTED
[03:38:19.312] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:16:0) after 9m40s
[03:47:59.476] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:16:0)
[03:47:59.507] Build status: STARTED
[03:47:59.507] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:17:0) after 4m41s
[03:52:40.535] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:17:0)
[03:52:40.560] Build status: STARTED
[03:52:40.560] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:18:0) after 1m6s
[03:53:46.572] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:18:0)
[03:53:46.591] Build status: STARTED
[03:53:46.591] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:19:0) after 6m45s
[04:00:31.707] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:19:0)
[04:00:31.725] Build status: STARTED
[04:00:31.725] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:20:0) after 6m58s
[04:07:29.799] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:20:0)
[04:07:29.818] Build status: STARTED
[04:07:29.818] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:21:0) after 9m32s
[04:17:01.954] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:21:0)
[04:17:01.973] Build status: STARTED
[04:17:01.973] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:22:0) after 2m49s
[04:19:50.996] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:22:0)
[04:19:51.038] Build status: STARTED
[04:19:51.038] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:23:0) after 2m50s
[04:22:41.055] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:23:0)
[04:22:41.089] Build status: STARTED
[04:22:41.089] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:24:0) after 7m6s
[04:29:47.104] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:24:0)
[04:29:47.138] Build status: STARTED
[04:29:47.138] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:25:0) after 3m12s
[04:32:59.152] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:25:0)
[04:32:59.173] Build status: STARTED
[04:32:59.173] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:26:0) after 7m5s
[04:40:04.279] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:26:0)
[04:40:04.301] Build status: STARTED
[04:40:04.301] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:27:0) after 4m30s
[04:44:34.316] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:27:0)
[04:44:34.346] Build status: STARTED
[04:44:34.346] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:28:0) after 5m50s
[04:50:24.433] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:28:0)
[04:50:24.446] Build status: STARTED
[04:50:24.446] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:29:0) after 6m8s
[04:56:32.709] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:29:0)
[04:56:32.727] Build status: STARTED
[04:56:32.727] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:30:0) after 9m11s
[05:05:44.092] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:30:0)
[05:05:44.200] Build status: STARTED
[05:05:44.200] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:31:0) after 5m11s
[05:10:55.215] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:31:0)
[05:10:55.236] Build status: STARTED
[05:10:55.236] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:32:0) after 4m26s
[05:15:21.253] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:32:0)
[05:15:21.270] Build status: STARTED
[05:15:21.270] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:33:0) after 4m41s
[05:20:02.344] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:33:0)
[05:20:02.377] Build status: STARTED
[05:20:02.377] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:34:0) after 8m30s
[05:28:32.476] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:34:0)
[05:28:32.492] Build status: STARTED
[05:28:32.493] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:35:0) after 8m7s
[05:36:39.511] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:35:0)
[05:36:39.534] Build status: STARTED
[05:36:39.534] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:36:0) after 4m22s
[05:41:01.719] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:36:0)
[05:41:01.822] Build status: STARTED
[05:41:01.822] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:37:0) after 6m11s
[05:47:13.027] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:37:0)
[05:47:13.154] Build status: STARTED
[05:47:13.154] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:38:0) after 9m11s
[05:56:24.171] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:38:0)
[05:56:24.250] Build status: STARTED
[05:56:24.250] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:39:0) after 5m53s
[06:02:17.275] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:39:0)
[06:02:17.293] Build status: STARTED
[06:02:17.293] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:40:0) after 5m43s
[06:08:00.309] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:40:0)
[06:08:00.337] Build status: STARTED
[06:08:00.337] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:41:0) after 8m9s
[06:16:09.585] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:41:0)
[06:16:09.621] Build status: STARTED
[06:16:09.622] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:42:0) after 3m38s
[06:19:47.682] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:42:0)
[06:19:47.733] Build status: STARTED
[06:19:47.733] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:43:0) after 1m19s
[06:21:06.966] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:43:0)
[06:21:06.995] Build status: STARTED
[06:21:06.995] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:44:0) after 6m59s
[06:28:06.014] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:44:0)
[06:28:06.034] Build status: STARTED
[06:28:06.034] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:45:0) after 2m44s
[06:30:50.268] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:45:0)
[06:30:50.329] Build status: STARTED
[06:30:50.329] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:46:0) after 2m4s
[06:32:54.347] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:46:0)
[06:32:54.368] Build status: STARTED
[06:32:54.368] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:47:0) after 2m38s
[06:35:32.385] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:47:0)
[06:35:32.407] Build status: STARTED
[06:35:32.408] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:48:0) after 5m56s
[06:41:28.422] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:48:0)
[06:41:28.441] Build status: STARTED
[06:41:28.441] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:49:0) after 1m55s
[06:43:23.466] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:49:0)
[06:43:23.490] Build status: STARTED
[06:43:23.490] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:50:0) after 5m40s
[06:49:03.508] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:50:0)
[06:49:03.529] Build status: STARTED
[06:49:03.529] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:51:0) after 5m59s
[06:55:02.650] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:51:0)
[06:55:02.680] Build status: STARTED
[06:55:02.680] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:52:0) after 4m59s
[07:00:01.796] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:52:0)
[07:00:01.830] Build status: STARTED
[07:00:01.830] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:53:0) after 7m37s
[07:07:39.087] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:53:0)
[07:07:39.224] Build status: STARTED
[07:07:39.224] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:54:0) after 9m2s
[07:16:41.405] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:54:0)
[07:16:41.425] Build status: STARTED
[07:16:41.426] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:55:0) after 9m16s
[07:25:57.682] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:55:0)
[07:25:57.704] Build status: STARTED
[07:25:57.704] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:56:0) after 7m31s
[07:33:28.741] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:56:0)
[07:33:28.784] Build status: STARTED
[07:33:28.784] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:57:0) after 9m29s
[07:42:57.799] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:57:0)
[07:42:57.821] Build status: STARTED
[07:42:57.821] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:58:0) after 3m39s
[07:46:36.887] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:58:0)
[07:46:36.917] Build status: STARTED
[07:46:36.917] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:59:0) after 9m27s
[07:56:03.937] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:59:0)
[07:56:03.951] Build status: STARTED
[07:56:03.951] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:60:0) after 5m35s
[08:01:38.991] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:60:0)
[08:01:39.012] Build status: STARTED
[08:01:39.012] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:61:0) after 9m45s
[08:11:24.031] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:61:0)
[08:11:24.058] Build status: STARTED
[08:11:24.058] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:62:0) after 3m15s
[08:14:39.219] Handling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:62:0)
[08:14:39.244] Build status: STARTED
[08:14:39.244] Scheduling timer "check-buildbucket-build-status" (chromium/win-build-perf-ci-siso:8897847399823647936:63:0) after 6m44s
[08:15:08.328] Received PubSub notification, asking Buildbucket for the build status
[08:15:08.353] Build:
{
  "id": "8732417337743251537",
  "builder": {
    "project": "chromium",
    "bucket": "build",
    "builder": "win-build-perf-ci-siso"
  },
  "number": 84,
  "createdBy": "project:chromium",
  "createTime": "2024-11-02T02:34:47.061570150Z",
  "startTime": "2024-11-02T02:34:56.082085533Z",
  "endTime": "2024-11-02T08:15:07.956714858Z",
  "updateTime": "2024-11-02T08:15:07.956714858Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "65acb19403bd05c5c2f1e9463bafea2e3584de21",
      "ref": "refs/heads/main"
    }
  }
}
[08:15:08.353] Invocation finished in 5h40m22.658644677s with status FAILED