Status: FAILED
Triggered by: -
Duration: 6 hours
Actions:

Definition (rev 36bc28844ffd5e030af6e41e2e0e4ddc5c92b699)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "ToTiOSDevice"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "476bbf7872fcf51acb987a62b44efffc899f9bc6"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/476bbf7872fcf51acb987a62b44efffc899f9bc6
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[09:13:45.046] New invocation is queued and will start shortly
[09:13:46.214] Starting the invocation (attempt 1)
[09:13:46.241] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/476bbf7872fcf51acb987a62b44efffc899f9bc6"
[09:13:46.241] Popped gitiles commit info from properties and tags
[09:13:46.241] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[09:13:46.241] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[09:13:46.241] Buildbucket request:
{
  "requestId": "8931071384301195456",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "ToTiOSDevice"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "476bbf7872fcf51acb987a62b44efffc899f9bc6",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8931071384301195456"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/ToTiOSDevice"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[09:13:47.239] Scheduled build:
{
  "id": "8765641322295193793",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "ToTiOSDevice"
  },
  "number": 30852,
  "createdBy": "project:chromium",
  "createTime": "2023-11-01T09:13:46.305832290Z",
  "updateTime": "2023-11-01T09:13:46.305832290Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "476bbf7872fcf51acb987a62b44efffc899f9bc6",
      "ref": "refs/heads/main"
    }
  }
}
[09:13:47.239] Task URL: https://cr-buildbucket.appspot.com/build/8765641322295193793
[09:13:47.239] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:2:0) after 4m43s
[09:13:48.213] Received PubSub notification, asking Buildbucket for the build status
[09:13:48.235] Build status: SCHEDULED
[09:14:53.560] Received PubSub notification, asking Buildbucket for the build status
[09:14:53.599] Build status: STARTED
[09:18:30.457] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:2:0)
[09:18:30.473] Build status: STARTED
[09:18:30.473] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:5:0) after 6m35s
[09:25:05.493] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:5:0)
[09:25:05.511] Build status: STARTED
[09:25:05.511] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:6:0) after 2m36s
[09:27:41.580] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:6:0)
[09:27:41.628] Build status: STARTED
[09:27:41.628] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:7:0) after 2m1s
[09:29:42.654] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:7:0)
[09:29:42.675] Build status: STARTED
[09:29:42.675] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:8:0) after 2m41s
[09:32:23.696] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:8:0)
[09:32:23.720] Build status: STARTED
[09:32:23.720] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:9:0) after 3m22s
[09:35:45.985] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:9:0)
[09:35:46.008] Build status: STARTED
[09:35:46.008] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:10:0) after 6m57s
[09:42:43.177] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:10:0)
[09:42:43.205] Build status: STARTED
[09:42:43.205] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:11:0) after 4m24s
[09:47:07.228] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:11:0)
[09:47:07.247] Build status: STARTED
[09:47:07.247] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:12:0) after 1m53s
[09:49:00.603] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:12:0)
[09:49:00.627] Build status: STARTED
[09:49:00.627] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:13:0) after 5m50s
[09:54:50.976] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:13:0)
[09:54:50.995] Build status: STARTED
[09:54:50.995] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:14:0) after 8m23s
[10:03:14.087] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:14:0)
[10:03:14.106] Build status: STARTED
[10:03:14.106] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:15:0) after 1m49s
[10:05:03.122] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:15:0)
[10:05:03.143] Build status: STARTED
[10:05:03.143] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:16:0) after 8m41s
[10:13:44.182] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:16:0)
[10:13:44.201] Build status: STARTED
[10:13:44.201] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:17:0) after 9m52s
[10:23:36.467] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:17:0)
[10:23:36.483] Build status: STARTED
[10:23:36.483] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:18:0) after 6m14s
[10:29:50.542] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:18:0)
[10:29:50.559] Build status: STARTED
[10:29:50.559] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:19:0) after 4m43s
[10:34:33.687] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:19:0)
[10:34:33.709] Build status: STARTED
[10:34:33.709] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:20:0) after 9m55s
[10:44:28.947] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:20:0)
[10:44:28.963] Build status: STARTED
[10:44:28.963] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:21:0) after 3m15s
[10:47:44.049] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:21:0)
[10:47:44.069] Build status: STARTED
[10:47:44.069] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:22:0) after 6m35s
[10:54:19.087] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:22:0)
[10:54:19.120] Build status: STARTED
[10:54:19.120] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:23:0) after 9m59s
[11:04:18.238] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:23:0)
[11:04:18.260] Build status: STARTED
[11:04:18.260] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:24:0) after 7m45s
[11:12:03.497] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:24:0)
[11:12:03.529] Build status: STARTED
[11:12:03.529] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:25:0) after 2m51s
[11:14:54.625] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:25:0)
[11:14:54.647] Build status: STARTED
[11:14:54.647] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:26:0) after 3m46s
[11:18:40.629] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:26:0)
[11:18:40.648] Build status: STARTED
[11:18:40.648] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:27:0) after 8m58s
[11:27:38.654] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:27:0)
[11:27:38.677] Build status: STARTED
[11:27:38.677] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:28:0) after 7m52s
[11:35:30.695] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:28:0)
[11:35:30.719] Build status: STARTED
[11:35:30.719] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:29:0) after 7m43s
[11:43:13.968] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:29:0)
[11:43:13.998] Build status: STARTED
[11:43:13.998] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:30:0) after 9m51s
[11:53:05.009] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:30:0)
[11:53:05.026] Build status: STARTED
[11:53:05.026] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:31:0) after 2m47s
[11:55:52.401] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:31:0)
[11:55:52.512] Build status: STARTED
[11:55:52.512] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:32:0) after 9m33s
[12:05:25.619] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:32:0)
[12:05:25.641] Build status: STARTED
[12:05:25.641] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:33:0) after 1m17s
[12:06:42.672] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:33:0)
[12:06:42.691] Build status: STARTED
[12:06:42.691] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:34:0) after 8m39s
[12:15:22.064] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:34:0)
[12:15:22.083] Build status: STARTED
[12:15:22.083] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:35:0) after 6m39s
[12:22:01.194] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:35:0)
[12:22:01.249] Build status: STARTED
[12:22:01.249] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:36:0) after 5m3s
[12:27:04.282] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:36:0)
[12:27:04.306] Build status: STARTED
[12:27:04.306] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:37:0) after 3m18s
[12:30:22.366] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:37:0)
[12:30:22.432] Build status: STARTED
[12:30:22.432] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:38:0) after 4m7s
[12:34:29.536] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:38:0)
[12:34:29.644] Build status: STARTED
[12:34:29.644] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:39:0) after 8m57s
[12:43:26.697] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:39:0)
[12:43:26.805] Build status: STARTED
[12:43:26.805] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:40:0) after 7m28s
[12:50:54.942] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:40:0)
[12:50:54.963] Build status: STARTED
[12:50:54.963] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:41:0) after 8m17s
[12:59:12.029] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:41:0)
[12:59:12.061] Build status: STARTED
[12:59:12.061] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:42:0) after 4m56s
[13:04:08.518] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:42:0)
[13:04:08.535] Build status: STARTED
[13:04:08.535] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:43:0) after 7m10s
[13:11:18.671] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:43:0)
[13:11:18.689] Build status: STARTED
[13:11:18.689] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:44:0) after 3m25s
[13:14:43.778] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:44:0)
[13:14:43.806] Build status: STARTED
[13:14:43.806] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:45:0) after 7m33s
[13:22:16.981] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:45:0)
[13:22:17.002] Build status: STARTED
[13:22:17.002] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:46:0) after 4m2s
[13:26:19.526] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:46:0)
[13:26:19.575] Build status: STARTED
[13:26:19.575] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:47:0) after 3m52s
[13:30:11.619] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:47:0)
[13:30:11.727] Build status: STARTED
[13:30:11.727] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:48:0) after 5m4s
[13:35:15.941] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:48:0)
[13:35:15.966] Build status: STARTED
[13:35:15.966] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:49:0) after 5m51s
[13:41:07.041] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:49:0)
[13:41:07.067] Build status: STARTED
[13:41:07.067] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:50:0) after 7m14s
[13:48:21.222] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:50:0)
[13:48:21.241] Build status: STARTED
[13:48:21.241] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:51:0) after 7m41s
[13:56:02.414] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:51:0)
[13:56:02.443] Build status: STARTED
[13:56:02.443] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:52:0) after 6m15s
[14:02:17.513] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:52:0)
[14:02:17.536] Build status: STARTED
[14:02:17.536] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:53:0) after 7m54s
[14:10:11.697] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:53:0)
[14:10:11.726] Build status: STARTED
[14:10:11.726] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:54:0) after 1m6s
[14:11:17.753] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:54:0)
[14:11:17.802] Build status: STARTED
[14:11:17.802] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:55:0) after 9m57s
[14:21:15.011] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:55:0)
[14:21:15.044] Build status: STARTED
[14:21:15.044] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:56:0) after 4m42s
[14:25:57.387] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:56:0)
[14:25:57.476] Build status: STARTED
[14:25:57.476] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:57:0) after 6m28s
[14:32:25.544] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:57:0)
[14:32:25.587] Build status: STARTED
[14:32:25.587] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:58:0) after 8m50s
[14:41:15.623] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:58:0)
[14:41:15.710] Build status: STARTED
[14:41:15.710] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:59:0) after 5m31s
[14:46:47.076] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:59:0)
[14:46:47.189] Build status: STARTED
[14:46:47.189] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:60:0) after 3m11s
[14:49:58.354] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:60:0)
[14:49:58.372] Build status: STARTED
[14:49:58.372] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:61:0) after 6m40s
[14:56:38.482] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:61:0)
[14:56:38.497] Build status: STARTED
[14:56:38.497] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:62:0) after 6m37s
[15:03:15.623] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:62:0)
[15:03:15.642] Build status: STARTED
[15:03:15.642] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:63:0) after 3m48s
[15:07:03.687] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:63:0)
[15:07:03.705] Build status: STARTED
[15:07:03.705] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:64:0) after 5m24s
[15:12:27.994] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:64:0)
[15:12:28.101] Build status: STARTED
[15:12:28.101] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:65:0) after 9m21s
[15:21:49.631] Handling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:65:0)
[15:21:49.657] Build status: STARTED
[15:21:49.657] Scheduling timer "check-buildbucket-build-status" (chromium/ToTiOSDevice:8931071384301195456:66:0) after 4m11s
[15:25:45.544] Received PubSub notification, asking Buildbucket for the build status
[15:25:45.575] Build:
{
  "id": "8765641322295193793",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "ToTiOSDevice"
  },
  "number": 30852,
  "createdBy": "project:chromium",
  "createTime": "2023-11-01T09:13:46.305832290Z",
  "startTime": "2023-11-01T09:14:52.358180Z",
  "endTime": "2023-11-01T15:25:44.821599Z",
  "updateTime": "2023-11-01T15:25:44.821599Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "476bbf7872fcf51acb987a62b44efffc899f9bc6",
      "ref": "refs/heads/main"
    }
  }
}
[15:25:45.575] Invocation finished in 6h12m0.565311985s with status FAILED