Status: FAILED
Triggered by: -
Duration: 3 hours
Actions:

Definition (rev 66e277fee0cff54a12048d87e80b49373e18ab9f)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "matter.dev.ci"
  builder: "matter-linux-arm64-light-clang-rpc"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/master",
  "repository": "https://pigweed.googlesource.com/pigweed/pigweed",
  "revision": "d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5"
}

Incoming tags

  • buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5
  • gitiles_ref:refs/heads/master

Debug log (UTC)

[10:08:04.262] New invocation is queued and will start shortly
[10:08:05.300] Starting the invocation (attempt 1)
[10:08:05.325] Popped buildset tag "buildset:commit/gitiles/pigweed.googlesource.com/pigweed/pigweed/+/d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5"
[10:08:05.325] Popped gitiles commit info from properties and tags
[10:08:05.325] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[10:08:05.326] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[10:08:05.326] Buildbucket request:
{
  "requestId": "8867468896326802144",
  "builder": {
    "project": "pigweed",
    "bucket": "matter.dev.ci",
    "builder": "matter-linux-arm64-light-clang-rpc"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8867468896326802144",
            "job": "pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc",
            "triggers": [
                  {
                        "gitiles": {
                              "ref": "refs/heads/main",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main@d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5",
                        "title": "d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5"
                      },
                  {
                        "gitiles": {
                              "ref": "refs/heads/master",
                              "repo": "https://pigweed.googlesource.com/pigweed/pigweed",
                              "revision": "d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5"
                            },
                        "id": "https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/master@d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5",
                        "title": "d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5",
                        "url": "https://pigweed.googlesource.com/pigweed/pigweed/+/d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5"
                      }
                ]
          }
    },
  "gitilesCommit": {
    "host": "pigweed.googlesource.com",
    "project": "pigweed/pigweed",
    "id": "d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5",
    "ref": "refs/heads/master"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8867468896326802144"
    },
    {
      "key": "scheduler_job_id",
      "value": "pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[10:08:05.778] Scheduled build:
{
  "id": "8702038834504035425",
  "builder": {
    "project": "pigweed",
    "bucket": "matter.dev.ci",
    "builder": "matter-linux-arm64-light-clang-rpc"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-10-03T10:08:05.371707389Z",
  "updateTime": "2025-10-03T10:08:05.371707389Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5",
      "ref": "refs/heads/master"
    }
  }
}
[10:08:05.779] Task URL: https://cr-buildbucket.appspot.com/build/8702038834504035425
[10:08:05.779] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:2:0) after 7m31s
[10:08:06.712] Received PubSub notification, asking Buildbucket for the build status
[10:08:06.823] Build status: SCHEDULED
[10:15:36.842] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:2:0)
[10:15:36.866] Build status: SCHEDULED
[10:15:36.866] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:4:0) after 4m39s
[10:20:16.068] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:4:0)
[10:20:16.086] Build status: SCHEDULED
[10:20:16.086] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:5:0) after 5m27s
[10:25:43.100] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:5:0)
[10:25:43.157] Build status: SCHEDULED
[10:25:43.158] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:6:0) after 4m48s
[10:30:31.168] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:6:0)
[10:30:31.204] Build status: SCHEDULED
[10:30:31.204] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:7:0) after 1m4s
[10:31:35.229] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:7:0)
[10:31:35.257] Build status: SCHEDULED
[10:31:35.257] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:8:0) after 1m39s
[10:33:14.268] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:8:0)
[10:33:14.299] Build status: SCHEDULED
[10:33:14.299] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:9:0) after 3m1s
[10:36:15.319] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:9:0)
[10:36:15.349] Build status: SCHEDULED
[10:36:15.349] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:10:0) after 3m49s
[10:40:04.430] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:10:0)
[10:40:04.646] Build status: SCHEDULED
[10:40:04.646] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:11:0) after 7m58s
[10:48:02.629] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:11:0)
[10:48:02.646] Build status: SCHEDULED
[10:48:02.646] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:12:0) after 4m9s
[10:52:11.639] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:12:0)
[10:52:11.654] Build status: SCHEDULED
[10:52:11.654] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:13:0) after 1m18s
[10:53:29.669] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:13:0)
[10:53:29.713] Build status: SCHEDULED
[10:53:29.713] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:14:0) after 7m1s
[11:00:30.775] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:14:0)
[11:00:30.836] Build status: SCHEDULED
[11:00:30.836] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:15:0) after 5m17s
[11:05:47.933] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:15:0)
[11:05:47.949] Build status: SCHEDULED
[11:05:47.949] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:16:0) after 5m19s
[11:11:07.033] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:16:0)
[11:11:07.053] Build status: SCHEDULED
[11:11:07.053] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:17:0) after 5m57s
[11:17:04.068] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:17:0)
[11:17:04.123] Build status: SCHEDULED
[11:17:04.123] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:18:0) after 9m32s
[11:26:36.196] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:18:0)
[11:26:36.212] Build status: SCHEDULED
[11:26:36.212] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:19:0) after 9m53s
[11:36:29.244] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:19:0)
[11:36:29.267] Build status: SCHEDULED
[11:36:29.267] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:20:0) after 5m12s
[11:41:41.281] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:20:0)
[11:41:41.303] Build status: SCHEDULED
[11:41:41.303] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:21:0) after 3m51s
[11:45:32.436] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:21:0)
[11:45:32.459] Build status: SCHEDULED
[11:45:32.459] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:22:0) after 1m10s
[11:46:42.434] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:22:0)
[11:46:42.480] Build status: SCHEDULED
[11:46:42.480] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:23:0) after 2m8s
[11:48:50.447] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:23:0)
[11:48:50.466] Build status: SCHEDULED
[11:48:50.466] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:24:0) after 3m27s
[11:52:17.481] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:24:0)
[11:52:17.498] Build status: SCHEDULED
[11:52:17.498] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:25:0) after 5m53s
[11:58:10.563] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:25:0)
[11:58:10.589] Build status: SCHEDULED
[11:58:10.589] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:26:0) after 8m54s
[12:07:04.603] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:26:0)
[12:07:04.632] Build status: SCHEDULED
[12:07:04.632] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:27:0) after 8m23s
[12:15:27.656] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:27:0)
[12:15:27.673] Build status: SCHEDULED
[12:15:27.673] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:28:0) after 3m59s
[12:19:26.693] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:28:0)
[12:19:26.718] Build status: SCHEDULED
[12:19:26.718] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:29:0) after 3m53s
[12:23:19.751] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:29:0)
[12:23:19.782] Build status: SCHEDULED
[12:23:19.782] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:30:0) after 8m50s
[12:32:09.870] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:30:0)
[12:32:09.891] Build status: SCHEDULED
[12:32:09.891] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:31:0) after 9m14s
[12:41:24.545] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:31:0)
[12:41:24.588] Build status: SCHEDULED
[12:41:24.588] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:32:0) after 8m13s
[12:49:37.682] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:32:0)
[12:49:37.705] Build status: SCHEDULED
[12:49:37.705] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:33:0) after 6m24s
[12:56:01.769] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:33:0)
[12:56:03.387] Build status: SCHEDULED
[12:56:03.387] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:34:0) after 1m47s
[12:57:50.410] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:34:0)
[12:57:50.445] Build status: SCHEDULED
[12:57:50.445] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:35:0) after 8m27s
[13:06:17.460] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:35:0)
[13:06:17.480] Build status: SCHEDULED
[13:06:17.480] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:36:0) after 3m14s
[13:09:31.546] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:36:0)
[13:09:31.566] Build status: SCHEDULED
[13:09:31.566] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:37:0) after 4m37s
[13:14:08.636] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:37:0)
[13:14:08.654] Build status: SCHEDULED
[13:14:08.654] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:38:0) after 8m4s
[13:22:12.730] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:38:0)
[13:22:12.752] Build status: SCHEDULED
[13:22:12.752] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:39:0) after 5m45s
[13:27:57.751] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:39:0)
[13:27:57.776] Build status: SCHEDULED
[13:27:57.776] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:40:0) after 7m27s
[13:35:24.823] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:40:0)
[13:35:24.858] Build status: SCHEDULED
[13:35:24.858] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:41:0) after 5m12s
[13:40:36.910] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:41:0)
[13:40:36.959] Build status: SCHEDULED
[13:40:36.959] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:42:0) after 3m38s
[13:44:15.033] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:42:0)
[13:44:15.081] Build status: SCHEDULED
[13:44:15.081] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:43:0) after 4m18s
[13:48:33.145] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:43:0)
[13:48:33.170] Build status: SCHEDULED
[13:48:33.170] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:44:0) after 3m43s
[13:49:49.133] Received PubSub notification, asking Buildbucket for the build status
[13:49:49.147] Build status: STARTED
[13:52:16.187] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:44:0)
[13:52:16.207] Build status: STARTED
[13:52:16.207] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:46:0) after 8m10s
[14:00:26.286] Handling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:46:0)
[14:00:26.301] Build status: STARTED
[14:00:26.301] Scheduling timer "check-buildbucket-build-status" (pigweed/matter.dev.ci-matter-linux-arm64-light-clang-rpc:8867468896326802144:47:0) after 7m30s
[14:01:58.815] Received PubSub notification, asking Buildbucket for the build status
[14:01:58.870] Build:
{
  "id": "8702038834504035425",
  "builder": {
    "project": "pigweed",
    "bucket": "matter.dev.ci",
    "builder": "matter-linux-arm64-light-clang-rpc"
  },
  "createdBy": "project:pigweed",
  "createTime": "2025-10-03T10:08:05.371707389Z",
  "startTime": "2025-10-03T13:49:48.963154136Z",
  "endTime": "2025-10-03T14:01:58.198748854Z",
  "updateTime": "2025-10-03T14:01:58.198748854Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "pigweed.googlesource.com",
      "project": "pigweed/pigweed",
      "id": "d7b8ed3f82c75d7f9ba8503983f220ffbb09b3e5",
      "ref": "refs/heads/master"
    }
  }
}
[14:01:58.870] Invocation finished in 3h53m54.62144578s with status FAILED