Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 3 hours
Actions:

Definition (rev 36bc28844ffd5e030af6e41e2e0e4ddc5c92b699)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Mac FYI ASAN (Intel)"
>

Incoming triggers

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "b870da7099cbfe50b02672ff9503d29030c31888",
        "project": "chromium/src",
        "ref": "refs/heads/main"
      }
    ]
  },
  "branch": "refs/heads/main",
  "deps_revision_overrides": {
    "src": "b870da7099cbfe50b02672ff9503d29030c31888"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8713911804053210673"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8713911804053210673"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8713911804053210673"
    }
  ],
  "parent_build_id": "8713911804053210673",
  "parent_builder_group": "chromium.gpu.fyi",
  "parent_buildername": "GPU FYI Mac Builder (asan)",
  "parent_got_angle_revision": "ec4d8f8e4d3f1a9e64430abd8b6b436253732adf",
  "parent_got_dawn_revision": "ec871f2da1702386bfb22c52f29fd527dad70b76",
  "parent_got_revision": "b870da7099cbfe50b02672ff9503d29030c31888",
  "parent_got_revision_cp": "refs/heads/main@{#1465219}",
  "parent_got_swiftshader_revision": "a8133cbb3c8969e3c1e6b3cea2c02ec8312ef9ca",
  "parent_got_v8_revision": "54414b2fc025cb385c0904c160c5e6f7f3204d2a",
  "parent_got_v8_revision_cp": "refs/heads/13.8.256@{#1}",
  "parent_got_webrtc_revision": "5400eaf45a36a0b5db81fe278e5e10963f938800",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#44750}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "b870da7099cbfe50b02672ff9503d29030c31888",
  "swarm_hashes": {
    "angle_unittests": "db1821c44ed671eb115ceaa5197bd50f2ae19617a611f4cd276bd6f2db194733/489",
    "browser_tests": "1d6650c9259d86449e192057a778097856c28a8c3addf84ced3e2a009f8430c0/1220",
    "gl_tests": "552ea77fd61d1a5b162fc0931ca7ceb9098fe166dd4f53eaae023e4f42370e75/488",
    "gl_unittests": "c87843f684f6bcb826c7cc81d031e4347ac14b380a4d7559be5865ce71de5599/488",
    "gpu_unittests": "718145a061ee8880ca65377b0cef3d4611af6228e1656b3a88361253adf2d17e/565",
    "services_unittests": "8c41b22b7571add5c582f87b9d3b30d3d949a8bff0c66b5fc42414a1141a253d/734",
    "telemetry_gpu_integration_test": "9ff3681beff279d70d1f3559015711b0d79c1e4531bff31d03ab53e3fd874d9f/815"
  },
  "swarming_command_lines_cwd": "out/49df-GPU_FYI_Mac_Bui",
  "swarming_command_lines_digest": "107fa9d3e7427248613763ade0743d4316da28b528c9cf3828114368cecb8374/93",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/b870da7099cbfe50b02672ff9503d29030c31888
  • gitiles_ref:refs/heads/main
  • parent_buildername:GPU FYI Mac Builder (asan)
  • parent_buildnumber:65385
  • user_agent:recipe

Debug log (UTC)

[09:22:44.732] New invocation is queued and will start shortly
[09:22:44.732] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[09:22:45.822] Starting the invocation (attempt 1)
[09:22:45.850] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/b870da7099cbfe50b02672ff9503d29030c31888"
[09:22:45.850] Popped gitiles commit info from properties and tags
[09:22:45.850] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[09:22:45.850] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[09:22:45.850] Buildbucket request:
{
  "requestId": "8879339950562150032",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Mac FYI ASAN (Intel)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "b870da7099cbfe50b02672ff9503d29030c31888",
                        "project": "chromium/src",
                        "ref": "refs/heads/main"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "b870da7099cbfe50b02672ff9503d29030c31888"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8713911804053210673"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8713911804053210673"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8713911804053210673"
                }
          ],
      "parent_build_id": "8713911804053210673",
      "parent_builder_group": "chromium.gpu.fyi",
      "parent_buildername": "GPU FYI Mac Builder (asan)",
      "parent_got_angle_revision": "ec4d8f8e4d3f1a9e64430abd8b6b436253732adf",
      "parent_got_dawn_revision": "ec871f2da1702386bfb22c52f29fd527dad70b76",
      "parent_got_revision": "b870da7099cbfe50b02672ff9503d29030c31888",
      "parent_got_revision_cp": "refs/heads/main@{#1465219}",
      "parent_got_swiftshader_revision": "a8133cbb3c8969e3c1e6b3cea2c02ec8312ef9ca",
      "parent_got_v8_revision": "54414b2fc025cb385c0904c160c5e6f7f3204d2a",
      "parent_got_v8_revision_cp": "refs/heads/13.8.256@{#1}",
      "parent_got_webrtc_revision": "5400eaf45a36a0b5db81fe278e5e10963f938800",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#44750}",
      "swarm_hashes": {
            "angle_unittests": "db1821c44ed671eb115ceaa5197bd50f2ae19617a611f4cd276bd6f2db194733/489",
            "browser_tests": "1d6650c9259d86449e192057a778097856c28a8c3addf84ced3e2a009f8430c0/1220",
            "gl_tests": "552ea77fd61d1a5b162fc0931ca7ceb9098fe166dd4f53eaae023e4f42370e75/488",
            "gl_unittests": "c87843f684f6bcb826c7cc81d031e4347ac14b380a4d7559be5865ce71de5599/488",
            "gpu_unittests": "718145a061ee8880ca65377b0cef3d4611af6228e1656b3a88361253adf2d17e/565",
            "services_unittests": "8c41b22b7571add5c582f87b9d3b30d3d949a8bff0c66b5fc42414a1141a253d/734",
            "telemetry_gpu_integration_test": "9ff3681beff279d70d1f3559015711b0d79c1e4531bff31d03ab53e3fd874d9f/815"
          },
      "swarming_command_lines_cwd": "out/49df-GPU_FYI_Mac_Bui",
      "swarming_command_lines_digest": "107fa9d3e7427248613763ade0743d4316da28b528c9cf3828114368cecb8374/93",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "b870da7099cbfe50b02672ff9503d29030c31888",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8879339950562150032"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/Mac FYI ASAN (Intel)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "GPU FYI Mac Builder (asan)"
    },
    {
      "key": "parent_buildnumber",
      "value": "65385"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[09:22:46.515] Scheduled build:
{
  "id": "8713909888618314481",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Mac FYI ASAN (Intel)"
  },
  "number": 16735,
  "createdBy": "project:chromium",
  "createTime": "2025-05-25T09:22:45.954299052Z",
  "updateTime": "2025-05-25T09:22:45.954299052Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "b870da7099cbfe50b02672ff9503d29030c31888",
      "ref": "refs/heads/main"
    }
  }
}
[09:22:46.515] Task URL: https://cr-buildbucket.appspot.com/build/8713909888618314481
[09:22:46.515] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:2:0) after 2m54s
[09:22:47.494] Received PubSub notification, asking Buildbucket for the build status
[09:22:47.518] Build status: SCHEDULED
[09:22:56.702] Received PubSub notification, asking Buildbucket for the build status
[09:22:56.730] Build status: STARTED
[09:25:40.656] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:2:0)
[09:25:40.676] Build status: STARTED
[09:25:40.676] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:5:0) after 9m1s
[09:34:41.719] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:5:0)
[09:34:41.749] Build status: STARTED
[09:34:41.749] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:6:0) after 5m21s
[09:40:02.778] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:6:0)
[09:40:02.798] Build status: STARTED
[09:40:02.798] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:7:0) after 1m19s
[09:41:21.822] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:7:0)
[09:41:21.845] Build status: STARTED
[09:41:21.845] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:8:0) after 2m18s
[09:43:39.963] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:8:0)
[09:43:39.978] Build status: STARTED
[09:43:39.978] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:9:0) after 4m58s
[09:48:38.102] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:9:0)
[09:48:38.122] Build status: STARTED
[09:48:38.122] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:10:0) after 2m32s
[09:51:10.312] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:10:0)
[09:51:10.362] Build status: STARTED
[09:51:10.362] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:11:0) after 7m15s
[09:58:25.672] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:11:0)
[09:58:25.736] Build status: STARTED
[09:58:25.736] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:12:0) after 2m55s
[10:01:20.762] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:12:0)
[10:01:20.779] Build status: STARTED
[10:01:20.779] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:13:0) after 6m20s
[10:07:40.805] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:13:0)
[10:07:40.850] Build status: STARTED
[10:07:40.850] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:14:0) after 1m26s
[10:09:06.909] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:14:0)
[10:09:06.928] Build status: STARTED
[10:09:06.928] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:15:0) after 5m19s
[10:14:25.955] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:15:0)
[10:14:25.970] Build status: STARTED
[10:14:25.970] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:16:0) after 8m21s
[10:22:46.994] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:16:0)
[10:22:47.009] Build status: STARTED
[10:22:47.009] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:17:0) after 2m24s
[10:25:11.085] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:17:0)
[10:25:11.105] Build status: STARTED
[10:25:11.105] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:18:0) after 2m7s
[10:27:18.235] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:18:0)
[10:27:18.255] Build status: STARTED
[10:27:18.255] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:19:0) after 1m47s
[10:29:05.478] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:19:0)
[10:29:05.530] Build status: STARTED
[10:29:05.530] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:20:0) after 2m3s
[10:31:08.882] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:20:0)
[10:31:08.898] Build status: STARTED
[10:31:08.898] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:21:0) after 6m15s
[10:37:23.931] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:21:0)
[10:37:23.966] Build status: STARTED
[10:37:23.966] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:22:0) after 6m34s
[10:43:58.150] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:22:0)
[10:43:58.164] Build status: STARTED
[10:43:58.164] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:23:0) after 2m20s
[10:46:18.168] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:23:0)
[10:46:18.236] Build status: STARTED
[10:46:18.236] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:24:0) after 4m54s
[10:51:12.237] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:24:0)
[10:51:12.263] Build status: STARTED
[10:51:12.263] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:25:0) after 2m26s
[10:53:38.471] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:25:0)
[10:53:38.489] Build status: STARTED
[10:53:38.489] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:26:0) after 7m19s
[11:00:57.890] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:26:0)
[11:00:57.915] Build status: STARTED
[11:00:57.915] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:27:0) after 4m25s
[11:05:23.068] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:27:0)
[11:05:23.092] Build status: STARTED
[11:05:23.092] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:28:0) after 9m6s
[11:14:29.209] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:28:0)
[11:14:29.228] Build status: STARTED
[11:14:29.228] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:29:0) after 7m16s
[11:21:45.309] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:29:0)
[11:21:45.329] Build status: STARTED
[11:21:45.329] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:30:0) after 9m31s
[11:31:16.356] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:30:0)
[11:31:16.373] Build status: STARTED
[11:31:16.373] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:31:0) after 9m7s
[11:40:23.396] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:31:0)
[11:40:23.424] Build status: STARTED
[11:40:23.424] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:32:0) after 2m1s
[11:42:24.555] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:32:0)
[11:42:24.580] Build status: STARTED
[11:42:24.580] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:33:0) after 1m4s
[11:43:28.601] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:33:0)
[11:43:28.649] Build status: STARTED
[11:43:28.649] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:34:0) after 7m55s
[11:51:23.648] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:34:0)
[11:51:23.689] Build status: STARTED
[11:51:23.689] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:35:0) after 1m55s
[11:53:18.914] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:35:0)
[11:53:18.973] Build status: STARTED
[11:53:18.973] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:36:0) after 2m22s
[11:55:40.994] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:36:0)
[11:55:41.010] Build status: STARTED
[11:55:41.010] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:37:0) after 8m31s
[12:04:12.076] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:37:0)
[12:04:12.093] Build status: STARTED
[12:04:12.093] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:38:0) after 8m14s
[12:12:26.245] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:38:0)
[12:12:26.350] Build status: STARTED
[12:12:26.350] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:39:0) after 7m11s
[12:19:37.352] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:39:0)
[12:19:37.371] Build status: STARTED
[12:19:37.371] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:40:0) after 1m42s
[12:21:19.356] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:40:0)
[12:21:19.420] Build status: STARTED
[12:21:19.420] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:41:0) after 9m15s
[12:30:34.427] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:41:0)
[12:30:34.440] Build status: STARTED
[12:30:34.440] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8879339950562150032:42:0) after 5m11s
[12:32:14.823] Received PubSub notification, asking Buildbucket for the build status
[12:32:14.844] Build:
{
  "id": "8713909888618314481",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Mac FYI ASAN (Intel)"
  },
  "number": 16735,
  "createdBy": "project:chromium",
  "createTime": "2025-05-25T09:22:45.954299052Z",
  "startTime": "2025-05-25T09:22:56.501766288Z",
  "endTime": "2025-05-25T12:31:52.928560948Z",
  "updateTime": "2025-05-25T12:31:52.928560948Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "b870da7099cbfe50b02672ff9503d29030c31888",
      "ref": "refs/heads/main"
    }
  }
}
[12:32:14.844] Invocation finished in 3h9m30.128406733s with status FAILED