[16:21:52.468] New invocation is queued and will start shortly
[16:21:52.468] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[16:21:53.480] Starting the invocation (attempt 1)
[16:21:53.503] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/4d9e03c7bdc9dd56c88a3077ea69fa2bea504371"
[16:21:53.503] Popped gitiles commit info from properties and tags
[16:21:53.503] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[16:21:53.503] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[16:21:53.504] Buildbucket request:
{
"requestId": "8863187321217592080",
"builder": {
"project": "chromium-m143",
"bucket": "ci",
"builder": "Dawn Mac x64 DEPS Release (Intel)"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "4d9e03c7bdc9dd56c88a3077ea69fa2bea504371",
"project": "chromium/src",
"ref": "refs/branch-heads/7499"
}
]
},
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8863187321217592080",
"job": "chromium-m143/Dawn Mac x64 DEPS Release (Intel)",
"triggers": [
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "fc5717798d2710b6f60322e7fdf8f733ec6297b1",
"project": "chromium/src",
"ref": "refs/branch-heads/7499"
}
]
},
"deps_revision_overrides": {
"src": "fc5717798d2710b6f60322e7fdf8f733ec6297b1"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8697762803351583265"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8697762803351583265"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8697762803351583265"
}
],
"parent_build_id": "8697762803351583265",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Mac x64 DEPS Builder",
"parent_got_angle_revision": "9b75aecb1fa769e357d1dad2614228b6a04ddee6",
"parent_got_dawn_revision": "e1ce227ebf75378c5f60a9d531579982bcdd93ee",
"parent_got_revision": "fc5717798d2710b6f60322e7fdf8f733ec6297b1",
"parent_got_revision_cp": "refs/branch-heads/7499@{#2355}",
"parent_got_swiftshader_revision": "3d536c0fc62b1cdea0f78c3c38d79be559855b88",
"parent_got_v8_revision": "beee9f5cafde91bbd086077a11db16cb9768e62a",
"parent_got_v8_revision_cp": "refs/branch-heads/14.3@{#31}",
"parent_got_webrtc_revision": "a5751574a386ba0ba80b8c62201977f6aab6c225",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7499@{#1}",
"revision": "fc5717798d2710b6f60322e7fdf8f733ec6297b1",
"swarm_hashes": {
"dawn_end2end_tests": "e0b09013244cffe8cf19dcc660cf2632b078206ad45997c91f6235e0bfd410c6/323",
"dawn_perf_tests": "4616c80fd97b3db6a0a77a94c50e67d147904554c8733fcad7cc6727c240b012/487",
"gl_tests": "953e60e7e413fbee5e093b8681686ad06edf41e9f2f5ccda11f54358e1f9f123/323",
"gl_unittests": "609afdc4c3a84dd2557701f97567ec592a8fd5983f142261441e8ccd0484cd32/323",
"telemetry_gpu_integration_test": "8404b99a55d71a40a2701cfd7ca799fbb379c9aaee7dd0baa378c3f730e96681/815",
"telemetry_gpu_unittests": "fa354d56faed289543343481d4a3a49f0a580b5e1aab6c62020e25f166877813/736",
"webgpu_blink_web_tests": "005346a3c9d9b5afdf19a04a638609db1b18cdef70133887d3daac2fae7105e6/566"
},
"swarming_command_lines_cwd": "out/aeae-Dawn_Mac_x64_DE",
"swarming_command_lines_digest": "60de9dd905cce771078a83e00991e4c89813c7ce0fd35ffdc98650f989deaf14/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/7499",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "fc5717798d2710b6f60322e7fdf8f733ec6297b1",
"tags": [
"parent_buildername:Dawn Mac x64 DEPS Builder",
"parent_buildnumber:1744",
"user_agent:recipe"
]
},
"id": "3905151f-d3e7-4cf5-b434-ed4e17059e1e",
"title": "Dawn Mac x64 DEPS Builder/1744",
"url": "https://cr-buildbucket.appspot.com/build/8697762803351583265"
},
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "2b4e02eefe0649748b8011438849e04955d1c6f5",
"project": "chromium/src",
"ref": "refs/branch-heads/7499"
}
]
},
"deps_revision_overrides": {
"src": "2b4e02eefe0649748b8011438849e04955d1c6f5"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8697761707871281393"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8697761707871281393"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8697761707871281393"
}
],
"parent_build_id": "8697761707871281393",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Mac x64 DEPS Builder",
"parent_got_angle_revision": "9b75aecb1fa769e357d1dad2614228b6a04ddee6",
"parent_got_dawn_revision": "e1ce227ebf75378c5f60a9d531579982bcdd93ee",
"parent_got_revision": "2b4e02eefe0649748b8011438849e04955d1c6f5",
"parent_got_revision_cp": "refs/branch-heads/7499@{#2356}",
"parent_got_swiftshader_revision": "3d536c0fc62b1cdea0f78c3c38d79be559855b88",
"parent_got_v8_revision": "beee9f5cafde91bbd086077a11db16cb9768e62a",
"parent_got_v8_revision_cp": "refs/branch-heads/14.3@{#31}",
"parent_got_webrtc_revision": "a5751574a386ba0ba80b8c62201977f6aab6c225",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7499@{#1}",
"revision": "2b4e02eefe0649748b8011438849e04955d1c6f5",
"swarm_hashes": {
"dawn_end2end_tests": "e0b09013244cffe8cf19dcc660cf2632b078206ad45997c91f6235e0bfd410c6/323",
"dawn_perf_tests": "4616c80fd97b3db6a0a77a94c50e67d147904554c8733fcad7cc6727c240b012/487",
"gl_tests": "953e60e7e413fbee5e093b8681686ad06edf41e9f2f5ccda11f54358e1f9f123/323",
"gl_unittests": "609afdc4c3a84dd2557701f97567ec592a8fd5983f142261441e8ccd0484cd32/323",
"telemetry_gpu_integration_test": "19eea8b9b9ad068849e7941073ead3c969019ed618a8ad5ba303099f79f6d9a5/815",
"telemetry_gpu_unittests": "fa354d56faed289543343481d4a3a49f0a580b5e1aab6c62020e25f166877813/736",
"webgpu_blink_web_tests": "38dea23fec952a687f300a36b22063b20a2c739b0c02a44397206129323a192a/566"
},
"swarming_command_lines_cwd": "out/aeae-Dawn_Mac_x64_DE",
"swarming_command_lines_digest": "60de9dd905cce771078a83e00991e4c89813c7ce0fd35ffdc98650f989deaf14/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/7499",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "2b4e02eefe0649748b8011438849e04955d1c6f5",
"tags": [
"parent_buildername:Dawn Mac x64 DEPS Builder",
"parent_buildnumber:1745",
"user_agent:recipe"
]
},
"id": "d5c4e1ab-1bf9-4aa4-b8d9-3eb30c750023",
"title": "Dawn Mac x64 DEPS Builder/1745",
"url": "https://cr-buildbucket.appspot.com/build/8697761707871281393"
},
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "4d9e03c7bdc9dd56c88a3077ea69fa2bea504371",
"project": "chromium/src",
"ref": "refs/branch-heads/7499"
}
]
},
"deps_revision_overrides": {
"src": "4d9e03c7bdc9dd56c88a3077ea69fa2bea504371"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8697760030050845617"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8697760030050845617"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8697760030050845617"
}
],
"parent_build_id": "8697760030050845617",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Mac x64 DEPS Builder",
"parent_got_angle_revision": "9b75aecb1fa769e357d1dad2614228b6a04ddee6",
"parent_got_dawn_revision": "e1ce227ebf75378c5f60a9d531579982bcdd93ee",
"parent_got_revision": "4d9e03c7bdc9dd56c88a3077ea69fa2bea504371",
"parent_got_revision_cp": "refs/branch-heads/7499@{#2357}",
"parent_got_swiftshader_revision": "3d536c0fc62b1cdea0f78c3c38d79be559855b88",
"parent_got_v8_revision": "beee9f5cafde91bbd086077a11db16cb9768e62a",
"parent_got_v8_revision_cp": "refs/branch-heads/14.3@{#31}",
"parent_got_webrtc_revision": "a5751574a386ba0ba80b8c62201977f6aab6c225",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7499@{#1}",
"revision": "4d9e03c7bdc9dd56c88a3077ea69fa2bea504371",
"swarm_hashes": {
"dawn_end2end_tests": "e0b09013244cffe8cf19dcc660cf2632b078206ad45997c91f6235e0bfd410c6/323",
"dawn_perf_tests": "4616c80fd97b3db6a0a77a94c50e67d147904554c8733fcad7cc6727c240b012/487",
"gl_tests": "953e60e7e413fbee5e093b8681686ad06edf41e9f2f5ccda11f54358e1f9f123/323",
"gl_unittests": "609afdc4c3a84dd2557701f97567ec592a8fd5983f142261441e8ccd0484cd32/323",
"telemetry_gpu_integration_test": "72207ac7479b45d57e6ee0d21cf4a3b74a67b6306ca4b6098171dca310a060b7/815",
"telemetry_gpu_unittests": "fa354d56faed289543343481d4a3a49f0a580b5e1aab6c62020e25f166877813/736",
"webgpu_blink_web_tests": "3d013d858c2ba2dee8df5c500a8ea66a0ca301110d4a7eb894b34971d04a0e82/566"
},
"swarming_command_lines_cwd": "out/aeae-Dawn_Mac_x64_DE",
"swarming_command_lines_digest": "60de9dd905cce771078a83e00991e4c89813c7ce0fd35ffdc98650f989deaf14/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/7499",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "4d9e03c7bdc9dd56c88a3077ea69fa2bea504371",
"tags": [
"parent_buildername:Dawn Mac x64 DEPS Builder",
"parent_buildnumber:1746",
"user_agent:recipe"
]
},
"id": "2e64106e-035c-45f3-ab8f-3cf498e4d8be",
"title": "Dawn Mac x64 DEPS Builder/1746",
"url": "https://cr-buildbucket.appspot.com/build/8697760030050845617"
}
]
},
"deps_revision_overrides": {
"src": "4d9e03c7bdc9dd56c88a3077ea69fa2bea504371"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8697760030050845617"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8697760030050845617"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8697760030050845617"
}
],
"parent_build_id": "8697760030050845617",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Mac x64 DEPS Builder",
"parent_got_angle_revision": "9b75aecb1fa769e357d1dad2614228b6a04ddee6",
"parent_got_dawn_revision": "e1ce227ebf75378c5f60a9d531579982bcdd93ee",
"parent_got_revision": "4d9e03c7bdc9dd56c88a3077ea69fa2bea504371",
"parent_got_revision_cp": "refs/branch-heads/7499@{#2357}",
"parent_got_swiftshader_revision": "3d536c0fc62b1cdea0f78c3c38d79be559855b88",
"parent_got_v8_revision": "beee9f5cafde91bbd086077a11db16cb9768e62a",
"parent_got_v8_revision_cp": "refs/branch-heads/14.3@{#31}",
"parent_got_webrtc_revision": "a5751574a386ba0ba80b8c62201977f6aab6c225",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7499@{#1}",
"swarm_hashes": {
"dawn_end2end_tests": "e0b09013244cffe8cf19dcc660cf2632b078206ad45997c91f6235e0bfd410c6/323",
"dawn_perf_tests": "4616c80fd97b3db6a0a77a94c50e67d147904554c8733fcad7cc6727c240b012/487",
"gl_tests": "953e60e7e413fbee5e093b8681686ad06edf41e9f2f5ccda11f54358e1f9f123/323",
"gl_unittests": "609afdc4c3a84dd2557701f97567ec592a8fd5983f142261441e8ccd0484cd32/323",
"telemetry_gpu_integration_test": "72207ac7479b45d57e6ee0d21cf4a3b74a67b6306ca4b6098171dca310a060b7/815",
"telemetry_gpu_unittests": "fa354d56faed289543343481d4a3a49f0a580b5e1aab6c62020e25f166877813/736",
"webgpu_blink_web_tests": "3d013d858c2ba2dee8df5c500a8ea66a0ca301110d4a7eb894b34971d04a0e82/566"
},
"swarming_command_lines_cwd": "out/aeae-Dawn_Mac_x64_DE",
"swarming_command_lines_digest": "60de9dd905cce771078a83e00991e4c89813c7ce0fd35ffdc98650f989deaf14/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "4d9e03c7bdc9dd56c88a3077ea69fa2bea504371",
"ref": "refs/branch-heads/7499"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8863187321217592080"
},
{
"key": "scheduler_job_id",
"value": "chromium-m143/Dawn Mac x64 DEPS Release (Intel)"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Dawn Mac x64 DEPS Builder"
},
{
"key": "parent_buildnumber",
"value": "1746"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[16:21:54.013] Scheduled build:
{
"id": "8697757259425219633",
"builder": {
"project": "chromium-m143",
"bucket": "ci",
"builder": "Dawn Mac x64 DEPS Release (Intel)"
},
"number": 382,
"createdBy": "project:chromium-m143",
"createTime": "2025-11-19T16:21:53.558810537Z",
"updateTime": "2025-11-19T16:21:53.558810537Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "4d9e03c7bdc9dd56c88a3077ea69fa2bea504371",
"ref": "refs/branch-heads/7499"
}
}
}
[16:21:54.013] Task URL: https://cr-buildbucket.appspot.com/build/8697757259425219633
[16:21:54.013] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:2:0) after 2m47s
[16:21:55.034] Received PubSub notification, asking Buildbucket for the build status
[16:21:55.061] Build status: SCHEDULED
[16:22:01.737] Received PubSub notification, asking Buildbucket for the build status
[16:22:01.752] Build status: STARTED
[16:24:41.033] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:2:0)
[16:24:41.077] Build status: STARTED
[16:24:41.077] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:5:0) after 7m43s
[16:32:24.095] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:5:0)
[16:32:24.113] Build status: STARTED
[16:32:24.113] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:6:0) after 2m27s
[16:34:51.116] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:6:0)
[16:34:51.148] Build status: STARTED
[16:34:51.148] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:7:0) after 1m9s
[16:36:00.258] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:7:0)
[16:36:00.278] Build status: STARTED
[16:36:00.278] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:8:0) after 8m0s
[16:44:00.292] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:8:0)
[16:44:00.339] Build status: STARTED
[16:44:00.339] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:9:0) after 7m40s
[16:51:40.367] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:9:0)
[16:51:40.403] Build status: STARTED
[16:51:40.403] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:10:0) after 3m45s
[16:55:25.415] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:10:0)
[16:55:25.430] Build status: STARTED
[16:55:25.430] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:11:0) after 6m48s
[17:02:13.492] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:11:0)
[17:02:13.598] Build status: STARTED
[17:02:13.598] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:12:0) after 9m52s
[17:12:05.617] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:12:0)
[17:12:05.690] Build status: STARTED
[17:12:05.690] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:13:0) after 8m26s
[17:20:31.707] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:13:0)
[17:20:31.734] Build status: STARTED
[17:20:31.734] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:14:0) after 3m40s
[17:24:11.749] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:14:0)
[17:24:11.794] Build status: STARTED
[17:24:11.794] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:15:0) after 5m25s
[17:29:36.813] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:15:0)
[17:29:36.838] Build status: STARTED
[17:29:36.838] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:16:0) after 4m28s
[17:34:04.819] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:16:0)
[17:34:04.834] Build status: STARTED
[17:34:04.834] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:17:0) after 4m41s
[17:38:45.925] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:17:0)
[17:38:45.940] Build status: STARTED
[17:38:45.940] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:18:0) after 9m59s
[17:48:45.006] Handling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:18:0)
[17:48:45.024] Build status: STARTED
[17:48:45.024] Scheduling timer "check-buildbucket-build-status" (chromium-m143/Dawn Mac x64 DEPS Release (Intel):8863187321217592080:19:0) after 5m58s
[17:51:28.909] Received PubSub notification, asking Buildbucket for the build status
[17:51:29.000] Build:
{
"id": "8697757259425219633",
"builder": {
"project": "chromium-m143",
"bucket": "ci",
"builder": "Dawn Mac x64 DEPS Release (Intel)"
},
"number": 382,
"createdBy": "project:chromium-m143",
"createTime": "2025-11-19T16:21:53.558810537Z",
"startTime": "2025-11-19T16:22:01.453365375Z",
"endTime": "2025-11-19T17:51:28.629891121Z",
"updateTime": "2025-11-19T17:51:28.629891121Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "4d9e03c7bdc9dd56c88a3077ea69fa2bea504371",
"ref": "refs/branch-heads/7499"
}
}
}
[17:51:29.000] Invocation finished in 1h29m36.53461079s with status FAILED