[02:02:08.755] New invocation is queued and will start shortly
[02:02:08.755] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[02:02:09.911] Starting the invocation (attempt 1)
[02:02:09.941] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/b6f9edd398a61b46f8bb2699ee6220a320ea8feb"
[02:02:09.941] Popped gitiles commit info from properties and tags
[02:02:09.941] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[02:02:09.941] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[02:02:09.942] Buildbucket request:
{
"requestId": "8865143946965246320",
"builder": {
"project": "chromium-m142",
"bucket": "ci",
"builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "b6f9edd398a61b46f8bb2699ee6220a320ea8feb",
"project": "chromium/src",
"ref": "refs/branch-heads/7444"
}
]
},
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8865143946965246320",
"job": "chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA)",
"triggers": [
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "5c999a13cd892ad2d05d74be73971c98e2ddaebc",
"project": "chromium/src",
"ref": "refs/branch-heads/7444"
}
]
},
"deps_revision_overrides": {
"src": "5c999a13cd892ad2d05d74be73971c98e2ddaebc"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8699718715550028561"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8699718715550028561"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8699718715550028561"
}
],
"parent_build_id": "8699718715550028561",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Linux x64 DEPS Builder",
"parent_got_angle_revision": "e5b338502c7c65bd38d4695bc557938ca4cc3aa7",
"parent_got_dawn_revision": "cee9cb0d67e749bf42f5e90cb3b8a6f525dbb920",
"parent_got_revision": "5c999a13cd892ad2d05d74be73971c98e2ddaebc",
"parent_got_revision_cp": "refs/branch-heads/7444@{#2259}",
"parent_got_swiftshader_revision": "5f1c459a11bb4899301e89609cbf7547f9f31e20",
"parent_got_v8_revision": "fd9c56506b6d879cab4ee5fd6349406eedb0cd24",
"parent_got_v8_revision_cp": "refs/branch-heads/14.2@{#30}",
"parent_got_webrtc_revision": "29d6eabaf0b5c1a62af6e7fd03adebf5e1a446e8",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7444@{#2}",
"revision": "5c999a13cd892ad2d05d74be73971c98e2ddaebc",
"swarm_hashes": {
"dawn_end2end_tests": "209e0d5fea9905e438084dfecd1f33f7b16ab377947d04c714b4f2abffca3f1e/323",
"dawn_perf_tests": "f5ced29c298e7fdf4d5f6409fb438321b98a4410d29b2818fb29c0e80be89078/487",
"gl_tests": "973af978440d3f86d62637d699d96964cc8b9a6ac622cb6c7ed63d303eb3a6a4/323",
"gl_unittests": "2d80145e9f6b94efd84c0b37b3c30297dccb95ea42031fa28dff83b290f9f98a/323",
"telemetry_gpu_integration_test": "35cafe9ad1f8887fcf32856632d5d0d34ae3b00d42818ba6ed592de0dfbc008b/815",
"telemetry_gpu_unittests": "eb4db0917da8b8895b60285fda5b7ee334ad054baf385cc89faacfacf869fffa/736",
"webgpu_blink_web_tests": "bbf280c7c26c9aecf68a70a3068602c35d55a7af7ad54c31e7caa5aa38187862/566"
},
"swarming_command_lines_cwd": "out/ee60-Dawn_Linux_x64_",
"swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/7444",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "5c999a13cd892ad2d05d74be73971c98e2ddaebc",
"tags": [
"parent_buildername:Dawn Linux x64 DEPS Builder",
"parent_buildnumber:1708",
"user_agent:recipe"
]
},
"id": "c397f488-c362-4da1-bf26-fc27730458d5",
"title": "Dawn Linux x64 DEPS Builder/1708",
"url": "https://cr-buildbucket.appspot.com/build/8699718715550028561"
},
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "f3bf0684d519fdb3d43661bee9f8b52533282d39",
"project": "chromium/src",
"ref": "refs/branch-heads/7444"
}
]
},
"deps_revision_overrides": {
"src": "f3bf0684d519fdb3d43661bee9f8b52533282d39"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8699717502670413633"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8699717502670413633"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8699717502670413633"
}
],
"parent_build_id": "8699717502670413633",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Linux x64 DEPS Builder",
"parent_got_angle_revision": "e5b338502c7c65bd38d4695bc557938ca4cc3aa7",
"parent_got_dawn_revision": "cee9cb0d67e749bf42f5e90cb3b8a6f525dbb920",
"parent_got_revision": "f3bf0684d519fdb3d43661bee9f8b52533282d39",
"parent_got_revision_cp": "refs/branch-heads/7444@{#2260}",
"parent_got_swiftshader_revision": "5f1c459a11bb4899301e89609cbf7547f9f31e20",
"parent_got_v8_revision": "fd9c56506b6d879cab4ee5fd6349406eedb0cd24",
"parent_got_v8_revision_cp": "refs/branch-heads/14.2@{#30}",
"parent_got_webrtc_revision": "29d6eabaf0b5c1a62af6e7fd03adebf5e1a446e8",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7444@{#2}",
"revision": "f3bf0684d519fdb3d43661bee9f8b52533282d39",
"swarm_hashes": {
"dawn_end2end_tests": "209e0d5fea9905e438084dfecd1f33f7b16ab377947d04c714b4f2abffca3f1e/323",
"dawn_perf_tests": "f5ced29c298e7fdf4d5f6409fb438321b98a4410d29b2818fb29c0e80be89078/487",
"gl_tests": "973af978440d3f86d62637d699d96964cc8b9a6ac622cb6c7ed63d303eb3a6a4/323",
"gl_unittests": "2d80145e9f6b94efd84c0b37b3c30297dccb95ea42031fa28dff83b290f9f98a/323",
"telemetry_gpu_integration_test": "4c25f4836085747167c80694fc90436460669e25e9e73d8ddcb8074f3c9eaba4/815",
"telemetry_gpu_unittests": "eb4db0917da8b8895b60285fda5b7ee334ad054baf385cc89faacfacf869fffa/736",
"webgpu_blink_web_tests": "effa863300a2026d3f1c92040d7346ac85cfa19742ab6dedae6f871ef33349bc/566"
},
"swarming_command_lines_cwd": "out/ee60-Dawn_Linux_x64_",
"swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/7444",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "f3bf0684d519fdb3d43661bee9f8b52533282d39",
"tags": [
"parent_buildername:Dawn Linux x64 DEPS Builder",
"parent_buildnumber:1709",
"user_agent:recipe"
]
},
"id": "d922b6a6-ad28-4665-aae5-a689c8ea678d",
"title": "Dawn Linux x64 DEPS Builder/1709",
"url": "https://cr-buildbucket.appspot.com/build/8699717502670413633"
},
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "b6f9edd398a61b46f8bb2699ee6220a320ea8feb",
"project": "chromium/src",
"ref": "refs/branch-heads/7444"
}
]
},
"deps_revision_overrides": {
"src": "b6f9edd398a61b46f8bb2699ee6220a320ea8feb"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8699716738893316833"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8699716738893316833"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8699716738893316833"
}
],
"parent_build_id": "8699716738893316833",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Linux x64 DEPS Builder",
"parent_got_angle_revision": "e5b338502c7c65bd38d4695bc557938ca4cc3aa7",
"parent_got_dawn_revision": "cee9cb0d67e749bf42f5e90cb3b8a6f525dbb920",
"parent_got_revision": "b6f9edd398a61b46f8bb2699ee6220a320ea8feb",
"parent_got_revision_cp": "refs/branch-heads/7444@{#2261}",
"parent_got_swiftshader_revision": "5f1c459a11bb4899301e89609cbf7547f9f31e20",
"parent_got_v8_revision": "fd9c56506b6d879cab4ee5fd6349406eedb0cd24",
"parent_got_v8_revision_cp": "refs/branch-heads/14.2@{#30}",
"parent_got_webrtc_revision": "29d6eabaf0b5c1a62af6e7fd03adebf5e1a446e8",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7444@{#2}",
"revision": "b6f9edd398a61b46f8bb2699ee6220a320ea8feb",
"swarm_hashes": {
"dawn_end2end_tests": "209e0d5fea9905e438084dfecd1f33f7b16ab377947d04c714b4f2abffca3f1e/323",
"dawn_perf_tests": "f5ced29c298e7fdf4d5f6409fb438321b98a4410d29b2818fb29c0e80be89078/487",
"gl_tests": "38c9594327373dbba32d5d6a1fc5068f4f2a733ca678729caf655d3c43e728bc/323",
"gl_unittests": "2d80145e9f6b94efd84c0b37b3c30297dccb95ea42031fa28dff83b290f9f98a/323",
"telemetry_gpu_integration_test": "dd20f761c4fdb61db829f0419f07ac79565df52c3c4218ff215bd729cd61e29f/815",
"telemetry_gpu_unittests": "fc6e8b6eeedfa04cc5d05689de1d7acf495bb84a31b57f6c9b85aebe971aba88/736",
"webgpu_blink_web_tests": "012700e268aa199686debbf50b4f32460d26b8f768460c98f4ea6e3eeca0522c/566"
},
"swarming_command_lines_cwd": "out/ee60-Dawn_Linux_x64_",
"swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/7444",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "b6f9edd398a61b46f8bb2699ee6220a320ea8feb",
"tags": [
"parent_buildername:Dawn Linux x64 DEPS Builder",
"parent_buildnumber:1710",
"user_agent:recipe"
]
},
"id": "3a6abe84-89fd-4fbf-bd20-f307f75beb31",
"title": "Dawn Linux x64 DEPS Builder/1710",
"url": "https://cr-buildbucket.appspot.com/build/8699716738893316833"
}
]
},
"deps_revision_overrides": {
"src": "b6f9edd398a61b46f8bb2699ee6220a320ea8feb"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8699716738893316833"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8699716738893316833"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8699716738893316833"
}
],
"parent_build_id": "8699716738893316833",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Linux x64 DEPS Builder",
"parent_got_angle_revision": "e5b338502c7c65bd38d4695bc557938ca4cc3aa7",
"parent_got_dawn_revision": "cee9cb0d67e749bf42f5e90cb3b8a6f525dbb920",
"parent_got_revision": "b6f9edd398a61b46f8bb2699ee6220a320ea8feb",
"parent_got_revision_cp": "refs/branch-heads/7444@{#2261}",
"parent_got_swiftshader_revision": "5f1c459a11bb4899301e89609cbf7547f9f31e20",
"parent_got_v8_revision": "fd9c56506b6d879cab4ee5fd6349406eedb0cd24",
"parent_got_v8_revision_cp": "refs/branch-heads/14.2@{#30}",
"parent_got_webrtc_revision": "29d6eabaf0b5c1a62af6e7fd03adebf5e1a446e8",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7444@{#2}",
"swarm_hashes": {
"dawn_end2end_tests": "209e0d5fea9905e438084dfecd1f33f7b16ab377947d04c714b4f2abffca3f1e/323",
"dawn_perf_tests": "f5ced29c298e7fdf4d5f6409fb438321b98a4410d29b2818fb29c0e80be89078/487",
"gl_tests": "38c9594327373dbba32d5d6a1fc5068f4f2a733ca678729caf655d3c43e728bc/323",
"gl_unittests": "2d80145e9f6b94efd84c0b37b3c30297dccb95ea42031fa28dff83b290f9f98a/323",
"telemetry_gpu_integration_test": "dd20f761c4fdb61db829f0419f07ac79565df52c3c4218ff215bd729cd61e29f/815",
"telemetry_gpu_unittests": "fc6e8b6eeedfa04cc5d05689de1d7acf495bb84a31b57f6c9b85aebe971aba88/736",
"webgpu_blink_web_tests": "012700e268aa199686debbf50b4f32460d26b8f768460c98f4ea6e3eeca0522c/566"
},
"swarming_command_lines_cwd": "out/ee60-Dawn_Linux_x64_",
"swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "b6f9edd398a61b46f8bb2699ee6220a320ea8feb",
"ref": "refs/branch-heads/7444"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8865143946965246320"
},
{
"key": "scheduler_job_id",
"value": "chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA)"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Dawn Linux x64 DEPS Builder"
},
{
"key": "parent_buildnumber",
"value": "1710"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[02:02:10.414] Scheduled build:
{
"id": "8699713885015108689",
"builder": {
"project": "chromium-m142",
"bucket": "ci",
"builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
},
"number": 544,
"createdBy": "project:chromium-m142",
"createTime": "2025-10-29T02:02:09.986838054Z",
"updateTime": "2025-10-29T02:02:09.986838054Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "b6f9edd398a61b46f8bb2699ee6220a320ea8feb",
"ref": "refs/branch-heads/7444"
}
}
}
[02:02:10.414] Task URL: https://cr-buildbucket.appspot.com/build/8699713885015108689
[02:02:10.414] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:2:0) after 3m32s
[02:02:14.429] Received PubSub notification, asking Buildbucket for the build status
[02:02:14.488] Build status: SCHEDULED
[02:03:06.026] Received PubSub notification, asking Buildbucket for the build status
[02:03:06.089] Build status: STARTED
[02:05:42.475] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:2:0)
[02:05:42.543] Build status: STARTED
[02:05:42.543] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:5:0) after 8m58s
[02:14:40.609] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:5:0)
[02:14:40.625] Build status: STARTED
[02:14:40.625] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:6:0) after 1m22s
[02:16:02.640] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:6:0)
[02:16:02.703] Build status: STARTED
[02:16:02.703] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:7:0) after 1m37s
[02:17:39.718] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:7:0)
[02:17:39.734] Build status: STARTED
[02:17:39.734] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:8:0) after 8m32s
[02:26:11.750] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:8:0)
[02:26:11.771] Build status: STARTED
[02:26:11.771] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:9:0) after 6m57s
[02:33:08.816] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:9:0)
[02:33:08.855] Build status: STARTED
[02:33:08.855] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:10:0) after 6m0s
[02:39:08.856] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:10:0)
[02:39:08.882] Build status: STARTED
[02:39:08.883] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:11:0) after 4m11s
[02:43:19.900] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:11:0)
[02:43:20.015] Build status: STARTED
[02:43:20.016] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:12:0) after 1m56s
[02:45:16.161] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:12:0)
[02:45:16.223] Build status: STARTED
[02:45:16.223] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:13:0) after 9m11s
[02:54:27.256] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:13:0)
[02:54:27.270] Build status: STARTED
[02:54:27.270] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:14:0) after 2m21s
[02:56:48.297] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:14:0)
[02:56:48.363] Build status: STARTED
[02:56:48.363] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:15:0) after 4m21s
[03:01:09.509] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:15:0)
[03:01:09.539] Build status: STARTED
[03:01:09.539] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:16:0) after 3m6s
[03:04:15.558] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:16:0)
[03:04:15.617] Build status: STARTED
[03:04:15.617] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:17:0) after 4m8s
[03:08:23.612] Handling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:17:0)
[03:08:23.632] Build status: STARTED
[03:08:23.633] Scheduling timer "check-buildbucket-build-status" (chromium-m142/Dawn Linux x64 DEPS Release (NVIDIA):8865143946965246320:18:0) after 8m39s
[03:12:51.996] Received PubSub notification, asking Buildbucket for the build status
[03:12:52.019] Build:
{
"id": "8699713885015108689",
"builder": {
"project": "chromium-m142",
"bucket": "ci",
"builder": "Dawn Linux x64 DEPS Release (NVIDIA)"
},
"number": 544,
"createdBy": "project:chromium-m142",
"createTime": "2025-10-29T02:02:09.986838054Z",
"startTime": "2025-10-29T02:03:05.726963088Z",
"endTime": "2025-10-29T03:12:51.522065747Z",
"updateTime": "2025-10-29T03:12:51.522065747Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "b6f9edd398a61b46f8bb2699ee6220a320ea8feb",
"ref": "refs/branch-heads/7444"
}
}
}
[03:12:52.019] Invocation finished in 1h10m43.276358132s with status FAILED