[05:29:33.383] New invocation is queued and will start shortly
[05:29:33.383] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[05:29:34.616] Starting the invocation (attempt 1)
[05:29:34.642] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/b2f440ad2c279fb46a9d5d54fe8f8c4938bc09de"
[05:29:34.642] Popped gitiles commit info from properties and tags
[05:29:34.642] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[05:29:34.642] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[05:29:34.643] Buildbucket request:
{
"requestId": "8867848806817481296",
"builder": {
"project": "chromium-m140",
"bucket": "ci",
"builder": "Dawn Win10 x64 DEPS Release (Intel)"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "b2f440ad2c279fb46a9d5d54fe8f8c4938bc09de",
"project": "chromium/src",
"ref": "refs/branch-heads/7339"
}
]
},
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8867848806817481296",
"job": "chromium-m140/Dawn Win10 x64 DEPS Release (Intel)",
"triggers": [
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "cb080f1b71737fb7434c6f4897570fbf055f5fd2",
"project": "chromium/src",
"ref": "refs/branch-heads/7339"
}
]
},
"deps_revision_overrides": {
"src": "cb080f1b71737fb7434c6f4897570fbf055f5fd2"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8702427800027189057"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8702427800027189057"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8702427800027189057"
}
],
"parent_build_id": "8702427800027189057",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Win10 x64 DEPS Builder",
"parent_got_angle_revision": "a8c8a6febe630c6239a5e207530e9fac651ae373",
"parent_got_dawn_revision": "67be7fddacc4f4bcb21d0cf7bf8bb18752d8fb08",
"parent_got_revision": "cb080f1b71737fb7434c6f4897570fbf055f5fd2",
"parent_got_revision_cp": "refs/branch-heads/7339@{#2801}",
"parent_got_swiftshader_revision": "fdb6700ecb04103b658d2e4623d6bc663ba80ea8",
"parent_got_v8_revision": "b7ed978e41b4bac7802b206404d0e2f3d09f31ac",
"parent_got_v8_revision_cp": "refs/branch-heads/14.0@{#19}",
"parent_got_webrtc_revision": "36ea4535a500ac137dbf1f577ce40dc1aaa774ef",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7339@{#2}",
"revision": "cb080f1b71737fb7434c6f4897570fbf055f5fd2",
"swarm_hashes": {
"dawn_end2end_tests": "7b8d74427b4c5efc0237f914f5717e671c23d4971356997bbb3a771dd51463fe/323",
"dawn_perf_tests": "e072dc2b37dd38f2f0f396a71de6edf6b03f55cdb76d5cd02624d11c9fa7579b/487",
"gl_tests": "a87f07d58af26c1f5575c0ca86f3f7bb7dadc4b03984f0f2163f5e0cddd9872f/323",
"gl_unittests": "5beb75948bec18c5792df2b4ec0548fe0ad333e721f05d662c64a6c2b2b7ccf8/402",
"telemetry_gpu_integration_test": "0ebee37fe8417d2a86713008213ae277dbfb4267b8f7fd6bc86227dd340b717e/815",
"telemetry_gpu_unittests": "7015979086e50748288d9d99b86934066a2bc5f206d1249065f3da920394a7bd/736",
"webgpu_blink_web_tests": "bf660ad668d39746cafef2b9dbcbbb36782bcf5992292b0f3e6764da091a7026/566"
},
"swarming_command_lines_cwd": "out\\8c34-Dawn_Win10_x64_",
"swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/7339",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "cb080f1b71737fb7434c6f4897570fbf055f5fd2",
"tags": [
"parent_buildername:Dawn Win10 x64 DEPS Builder",
"parent_buildnumber:2106",
"user_agent:recipe"
]
},
"id": "bdfb91ab-fc9a-4753-bd72-125fe93d7365",
"title": "Dawn Win10 x64 DEPS Builder/2106",
"url": "https://cr-buildbucket.appspot.com/build/8702427800027189057"
},
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "54fbb7a4dea54cde11bdcee4960ee8df6ad4f86f",
"project": "chromium/src",
"ref": "refs/branch-heads/7339"
}
]
},
"deps_revision_overrides": {
"src": "54fbb7a4dea54cde11bdcee4960ee8df6ad4f86f"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8702424918987721585"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8702424918987721585"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8702424918987721585"
}
],
"parent_build_id": "8702424918987721585",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Win10 x64 DEPS Builder",
"parent_got_angle_revision": "a8c8a6febe630c6239a5e207530e9fac651ae373",
"parent_got_dawn_revision": "67be7fddacc4f4bcb21d0cf7bf8bb18752d8fb08",
"parent_got_revision": "54fbb7a4dea54cde11bdcee4960ee8df6ad4f86f",
"parent_got_revision_cp": "refs/branch-heads/7339@{#2803}",
"parent_got_swiftshader_revision": "fdb6700ecb04103b658d2e4623d6bc663ba80ea8",
"parent_got_v8_revision": "b7ed978e41b4bac7802b206404d0e2f3d09f31ac",
"parent_got_v8_revision_cp": "refs/branch-heads/14.0@{#19}",
"parent_got_webrtc_revision": "36ea4535a500ac137dbf1f577ce40dc1aaa774ef",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7339@{#2}",
"revision": "54fbb7a4dea54cde11bdcee4960ee8df6ad4f86f",
"swarm_hashes": {
"dawn_end2end_tests": "7b8d74427b4c5efc0237f914f5717e671c23d4971356997bbb3a771dd51463fe/323",
"dawn_perf_tests": "e072dc2b37dd38f2f0f396a71de6edf6b03f55cdb76d5cd02624d11c9fa7579b/487",
"gl_tests": "a87f07d58af26c1f5575c0ca86f3f7bb7dadc4b03984f0f2163f5e0cddd9872f/323",
"gl_unittests": "5beb75948bec18c5792df2b4ec0548fe0ad333e721f05d662c64a6c2b2b7ccf8/402",
"telemetry_gpu_integration_test": "6003c965dc2f635d7c0e135ec34b84131762ff3f83d9130a54f5ca8f6ffb1623/815",
"telemetry_gpu_unittests": "7015979086e50748288d9d99b86934066a2bc5f206d1249065f3da920394a7bd/736",
"webgpu_blink_web_tests": "38e6a00781496eeeaa8091758d935e369b6c2635356ec7a974ce33bfd962c965/566"
},
"swarming_command_lines_cwd": "out\\8c34-Dawn_Win10_x64_",
"swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/7339",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "54fbb7a4dea54cde11bdcee4960ee8df6ad4f86f",
"tags": [
"parent_buildername:Dawn Win10 x64 DEPS Builder",
"parent_buildnumber:2107",
"user_agent:recipe"
]
},
"id": "6e5faabf-bd25-4848-a049-4ff34a6dbdfe",
"title": "Dawn Win10 x64 DEPS Builder/2107",
"url": "https://cr-buildbucket.appspot.com/build/8702424918987721585"
},
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "b2f440ad2c279fb46a9d5d54fe8f8c4938bc09de",
"project": "chromium/src",
"ref": "refs/branch-heads/7339"
}
]
},
"deps_revision_overrides": {
"src": "b2f440ad2c279fb46a9d5d54fe8f8c4938bc09de"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8702421944952570065"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8702421944952570065"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8702421944952570065"
}
],
"parent_build_id": "8702421944952570065",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Win10 x64 DEPS Builder",
"parent_got_angle_revision": "a8c8a6febe630c6239a5e207530e9fac651ae373",
"parent_got_dawn_revision": "67be7fddacc4f4bcb21d0cf7bf8bb18752d8fb08",
"parent_got_revision": "b2f440ad2c279fb46a9d5d54fe8f8c4938bc09de",
"parent_got_revision_cp": "refs/branch-heads/7339@{#2804}",
"parent_got_swiftshader_revision": "fdb6700ecb04103b658d2e4623d6bc663ba80ea8",
"parent_got_v8_revision": "b7ed978e41b4bac7802b206404d0e2f3d09f31ac",
"parent_got_v8_revision_cp": "refs/branch-heads/14.0@{#19}",
"parent_got_webrtc_revision": "36ea4535a500ac137dbf1f577ce40dc1aaa774ef",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7339@{#2}",
"revision": "b2f440ad2c279fb46a9d5d54fe8f8c4938bc09de",
"swarm_hashes": {
"dawn_end2end_tests": "7b8d74427b4c5efc0237f914f5717e671c23d4971356997bbb3a771dd51463fe/323",
"dawn_perf_tests": "e072dc2b37dd38f2f0f396a71de6edf6b03f55cdb76d5cd02624d11c9fa7579b/487",
"gl_tests": "a87f07d58af26c1f5575c0ca86f3f7bb7dadc4b03984f0f2163f5e0cddd9872f/323",
"gl_unittests": "5beb75948bec18c5792df2b4ec0548fe0ad333e721f05d662c64a6c2b2b7ccf8/402",
"telemetry_gpu_integration_test": "322f5fe8919131f56022caef12777e342b39546f494ae501f002d1bf58e90bb5/815",
"telemetry_gpu_unittests": "7015979086e50748288d9d99b86934066a2bc5f206d1249065f3da920394a7bd/736",
"webgpu_blink_web_tests": "bde6aa6e5f798308dcc0b441438505e13e3b3bb86e88e513398718c52ff7721e/566"
},
"swarming_command_lines_cwd": "out\\8c34-Dawn_Win10_x64_",
"swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/7339",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "b2f440ad2c279fb46a9d5d54fe8f8c4938bc09de",
"tags": [
"parent_buildername:Dawn Win10 x64 DEPS Builder",
"parent_buildnumber:2108",
"user_agent:recipe"
]
},
"id": "237aeb0c-af1f-426e-b9fe-ac1adfb8e194",
"title": "Dawn Win10 x64 DEPS Builder/2108",
"url": "https://cr-buildbucket.appspot.com/build/8702421944952570065"
}
]
},
"deps_revision_overrides": {
"src": "b2f440ad2c279fb46a9d5d54fe8f8c4938bc09de"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8702421944952570065"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8702421944952570065"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8702421944952570065"
}
],
"parent_build_id": "8702421944952570065",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Win10 x64 DEPS Builder",
"parent_got_angle_revision": "a8c8a6febe630c6239a5e207530e9fac651ae373",
"parent_got_dawn_revision": "67be7fddacc4f4bcb21d0cf7bf8bb18752d8fb08",
"parent_got_revision": "b2f440ad2c279fb46a9d5d54fe8f8c4938bc09de",
"parent_got_revision_cp": "refs/branch-heads/7339@{#2804}",
"parent_got_swiftshader_revision": "fdb6700ecb04103b658d2e4623d6bc663ba80ea8",
"parent_got_v8_revision": "b7ed978e41b4bac7802b206404d0e2f3d09f31ac",
"parent_got_v8_revision_cp": "refs/branch-heads/14.0@{#19}",
"parent_got_webrtc_revision": "36ea4535a500ac137dbf1f577ce40dc1aaa774ef",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7339@{#2}",
"swarm_hashes": {
"dawn_end2end_tests": "7b8d74427b4c5efc0237f914f5717e671c23d4971356997bbb3a771dd51463fe/323",
"dawn_perf_tests": "e072dc2b37dd38f2f0f396a71de6edf6b03f55cdb76d5cd02624d11c9fa7579b/487",
"gl_tests": "a87f07d58af26c1f5575c0ca86f3f7bb7dadc4b03984f0f2163f5e0cddd9872f/323",
"gl_unittests": "5beb75948bec18c5792df2b4ec0548fe0ad333e721f05d662c64a6c2b2b7ccf8/402",
"telemetry_gpu_integration_test": "322f5fe8919131f56022caef12777e342b39546f494ae501f002d1bf58e90bb5/815",
"telemetry_gpu_unittests": "7015979086e50748288d9d99b86934066a2bc5f206d1249065f3da920394a7bd/736",
"webgpu_blink_web_tests": "bde6aa6e5f798308dcc0b441438505e13e3b3bb86e88e513398718c52ff7721e/566"
},
"swarming_command_lines_cwd": "out\\8c34-Dawn_Win10_x64_",
"swarming_command_lines_digest": "7aeb6d1c6a48a8535af5116a806064295f19d4c0a17b169a6156304caa1fbe77/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "b2f440ad2c279fb46a9d5d54fe8f8c4938bc09de",
"ref": "refs/branch-heads/7339"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8867848806817481296"
},
{
"key": "scheduler_job_id",
"value": "chromium-m140/Dawn Win10 x64 DEPS Release (Intel)"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Dawn Win10 x64 DEPS Builder"
},
{
"key": "parent_buildnumber",
"value": "2108"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[05:29:35.325] Scheduled build:
{
"id": "8702418744753698609",
"builder": {
"project": "chromium-m140",
"bucket": "ci",
"builder": "Dawn Win10 x64 DEPS Release (Intel)"
},
"number": 888,
"createdBy": "project:chromium-m140",
"createTime": "2025-09-29T05:29:34.724012491Z",
"updateTime": "2025-09-29T05:29:34.724012491Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "b2f440ad2c279fb46a9d5d54fe8f8c4938bc09de",
"ref": "refs/branch-heads/7339"
}
}
}
[05:29:35.325] Task URL: https://cr-buildbucket.appspot.com/build/8702418744753698609
[05:29:35.325] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:2:0) after 9m0s
[05:29:36.229] Received PubSub notification, asking Buildbucket for the build status
[05:29:36.276] Build status: SCHEDULED
[05:29:42.411] Received PubSub notification, asking Buildbucket for the build status
[05:29:42.433] Build status: STARTED
[05:38:35.502] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:2:0)
[05:38:35.627] Build status: STARTED
[05:38:35.627] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:5:0) after 1m14s
[05:39:49.642] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:5:0)
[05:39:49.657] Build status: STARTED
[05:39:49.657] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:6:0) after 5m3s
[05:44:52.786] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:6:0)
[05:44:52.877] Build status: STARTED
[05:44:52.877] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:7:0) after 5m15s
[05:50:08.107] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:7:0)
[05:50:08.203] Build status: STARTED
[05:50:08.203] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:8:0) after 6m58s
[05:57:06.547] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:8:0)
[05:57:06.563] Build status: STARTED
[05:57:06.563] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:9:0) after 5m12s
[06:02:18.711] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:9:0)
[06:02:18.731] Build status: STARTED
[06:02:18.731] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:10:0) after 3m29s
[06:05:48.110] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:10:0)
[06:05:48.128] Build status: STARTED
[06:05:48.128] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:11:0) after 7m30s
[06:13:18.243] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:11:0)
[06:13:18.265] Build status: STARTED
[06:13:18.265] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:12:0) after 4m50s
[06:18:08.320] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:12:0)
[06:18:08.345] Build status: STARTED
[06:18:08.345] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:13:0) after 6m24s
[06:24:32.358] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:13:0)
[06:24:32.417] Build status: STARTED
[06:24:32.418] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:14:0) after 8m36s
[06:33:08.432] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:14:0)
[06:33:08.457] Build status: STARTED
[06:33:08.458] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:15:0) after 1m13s
[06:34:21.478] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:15:0)
[06:34:21.496] Build status: STARTED
[06:34:21.496] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:16:0) after 4m12s
[06:38:33.554] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:16:0)
[06:38:33.577] Build status: STARTED
[06:38:33.577] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:17:0) after 4m12s
[06:42:45.895] Handling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:17:0)
[06:42:45.910] Build status: STARTED
[06:42:45.910] Scheduling timer "check-buildbucket-build-status" (chromium-m140/Dawn Win10 x64 DEPS Release (Intel):8867848806817481296:18:0) after 7m53s
[06:43:17.076] Received PubSub notification, asking Buildbucket for the build status
[06:43:17.092] Build:
{
"id": "8702418744753698609",
"builder": {
"project": "chromium-m140",
"bucket": "ci",
"builder": "Dawn Win10 x64 DEPS Release (Intel)"
},
"number": 888,
"createdBy": "project:chromium-m140",
"createTime": "2025-09-29T05:29:34.724012491Z",
"startTime": "2025-09-29T05:29:41.940769628Z",
"endTime": "2025-09-29T06:43:16.521380031Z",
"updateTime": "2025-09-29T06:43:16.521380031Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "b2f440ad2c279fb46a9d5d54fe8f8c4938bc09de",
"ref": "refs/branch-heads/7339"
}
}
}
[06:43:17.092] Invocation finished in 1h13m43.720383745s with status FAILED