[12:13:04.304] New invocation is queued and will start shortly
[12:13:04.304] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[12:13:05.410] Starting the invocation (attempt 1)
[12:13:05.443] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/f6c517a6c7a4fcb3b2c0285ab69a718992d62157"
[12:13:05.443] Popped gitiles commit info from properties and tags
[12:13:05.443] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[12:13:05.443] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[12:13:05.444] Buildbucket request:
{
"requestId": "8893734152212875344",
"builder": {
"project": "chromium-m132",
"bucket": "ci",
"builder": "Dawn Mac x64 DEPS Release (AMD)"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "f6c517a6c7a4fcb3b2c0285ab69a718992d62157",
"project": "chromium/src",
"ref": "refs/branch-heads/6834"
}
]
},
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8893734152212875344",
"job": "chromium-m132/Dawn Mac x64 DEPS Release (AMD)",
"triggers": [
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "4b5ddb70fe33d8948bde56b9846d697d01272225",
"project": "chromium/src",
"ref": "refs/branch-heads/6834"
}
]
},
"deps_revision_overrides": {
"src": "4b5ddb70fe33d8948bde56b9846d697d01272225"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8728307789819450065"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8728307789819450065"
}
],
"parent_build_id": "8728307789819450065",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Mac x64 DEPS Builder",
"parent_got_angle_revision": "ce13a00a2b049a1ef5e0e70a3d333ce70838ef7b",
"parent_got_dawn_revision": "73dbf565079c89a531e6e01c4e8fc048a8a9660b",
"parent_got_revision": "4b5ddb70fe33d8948bde56b9846d697d01272225",
"parent_got_revision_cp": "refs/branch-heads/6834@{#2299}",
"parent_got_swiftshader_revision": "d5c4284774115bb1e32c012a2be1b5fbeb1ab1f9",
"parent_got_v8_revision": "14e9f5013ad9e04c3c9d6b4a662ce58a2122b6a6",
"parent_got_v8_revision_cp": "refs/branch-heads/13.2@{#45}",
"parent_got_webrtc_revision": "f632fe9be9ce5cef5fcbfd42995271cb3fbb077d",
"parent_got_webrtc_revision_cp": "refs/branch-heads/6834@{#4}",
"revision": "4b5ddb70fe33d8948bde56b9846d697d01272225",
"swarm_hashes": {
"dawn_end2end_tests": "9f8719340cb64d0eb3f17575818d87696a64b00ae29faaf10796d3eb46881c64/323",
"dawn_perf_tests": "6c64a5c1a400c3f4b95ad6f19c4d93d0f34a1f9eb549579b5f143c10c50ab739/487",
"gl_tests": "383987521ee31332b344b0a4537e4ea820d467c08a9e7935ba213feb14079cd3/323",
"gl_unittests": "16e68f4a924b82a75ad6cffd770572f4b1e8fa09e54675271d7e8c50e9dea539/323",
"telemetry_gpu_integration_test": "09c52fe9469c69576310640c24394c78159d5071ac520929ea39a47b9e816a7d/815",
"telemetry_gpu_unittests": "37801160eb204ef3a379079038a4e731d45418724ae96d30320ec7e3e334cbf0/736",
"webgpu_blink_web_tests": "cee4bb78d6c9aa32434ac2c2c2a215780e6b8837c8f918fb8294da232aa99f09/566"
},
"swarming_command_lines_cwd": "out/9729-Dawn_Mac_x64_DE",
"swarming_command_lines_digest": "60de9dd905cce771078a83e00991e4c89813c7ce0fd35ffdc98650f989deaf14/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/6834",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "4b5ddb70fe33d8948bde56b9846d697d01272225",
"tags": [
"parent_buildername:Dawn Mac x64 DEPS Builder",
"parent_buildnumber:2133",
"user_agent:recipe"
]
},
"id": "1b8e9b64-adce-486b-a5fd-268a8c3a8f50",
"title": "Dawn Mac x64 DEPS Builder/2133"
},
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "0b0e26106519e828e94b37a1b16997e4de5b8775",
"project": "chromium/src",
"ref": "refs/branch-heads/6834"
}
]
},
"deps_revision_overrides": {
"src": "0b0e26106519e828e94b37a1b16997e4de5b8775"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8728306431478012017"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8728306431478012017"
}
],
"parent_build_id": "8728306431478012017",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Mac x64 DEPS Builder",
"parent_got_angle_revision": "ce13a00a2b049a1ef5e0e70a3d333ce70838ef7b",
"parent_got_dawn_revision": "73dbf565079c89a531e6e01c4e8fc048a8a9660b",
"parent_got_revision": "0b0e26106519e828e94b37a1b16997e4de5b8775",
"parent_got_revision_cp": "refs/branch-heads/6834@{#2300}",
"parent_got_swiftshader_revision": "d5c4284774115bb1e32c012a2be1b5fbeb1ab1f9",
"parent_got_v8_revision": "14e9f5013ad9e04c3c9d6b4a662ce58a2122b6a6",
"parent_got_v8_revision_cp": "refs/branch-heads/13.2@{#45}",
"parent_got_webrtc_revision": "f632fe9be9ce5cef5fcbfd42995271cb3fbb077d",
"parent_got_webrtc_revision_cp": "refs/branch-heads/6834@{#4}",
"revision": "0b0e26106519e828e94b37a1b16997e4de5b8775",
"swarm_hashes": {
"dawn_end2end_tests": "9f8719340cb64d0eb3f17575818d87696a64b00ae29faaf10796d3eb46881c64/323",
"dawn_perf_tests": "6c64a5c1a400c3f4b95ad6f19c4d93d0f34a1f9eb549579b5f143c10c50ab739/487",
"gl_tests": "383987521ee31332b344b0a4537e4ea820d467c08a9e7935ba213feb14079cd3/323",
"gl_unittests": "16e68f4a924b82a75ad6cffd770572f4b1e8fa09e54675271d7e8c50e9dea539/323",
"telemetry_gpu_integration_test": "c70b1f7dca630e773774064719d5a8deec4e645171b6feeebecadd257fe036c9/815",
"telemetry_gpu_unittests": "77d1c60ece0be64a48bd219c3a44de932695657362b1726dfb78347e1065f0b8/736",
"webgpu_blink_web_tests": "f089b7c10d4cf9d417163f9f45b11fdccf5d1330fe5f0a47715ff85fc743d877/566"
},
"swarming_command_lines_cwd": "out/9729-Dawn_Mac_x64_DE",
"swarming_command_lines_digest": "60de9dd905cce771078a83e00991e4c89813c7ce0fd35ffdc98650f989deaf14/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/6834",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "0b0e26106519e828e94b37a1b16997e4de5b8775",
"tags": [
"parent_buildername:Dawn Mac x64 DEPS Builder",
"parent_buildnumber:2134",
"user_agent:recipe"
]
},
"id": "a8cc03c4-c344-456b-a68f-d3642b3ab574",
"title": "Dawn Mac x64 DEPS Builder/2134"
},
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "70160f0bdff0e36c1d60d4290390df097daf97d3",
"project": "chromium/src",
"ref": "refs/branch-heads/6834"
}
]
},
"deps_revision_overrides": {
"src": "70160f0bdff0e36c1d60d4290390df097daf97d3"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8728305704790750449"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8728305704790750449"
}
],
"parent_build_id": "8728305704790750449",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Mac x64 DEPS Builder",
"parent_got_angle_revision": "ce13a00a2b049a1ef5e0e70a3d333ce70838ef7b",
"parent_got_dawn_revision": "73dbf565079c89a531e6e01c4e8fc048a8a9660b",
"parent_got_revision": "70160f0bdff0e36c1d60d4290390df097daf97d3",
"parent_got_revision_cp": "refs/branch-heads/6834@{#2301}",
"parent_got_swiftshader_revision": "d5c4284774115bb1e32c012a2be1b5fbeb1ab1f9",
"parent_got_v8_revision": "14e9f5013ad9e04c3c9d6b4a662ce58a2122b6a6",
"parent_got_v8_revision_cp": "refs/branch-heads/13.2@{#45}",
"parent_got_webrtc_revision": "f632fe9be9ce5cef5fcbfd42995271cb3fbb077d",
"parent_got_webrtc_revision_cp": "refs/branch-heads/6834@{#4}",
"revision": "70160f0bdff0e36c1d60d4290390df097daf97d3",
"swarm_hashes": {
"dawn_end2end_tests": "9f8719340cb64d0eb3f17575818d87696a64b00ae29faaf10796d3eb46881c64/323",
"dawn_perf_tests": "6c64a5c1a400c3f4b95ad6f19c4d93d0f34a1f9eb549579b5f143c10c50ab739/487",
"gl_tests": "383987521ee31332b344b0a4537e4ea820d467c08a9e7935ba213feb14079cd3/323",
"gl_unittests": "16e68f4a924b82a75ad6cffd770572f4b1e8fa09e54675271d7e8c50e9dea539/323",
"telemetry_gpu_integration_test": "fbbee6e8487568df220b845c3d7bae183ce6a40819d83c494f6ca688a6c4d6aa/815",
"telemetry_gpu_unittests": "37801160eb204ef3a379079038a4e731d45418724ae96d30320ec7e3e334cbf0/736",
"webgpu_blink_web_tests": "dad2698469715f26207ea242c60c95a6d1dd8fb2eb68e2cc78475887faa86b93/566"
},
"swarming_command_lines_cwd": "out/9729-Dawn_Mac_x64_DE",
"swarming_command_lines_digest": "60de9dd905cce771078a83e00991e4c89813c7ce0fd35ffdc98650f989deaf14/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/6834",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "70160f0bdff0e36c1d60d4290390df097daf97d3",
"tags": [
"parent_buildername:Dawn Mac x64 DEPS Builder",
"parent_buildnumber:2135",
"user_agent:recipe"
]
},
"id": "40ce6c25-87d9-423b-9f05-1a2f162fa0cc",
"title": "Dawn Mac x64 DEPS Builder/2135"
},
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "f6c517a6c7a4fcb3b2c0285ab69a718992d62157",
"project": "chromium/src",
"ref": "refs/branch-heads/6834"
}
]
},
"deps_revision_overrides": {
"src": "f6c517a6c7a4fcb3b2c0285ab69a718992d62157"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8728305066755097873"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8728305066755097873"
}
],
"parent_build_id": "8728305066755097873",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Mac x64 DEPS Builder",
"parent_got_angle_revision": "ce13a00a2b049a1ef5e0e70a3d333ce70838ef7b",
"parent_got_dawn_revision": "73dbf565079c89a531e6e01c4e8fc048a8a9660b",
"parent_got_revision": "f6c517a6c7a4fcb3b2c0285ab69a718992d62157",
"parent_got_revision_cp": "refs/branch-heads/6834@{#2302}",
"parent_got_swiftshader_revision": "d5c4284774115bb1e32c012a2be1b5fbeb1ab1f9",
"parent_got_v8_revision": "14e9f5013ad9e04c3c9d6b4a662ce58a2122b6a6",
"parent_got_v8_revision_cp": "refs/branch-heads/13.2@{#45}",
"parent_got_webrtc_revision": "f632fe9be9ce5cef5fcbfd42995271cb3fbb077d",
"parent_got_webrtc_revision_cp": "refs/branch-heads/6834@{#4}",
"revision": "f6c517a6c7a4fcb3b2c0285ab69a718992d62157",
"swarm_hashes": {
"dawn_end2end_tests": "9f8719340cb64d0eb3f17575818d87696a64b00ae29faaf10796d3eb46881c64/323",
"dawn_perf_tests": "6c64a5c1a400c3f4b95ad6f19c4d93d0f34a1f9eb549579b5f143c10c50ab739/487",
"gl_tests": "383987521ee31332b344b0a4537e4ea820d467c08a9e7935ba213feb14079cd3/323",
"gl_unittests": "16e68f4a924b82a75ad6cffd770572f4b1e8fa09e54675271d7e8c50e9dea539/323",
"telemetry_gpu_integration_test": "9a23eff8dfd726db41e8f56a04e9e076a3abdbb36618ed6037fb510c6a7c9dc6/815",
"telemetry_gpu_unittests": "37801160eb204ef3a379079038a4e731d45418724ae96d30320ec7e3e334cbf0/736",
"webgpu_blink_web_tests": "67bd34f9e3d618698c295c2ec6c5fbbed936eac96522921f9cf5103af6bd056b/566"
},
"swarming_command_lines_cwd": "out/9729-Dawn_Mac_x64_DE",
"swarming_command_lines_digest": "60de9dd905cce771078a83e00991e4c89813c7ce0fd35ffdc98650f989deaf14/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/6834",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "f6c517a6c7a4fcb3b2c0285ab69a718992d62157",
"tags": [
"parent_buildername:Dawn Mac x64 DEPS Builder",
"parent_buildnumber:2136",
"user_agent:recipe"
]
},
"id": "108f4bf3-3e8e-4a51-8c5f-6570b68b44bb",
"title": "Dawn Mac x64 DEPS Builder/2136"
}
]
},
"deps_revision_overrides": {
"src": "f6c517a6c7a4fcb3b2c0285ab69a718992d62157"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8728305066755097873"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8728305066755097873"
}
],
"parent_build_id": "8728305066755097873",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Mac x64 DEPS Builder",
"parent_got_angle_revision": "ce13a00a2b049a1ef5e0e70a3d333ce70838ef7b",
"parent_got_dawn_revision": "73dbf565079c89a531e6e01c4e8fc048a8a9660b",
"parent_got_revision": "f6c517a6c7a4fcb3b2c0285ab69a718992d62157",
"parent_got_revision_cp": "refs/branch-heads/6834@{#2302}",
"parent_got_swiftshader_revision": "d5c4284774115bb1e32c012a2be1b5fbeb1ab1f9",
"parent_got_v8_revision": "14e9f5013ad9e04c3c9d6b4a662ce58a2122b6a6",
"parent_got_v8_revision_cp": "refs/branch-heads/13.2@{#45}",
"parent_got_webrtc_revision": "f632fe9be9ce5cef5fcbfd42995271cb3fbb077d",
"parent_got_webrtc_revision_cp": "refs/branch-heads/6834@{#4}",
"swarm_hashes": {
"dawn_end2end_tests": "9f8719340cb64d0eb3f17575818d87696a64b00ae29faaf10796d3eb46881c64/323",
"dawn_perf_tests": "6c64a5c1a400c3f4b95ad6f19c4d93d0f34a1f9eb549579b5f143c10c50ab739/487",
"gl_tests": "383987521ee31332b344b0a4537e4ea820d467c08a9e7935ba213feb14079cd3/323",
"gl_unittests": "16e68f4a924b82a75ad6cffd770572f4b1e8fa09e54675271d7e8c50e9dea539/323",
"telemetry_gpu_integration_test": "9a23eff8dfd726db41e8f56a04e9e076a3abdbb36618ed6037fb510c6a7c9dc6/815",
"telemetry_gpu_unittests": "37801160eb204ef3a379079038a4e731d45418724ae96d30320ec7e3e334cbf0/736",
"webgpu_blink_web_tests": "67bd34f9e3d618698c295c2ec6c5fbbed936eac96522921f9cf5103af6bd056b/566"
},
"swarming_command_lines_cwd": "out/9729-Dawn_Mac_x64_DE",
"swarming_command_lines_digest": "60de9dd905cce771078a83e00991e4c89813c7ce0fd35ffdc98650f989deaf14/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "f6c517a6c7a4fcb3b2c0285ab69a718992d62157",
"ref": "refs/branch-heads/6834"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8893734152212875344"
},
{
"key": "scheduler_job_id",
"value": "chromium-m132/Dawn Mac x64 DEPS Release (AMD)"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Dawn Mac x64 DEPS Builder"
},
{
"key": "parent_buildnumber",
"value": "2136"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[12:13:05.972] Scheduled build:
{
"id": "8728304090293271841",
"builder": {
"project": "chromium-m132",
"bucket": "ci",
"builder": "Dawn Mac x64 DEPS Release (AMD)"
},
"number": 948,
"createdBy": "project:chromium-m132",
"createTime": "2024-12-17T12:13:05.511594555Z",
"updateTime": "2024-12-17T12:13:05.511594555Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "f6c517a6c7a4fcb3b2c0285ab69a718992d62157",
"ref": "refs/branch-heads/6834"
}
}
}
[12:13:05.972] Task URL: https://cr-buildbucket.appspot.com/build/8728304090293271841
[12:13:05.972] Scheduling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:2:0) after 4m34s
[12:13:06.966] Received PubSub notification, asking Buildbucket for the build status
[12:13:06.990] Build status: SCHEDULED
[12:13:14.902] Received PubSub notification, asking Buildbucket for the build status
[12:13:14.932] Build status: STARTED
[12:17:39.990] Handling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:2:0)
[12:17:40.027] Build status: STARTED
[12:17:40.027] Scheduling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:5:0) after 3m17s
[12:20:57.006] Handling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:5:0)
[12:20:57.027] Build status: STARTED
[12:20:57.027] Scheduling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:6:0) after 9m5s
[12:30:02.072] Handling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:6:0)
[12:30:02.118] Build status: STARTED
[12:30:02.118] Scheduling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:7:0) after 1m10s
[12:31:12.229] Handling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:7:0)
[12:31:12.252] Build status: STARTED
[12:31:12.252] Scheduling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:8:0) after 9m5s
[12:40:17.274] Handling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:8:0)
[12:40:17.296] Build status: STARTED
[12:40:17.296] Scheduling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:9:0) after 6m20s
[12:46:37.689] Handling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:9:0)
[12:46:37.781] Build status: STARTED
[12:46:37.781] Scheduling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:10:0) after 8m23s
[12:55:01.020] Handling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:10:0)
[12:55:01.044] Build status: STARTED
[12:55:01.044] Scheduling timer "check-buildbucket-build-status" (chromium-m132/Dawn Mac x64 DEPS Release (AMD):8893734152212875344:11:0) after 6m50s
[13:01:44.493] Received PubSub notification, asking Buildbucket for the build status
[13:01:44.513] Build:
{
"id": "8728304090293271841",
"builder": {
"project": "chromium-m132",
"bucket": "ci",
"builder": "Dawn Mac x64 DEPS Release (AMD)"
},
"number": 948,
"createdBy": "project:chromium-m132",
"createTime": "2024-12-17T12:13:05.511594555Z",
"startTime": "2024-12-17T12:13:14.590332195Z",
"endTime": "2024-12-17T13:01:43.535809997Z",
"updateTime": "2024-12-17T13:01:43.535809997Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "f6c517a6c7a4fcb3b2c0285ab69a718992d62157",
"ref": "refs/branch-heads/6834"
}
}
}
[13:01:44.513] Invocation finished in 48m40.215798719s with status FAILED