[03:16:10.314] New invocation is queued and will start shortly
[03:16:10.314] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[03:16:11.448] Starting the invocation (attempt 1)
[03:16:11.474] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/efd67297c90ee1180a014ce5b706625e462dfe23"
[03:16:11.474] Popped gitiles commit info from properties and tags
[03:16:11.474] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[03:16:11.474] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[03:16:11.475] Buildbucket request:
{
"requestId": "8859522277729029424",
"builder": {
"project": "chromium-m144",
"bucket": "ci",
"builder": "Mac Release (Intel)"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "efd67297c90ee1180a014ce5b706625e462dfe23",
"project": "chromium/src",
"ref": "refs/branch-heads/7559"
}
]
},
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8859522277729029424",
"job": "chromium-m144/Mac Release (Intel)",
"triggers": [
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "9e0f253d2a7b970117b02d7934b63b596b914db9",
"project": "chromium/src",
"ref": "refs/branch-heads/7559"
}
]
},
"deps_revision_overrides": {
"src": "9e0f253d2a7b970117b02d7934b63b596b914db9"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8694095848898518033"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8694095848898518033"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8694095848898518033"
}
],
"parent_build_id": "8694095848898518033",
"parent_builder_group": "chromium.gpu",
"parent_buildername": "GPU Mac Builder",
"parent_got_angle_revision": "a6510bd7fd342b02ce8225beffb8b7f9478632e8",
"parent_got_dawn_revision": "9bd45159352393f34bf70b4558d1ce75c2b1a574",
"parent_got_revision": "9e0f253d2a7b970117b02d7934b63b596b914db9",
"parent_got_revision_cp": "refs/branch-heads/7559@{#2512}",
"parent_got_swiftshader_revision": "04fbb7daf5a53689e067190e7ef1047c5d49e292",
"parent_got_v8_revision": "ad25f9ae50a53bee50f459bfee25fb1e6f64adc3",
"parent_got_v8_revision_cp": "refs/branch-heads/14.4@{#31}",
"parent_got_webrtc_revision": "8f3537ef5b85b4c7dabed2676d4b72214c69c494",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7559@{#2}",
"revision": "9e0f253d2a7b970117b02d7934b63b596b914db9",
"swarm_hashes": {
"angle_unittests": "0d544c967bbd3ef78c5cb704a04ff01b734c6439e250cdc411954585d6c02c57/408",
"browser_tests": "421dffc7bf7d467a5a031562248c3e2421a6ec0d7f3ca6c46baadd429945bdea/1220",
"gl_tests": "7a60d4f2c7ea98aa3dce0a7249b7b04e9e672e190986b317731346c26d643922/323",
"gl_unittests": "233e397cbe590c9c7b5d7b007c1ec1dc78cd112f65fb6001501204fd80fa376d/323",
"telemetry_gpu_integration_test": "9b159bdaa290485c41f3a004c4ece9ebecf2cc8446f3a714c6bb38f5056293e7/815"
},
"swarming_command_lines_cwd": "out/fe13-GPU_Mac_Builder",
"swarming_command_lines_digest": "4909342d6db616bd9d6a7548123a0293d804ef8053110f8dc0903eeef65b2ae9/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/7559",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "9e0f253d2a7b970117b02d7934b63b596b914db9",
"tags": [
"parent_buildername:GPU Mac Builder",
"parent_buildnumber:1902",
"user_agent:recipe"
]
},
"id": "d78b9732-52b5-48ad-a330-c7810788aabd",
"title": "GPU Mac Builder/1902",
"url": "https://cr-buildbucket.appspot.com/build/8694095848898518033"
},
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "94ba5ce09f735cb8d6dbd8292be3e368cb5d2108",
"project": "chromium/src",
"ref": "refs/branch-heads/7559"
}
]
},
"deps_revision_overrides": {
"src": "94ba5ce09f735cb8d6dbd8292be3e368cb5d2108"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8694094843588671345"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8694094843588671345"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8694094843588671345"
}
],
"parent_build_id": "8694094843588671345",
"parent_builder_group": "chromium.gpu",
"parent_buildername": "GPU Mac Builder",
"parent_got_angle_revision": "a6510bd7fd342b02ce8225beffb8b7f9478632e8",
"parent_got_dawn_revision": "9bd45159352393f34bf70b4558d1ce75c2b1a574",
"parent_got_revision": "94ba5ce09f735cb8d6dbd8292be3e368cb5d2108",
"parent_got_revision_cp": "refs/branch-heads/7559@{#2513}",
"parent_got_swiftshader_revision": "04fbb7daf5a53689e067190e7ef1047c5d49e292",
"parent_got_v8_revision": "ad25f9ae50a53bee50f459bfee25fb1e6f64adc3",
"parent_got_v8_revision_cp": "refs/branch-heads/14.4@{#31}",
"parent_got_webrtc_revision": "8f3537ef5b85b4c7dabed2676d4b72214c69c494",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7559@{#2}",
"revision": "94ba5ce09f735cb8d6dbd8292be3e368cb5d2108",
"swarm_hashes": {
"angle_unittests": "0d544c967bbd3ef78c5cb704a04ff01b734c6439e250cdc411954585d6c02c57/408",
"browser_tests": "b6db814430fb18575b2c2c5ba68a47f1f4690b000675c9a90f7697357f15e9f4/1220",
"gl_tests": "7a60d4f2c7ea98aa3dce0a7249b7b04e9e672e190986b317731346c26d643922/323",
"gl_unittests": "233e397cbe590c9c7b5d7b007c1ec1dc78cd112f65fb6001501204fd80fa376d/323",
"telemetry_gpu_integration_test": "bbc93309b79467aa051f0b45a6c22b891ba13f52a182785b8e31628676424062/815"
},
"swarming_command_lines_cwd": "out/fe13-GPU_Mac_Builder",
"swarming_command_lines_digest": "4909342d6db616bd9d6a7548123a0293d804ef8053110f8dc0903eeef65b2ae9/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/7559",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "94ba5ce09f735cb8d6dbd8292be3e368cb5d2108",
"tags": [
"parent_buildername:GPU Mac Builder",
"parent_buildnumber:1903",
"user_agent:recipe"
]
},
"id": "1e7c0588-d20c-4b7a-8f4b-77e8eb6ab363",
"title": "GPU Mac Builder/1903",
"url": "https://cr-buildbucket.appspot.com/build/8694094843588671345"
},
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "a10c0d0055e0cce2d4c2861ceff46d772b22bb0b",
"project": "chromium/src",
"ref": "refs/branch-heads/7559"
}
]
},
"deps_revision_overrides": {
"src": "a10c0d0055e0cce2d4c2861ceff46d772b22bb0b"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8694094252140843553"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8694094252140843553"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8694094252140843553"
}
],
"parent_build_id": "8694094252140843553",
"parent_builder_group": "chromium.gpu",
"parent_buildername": "GPU Mac Builder",
"parent_got_angle_revision": "a6510bd7fd342b02ce8225beffb8b7f9478632e8",
"parent_got_dawn_revision": "9bd45159352393f34bf70b4558d1ce75c2b1a574",
"parent_got_revision": "a10c0d0055e0cce2d4c2861ceff46d772b22bb0b",
"parent_got_revision_cp": "refs/branch-heads/7559@{#2528}",
"parent_got_swiftshader_revision": "04fbb7daf5a53689e067190e7ef1047c5d49e292",
"parent_got_v8_revision": "ad25f9ae50a53bee50f459bfee25fb1e6f64adc3",
"parent_got_v8_revision_cp": "refs/branch-heads/14.4@{#31}",
"parent_got_webrtc_revision": "8f3537ef5b85b4c7dabed2676d4b72214c69c494",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7559@{#2}",
"revision": "a10c0d0055e0cce2d4c2861ceff46d772b22bb0b",
"swarm_hashes": {
"angle_unittests": "0d544c967bbd3ef78c5cb704a04ff01b734c6439e250cdc411954585d6c02c57/408",
"browser_tests": "7b2d7fdfef380278cb1b4c2b1250724fad976f13c777d32f6c76977286305a25/1220",
"gl_tests": "7a60d4f2c7ea98aa3dce0a7249b7b04e9e672e190986b317731346c26d643922/323",
"gl_unittests": "233e397cbe590c9c7b5d7b007c1ec1dc78cd112f65fb6001501204fd80fa376d/323",
"telemetry_gpu_integration_test": "610836956570e0850cba4e75933e7cb8ea5e26c18494522508f54dc00ca22dd3/815"
},
"swarming_command_lines_cwd": "out/fe13-GPU_Mac_Builder",
"swarming_command_lines_digest": "4909342d6db616bd9d6a7548123a0293d804ef8053110f8dc0903eeef65b2ae9/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/7559",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "a10c0d0055e0cce2d4c2861ceff46d772b22bb0b",
"tags": [
"parent_buildername:GPU Mac Builder",
"parent_buildnumber:1904",
"user_agent:recipe"
]
},
"id": "cc269bc3-0824-4866-a652-29cc41b673da",
"title": "GPU Mac Builder/1904",
"url": "https://cr-buildbucket.appspot.com/build/8694094252140843553"
},
{
"gitiles": {
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "efd67297c90ee1180a014ce5b706625e462dfe23",
"project": "chromium/src",
"ref": "refs/branch-heads/7559"
}
]
},
"deps_revision_overrides": {
"src": "efd67297c90ee1180a014ce5b706625e462dfe23"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8694093292287290049"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8694093292287290049"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8694093292287290049"
}
],
"parent_build_id": "8694093292287290049",
"parent_builder_group": "chromium.gpu",
"parent_buildername": "GPU Mac Builder",
"parent_got_angle_revision": "a6510bd7fd342b02ce8225beffb8b7f9478632e8",
"parent_got_dawn_revision": "9bd45159352393f34bf70b4558d1ce75c2b1a574",
"parent_got_revision": "efd67297c90ee1180a014ce5b706625e462dfe23",
"parent_got_revision_cp": "refs/branch-heads/7559@{#2529}",
"parent_got_swiftshader_revision": "04fbb7daf5a53689e067190e7ef1047c5d49e292",
"parent_got_v8_revision": "ad25f9ae50a53bee50f459bfee25fb1e6f64adc3",
"parent_got_v8_revision_cp": "refs/branch-heads/14.4@{#31}",
"parent_got_webrtc_revision": "8f3537ef5b85b4c7dabed2676d4b72214c69c494",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7559@{#2}",
"revision": "efd67297c90ee1180a014ce5b706625e462dfe23",
"swarm_hashes": {
"angle_unittests": "0d544c967bbd3ef78c5cb704a04ff01b734c6439e250cdc411954585d6c02c57/408",
"browser_tests": "c00181dd5feafcd5dd4b21d89c4ad2294836e2031ad40afc154762cdd2856113/1220",
"gl_tests": "7a60d4f2c7ea98aa3dce0a7249b7b04e9e672e190986b317731346c26d643922/323",
"gl_unittests": "233e397cbe590c9c7b5d7b007c1ec1dc78cd112f65fb6001501204fd80fa376d/323",
"telemetry_gpu_integration_test": "3fff91f74d8acbf762770fc53855a69b9cf846ab7a1cceb1af9c8871d8452c9b/815"
},
"swarming_command_lines_cwd": "out/fe13-GPU_Mac_Builder",
"swarming_command_lines_digest": "4909342d6db616bd9d6a7548123a0293d804ef8053110f8dc0903eeef65b2ae9/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"ref": "refs/branch-heads/7559",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "efd67297c90ee1180a014ce5b706625e462dfe23",
"tags": [
"parent_buildername:GPU Mac Builder",
"parent_buildnumber:1905",
"user_agent:recipe"
]
},
"id": "264ce50d-2bc4-48b8-8be6-458416ea67fd",
"title": "GPU Mac Builder/1905",
"url": "https://cr-buildbucket.appspot.com/build/8694093292287290049"
}
]
},
"deps_revision_overrides": {
"src": "efd67297c90ee1180a014ce5b706625e462dfe23"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8694093292287290049"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8694093292287290049"
},
{
"instruction_id": "compile_repro_instructions",
"invocation_id": "build-8694093292287290049"
}
],
"parent_build_id": "8694093292287290049",
"parent_builder_group": "chromium.gpu",
"parent_buildername": "GPU Mac Builder",
"parent_got_angle_revision": "a6510bd7fd342b02ce8225beffb8b7f9478632e8",
"parent_got_dawn_revision": "9bd45159352393f34bf70b4558d1ce75c2b1a574",
"parent_got_revision": "efd67297c90ee1180a014ce5b706625e462dfe23",
"parent_got_revision_cp": "refs/branch-heads/7559@{#2529}",
"parent_got_swiftshader_revision": "04fbb7daf5a53689e067190e7ef1047c5d49e292",
"parent_got_v8_revision": "ad25f9ae50a53bee50f459bfee25fb1e6f64adc3",
"parent_got_v8_revision_cp": "refs/branch-heads/14.4@{#31}",
"parent_got_webrtc_revision": "8f3537ef5b85b4c7dabed2676d4b72214c69c494",
"parent_got_webrtc_revision_cp": "refs/branch-heads/7559@{#2}",
"swarm_hashes": {
"angle_unittests": "0d544c967bbd3ef78c5cb704a04ff01b734c6439e250cdc411954585d6c02c57/408",
"browser_tests": "c00181dd5feafcd5dd4b21d89c4ad2294836e2031ad40afc154762cdd2856113/1220",
"gl_tests": "7a60d4f2c7ea98aa3dce0a7249b7b04e9e672e190986b317731346c26d643922/323",
"gl_unittests": "233e397cbe590c9c7b5d7b007c1ec1dc78cd112f65fb6001501204fd80fa376d/323",
"telemetry_gpu_integration_test": "3fff91f74d8acbf762770fc53855a69b9cf846ab7a1cceb1af9c8871d8452c9b/815"
},
"swarming_command_lines_cwd": "out/fe13-GPU_Mac_Builder",
"swarming_command_lines_digest": "4909342d6db616bd9d6a7548123a0293d804ef8053110f8dc0903eeef65b2ae9/93",
"swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "efd67297c90ee1180a014ce5b706625e462dfe23",
"ref": "refs/branch-heads/7559"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8859522277729029424"
},
{
"key": "scheduler_job_id",
"value": "chromium-m144/Mac Release (Intel)"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "GPU Mac Builder"
},
{
"key": "parent_buildnumber",
"value": "1905"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[03:16:11.984] Scheduled build:
{
"id": "8694092215805399697",
"builder": {
"project": "chromium-m144",
"bucket": "ci",
"builder": "Mac Release (Intel)"
},
"number": 1616,
"createdBy": "project:chromium-m144",
"createTime": "2025-12-30T03:16:11.526619213Z",
"updateTime": "2025-12-30T03:16:11.526619213Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "efd67297c90ee1180a014ce5b706625e462dfe23",
"ref": "refs/branch-heads/7559"
}
}
}
[03:16:11.984] Task URL: https://cr-buildbucket.appspot.com/build/8694092215805399697
[03:16:11.984] Scheduling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:2:0) after 8m10s
[03:16:13.046] Received PubSub notification, asking Buildbucket for the build status
[03:16:13.092] Build status: SCHEDULED
[03:17:12.068] Received PubSub notification, asking Buildbucket for the build status
[03:17:12.105] Build status: STARTED
[03:24:22.051] Handling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:2:0)
[03:24:22.104] Build status: STARTED
[03:24:22.104] Scheduling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:5:0) after 3m58s
[03:28:20.129] Handling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:5:0)
[03:28:20.151] Build status: STARTED
[03:28:20.151] Scheduling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:6:0) after 6m59s
[03:35:19.169] Handling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:6:0)
[03:35:19.188] Build status: STARTED
[03:35:19.188] Scheduling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:7:0) after 6m19s
[03:41:38.205] Handling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:7:0)
[03:41:38.227] Build status: STARTED
[03:41:38.228] Scheduling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:8:0) after 7m14s
[03:48:52.291] Handling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:8:0)
[03:48:52.317] Build status: STARTED
[03:48:52.318] Scheduling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:9:0) after 3m10s
[03:52:02.347] Handling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:9:0)
[03:52:02.371] Build status: STARTED
[03:52:02.371] Scheduling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:10:0) after 3m46s
[03:55:48.396] Handling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:10:0)
[03:55:48.442] Build status: STARTED
[03:55:48.442] Scheduling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:11:0) after 3m57s
[03:59:45.592] Handling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:11:0)
[03:59:45.609] Build status: STARTED
[03:59:45.609] Scheduling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:12:0) after 8m17s
[04:08:02.631] Handling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:12:0)
[04:08:02.705] Build status: STARTED
[04:08:02.705] Scheduling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:13:0) after 4m40s
[04:12:42.777] Handling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:13:0)
[04:12:42.790] Build status: STARTED
[04:12:42.790] Scheduling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:14:0) after 3m42s
[04:16:24.886] Handling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:14:0)
[04:16:24.901] Build status: STARTED
[04:16:24.901] Scheduling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:15:0) after 4m35s
[04:20:59.920] Handling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:15:0)
[04:20:59.933] Build status: STARTED
[04:20:59.933] Scheduling timer "check-buildbucket-build-status" (chromium-m144/Mac Release (Intel):8859522277729029424:16:0) after 7m41s
[04:21:51.712] Received PubSub notification, asking Buildbucket for the build status
[04:21:51.729] Build:
{
"id": "8694092215805399697",
"builder": {
"project": "chromium-m144",
"bucket": "ci",
"builder": "Mac Release (Intel)"
},
"number": 1616,
"createdBy": "project:chromium-m144",
"createTime": "2025-12-30T03:16:11.526619213Z",
"startTime": "2025-12-30T03:17:11.767457194Z",
"endTime": "2025-12-30T04:21:50.670009314Z",
"updateTime": "2025-12-30T04:21:50.670009314Z",
"status": "INFRA_FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "efd67297c90ee1180a014ce5b706625e462dfe23",
"ref": "refs/branch-heads/7559"
}
}
}
[04:21:51.729] Invocation finished in 1h5m41.421146571s with status FAILED