[15:51:31.318] New invocation is queued and will start shortly
[15:51:31.318] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[15:51:32.370] Starting the invocation (attempt 1)
[15:51:32.397] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/1f5a7adbfdaa17f8e03d9d92ee2f4aff6219499f"
[15:51:32.397] Popped gitiles commit info from properties and tags
[15:51:32.397] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[15:51:32.397] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[15:51:32.398] Buildbucket request:
{
"requestId": "8896710108406222848",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Dawn Win10 x64 ASAN Release (NVIDIA)"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "1f5a7adbfdaa17f8e03d9d92ee2f4aff6219499f",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "1f5a7adbfdaa17f8e03d9d92ee2f4aff6219499f",
"src/third_party/dawn": "ece2ced03079e0a58d55e46b6b412c694395cfdf"
},
"instruction_dependencies": [
{
"instruction_id": "bot_update_repro_instructions",
"invocation_id": "build-8731289180509070321"
},
{
"instruction_id": "generate_build_files_repro_instructions",
"invocation_id": "build-8731289180509070321"
}
],
"parent_build_id": "8731289180509070321",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Win10 x64 ASAN Builder",
"parent_got_angle_revision": "924ee1ba78dc8e5828b679d37021fada878d92d0",
"parent_got_dawn_revision": "ece2ced03079e0a58d55e46b6b412c694395cfdf",
"parent_got_revision": "1f5a7adbfdaa17f8e03d9d92ee2f4aff6219499f",
"parent_got_revision_cp": "refs/heads/main@{#1382888}",
"parent_got_swiftshader_revision": "d5c4284774115bb1e32c012a2be1b5fbeb1ab1f9",
"parent_got_v8_revision": "94b4f297fd4596f80d39b13d37ea080403aa99a9",
"parent_got_v8_revision_cp": "refs/heads/13.3.40@{#1}",
"parent_got_webrtc_revision": "d4a3002b9bd0947a19ecb1bcb6df45ce884b7be6",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#43397}",
"swarm_hashes": {
"dawn_end2end_tests": "e64fbae510a389238e95bf7f1c320062a5383c4777a2d13dabd0944d343851f2/488",
"dawn_perf_tests": "e36257d65974a614ee8f14f0a8b1afeea4acd1759db861f8ee67eba5cd4bd087/489",
"gl_tests": "e250aa9a47f5250d766958bc6d5df7eced093fb7999679a860da2a687a5204a9/488",
"gl_unittests": "dc33c373013caa9f8fe60fdde8c4b4d1e322bb37039b9a947a9879f318c5fb7e/488",
"telemetry_gpu_integration_test": "e22ca093abd65f171d09129fb7c0332728dc7c5395759bd23250ba0e8e1c640c/815",
"telemetry_gpu_unittests": "7453a80e6dea6d66158466f918705206be4a181e22c49537a8c062ce8eeac688/736",
"webgpu_blink_web_tests": "fba62ab22b39e5fdfad26b1cd60f0efb5ca8b6bc299a398a5c5ba162cd8726a5/567"
},
"swarming_command_lines_cwd": "out\\59dc-Dawn_Win10_x64_",
"swarming_command_lines_digest": "241fcb002f7cd4a5f15c614bfa0f9cb4f8170a72e39dc8fb0659328aadd8d894/93"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "1f5a7adbfdaa17f8e03d9d92ee2f4aff6219499f",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8896710108406222848"
},
{
"key": "scheduler_job_id",
"value": "chromium/Dawn Win10 x64 ASAN Release (NVIDIA)"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Dawn Win10 x64 ASAN Builder"
},
{
"key": "parent_buildnumber",
"value": "11182"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[15:51:32.979] Scheduled build:
{
"id": "8731280046559720689",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Dawn Win10 x64 ASAN Release (NVIDIA)"
},
"number": 4759,
"createdBy": "project:chromium",
"createTime": "2024-11-14T15:51:32.453661078Z",
"updateTime": "2024-11-14T15:51:32.453661078Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "1f5a7adbfdaa17f8e03d9d92ee2f4aff6219499f",
"ref": "refs/heads/main"
}
}
}
[15:51:32.979] Task URL: https://cr-buildbucket.appspot.com/build/8731280046559720689
[15:51:32.979] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:2:0) after 4m19s
[15:51:33.070] Received PubSub notification, asking Buildbucket for the build status
[15:51:33.089] Build status: SCHEDULED
[15:51:41.275] Received PubSub notification, asking Buildbucket for the build status
[15:51:41.313] Build status: STARTED
[15:55:52.043] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:2:0)
[15:55:52.064] Build status: STARTED
[15:55:52.064] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:5:0) after 3m0s
[15:58:52.098] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:5:0)
[15:58:52.123] Build status: STARTED
[15:58:52.123] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:6:0) after 6m47s
[16:05:39.147] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:6:0)
[16:05:39.195] Build status: STARTED
[16:05:39.195] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:7:0) after 7m15s
[16:12:54.264] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:7:0)
[16:12:54.350] Build status: STARTED
[16:12:54.350] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:8:0) after 5m42s
[16:18:36.368] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:8:0)
[16:18:36.387] Build status: STARTED
[16:18:36.387] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:9:0) after 9m37s
[16:28:13.426] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:9:0)
[16:28:13.444] Build status: STARTED
[16:28:13.444] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:10:0) after 7m26s
[16:35:39.483] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:10:0)
[16:35:39.503] Build status: STARTED
[16:35:39.503] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:11:0) after 2m1s
[16:37:40.522] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:11:0)
[16:37:40.807] Build status: STARTED
[16:37:40.807] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:12:0) after 7m32s
[16:45:12.952] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:12:0)
[16:45:12.976] Build status: STARTED
[16:45:12.976] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:13:0) after 6m27s
[16:51:40.046] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:13:0)
[16:51:40.066] Build status: STARTED
[16:51:40.066] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:14:0) after 4m36s
[16:56:16.148] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:14:0)
[16:56:16.171] Build status: STARTED
[16:56:16.171] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:15:0) after 4m55s
[17:01:11.189] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:15:0)
[17:01:11.206] Build status: STARTED
[17:01:11.206] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:16:0) after 5m5s
[17:06:16.204] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:16:0)
[17:06:16.222] Build status: STARTED
[17:06:16.222] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:17:0) after 1m39s
[17:07:55.242] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:17:0)
[17:07:55.266] Build status: STARTED
[17:07:55.266] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:18:0) after 7m4s
[17:14:59.280] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:18:0)
[17:14:59.335] Build status: STARTED
[17:14:59.335] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:19:0) after 9m47s
[17:24:46.355] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:19:0)
[17:24:46.401] Build status: STARTED
[17:24:46.402] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:20:0) after 9m0s
[17:33:46.457] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:20:0)
[17:33:46.476] Build status: STARTED
[17:33:46.476] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:21:0) after 3m49s
[17:37:35.491] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:21:0)
[17:37:35.509] Build status: STARTED
[17:37:35.509] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:22:0) after 9m29s
[17:47:04.528] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:22:0)
[17:47:04.643] Build status: STARTED
[17:47:04.643] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:23:0) after 5m11s
[17:52:15.665] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:23:0)
[17:52:15.685] Build status: STARTED
[17:52:15.685] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:24:0) after 8m7s
[18:00:22.709] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:24:0)
[18:00:22.725] Build status: STARTED
[18:00:22.725] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:25:0) after 9m19s
[18:09:41.742] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:25:0)
[18:09:41.761] Build status: STARTED
[18:09:41.761] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:26:0) after 7m26s
[18:17:07.821] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:26:0)
[18:17:07.847] Build status: STARTED
[18:17:07.847] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:27:0) after 3m26s
[18:20:33.867] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:27:0)
[18:20:33.886] Build status: STARTED
[18:20:33.886] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:28:0) after 1m45s
[18:22:18.912] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:28:0)
[18:22:18.931] Build status: STARTED
[18:22:18.931] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:29:0) after 4m12s
[18:26:30.952] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:29:0)
[18:26:30.975] Build status: STARTED
[18:26:30.975] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:30:0) after 6m46s
[18:33:16.978] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:30:0)
[18:33:16.997] Build status: STARTED
[18:33:16.997] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:31:0) after 3m10s
[18:36:27.017] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:31:0)
[18:36:27.040] Build status: STARTED
[18:36:27.040] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:32:0) after 1m21s
[18:37:48.169] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:32:0)
[18:37:48.202] Build status: STARTED
[18:37:48.202] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:33:0) after 6m38s
[18:44:26.222] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:33:0)
[18:44:26.242] Build status: STARTED
[18:44:26.242] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:34:0) after 2m45s
[18:47:11.310] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:34:0)
[18:47:11.329] Build status: STARTED
[18:47:11.329] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:35:0) after 8m13s
[18:55:24.370] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:35:0)
[18:55:24.409] Build status: STARTED
[18:55:24.409] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:36:0) after 6m6s
[19:01:30.493] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:36:0)
[19:01:30.511] Build status: STARTED
[19:01:30.511] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:37:0) after 4m39s
[19:06:09.534] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:37:0)
[19:06:09.561] Build status: STARTED
[19:06:09.561] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:38:0) after 2m22s
[19:08:31.619] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:38:0)
[19:08:31.641] Build status: STARTED
[19:08:31.641] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:39:0) after 1m44s
[19:10:15.659] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:39:0)
[19:10:15.675] Build status: STARTED
[19:10:15.675] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:40:0) after 4m28s
[19:14:43.737] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:40:0)
[19:14:43.767] Build status: STARTED
[19:14:43.767] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:41:0) after 7m18s
[19:22:01.792] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:41:0)
[19:22:01.812] Build status: STARTED
[19:22:01.812] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:42:0) after 2m50s
[19:24:51.856] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:42:0)
[19:24:51.907] Build status: STARTED
[19:24:51.907] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:43:0) after 1m53s
[19:26:44.924] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:43:0)
[19:26:44.951] Build status: STARTED
[19:26:44.951] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:44:0) after 3m6s
[19:29:50.996] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:44:0)
[19:29:51.030] Build status: STARTED
[19:29:51.030] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:45:0) after 8m21s
[19:38:12.033] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:45:0)
[19:38:12.048] Build status: STARTED
[19:38:12.048] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:46:0) after 9m2s
[19:47:14.067] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:46:0)
[19:47:14.086] Build status: STARTED
[19:47:14.086] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:47:0) after 6m20s
[19:53:34.061] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:47:0)
[19:53:34.078] Build status: STARTED
[19:53:34.078] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8896710108406222848:48:0) after 1m41s
[19:54:55.185] Received PubSub notification, asking Buildbucket for the build status
[19:54:55.235] Build:
{
"id": "8731280046559720689",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Dawn Win10 x64 ASAN Release (NVIDIA)"
},
"number": 4759,
"createdBy": "project:chromium",
"createTime": "2024-11-14T15:51:32.453661078Z",
"startTime": "2024-11-14T15:51:40.995774265Z",
"endTime": "2024-11-14T19:54:54.439015519Z",
"updateTime": "2024-11-14T19:54:54.439015519Z",
"status": "INFRA_FAILURE",
"statusDetails": {
"timeout": {
}
},
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "1f5a7adbfdaa17f8e03d9d92ee2f4aff6219499f",
"ref": "refs/heads/main"
}
}
}
[19:54:55.236] Invocation finished in 4h3m23.92685574s with status FAILED