[13:17:36.217] New invocation is queued and will start shortly
[13:17:36.217] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[13:17:37.329] Starting the invocation (attempt 1)
[13:17:37.358] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/a528969a5e8fc371ed5548458736274fd73ab172"
[13:17:37.358] Popped gitiles commit info from properties and tags
[13:17:37.358] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[13:17:37.358] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[13:17:37.358] Buildbucket request:
{
"requestId": "8905145309990627312",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Dawn Win10 x64 ASAN Release (NVIDIA)"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "a528969a5e8fc371ed5548458736274fd73ab172",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "a528969a5e8fc371ed5548458736274fd73ab172",
"src/third_party/dawn": "1eca38fa52364bf66c0d288a0537a2813d72b39b"
},
"parent_build_id": "8739726444188502785",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Win10 x64 ASAN Builder",
"parent_got_angle_revision": "3da39c326365476eca5db9526425e940ffcea56b",
"parent_got_dawn_revision": "1eca38fa52364bf66c0d288a0537a2813d72b39b",
"parent_got_revision": "a528969a5e8fc371ed5548458736274fd73ab172",
"parent_got_revision_cp": "refs/heads/main@{#1340874}",
"parent_got_swiftshader_revision": "65157d32945d9a75fc9a657e878a1b2f61342f03",
"parent_got_v8_revision": "c2a73a70a70a9f6160c86bf6727c58ac63bd27d9",
"parent_got_v8_revision_cp": "refs/heads/12.9.164@{#1}",
"parent_got_webrtc_revision": "b16d8288950c60e64ea4aeac0846ba9c8f3c1bef",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#42765}",
"swarm_hashes": {
"dawn_end2end_tests": "77ccd36fec8c7c2d68f7fcab3364a8db813433ff602a310557f2ed83174a01c6/487",
"dawn_perf_tests": "a2179d769899a86553dcac4c87aa538f4c4e96b84811183fea63ec5c43418043/488",
"gl_tests": "46eca787227bea6ac1a93f5e55e4bb390edeaf4fa11637d7010fdb377dae0abc/487",
"gl_unittests": "c772cbb84890b8f0d853048682287fbef64ceb5f8ef43f058b4fbf6ce33b1837/487",
"telemetry_gpu_integration_test": "a0091752cbfad3fd3889caadcd64738d6c938e7db9e666aa976f13c351da75b6/814",
"telemetry_gpu_unittests": "dfc1d5d0e61df0750fc4b3221fa6985eb327395ee3175475241c08509a89e2f5/735",
"webgpu_blink_web_tests": "2040c8e7abe86229c26fe496170e31b1a6eb7093c5ef760bc7dced07c09b0626/566"
},
"swarming_command_lines_cwd": "out\\Release_x64",
"swarming_command_lines_digest": "241fcb002f7cd4a5f15c614bfa0f9cb4f8170a72e39dc8fb0659328aadd8d894/93"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "a528969a5e8fc371ed5548458736274fd73ab172",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8905145309990627312"
},
{
"key": "scheduler_job_id",
"value": "chromium/Dawn Win10 x64 ASAN Release (NVIDIA)"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Dawn Win10 x64 ASAN Builder"
},
{
"key": "parent_buildnumber",
"value": "9606"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[13:17:38.045] Scheduled build:
{
"id": "8739715248064849313",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Dawn Win10 x64 ASAN Release (NVIDIA)"
},
"number": 4218,
"createdBy": "project:chromium",
"createTime": "2024-08-13T13:17:37.424142779Z",
"updateTime": "2024-08-13T13:17:37.424142779Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "a528969a5e8fc371ed5548458736274fd73ab172",
"ref": "refs/heads/main"
}
},
"canary": true
}
[13:17:38.046] Task URL: https://cr-buildbucket.appspot.com/build/8739715248064849313
[13:17:38.046] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:2:0) after 7m53s
[13:17:38.875] Received PubSub notification, asking Buildbucket for the build status
[13:17:38.898] Build status: SCHEDULED
[13:19:28.602] Received PubSub notification, asking Buildbucket for the build status
[13:19:28.625] Build status: STARTED
[13:25:31.069] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:2:0)
[13:25:31.088] Build status: STARTED
[13:25:31.088] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:5:0) after 2m43s
[13:28:14.203] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:5:0)
[13:28:14.291] Build status: STARTED
[13:28:14.291] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:6:0) after 2m22s
[13:30:36.322] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:6:0)
[13:30:36.346] Build status: STARTED
[13:30:36.346] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:7:0) after 2m9s
[13:32:45.593] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:7:0)
[13:32:45.724] Build status: STARTED
[13:32:45.724] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:8:0) after 7m41s
[13:40:26.738] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:8:0)
[13:40:26.757] Build status: STARTED
[13:40:26.757] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:9:0) after 5m6s
[13:45:32.878] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:9:0)
[13:45:32.896] Build status: STARTED
[13:45:32.896] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:10:0) after 4m36s
[13:50:09.097] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:10:0)
[13:50:09.210] Build status: STARTED
[13:50:09.210] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:11:0) after 9m15s
[13:59:24.299] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:11:0)
[13:59:24.318] Build status: STARTED
[13:59:24.318] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:12:0) after 8m52s
[14:08:16.347] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:12:0)
[14:08:16.440] Build status: STARTED
[14:08:16.440] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:13:0) after 2m33s
[14:10:49.560] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:13:0)
[14:10:49.590] Build status: STARTED
[14:10:49.590] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:14:0) after 9m29s
[14:20:18.782] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:14:0)
[14:20:18.803] Build status: STARTED
[14:20:18.803] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:15:0) after 6m44s
[14:27:02.925] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:15:0)
[14:27:03.119] Build status: STARTED
[14:27:03.119] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:16:0) after 5m19s
[14:32:22.230] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:16:0)
[14:32:22.248] Build status: STARTED
[14:32:22.248] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:17:0) after 2m23s
[14:34:45.565] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:17:0)
[14:34:45.618] Build status: STARTED
[14:34:45.618] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:18:0) after 3m36s
[14:38:21.675] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:18:0)
[14:38:21.730] Build status: STARTED
[14:38:21.730] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:19:0) after 5m34s
[14:43:55.804] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:19:0)
[14:43:55.821] Build status: STARTED
[14:43:55.821] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:20:0) after 9m28s
[14:53:23.857] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:20:0)
[14:53:23.875] Build status: STARTED
[14:53:23.875] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:21:0) after 2m58s
[14:56:21.946] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:21:0)
[14:56:22.036] Build status: STARTED
[14:56:22.036] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:22:0) after 2m25s
[14:58:47.057] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:22:0)
[14:58:47.106] Build status: STARTED
[14:58:47.106] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:23:0) after 4m39s
[15:03:26.109] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:23:0)
[15:03:26.128] Build status: STARTED
[15:03:26.128] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:24:0) after 5m42s
[15:09:08.176] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:24:0)
[15:09:08.192] Build status: STARTED
[15:09:08.192] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:25:0) after 7m38s
[15:16:46.218] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:25:0)
[15:16:46.307] Build status: STARTED
[15:16:46.307] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:26:0) after 1m31s
[15:18:17.388] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:26:0)
[15:18:17.405] Build status: STARTED
[15:18:17.405] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:27:0) after 1m11s
[15:19:28.564] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:27:0)
[15:19:28.586] Build status: STARTED
[15:19:28.586] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:28:0) after 8m51s
[15:28:19.627] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:28:0)
[15:28:19.663] Build status: STARTED
[15:28:19.663] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:29:0) after 6m27s
[15:34:46.690] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:29:0)
[15:34:46.711] Build status: STARTED
[15:34:46.711] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:30:0) after 6m56s
[15:41:42.731] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:30:0)
[15:41:42.806] Build status: STARTED
[15:41:42.806] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:31:0) after 7m26s
[15:49:08.873] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:31:0)
[15:49:08.894] Build status: STARTED
[15:49:08.894] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:32:0) after 9m40s
[15:58:48.881] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:32:0)
[15:58:48.903] Build status: STARTED
[15:58:48.903] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:33:0) after 2m37s
[16:01:25.969] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:33:0)
[16:01:26.016] Build status: STARTED
[16:01:26.016] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:34:0) after 9m39s
[16:11:05.039] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:34:0)
[16:11:05.074] Build status: STARTED
[16:11:05.074] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:35:0) after 1m56s
[16:13:01.099] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:35:0)
[16:13:01.116] Build status: STARTED
[16:13:01.116] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:36:0) after 6m25s
[16:19:26.137] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:36:0)
[16:19:26.156] Build status: STARTED
[16:19:26.156] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:37:0) after 2m52s
[16:22:18.242] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:37:0)
[16:22:18.263] Build status: STARTED
[16:22:18.263] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:38:0) after 7m34s
[16:29:52.337] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:38:0)
[16:29:52.361] Build status: STARTED
[16:29:52.361] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:39:0) after 1m47s
[16:31:39.378] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:39:0)
[16:31:39.401] Build status: STARTED
[16:31:39.401] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:40:0) after 7m36s
[16:39:15.429] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:40:0)
[16:39:15.448] Build status: STARTED
[16:39:15.448] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:41:0) after 2m29s
[16:41:44.476] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:41:0)
[16:41:44.513] Build status: STARTED
[16:41:44.513] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:42:0) after 3m56s
[16:45:40.538] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:42:0)
[16:45:40.627] Build status: STARTED
[16:45:40.627] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:43:0) after 1m51s
[16:47:31.700] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:43:0)
[16:47:31.717] Build status: STARTED
[16:47:31.717] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:44:0) after 7m1s
[16:54:32.748] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:44:0)
[16:54:32.766] Build status: STARTED
[16:54:32.766] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:45:0) after 7m38s
[17:02:10.792] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:45:0)
[17:02:10.807] Build status: STARTED
[17:02:10.807] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:46:0) after 7m19s
[17:09:29.824] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:46:0)
[17:09:29.844] Build status: STARTED
[17:09:29.844] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:47:0) after 6m46s
[17:16:15.895] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:47:0)
[17:16:15.911] Build status: STARTED
[17:16:15.911] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:48:0) after 5m1s
[17:21:16.935] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:48:0)
[17:21:16.953] Build status: STARTED
[17:21:16.953] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905145309990627312:49:0) after 8m58s
[17:23:31.702] Received PubSub notification, asking Buildbucket for the build status
[17:23:31.720] Build:
{
"id": "8739715248064849313",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Dawn Win10 x64 ASAN Release (NVIDIA)"
},
"number": 4218,
"createdBy": "project:chromium",
"createTime": "2024-08-13T13:17:37.424142779Z",
"startTime": "2024-08-13T13:19:28.205890926Z",
"endTime": "2024-08-13T17:23:30.941304324Z",
"updateTime": "2024-08-13T17:23:30.941304324Z",
"status": "INFRA_FAILURE",
"statusDetails": {
"timeout": {
}
},
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "a528969a5e8fc371ed5548458736274fd73ab172",
"ref": "refs/heads/main"
}
},
"canary": true
}
[17:23:31.720] Invocation finished in 4h5m55.515988344s with status FAILED