[07:04:05.015] New invocation is queued and will start shortly
[07:04:05.015] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[07:04:06.527] Starting the invocation (attempt 1)
[07:04:06.605] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/3bd510c185958e770f6f7f14febae2ab36a4aca7"
[07:04:06.606] Popped gitiles commit info from properties and tags
[07:04:06.606] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[07:04:06.606] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[07:04:06.606] Buildbucket request:
{
"requestId": "8905712391629746864",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Dawn Win10 x64 ASAN Release (NVIDIA)"
},
"properties": {
"$bootstrap/trigger": {
"commits": [
{
"host": "chromium.googlesource.com",
"id": "3bd510c185958e770f6f7f14febae2ab36a4aca7",
"project": "chromium/src",
"ref": "refs/heads/main"
}
]
},
"deps_revision_overrides": {
"src": "3bd510c185958e770f6f7f14febae2ab36a4aca7",
"src/third_party/dawn": "d07d6ba280ba2e4a6340bcd39eb94249eba986d4"
},
"parent_build_id": "8740293626910981889",
"parent_builder_group": "chromium.dawn",
"parent_buildername": "Dawn Win10 x64 ASAN Builder",
"parent_got_angle_revision": "70f2e2033b17151afca079ef63a9898ff705c995",
"parent_got_dawn_revision": "d07d6ba280ba2e4a6340bcd39eb94249eba986d4",
"parent_got_revision": "3bd510c185958e770f6f7f14febae2ab36a4aca7",
"parent_got_revision_cp": "refs/heads/main@{#1338256}",
"parent_got_swiftshader_revision": "65157d32945d9a75fc9a657e878a1b2f61342f03",
"parent_got_v8_revision": "0655cd60418538e725859ac55e4fa58e4db85e19",
"parent_got_v8_revision_cp": "refs/heads/12.9.116@{#1}",
"parent_got_webrtc_revision": "843e14b36a9c725278f4c5f584fda4313a0e54e7",
"parent_got_webrtc_revision_cp": "refs/heads/main@{#42730}",
"swarm_hashes": {
"dawn_end2end_tests": "9adf4ceb283999abd418b7d9bdf98a159d10881266e687debf72c2695ccc3d95/487",
"dawn_perf_tests": "169e6b2145d3de5b9db04ada7a3df8adbe5c03eabbf777ca32ce4175622cfd14/488",
"gl_tests": "1a909bd2b7fa03f1257fdd5123c30ff5b5e2449370b2d63f021d24742ea108da/487",
"gl_unittests": "007367f755d1055b3b926e701ced318b940654b712214fabe47eee0cc323107c/487",
"telemetry_gpu_integration_test": "713250bab70ddd43b7cd8fbd19cebeca3b3e67faf4dc076a7fea720701a07e7a/814",
"telemetry_gpu_unittests": "f37f3e6cc953c82be53e724954ba3fa236d9817e8cf9e124398eb4db6af6436c/735",
"webgpu_blink_web_tests": "9712a028e195956ab543679c234e8d7b73b82c2a3311174dc207a14535a1f328/566"
},
"swarming_command_lines_cwd": "out\\Release_x64",
"swarming_command_lines_digest": "241fcb002f7cd4a5f15c614bfa0f9cb4f8170a72e39dc8fb0659328aadd8d894/93"
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "3bd510c185958e770f6f7f14febae2ab36a4aca7",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8905712391629746864"
},
{
"key": "scheduler_job_id",
"value": "chromium/Dawn Win10 x64 ASAN Release (NVIDIA)"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "Dawn Win10 x64 ASAN Builder"
},
{
"key": "parent_buildnumber",
"value": "9486"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[07:04:07.309] Scheduled build:
{
"id": "8740282329179878369",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Dawn Win10 x64 ASAN Release (NVIDIA)"
},
"number": 4181,
"createdBy": "project:chromium",
"createTime": "2024-08-07T07:04:06.729956666Z",
"updateTime": "2024-08-07T07:04:06.729956666Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "3bd510c185958e770f6f7f14febae2ab36a4aca7",
"ref": "refs/heads/main"
}
}
}
[07:04:07.310] Task URL: https://cr-buildbucket.appspot.com/build/8740282329179878369
[07:04:07.310] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:2:0) after 5m2s
[07:04:09.816] Received PubSub notification, asking Buildbucket for the build status
[07:04:09.835] Build status: SCHEDULED
[07:05:33.469] Received PubSub notification, asking Buildbucket for the build status
[07:05:33.496] Build status: STARTED
[07:09:09.324] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:2:0)
[07:09:09.342] Build status: STARTED
[07:09:09.342] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:5:0) after 5m52s
[07:15:01.366] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:5:0)
[07:15:01.392] Build status: STARTED
[07:15:01.392] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:6:0) after 5m56s
[07:20:57.460] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:6:0)
[07:20:57.490] Build status: STARTED
[07:20:57.490] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:7:0) after 2m0s
[07:22:57.475] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:7:0)
[07:22:57.496] Build status: STARTED
[07:22:57.496] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:8:0) after 2m46s
[07:25:43.526] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:8:0)
[07:25:43.549] Build status: STARTED
[07:25:43.549] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:9:0) after 8m43s
[07:34:26.697] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:9:0)
[07:34:26.806] Build status: STARTED
[07:34:26.806] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:10:0) after 2m38s
[07:37:04.851] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:10:0)
[07:37:04.881] Build status: STARTED
[07:37:04.881] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:11:0) after 2m28s
[07:39:32.858] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:11:0)
[07:39:32.888] Build status: STARTED
[07:39:32.888] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:12:0) after 8m39s
[07:48:12.210] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:12:0)
[07:48:12.232] Build status: STARTED
[07:48:12.232] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:13:0) after 7m29s
[07:55:41.680] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:13:0)
[07:55:41.698] Build status: STARTED
[07:55:41.698] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:14:0) after 8m58s
[08:04:39.717] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:14:0)
[08:04:39.737] Build status: STARTED
[08:04:39.737] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:15:0) after 8m40s
[08:13:19.718] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:15:0)
[08:13:19.743] Build status: STARTED
[08:13:19.743] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:16:0) after 7m14s
[08:20:33.864] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:16:0)
[08:20:33.878] Build status: STARTED
[08:20:33.878] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:17:0) after 2m35s
[08:23:09.397] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:17:0)
[08:23:09.418] Build status: STARTED
[08:23:09.418] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:18:0) after 7m41s
[08:30:50.532] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:18:0)
[08:30:50.582] Build status: STARTED
[08:30:50.582] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:19:0) after 9m33s
[08:40:23.746] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:19:0)
[08:40:23.771] Build status: STARTED
[08:40:23.771] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:20:0) after 9m8s
[08:49:31.943] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:20:0)
[08:49:31.990] Build status: STARTED
[08:49:31.990] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:21:0) after 4m2s
[08:53:34.226] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:21:0)
[08:53:34.246] Build status: STARTED
[08:53:34.246] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:22:0) after 2m19s
[08:55:53.268] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:22:0)
[08:55:53.288] Build status: STARTED
[08:55:53.288] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:23:0) after 5m22s
[09:01:15.315] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:23:0)
[09:01:15.340] Build status: STARTED
[09:01:15.340] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:24:0) after 9m48s
[09:11:03.841] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:24:0)
[09:11:03.861] Build status: STARTED
[09:11:03.861] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:25:0) after 2m15s
[09:13:18.952] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:25:0)
[09:13:18.971] Build status: STARTED
[09:13:18.971] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:26:0) after 8m32s
[09:21:51.000] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:26:0)
[09:21:51.026] Build status: STARTED
[09:21:51.026] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:27:0) after 3m55s
[09:25:46.131] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:27:0)
[09:25:46.163] Build status: STARTED
[09:25:46.163] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:28:0) after 8m28s
[09:34:14.305] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:28:0)
[09:34:14.332] Build status: STARTED
[09:34:14.333] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:29:0) after 6m53s
[09:41:07.367] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:29:0)
[09:41:07.427] Build status: STARTED
[09:41:07.427] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:30:0) after 5m32s
[09:46:39.511] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:30:0)
[09:46:39.529] Build status: STARTED
[09:46:39.529] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:31:0) after 4m51s
[09:51:30.571] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:31:0)
[09:51:30.597] Build status: STARTED
[09:51:30.597] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:32:0) after 1m21s
[09:52:52.007] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:32:0)
[09:52:52.027] Build status: STARTED
[09:52:52.027] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:33:0) after 3m1s
[09:55:53.162] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:33:0)
[09:55:53.184] Build status: STARTED
[09:55:53.184] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:34:0) after 5m42s
[10:01:35.200] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:34:0)
[10:01:35.256] Build status: STARTED
[10:01:35.256] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:35:0) after 2m43s
[10:04:18.276] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:35:0)
[10:04:18.294] Build status: STARTED
[10:04:18.294] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:36:0) after 2m37s
[10:06:55.365] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:36:0)
[10:06:55.384] Build status: STARTED
[10:06:55.384] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:37:0) after 6m17s
[10:13:12.505] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:37:0)
[10:13:12.523] Build status: STARTED
[10:13:12.523] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:38:0) after 9m9s
[10:22:21.609] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:38:0)
[10:22:21.625] Build status: STARTED
[10:22:21.625] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:39:0) after 5m25s
[10:27:46.659] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:39:0)
[10:27:46.677] Build status: STARTED
[10:27:46.677] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:40:0) after 1m22s
[10:29:08.723] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:40:0)
[10:29:08.750] Build status: STARTED
[10:29:08.750] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:41:0) after 8m5s
[10:37:14.104] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:41:0)
[10:37:14.120] Build status: STARTED
[10:37:14.120] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:42:0) after 4m8s
[10:41:22.190] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:42:0)
[10:41:22.206] Build status: STARTED
[10:41:22.206] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:43:0) after 2m19s
[10:43:41.240] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:43:0)
[10:43:41.342] Build status: STARTED
[10:43:41.342] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:44:0) after 7m7s
[10:50:48.367] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:44:0)
[10:50:48.383] Build status: STARTED
[10:50:48.383] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:45:0) after 1m35s
[10:52:23.412] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:45:0)
[10:52:23.432] Build status: STARTED
[10:52:23.432] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:46:0) after 3m3s
[10:55:26.460] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:46:0)
[10:55:26.475] Build status: STARTED
[10:55:26.475] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:47:0) after 7m6s
[11:02:32.840] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:47:0)
[11:02:32.863] Build status: STARTED
[11:02:32.863] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:48:0) after 6m23s
[11:08:56.093] Handling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:48:0)
[11:08:56.128] Build status: STARTED
[11:08:56.128] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Win10 x64 ASAN Release (NVIDIA):8905712391629746864:49:0) after 6m9s
[11:09:10.450] Received PubSub notification, asking Buildbucket for the build status
[11:09:10.477] Build:
{
"id": "8740282329179878369",
"builder": {
"project": "chromium",
"bucket": "ci",
"builder": "Dawn Win10 x64 ASAN Release (NVIDIA)"
},
"number": 4181,
"createdBy": "project:chromium",
"createTime": "2024-08-07T07:04:06.729956666Z",
"startTime": "2024-08-07T07:05:33.090227387Z",
"endTime": "2024-08-07T11:09:04.206572839Z",
"updateTime": "2024-08-07T11:09:04.206572839Z",
"status": "INFRA_FAILURE",
"statusDetails": {
"timeout": {
}
},
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "3bd510c185958e770f6f7f14febae2ab36a4aca7",
"ref": "refs/heads/main"
}
}
}
[11:09:10.477] Invocation finished in 4h5m5.467295437s with status FAILED