Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 3 hours
Actions:

Definition (rev a1ec30b6d30b147f6a54afc2480db7b9bd8a01da)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Dawn Linux x64 Experimental Release (NVIDIA GTX 1660)"
>

Incoming triggers

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "3c05d8a3c041b0766028d3b12445fa7fa15d2a43",
        "project": "chromium/src",
        "ref": "refs/heads/main"
      }
    ]
  },
  "branch": "refs/heads/main",
  "deps_revision_overrides": {
    "src": "3c05d8a3c041b0766028d3b12445fa7fa15d2a43",
    "src/third_party/dawn": "0f93cd5124d8b749c622e1f4f6d4fe32df163023"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8707868676413874753"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8707868676413874753"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8707868676413874753"
    }
  ],
  "parent_build_id": "8707868676413874753",
  "parent_builder_group": "chromium.dawn",
  "parent_buildername": "Dawn Linux x64 Builder",
  "parent_got_angle_revision": "fa90dac8ab47d7f6bc2238a7685f21593bb0ee61",
  "parent_got_dawn_revision": "0f93cd5124d8b749c622e1f4f6d4fe32df163023",
  "parent_got_revision": "3c05d8a3c041b0766028d3b12445fa7fa15d2a43",
  "parent_got_revision_cp": "refs/heads/main@{#1494561}",
  "parent_got_swiftshader_revision": "681d07e76d8e03286b231766447800aab543bd27",
  "parent_got_v8_revision": "6f2788995b3eef031cfafb5c9b67dd58a16f382b",
  "parent_got_v8_revision_cp": "refs/heads/14.0.345@{#1}",
  "parent_got_webrtc_revision": "896a9289487032e56caec4d0207ab1dce854f878",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#45244}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "3c05d8a3c041b0766028d3b12445fa7fa15d2a43",
  "swarm_hashes": {
    "dawn_end2end_tests": "479c0b7284ec36bd7879d9caf00b90202418d06a84a4045cfa339416aabe7bda/323",
    "dawn_perf_tests": "a50eabcd575b3773443fad28750039fd28c45cb9adc580075bb4e4fbd35fdb3e/487",
    "gl_tests": "89867668e72bf92a2e5f7f8ecefe00ef7ce43933ec195488df6c1e68649e5b8d/323",
    "gl_unittests": "ebac2b3e3daa10bba9521b6d9da9cb976a88194bd2ddf219dd53d649020255cb/323",
    "telemetry_gpu_integration_test": "7e0a567d1284dc216c593b06b6554f5b73d4fad7398c61b778a3c005d2d66fa7/815",
    "telemetry_gpu_unittests": "d8c1dd7c97d44626aa412bfad2c687498fc39ed2dcd5e403bdba434297512de6/736",
    "webgpu_blink_web_tests": "c62cba7ce5148b3303f2fc022caece11dd7b4f28209adf5a360a67511962403f/566"
  },
  "swarming_command_lines_cwd": "out/8116-Dawn_Linux_x64_",
  "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/3c05d8a3c041b0766028d3b12445fa7fa15d2a43
  • gitiles_ref:refs/heads/main
  • parent_buildername:Dawn Linux x64 Builder
  • parent_buildnumber:252089
  • user_agent:recipe

Debug log (UTC)

[02:10:59.258] New invocation is queued and will start shortly
[02:10:59.258] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[02:11:00.412] Starting the invocation (attempt 1)
[02:11:00.436] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/3c05d8a3c041b0766028d3b12445fa7fa15d2a43"
[02:11:00.436] Popped gitiles commit info from properties and tags
[02:11:00.436] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[02:11:00.437] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[02:11:00.437] Buildbucket request:
{
  "requestId": "8873297117662437840",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Linux x64 Experimental Release (NVIDIA GTX 1660)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "3c05d8a3c041b0766028d3b12445fa7fa15d2a43",
                        "project": "chromium/src",
                        "ref": "refs/heads/main"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "3c05d8a3c041b0766028d3b12445fa7fa15d2a43",
            "src/third_party/dawn": "0f93cd5124d8b749c622e1f4f6d4fe32df163023"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8707868676413874753"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8707868676413874753"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8707868676413874753"
                }
          ],
      "parent_build_id": "8707868676413874753",
      "parent_builder_group": "chromium.dawn",
      "parent_buildername": "Dawn Linux x64 Builder",
      "parent_got_angle_revision": "fa90dac8ab47d7f6bc2238a7685f21593bb0ee61",
      "parent_got_dawn_revision": "0f93cd5124d8b749c622e1f4f6d4fe32df163023",
      "parent_got_revision": "3c05d8a3c041b0766028d3b12445fa7fa15d2a43",
      "parent_got_revision_cp": "refs/heads/main@{#1494561}",
      "parent_got_swiftshader_revision": "681d07e76d8e03286b231766447800aab543bd27",
      "parent_got_v8_revision": "6f2788995b3eef031cfafb5c9b67dd58a16f382b",
      "parent_got_v8_revision_cp": "refs/heads/14.0.345@{#1}",
      "parent_got_webrtc_revision": "896a9289487032e56caec4d0207ab1dce854f878",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#45244}",
      "swarm_hashes": {
            "dawn_end2end_tests": "479c0b7284ec36bd7879d9caf00b90202418d06a84a4045cfa339416aabe7bda/323",
            "dawn_perf_tests": "a50eabcd575b3773443fad28750039fd28c45cb9adc580075bb4e4fbd35fdb3e/487",
            "gl_tests": "89867668e72bf92a2e5f7f8ecefe00ef7ce43933ec195488df6c1e68649e5b8d/323",
            "gl_unittests": "ebac2b3e3daa10bba9521b6d9da9cb976a88194bd2ddf219dd53d649020255cb/323",
            "telemetry_gpu_integration_test": "7e0a567d1284dc216c593b06b6554f5b73d4fad7398c61b778a3c005d2d66fa7/815",
            "telemetry_gpu_unittests": "d8c1dd7c97d44626aa412bfad2c687498fc39ed2dcd5e403bdba434297512de6/736",
            "webgpu_blink_web_tests": "c62cba7ce5148b3303f2fc022caece11dd7b4f28209adf5a360a67511962403f/566"
          },
      "swarming_command_lines_cwd": "out/8116-Dawn_Linux_x64_",
      "swarming_command_lines_digest": "f8affc12a6026f3b1c44ce425f22dbf15309123519a892be12a890f5a3bee036/93",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "3c05d8a3c041b0766028d3b12445fa7fa15d2a43",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8873297117662437840"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Dawn Linux x64 Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "252089"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[02:11:00.943] Scheduled build:
{
  "id": "8707867055697224465",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Linux x64 Experimental Release (NVIDIA GTX 1660)"
  },
  "number": 7278,
  "createdBy": "project:chromium",
  "createTime": "2025-07-31T02:11:00.509221604Z",
  "updateTime": "2025-07-31T02:11:00.509221604Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "3c05d8a3c041b0766028d3b12445fa7fa15d2a43",
      "ref": "refs/heads/main"
    }
  }
}
[02:11:00.943] Task URL: https://cr-buildbucket.appspot.com/build/8707867055697224465
[02:11:00.943] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:2:0) after 9m36s
[02:11:01.971] Received PubSub notification, asking Buildbucket for the build status
[02:11:01.999] Build status: SCHEDULED
[02:12:13.799] Received PubSub notification, asking Buildbucket for the build status
[02:12:13.820] Build status: STARTED
[02:20:37.005] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:2:0)
[02:20:37.030] Build status: STARTED
[02:20:37.030] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:5:0) after 4m14s
[02:24:51.018] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:5:0)
[02:24:51.046] Build status: STARTED
[02:24:51.046] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:6:0) after 1m44s
[02:26:35.121] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:6:0)
[02:26:35.148] Build status: STARTED
[02:26:35.148] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:7:0) after 5m22s
[02:31:57.170] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:7:0)
[02:31:57.198] Build status: STARTED
[02:31:57.198] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:8:0) after 9m21s
[02:41:18.245] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:8:0)
[02:41:18.271] Build status: STARTED
[02:41:18.271] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:9:0) after 5m52s
[02:47:10.366] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:9:0)
[02:47:10.393] Build status: STARTED
[02:47:10.393] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:10:0) after 5m54s
[02:53:04.406] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:10:0)
[02:53:04.442] Build status: STARTED
[02:53:04.442] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:11:0) after 5m59s
[02:59:03.464] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:11:0)
[02:59:03.491] Build status: STARTED
[02:59:03.491] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:12:0) after 9m29s
[03:08:32.611] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:12:0)
[03:08:32.759] Build status: STARTED
[03:08:32.759] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:13:0) after 6m50s
[03:15:22.824] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:13:0)
[03:15:22.865] Build status: STARTED
[03:15:22.865] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:14:0) after 8m50s
[03:24:12.889] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:14:0)
[03:24:12.911] Build status: STARTED
[03:24:12.911] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:15:0) after 9m32s
[03:33:44.985] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:15:0)
[03:33:45.007] Build status: STARTED
[03:33:45.007] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:16:0) after 3m7s
[03:36:52.027] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:16:0)
[03:36:52.174] Build status: STARTED
[03:36:52.174] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:17:0) after 4m11s
[03:41:03.205] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:17:0)
[03:41:03.328] Build status: STARTED
[03:41:03.328] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:18:0) after 5m52s
[03:46:55.354] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:18:0)
[03:46:55.414] Build status: STARTED
[03:46:55.414] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:19:0) after 1m33s
[03:48:28.503] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:19:0)
[03:48:29.143] Build status: STARTED
[03:48:29.143] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:20:0) after 1m35s
[03:50:04.169] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:20:0)
[03:50:04.192] Build status: STARTED
[03:50:04.192] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:21:0) after 7m36s
[03:57:40.225] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:21:0)
[03:57:40.251] Build status: STARTED
[03:57:40.252] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:22:0) after 4m22s
[04:02:02.431] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:22:0)
[04:02:02.459] Build status: STARTED
[04:02:02.459] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:23:0) after 8m46s
[04:10:48.512] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:23:0)
[04:10:48.545] Build status: STARTED
[04:10:48.545] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:24:0) after 2m34s
[04:13:22.545] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:24:0)
[04:13:22.571] Build status: STARTED
[04:13:22.571] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:25:0) after 6m36s
[04:19:58.601] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:25:0)
[04:19:58.625] Build status: STARTED
[04:19:58.625] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:26:0) after 1m9s
[04:21:07.648] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:26:0)
[04:21:07.669] Build status: STARTED
[04:21:07.669] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:27:0) after 9m49s
[04:30:56.697] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:27:0)
[04:30:56.726] Build status: STARTED
[04:30:56.726] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:28:0) after 5m18s
[04:36:14.791] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:28:0)
[04:36:14.817] Build status: STARTED
[04:36:14.817] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:29:0) after 4m54s
[04:41:08.908] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:29:0)
[04:41:08.939] Build status: STARTED
[04:41:08.939] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:30:0) after 9m28s
[04:50:37.025] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:30:0)
[04:50:37.050] Build status: STARTED
[04:50:37.050] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:31:0) after 9m45s
[05:00:22.139] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:31:0)
[05:00:22.166] Build status: STARTED
[05:00:22.166] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:32:0) after 6m48s
[05:07:10.238] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:32:0)
[05:07:10.253] Build status: STARTED
[05:07:10.253] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:33:0) after 5m21s
[05:12:31.408] Handling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:33:0)
[05:12:31.451] Build status: STARTED
[05:12:31.452] Scheduling timer "check-buildbucket-build-status" (chromium/Dawn Linux x64 Experimental Release (NVIDIA GTX 1660):8873297117662437840:34:0) after 6m11s
[05:16:25.664] Received PubSub notification, asking Buildbucket for the build status
[05:16:25.682] Build:
{
  "id": "8707867055697224465",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Dawn Linux x64 Experimental Release (NVIDIA GTX 1660)"
  },
  "number": 7278,
  "createdBy": "project:chromium",
  "createTime": "2025-07-31T02:11:00.509221604Z",
  "startTime": "2025-07-31T02:12:13.605577958Z",
  "endTime": "2025-07-31T05:16:25.189339491Z",
  "updateTime": "2025-07-31T05:16:25.189339491Z",
  "status": "INFRA_FAILURE",
  "statusDetails": {
    "timeout": {

    }
  },
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "3c05d8a3c041b0766028d3b12445fa7fa15d2a43",
      "ref": "refs/heads/main"
    }
  }
}
[05:16:25.682] Invocation finished in 3h5m26.430991409s with status FAILED