1. All projects
  2. chromium
  3. Mac FYI ASAN (Intel)
  4. 8896949577454521216
Status: FAILED
Triggered by: chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 2 hours
Actions:

Definition (rev 36bc28844ffd5e030af6e41e2e0e4ddc5c92b699) 

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Mac FYI ASAN (Intel)"
>

Incoming triggers

  • GPU FYI Mac Builder (asan)/56750 (2 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • GPU FYI Mac Builder (asan)/56751 (2 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • GPU FYI Mac Builder (asan)/56752 (2 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • GPU FYI Mac Builder (asan)/56753 (2 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)
  • GPU FYI Mac Builder (asan)/56754 (2 months ago by chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "82832e834bed0388353fef9b279c720e7d86247e",
        "project": "chromium/src",
        "ref": "refs/heads/main"
      }
    ]
  },
  "branch": "refs/heads/main",
  "deps_revision_overrides": {
    "src": "82832e834bed0388353fef9b279c720e7d86247e"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8731520911169724561"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8731520911169724561"
    }
  ],
  "parent_build_id": "8731520911169724561",
  "parent_builder_group": "chromium.gpu.fyi",
  "parent_buildername": "GPU FYI Mac Builder (asan)",
  "parent_got_angle_revision": "ce13a00a2b049a1ef5e0e70a3d333ce70838ef7b",
  "parent_got_dawn_revision": "f910f6694eef26f5cda0314c911bd31a1f8074af",
  "parent_got_revision": "82832e834bed0388353fef9b279c720e7d86247e",
  "parent_got_revision_cp": "refs/heads/main@{#1381483}",
  "parent_got_swiftshader_revision": "d5c4284774115bb1e32c012a2be1b5fbeb1ab1f9",
  "parent_got_v8_revision": "24068c59cedad9ee976ddc05431f5f497b1ebd71",
  "parent_got_v8_revision_cp": "refs/heads/13.2.152@{#1}",
  "parent_got_webrtc_revision": "a5d71009ac1dce7da23813dc9413c03073cfa8ca",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#43387}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "82832e834bed0388353fef9b279c720e7d86247e",
  "swarm_hashes": {
    "angle_unittests": "4911c166ddec5bf955b7e6536665ff2bfce0d0c8273848d3d7128485611fbb9c/489",
    "browser_tests": "3704ec7ccd43292a496ced4d378abc267d028d18f062c15962a6a8519483b7ac/1215",
    "gl_tests": "3a4854f25efab5932e06dece3b99ac510b5d09b618e260a1fe395d4793e3a670/488",
    "gl_unittests": "96626cb07e2646f5d42f96cd33cc5381952aaf9c8a5f8b422c8d0f29e52ec5b9/488",
    "gpu_unittests": "f320c5edd3b5c8d736a02a6dc06de156d097f0d679cd980ccb1c5e207399e300/565",
    "services_unittests": "67672f3b0a60c6227c7a794d40627f6fce85f10bf98d22d4c559b279aaaba17a/734",
    "telemetry_gpu_integration_test": "2f9497f5720d6da852e2c290c848223c773bd7a461c82ce9d2ebd1a5eda4317a/815"
  },
  "swarming_command_lines_cwd": "out/49df-GPU_FYI_Mac_Bui",
  "swarming_command_lines_digest": "107fa9d3e7427248613763ade0743d4316da28b528c9cf3828114368cecb8374/93"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/82832e834bed0388353fef9b279c720e7d86247e
  • gitiles_ref:refs/heads/main
  • parent_buildername:GPU FYI Mac Builder (asan)
  • parent_buildnumber:56754
  • user_agent:recipe

Debug log (UTC)

[00:25:15.834] New invocation is queued and will start shortly
[00:25:15.834] Triggered by user:chromium-ci-gpu-builder@chops-service-accounts.iam.gserviceaccount.com
[00:25:16.912] Starting the invocation (attempt 1)
[00:25:16.943] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/82832e834bed0388353fef9b279c720e7d86247e"
[00:25:16.944] Popped gitiles commit info from properties and tags
[00:25:16.944] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[00:25:16.944] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[00:25:16.944] Buildbucket request:
{
  "requestId": "8896949577454521216",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Mac FYI ASAN (Intel)"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "82832e834bed0388353fef9b279c720e7d86247e",
                        "project": "chromium/src",
                        "ref": "refs/heads/main"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "82832e834bed0388353fef9b279c720e7d86247e"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8731520911169724561"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8731520911169724561"
                }
          ],
      "parent_build_id": "8731520911169724561",
      "parent_builder_group": "chromium.gpu.fyi",
      "parent_buildername": "GPU FYI Mac Builder (asan)",
      "parent_got_angle_revision": "ce13a00a2b049a1ef5e0e70a3d333ce70838ef7b",
      "parent_got_dawn_revision": "f910f6694eef26f5cda0314c911bd31a1f8074af",
      "parent_got_revision": "82832e834bed0388353fef9b279c720e7d86247e",
      "parent_got_revision_cp": "refs/heads/main@{#1381483}",
      "parent_got_swiftshader_revision": "d5c4284774115bb1e32c012a2be1b5fbeb1ab1f9",
      "parent_got_v8_revision": "24068c59cedad9ee976ddc05431f5f497b1ebd71",
      "parent_got_v8_revision_cp": "refs/heads/13.2.152@{#1}",
      "parent_got_webrtc_revision": "a5d71009ac1dce7da23813dc9413c03073cfa8ca",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#43387}",
      "swarm_hashes": {
            "angle_unittests": "4911c166ddec5bf955b7e6536665ff2bfce0d0c8273848d3d7128485611fbb9c/489",
            "browser_tests": "3704ec7ccd43292a496ced4d378abc267d028d18f062c15962a6a8519483b7ac/1215",
            "gl_tests": "3a4854f25efab5932e06dece3b99ac510b5d09b618e260a1fe395d4793e3a670/488",
            "gl_unittests": "96626cb07e2646f5d42f96cd33cc5381952aaf9c8a5f8b422c8d0f29e52ec5b9/488",
            "gpu_unittests": "f320c5edd3b5c8d736a02a6dc06de156d097f0d679cd980ccb1c5e207399e300/565",
            "services_unittests": "67672f3b0a60c6227c7a794d40627f6fce85f10bf98d22d4c559b279aaaba17a/734",
            "telemetry_gpu_integration_test": "2f9497f5720d6da852e2c290c848223c773bd7a461c82ce9d2ebd1a5eda4317a/815"
          },
      "swarming_command_lines_cwd": "out/49df-GPU_FYI_Mac_Bui",
      "swarming_command_lines_digest": "107fa9d3e7427248613763ade0743d4316da28b528c9cf3828114368cecb8374/93"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "82832e834bed0388353fef9b279c720e7d86247e",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8896949577454521216"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/Mac FYI ASAN (Intel)"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "GPU FYI Mac Builder (asan)"
    },
    {
      "key": "parent_buildnumber",
      "value": "56754"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[00:25:17.500] Scheduled build:
{
  "id": "8731519515567364849",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Mac FYI ASAN (Intel)"
  },
  "number": 15169,
  "createdBy": "project:chromium",
  "createTime": "2024-11-12T00:25:17.011704540Z",
  "updateTime": "2024-11-12T00:25:17.011704540Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "82832e834bed0388353fef9b279c720e7d86247e",
      "ref": "refs/heads/main"
    }
  }
}
[00:25:17.500] Task URL: https://cr-buildbucket.appspot.com/build/8731519515567364849
[00:25:17.500] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:2:0) after 8m30s
[00:25:18.488] Received PubSub notification, asking Buildbucket for the build status
[00:25:18.512] Build status: SCHEDULED
[00:25:25.388] Received PubSub notification, asking Buildbucket for the build status
[00:25:25.445] Build status: STARTED
[00:33:47.520] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:2:0)
[00:33:47.536] Build status: STARTED
[00:33:47.536] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:5:0) after 6m32s
[00:40:19.553] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:5:0)
[00:40:19.596] Build status: STARTED
[00:40:19.596] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:6:0) after 6m21s
[00:46:40.615] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:6:0)
[00:46:40.648] Build status: STARTED
[00:46:40.648] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:7:0) after 9m8s
[00:55:48.668] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:7:0)
[00:55:48.744] Build status: STARTED
[00:55:48.744] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:8:0) after 3m55s
[00:59:43.772] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:8:0)
[00:59:43.861] Build status: STARTED
[00:59:43.861] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:9:0) after 9m36s
[01:09:19.929] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:9:0)
[01:09:19.950] Build status: STARTED
[01:09:19.950] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:10:0) after 8m49s
[01:18:08.974] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:10:0)
[01:18:09.006] Build status: STARTED
[01:18:09.006] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:11:0) after 4m56s
[01:23:05.038] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:11:0)
[01:23:05.062] Build status: STARTED
[01:23:05.062] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:12:0) after 4m39s
[01:27:44.082] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:12:0)
[01:27:44.099] Build status: STARTED
[01:27:44.099] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:13:0) after 9m17s
[01:37:01.215] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:13:0)
[01:37:01.259] Build status: STARTED
[01:37:01.259] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:14:0) after 3m47s
[01:40:48.333] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:14:0)
[01:40:48.351] Build status: STARTED
[01:40:48.351] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:15:0) after 3m53s
[01:44:41.406] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:15:0)
[01:44:41.424] Build status: STARTED
[01:44:41.424] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:16:0) after 6m38s
[01:51:19.446] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:16:0)
[01:51:19.505] Build status: STARTED
[01:51:19.506] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:17:0) after 1m21s
[01:52:40.584] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:17:0)
[01:52:40.607] Build status: STARTED
[01:52:40.607] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:18:0) after 3m47s
[01:56:27.613] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:18:0)
[01:56:27.660] Build status: STARTED
[01:56:27.660] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:19:0) after 2m16s
[01:58:43.677] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:19:0)
[01:58:43.715] Build status: STARTED
[01:58:43.715] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:20:0) after 3m55s
[02:02:38.767] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:20:0)
[02:02:38.787] Build status: STARTED
[02:02:38.787] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:21:0) after 2m9s
[02:04:47.855] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:21:0)
[02:04:47.869] Build status: STARTED
[02:04:47.869] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:22:0) after 2m3s
[02:06:50.919] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:22:0)
[02:06:50.941] Build status: STARTED
[02:06:50.941] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:23:0) after 1m43s
[02:08:33.977] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:23:0)
[02:08:34.002] Build status: STARTED
[02:08:34.002] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:24:0) after 1m31s
[02:10:05.020] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:24:0)
[02:10:05.280] Build status: STARTED
[02:10:05.280] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:25:0) after 3m2s
[02:13:07.299] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:25:0)
[02:13:07.318] Build status: STARTED
[02:13:07.318] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:26:0) after 5m55s
[02:19:02.335] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:26:0)
[02:19:02.353] Build status: STARTED
[02:19:02.353] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:27:0) after 6m27s
[02:25:29.369] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:27:0)
[02:25:29.394] Build status: STARTED
[02:25:29.394] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:28:0) after 5m52s
[02:31:21.447] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:28:0)
[02:31:21.465] Build status: STARTED
[02:31:21.465] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:29:0) after 1m47s
[02:33:08.466] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:29:0)
[02:33:08.503] Build status: STARTED
[02:33:08.503] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:30:0) after 2m40s
[02:35:48.555] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:30:0)
[02:35:48.603] Build status: STARTED
[02:35:48.603] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:31:0) after 4m27s
[02:40:15.622] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:31:0)
[02:40:15.643] Build status: STARTED
[02:40:15.643] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:32:0) after 3m6s
[02:43:21.673] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:32:0)
[02:43:21.698] Build status: STARTED
[02:43:21.698] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:33:0) after 9m5s
[02:52:27.092] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:33:0)
[02:52:27.111] Build status: STARTED
[02:52:27.111] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:34:0) after 3m39s
[02:56:07.122] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:34:0)
[02:56:07.139] Build status: STARTED
[02:56:07.139] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:35:0) after 1m59s
[02:58:06.185] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:35:0)
[02:58:06.211] Build status: STARTED
[02:58:06.211] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:36:0) after 4m14s
[03:02:20.240] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:36:0)
[03:02:20.301] Build status: STARTED
[03:02:20.301] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:37:0) after 1m36s
[03:03:56.320] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:37:0)
[03:03:56.338] Build status: STARTED
[03:03:56.338] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:38:0) after 2m34s
[03:06:30.702] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:38:0)
[03:06:30.759] Build status: STARTED
[03:06:30.759] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:39:0) after 6m22s
[03:12:52.780] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:39:0)
[03:12:52.798] Build status: STARTED
[03:12:52.798] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:40:0) after 1m23s
[03:14:15.817] Handling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:40:0)
[03:14:15.841] Build status: STARTED
[03:14:15.841] Scheduling timer "check-buildbucket-build-status" (chromium/Mac FYI ASAN (Intel):8896949577454521216:41:0) after 5m13s
[03:15:24.554] Received PubSub notification, asking Buildbucket for the build status
[03:15:24.588] Build:
{
  "id": "8731519515567364849",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "Mac FYI ASAN (Intel)"
  },
  "number": 15169,
  "createdBy": "project:chromium",
  "createTime": "2024-11-12T00:25:17.011704540Z",
  "startTime": "2024-11-12T00:25:25.200990906Z",
  "endTime": "2024-11-12T03:15:11.746385062Z",
  "updateTime": "2024-11-12T03:15:11.746385062Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "82832e834bed0388353fef9b279c720e7d86247e",
      "ref": "refs/heads/main"
    }
  }
}
[03:15:24.588] Invocation finished in 2h50m8.760431401s with status FAILED