Status: FAILED
Triggered by: -
Duration: 40 minutes
Actions:

Definition (rev 4c3247f4ed8a00ea6dc37d6edf987ff8d0e003c5)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "luci.chromium.ci"
  builder: "linux-chromeos-rel"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/master",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d"
}

Incoming tags

  • buildset:commit/git/ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d
  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d
  • gitiles_ref:refs/heads/master

Debug log (UTC)

[16:59:24.500] New invocation is queued and will start shortly
[16:59:25.608] Starting the invocation (attempt 1)
[16:59:25.660] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[16:59:25.660] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[16:59:25.661] Buildbucket request:
{
  "bucket": "luci.chromium.ci",
  "client_operation_id": "9060595749589745552",
  "parameters_json": "{\"builder_name\":\"linux-chromeos-rel\",\"properties\":{\"branch\":\"refs/heads/master\",\"repository\":\"https://chromium.googlesource.com/chromium/src\",\"revision\":\"ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d\"}}",
  "pubsub_callback": {
    "auth_token": "...",
    "topic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  },
  "tags": [
    "builder:linux-chromeos-rel",
    "scheduler_invocation_id:9060595749589745552",
    "scheduler_job_id:chromium/linux-chromeos-rel",
    "user_agent:luci-scheduler",
    "buildset:commit/git/ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d",
    "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d",
    "gitiles_ref:refs/heads/master"
  ]
}
[16:59:28.622] Buildbucket response:
{
  "build": {
    "bucket": "luci.chromium.ci",
    "canary": true,
    "canary_preference": "CANARY",
    "created_by": "project:chromium",
    "created_ts": "1575305967401769",
    "id": "8895165685884435120",
    "parameters_json": "{\"builder_name\": \"linux-chromeos-rel\", \"properties\": {\"branch\": \"refs/heads/master\", \"repository\": \"https://chromium.googlesource.com/chromium/src\", \"revision\": \"ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d\"}}",
    "project": "chromium",
    "result_details_json": "{\"properties\": {}}",
    "service_account": "chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
    "status": "SCHEDULED",
    "status_changed_ts": "1575305968470059",
    "tags": [
      "build_address:luci.chromium.ci/linux-chromeos-rel/31936",
      "builder:linux-chromeos-rel",
      "buildset:commit/git/ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d",
      "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d",
      "gitiles_ref:refs/heads/master",
      "scheduler_invocation_id:9060595749589745552",
      "scheduler_job_id:chromium/linux-chromeos-rel",
      "swarming_hostname:chromium-swarm.appspot.com",
      "swarming_tag:log_location:logdog://logs.chromium.org/chromium/buildbucket/cr-buildbucket.appspot.com/8895165685884435120/+/annotations",
      "swarming_tag:luci_project:chromium",
      "swarming_tag:recipe_name:chromium",
      "swarming_tag:recipe_package:infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",
      "swarming_task_id:",
      "user_agent:luci-scheduler"
    ],
    "updated_ts": "1575305968470158",
    "url": "https://ci.chromium.org/b/8895165685884435120",
    "utcnow_ts": "1575305968612406"
  }
}
[16:59:28.622] Task URL: https://ci.chromium.org/b/8895165685884435120
[16:59:28.622] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:2:0) after 1m0s
[17:00:28.642] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:2:0)
[17:00:28.642] Timer tick, asking Buildbucket for the build status
[17:00:28.826] Build 8895165685884435120: status "SCHEDULED", result "", failure_reason "", cancelation_reason ""
[17:00:28.826] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:3:0) after 1m0s
[17:01:29.300] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:3:0)
[17:01:29.300] Timer tick, asking Buildbucket for the build status
[17:01:29.451] Build 8895165685884435120: status "SCHEDULED", result "", failure_reason "", cancelation_reason ""
[17:01:29.451] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:4:0) after 1m0s
[17:02:29.685] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:4:0)
[17:02:29.685] Timer tick, asking Buildbucket for the build status
[17:02:29.792] Build 8895165685884435120: status "SCHEDULED", result "", failure_reason "", cancelation_reason ""
[17:02:29.792] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:5:0) after 1m0s
[17:02:31.683] Received PubSub notification, asking Buildbucket for the build status
[17:02:31.974] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:03:29.924] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:5:0)
[17:03:29.924] Timer tick, asking Buildbucket for the build status
[17:03:30.082] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:03:30.082] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:7:0) after 1m0s
[17:04:30.222] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:7:0)
[17:04:30.222] Timer tick, asking Buildbucket for the build status
[17:04:30.358] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:04:30.358] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:8:0) after 1m0s
[17:05:30.410] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:8:0)
[17:05:30.410] Timer tick, asking Buildbucket for the build status
[17:05:30.578] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:05:30.578] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:9:0) after 1m0s
[17:06:30.624] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:9:0)
[17:06:30.624] Timer tick, asking Buildbucket for the build status
[17:06:30.999] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:06:30.999] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:10:0) after 1m0s
[17:07:31.250] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:10:0)
[17:07:31.250] Timer tick, asking Buildbucket for the build status
[17:07:31.400] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:07:31.400] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:11:0) after 1m0s
[17:08:31.600] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:11:0)
[17:08:31.600] Timer tick, asking Buildbucket for the build status
[17:08:31.756] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:08:31.756] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:12:0) after 1m0s
[17:09:31.842] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:12:0)
[17:09:31.842] Timer tick, asking Buildbucket for the build status
[17:09:32.382] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:09:32.382] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:13:0) after 1m0s
[17:10:32.442] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:13:0)
[17:10:32.442] Timer tick, asking Buildbucket for the build status
[17:10:32.661] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:10:32.661] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:14:0) after 1m0s
[17:11:32.937] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:14:0)
[17:11:32.937] Timer tick, asking Buildbucket for the build status
[17:11:33.274] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:11:33.274] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:15:0) after 1m0s
[17:12:33.675] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:15:0)
[17:12:33.675] Timer tick, asking Buildbucket for the build status
[17:12:34.186] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:12:34.186] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:16:0) after 1m0s
[17:13:34.190] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:16:0)
[17:13:34.190] Timer tick, asking Buildbucket for the build status
[17:13:34.362] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:13:34.362] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:17:0) after 1m0s
[17:14:34.380] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:17:0)
[17:14:34.380] Timer tick, asking Buildbucket for the build status
[17:14:34.531] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:14:34.531] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:18:0) after 1m0s
[17:15:34.623] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:18:0)
[17:15:34.623] Timer tick, asking Buildbucket for the build status
[17:15:34.726] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:15:34.726] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:19:0) after 1m0s
[17:16:35.220] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:19:0)
[17:16:35.220] Timer tick, asking Buildbucket for the build status
[17:16:35.440] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:16:35.440] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:20:0) after 1m0s
[17:17:35.575] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:20:0)
[17:17:35.575] Timer tick, asking Buildbucket for the build status
[17:17:35.710] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:17:35.710] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:21:0) after 1m0s
[17:18:36.306] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:21:0)
[17:18:36.306] Timer tick, asking Buildbucket for the build status
[17:18:36.723] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:18:36.723] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:22:0) after 1m0s
[17:19:37.253] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:22:0)
[17:19:37.253] Timer tick, asking Buildbucket for the build status
[17:19:37.800] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:19:37.800] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:23:0) after 1m0s
[17:20:37.860] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:23:0)
[17:20:37.860] Timer tick, asking Buildbucket for the build status
[17:20:38.069] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:20:38.069] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:24:0) after 1m0s
[17:21:38.113] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:24:0)
[17:21:38.113] Timer tick, asking Buildbucket for the build status
[17:21:38.282] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:21:38.282] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:25:0) after 1m0s
[17:22:38.386] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:25:0)
[17:22:38.386] Timer tick, asking Buildbucket for the build status
[17:22:38.546] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:22:38.546] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:26:0) after 1m0s
[17:23:38.561] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:26:0)
[17:23:38.561] Timer tick, asking Buildbucket for the build status
[17:23:38.765] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:23:38.765] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:27:0) after 1m0s
[17:24:38.939] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:27:0)
[17:24:38.939] Timer tick, asking Buildbucket for the build status
[17:24:39.937] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:24:39.937] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:28:0) after 1m0s
[17:25:40.046] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:28:0)
[17:25:40.046] Timer tick, asking Buildbucket for the build status
[17:25:40.491] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:25:40.491] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:29:0) after 1m0s
[17:26:40.508] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:29:0)
[17:26:40.508] Timer tick, asking Buildbucket for the build status
[17:26:41.243] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:26:41.243] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:30:0) after 1m0s
[17:27:41.390] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:30:0)
[17:27:41.390] Timer tick, asking Buildbucket for the build status
[17:27:41.562] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:27:41.562] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:31:0) after 1m0s
[17:28:41.730] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:31:0)
[17:28:41.730] Timer tick, asking Buildbucket for the build status
[17:28:42.751] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:28:42.751] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:32:0) after 1m0s
[17:29:42.844] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:32:0)
[17:29:42.844] Timer tick, asking Buildbucket for the build status
[17:29:43.044] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:29:43.044] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:33:0) after 1m0s
[17:30:43.173] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:33:0)
[17:30:43.173] Timer tick, asking Buildbucket for the build status
[17:30:43.388] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:30:43.388] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:34:0) after 1m0s
[17:31:43.466] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:34:0)
[17:31:43.466] Timer tick, asking Buildbucket for the build status
[17:31:43.805] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:31:43.805] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:35:0) after 1m0s
[17:32:43.863] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:35:0)
[17:32:43.863] Timer tick, asking Buildbucket for the build status
[17:32:44.117] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:32:44.117] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:36:0) after 1m0s
[17:33:44.264] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:36:0)
[17:33:44.264] Timer tick, asking Buildbucket for the build status
[17:33:45.706] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:33:45.706] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:37:0) after 1m0s
[17:34:45.857] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:37:0)
[17:34:45.857] Timer tick, asking Buildbucket for the build status
[17:34:46.176] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:34:46.176] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:38:0) after 1m0s
[17:35:46.241] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:38:0)
[17:35:46.241] Timer tick, asking Buildbucket for the build status
[17:35:46.676] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:35:46.676] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:39:0) after 1m0s
[17:36:46.779] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:39:0)
[17:36:46.779] Timer tick, asking Buildbucket for the build status
[17:36:48.371] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:36:48.371] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:40:0) after 1m0s
[17:37:48.489] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:40:0)
[17:37:48.489] Timer tick, asking Buildbucket for the build status
[17:37:48.810] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:37:48.810] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:41:0) after 1m0s
[17:38:48.862] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:41:0)
[17:38:48.862] Timer tick, asking Buildbucket for the build status
[17:38:49.253] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:38:49.253] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:42:0) after 1m0s
[17:39:49.365] Handling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:42:0)
[17:39:49.365] Timer tick, asking Buildbucket for the build status
[17:39:49.559] Build 8895165685884435120: status "STARTED", result "", failure_reason "", cancelation_reason ""
[17:39:49.559] Scheduling timer "check-buildbucket-build-status" (chromium/linux-chromeos-rel:9060595749589745552:43:0) after 1m0s
[17:39:56.121] Received PubSub notification, asking Buildbucket for the build status
[17:39:56.291] Build 8895165685884435120: status "COMPLETED", result "FAILURE", failure_reason "BUILD_FAILURE", cancelation_reason ""
[17:39:56.291] Buildbucket build:
{
  "build": {
    "bucket": "luci.chromium.ci",
    "canary": true,
    "canary_preference": "CANARY",
    "completed_ts": "1575308394069537",
    "created_by": "project:chromium",
    "created_ts": "1575305967401769",
    "failure_reason": "BUILD_FAILURE",
    "id": "8895165685884435120",
    "parameters_json": "{\"builder_name\": \"linux-chromeos-rel\", \"properties\": {\"branch\": \"refs/heads/master\", \"repository\": \"https://chromium.googlesource.com/chromium/src\", \"revision\": \"ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d\"}}",
    "project": "chromium",
    "result": "FAILURE",
    "result_details_json": "{\"error\": {\"message\": \"1 Test Suite(s) failed.\\n\\n**unit_tests** failed because of:\\n\\n- All/ArcAppModelBuilderTest.IconLoader/1\"}, \"properties\": {\"$build/goma\": {\"rpc_extra_params\": \"?prod\", \"server_host\": \"goma.chromium.org\"}, \"$recipe_engine/path\": {\"cache_dir\": \"/b/s/w/ir/cache\", \"temp_dir\": \"/b/s/w/ir/tmp/rt\"}, \"$recipe_engine/runtime\": {\"is_experimental\": false, \"is_luci\": true}, \"bot_id\": \"luci-chromium-ci-xenial-us-central1-b-8-40-t54q\", \"branch\": \"refs/heads/master\", \"buildername\": \"linux-chromeos-rel\", \"buildnumber\": 31936, \"got_angle_revision\": \"8784a9284f653b9bc6fa54d75217e4f2a190376c\", \"got_dawn_revision\": \"969df2b3a21c126b2e0015ce2392c135f32a6a08\", \"got_nacl_revision\": \"3651f6c52037c96e034cc422746768d59182bff3\", \"got_revision\": \"ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d\", \"got_revision_cp\": \"refs/heads/master@{#720526}\", \"got_swarming_client_revision\": \"885b3febcc170a60f25795304e60927b77d1e92d\", \"got_swiftshader_revision\": \"b64fbfec4dcd3f81c776198641b1fc71ac8d64a4\", \"got_v8_revision\": \"6d4b96eb7059cc04fcacff9dbd080de50ac74aeb\", \"got_v8_revision_cp\": \"refs/heads/8.0.422@{#1}\", \"got_webrtc_revision\": \"89313451d8e2338845716b0ab8760ed0552580f9\", \"got_webrtc_revision_cp\": \"refs/heads/master@{#29962}\", \"isolate_server\": \"https://isolateserver.appspot.com\", \"mastername\": \"chromium.chromiumos\", \"path_config\": \"generic\", \"recipe\": \"chromium\", \"repository\": \"https://chromium.googlesource.com/chromium/src\", \"revision\": \"ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d\", \"swarm_hashes\": {\"accessibility_unittests\": \"1d43b95449d005857ee63904e3e7e80af62e131d\", \"angle_unittests\": \"80fcc1b4c4b34e66c2d7823fd572b8ecc54b1ebb\", \"app_list_unittests\": \"4c152c196887f343d0ef3c1c11a1e97a46bee36d\", \"app_shell_unittests\": \"2f82d2353cdc22e038a8f8c1a1646f7bad1a9b81\", \"ash_unittests\": \"2df7901c8fd4e1a385842a7085f5552dd18df7f2\", \"aura_unittests\": \"42e773de1549d5e5cf5fdaf48e63896229d0d945\", \"base_unittests\": \"6abd7e4f706afcee5d0ce0623ed8fcf9e3104a21\", \"base_util_unittests\": \"5a8a3a8af9097a9d4309b106ad140ecae1a30c10\", \"blink_common_unittests\": \"94cfebd2ad9eaff2ed71aacb33bb402f84bb5a17\", \"blink_fuzzer_unittests\": \"fab9881110a69d2e513e520d3aa636bdc2fc4b37\", \"blink_heap_unittests\": \"68299031a838154dd20e103ae326302d5bbc49af\", \"blink_platform_unittests\": \"bd21ba63c2862f2aa280e954d6d6fdd844d70347\", \"blink_unittests\": \"586544402d92ade5b4edbc9cb00456cb020c84f9\", \"boringssl_crypto_tests\": \"ab14104257a8b2af53033ff396ac71ad25228bc4\", \"boringssl_ssl_tests\": \"c331397ed7f1de1f5d108669c2c1ee2a5af575a7\", \"browser_tests\": \"a24ffd3674fda112844b35d247d929c4fd37d985\", \"cacheinvalidation_unittests\": \"e5b9fc9e87a4c1bf2015b418ade0be979a32bd31\", \"capture_unittests\": \"a0d186121ffc24f412e52cbe4f9162694dc2bfaa\", \"cast_unittests\": \"97c05c2df5e21daf6fd853b57df9cfe3fb838c54\", \"cc_unittests\": \"9dc25d8f81aaf4ec854034bcdade64de4775662f\", \"chrome_app_unittests\": \"fb2dbcf0f9c6dbfee59b7d3fa226035002a616cd\", \"chromedriver_unittests\": \"f061cf60f334d1d83be54030b82581b520eef3ec\", \"chromeos_components_unittests\": \"a16be9bcfde8bcb1be910a0d17df313e84c7912b\", \"chromeos_unittests\": \"432d090bd3d4380e253062d55b6ce32bd6db7f10\", \"color_unittests\": \"bb11d04b6a6fb29c5ba07c590d60bcc105f25cca\", \"components_browsertests\": \"39c7b14d97f56ebe4f94e414b5291a8dfec374ef\", \"components_unittests\": \"748b8c427a88deb7991ded85b752ed98272410d7\", \"compositor_unittests\": \"2d404b2d62fd85c88c9efaa275cd4b728d7c26d0\", \"content_browsertests\": \"1c5905d2632b2137c0ce9fe2fbcc8cf5e1abb837\", \"content_unittests\": \"a385ef7cf9558386814ff5fa6a274988709484f3\", \"crypto_unittests\": \"6d3353555e6371b782734ba0d3a70789cd92e2e6\", \"dbus_unittests\": \"d338adac51705ba728b6bdfbd56b174e32666529\", \"device_unittests\": \"425b26750f1b0cfe78d9ffa33d62351c5e7116ee\", \"display_unittests\": \"991f5e402c8d590fd0e8f3d7bed8a61dd4d312f4\", \"events_unittests\": \"ab6b9303af967c7d11bd6462c67f5ca2fed2397e\", \"exo_unittests\": \"d0b0d8c9b2a996510ada40030dc3b12d3edd99ed\", \"extensions_browsertests\": \"047ab26e8b03d3cabcd7c5eadcb540d64f22c22f\", \"extensions_unittests\": \"5b0930594be47e6225ddbb6d1759fd62a95764cf\", \"filesystem_service_unittests\": \"7c4e4663d660cc820e727498a6d4e3f9060436c6\", \"gcm_unit_tests\": \"935898d9d4c95707c47a36b9f79d8f01be30211e\", \"gfx_unittests\": \"186f2398dfd4e2436aa2fc36a3b8298c9a18d03c\", \"gin_unittests\": \"38634acaade11a1bb24fe80cb3f6e7b8904346b4\", \"gl_unittests_ozone\": \"407f11dbb35dcd4c5380c22101f08bf0629b2edd\", \"google_apis_unittests\": \"73f085ffa1ada9eb33065b0d450f1468276dfa9a\", \"gpu_unittests\": \"d2d55a864053b1a5b97f44a7763a5a3b8574e371\", \"gwp_asan_unittests\": \"c3791bfa2621c97727c36682cca98bc630feed18\", \"interactive_ui_tests\": \"743a89467ef7c6b6e6a1392c904d80c4037df6ae\", \"ipc_tests\": \"9d02ed7376af4281c6dc35c067739ad752a96d36\", \"jingle_unittests\": \"2b20003577407aa2a017b04c186b67826b4c1743\", \"keyboard_unittests\": \"61d6d8645528e82909f8770b842af1ced40ddf59\", \"latency_unittests\": \"24de23d03728dacee925084e9e61e0ba58d70bf9\", \"libjingle_xmpp_unittests\": \"072709348cdb7bc4259b52ec9b5fa911aec55191\", \"media_blink_unittests\": \"d251ec1f3ba6f2799d763d66c359c1cbb62fc9eb\", \"media_service_unittests\": \"3d14dfb49e343f8922bd8dab3f94ed1b5263bb4b\", \"media_unittests\": \"8578e30f0ad5dcfaf9a7e98f76adaf363bc83c87\", \"message_center_unittests\": \"93927eee77ba5addc817cb82e7af2cd9d4111b10\", \"midi_unittests\": \"a0fbe7adaee5d90cf2b78c957502a542f50e84a8\", \"mojo_core_unittests\": \"9a1c811842988919eec92e2cf04939b9d4fd3d6d\", \"mojo_unittests\": \"8b1fa5d334ea2c0757b15fa3161b4c0b1219ce87\", \"nacl_helper_nonsfi_unittests\": \"b688c73a7a22d7ab09263b37c05bf4a578d1c9c4\", \"nacl_loader_unittests\": \"c9249f613cd7a2285e014ddd34aa81b1ed79ee65\", \"native_theme_unittests\": \"3a69095197359edb5e916d79f1e69f005273df78\", \"net_unittests\": \"779a9036b022211a20935993750bb2db0a44faa8\", \"ozone_gl_unittests\": \"638e146e45ba5a124218feea7ae6aaac988925f0\", \"ozone_unittests\": \"244026ba92775350abe77e935142f91e90c74841\", \"ozone_x11_unittests\": \"468f9d9dac25783887a8de5b81b3696cec4d4839\", \"pdf_unittests\": \"f9defd1fdee904d15ca349f252559d2bdff4cc32\", \"perfetto_unittests\": \"178596fa588358823504995f9b3f2a9b39a48acf\", \"ppapi_unittests\": \"87db64affec6d7f7a3c8fbabcb1bd03a8f2e253d\", \"printing_unittests\": \"0c9801ebc28cb3d7c560623a33bb33d654dea440\", \"remoting_unittests\": \"28e60294269c3e6431630f3ef4e3b44e892ea0d0\", \"sandbox_linux_unittests\": \"e7b8cc0875c9f3230f50e80f03b7acae578e7de8\", \"service_manager_unittests\": \"742132a88eda7bcee2bbf5812716cfadbe136d60\", \"services_unittests\": \"d0a44b80c7cd67c891cb92db2941f372981af1ff\", \"shell_dialogs_unittests\": \"41d6885afb3bcca0cc08d34c6f92d82c00becac2\", \"skia_unittests\": \"e89318b481a9f4a87c5dab3d03ba9e3f34b98fc5\", \"snapshot_unittests\": \"a1958f713e6210f837bd05ab4e442f78d48301ca\", \"sql_unittests\": \"97de59e3ce802db895e0a04526b11e32bf576495\", \"storage_unittests\": \"28a548c116eab5cf0618fdf9bc817e08ac4da879\", \"sync_integration_tests\": \"b6fc6faaaa81a37658b6aea7d18e8c56c7fd5bf4\", \"traffic_annotation_auditor_unittests\": \"fb219bcab1df29fe86fd759d16352a4a90cd323d\", \"ui_base_unittests\": \"29ea69f1ea4dc924449838c6e87495def0ecb1d7\", \"ui_chromeos_unittests\": \"8d35155d741f7470e1d51d1f1aa299ec792fd0e1\", \"ui_touch_selection_unittests\": \"d5e1b857bc6dcd441f4b2e48a8ba82e8e547bdd4\", \"unit_tests\": \"4c5d8f762df1e238e6374b47b9bd696453c63207\", \"url_unittests\": \"1e1e02ea3b428999da594a32296e1c7871979276\", \"usage_time_limit_unittests\": \"f137f5b335a97ac728f1f79fc8d0bc1c4de44b82\", \"views_unittests\": \"75c825d7b2f253b21aba36d5a213e3304cc16b1f\", \"viz_unittests\": \"3b457ff5dc587cb4c78212719b6fce2357d12734\", \"wayland_client_perftests\": \"9a0adf455a1a5e70735109526bdd825a5ba1eae5\", \"wm_unittests\": \"695c33bd4a19296d811ee61d5ca5e8c7a7f05e74\", \"wtf_unittests\": \"29edea258544e330a64122775bf99640ef485d47\"}, \"swarm_hashes_refs/heads/master(at){#720526}_without_patch\": {\"accessibility_unittests\": \"1d43b95449d005857ee63904e3e7e80af62e131d\", \"angle_unittests\": \"80fcc1b4c4b34e66c2d7823fd572b8ecc54b1ebb\", \"app_list_unittests\": \"4c152c196887f343d0ef3c1c11a1e97a46bee36d\", \"app_shell_unittests\": \"2f82d2353cdc22e038a8f8c1a1646f7bad1a9b81\", \"ash_unittests\": \"2df7901c8fd4e1a385842a7085f5552dd18df7f2\", \"aura_unittests\": \"42e773de1549d5e5cf5fdaf48e63896229d0d945\", \"base_unittests\": \"6abd7e4f706afcee5d0ce0623ed8fcf9e3104a21\", \"base_util_unittests\": \"5a8a3a8af9097a9d4309b106ad140ecae1a30c10\", \"blink_common_unittests\": \"94cfebd2ad9eaff2ed71aacb33bb402f84bb5a17\", \"blink_fuzzer_unittests\": \"fab9881110a69d2e513e520d3aa636bdc2fc4b37\", \"blink_heap_unittests\": \"68299031a838154dd20e103ae326302d5bbc49af\", \"blink_platform_unittests\": \"bd21ba63c2862f2aa280e954d6d6fdd844d70347\", \"blink_unittests\": \"586544402d92ade5b4edbc9cb00456cb020c84f9\", \"boringssl_crypto_tests\": \"ab14104257a8b2af53033ff396ac71ad25228bc4\", \"boringssl_ssl_tests\": \"c331397ed7f1de1f5d108669c2c1ee2a5af575a7\", \"browser_tests\": \"a24ffd3674fda112844b35d247d929c4fd37d985\", \"cacheinvalidation_unittests\": \"e5b9fc9e87a4c1bf2015b418ade0be979a32bd31\", \"capture_unittests\": \"a0d186121ffc24f412e52cbe4f9162694dc2bfaa\", \"cast_unittests\": \"97c05c2df5e21daf6fd853b57df9cfe3fb838c54\", \"cc_unittests\": \"9dc25d8f81aaf4ec854034bcdade64de4775662f\", \"chrome_app_unittests\": \"fb2dbcf0f9c6dbfee59b7d3fa226035002a616cd\", \"chromedriver_unittests\": \"f061cf60f334d1d83be54030b82581b520eef3ec\", \"chromeos_components_unittests\": \"a16be9bcfde8bcb1be910a0d17df313e84c7912b\", \"chromeos_unittests\": \"432d090bd3d4380e253062d55b6ce32bd6db7f10\", \"color_unittests\": \"bb11d04b6a6fb29c5ba07c590d60bcc105f25cca\", \"components_browsertests\": \"39c7b14d97f56ebe4f94e414b5291a8dfec374ef\", \"components_unittests\": \"748b8c427a88deb7991ded85b752ed98272410d7\", \"compositor_unittests\": \"2d404b2d62fd85c88c9efaa275cd4b728d7c26d0\", \"content_browsertests\": \"1c5905d2632b2137c0ce9fe2fbcc8cf5e1abb837\", \"content_unittests\": \"a385ef7cf9558386814ff5fa6a274988709484f3\", \"crypto_unittests\": \"6d3353555e6371b782734ba0d3a70789cd92e2e6\", \"dbus_unittests\": \"d338adac51705ba728b6bdfbd56b174e32666529\", \"device_unittests\": \"425b26750f1b0cfe78d9ffa33d62351c5e7116ee\", \"display_unittests\": \"991f5e402c8d590fd0e8f3d7bed8a61dd4d312f4\", \"events_unittests\": \"ab6b9303af967c7d11bd6462c67f5ca2fed2397e\", \"exo_unittests\": \"d0b0d8c9b2a996510ada40030dc3b12d3edd99ed\", \"extensions_browsertests\": \"047ab26e8b03d3cabcd7c5eadcb540d64f22c22f\", \"extensions_unittests\": \"5b0930594be47e6225ddbb6d1759fd62a95764cf\", \"filesystem_service_unittests\": \"7c4e4663d660cc820e727498a6d4e3f9060436c6\", \"gcm_unit_tests\": \"935898d9d4c95707c47a36b9f79d8f01be30211e\", \"gfx_unittests\": \"186f2398dfd4e2436aa2fc36a3b8298c9a18d03c\", \"gin_unittests\": \"38634acaade11a1bb24fe80cb3f6e7b8904346b4\", \"gl_unittests_ozone\": \"407f11dbb35dcd4c5380c22101f08bf0629b2edd\", \"google_apis_unittests\": \"73f085ffa1ada9eb33065b0d450f1468276dfa9a\", \"gpu_unittests\": \"d2d55a864053b1a5b97f44a7763a5a3b8574e371\", \"gwp_asan_unittests\": \"c3791bfa2621c97727c36682cca98bc630feed18\", \"interactive_ui_tests\": \"743a89467ef7c6b6e6a1392c904d80c4037df6ae\", \"ipc_tests\": \"9d02ed7376af4281c6dc35c067739ad752a96d36\", \"jingle_unittests\": \"2b20003577407aa2a017b04c186b67826b4c1743\", \"keyboard_unittests\": \"61d6d8645528e82909f8770b842af1ced40ddf59\", \"latency_unittests\": \"24de23d03728dacee925084e9e61e0ba58d70bf9\", \"libjingle_xmpp_unittests\": \"072709348cdb7bc4259b52ec9b5fa911aec55191\", \"media_blink_unittests\": \"d251ec1f3ba6f2799d763d66c359c1cbb62fc9eb\", \"media_service_unittests\": \"3d14dfb49e343f8922bd8dab3f94ed1b5263bb4b\", \"media_unittests\": \"8578e30f0ad5dcfaf9a7e98f76adaf363bc83c87\", \"message_center_unittests\": \"93927eee77ba5addc817cb82e7af2cd9d4111b10\", \"midi_unittests\": \"a0fbe7adaee5d90cf2b78c957502a542f50e84a8\", \"mojo_core_unittests\": \"9a1c811842988919eec92e2cf04939b9d4fd3d6d\", \"mojo_unittests\": \"8b1fa5d334ea2c0757b15fa3161b4c0b1219ce87\", \"nacl_helper_nonsfi_unittests\": \"b688c73a7a22d7ab09263b37c05bf4a578d1c9c4\", \"nacl_loader_unittests\": \"c9249f613cd7a2285e014ddd34aa81b1ed79ee65\", \"native_theme_unittests\": \"3a69095197359edb5e916d79f1e69f005273df78\", \"net_unittests\": \"779a9036b022211a20935993750bb2db0a44faa8\", \"ozone_gl_unittests\": \"638e146e45ba5a124218feea7ae6aaac988925f0\", \"ozone_unittests\": \"244026ba92775350abe77e935142f91e90c74841\", \"ozone_x11_unittests\": \"468f9d9dac25783887a8de5b81b3696cec4d4839\", \"pdf_unittests\": \"f9defd1fdee904d15ca349f252559d2bdff4cc32\", \"perfetto_unittests\": \"178596fa588358823504995f9b3f2a9b39a48acf\", \"ppapi_unittests\": \"87db64affec6d7f7a3c8fbabcb1bd03a8f2e253d\", \"printing_unittests\": \"0c9801ebc28cb3d7c560623a33bb33d654dea440\", \"remoting_unittests\": \"28e60294269c3e6431630f3ef4e3b44e892ea0d0\", \"sandbox_linux_unittests\": \"e7b8cc0875c9f3230f50e80f03b7acae578e7de8\", \"service_manager_unittests\": \"742132a88eda7bcee2bbf5812716cfadbe136d60\", \"services_unittests\": \"d0a44b80c7cd67c891cb92db2941f372981af1ff\", \"shell_dialogs_unittests\": \"41d6885afb3bcca0cc08d34c6f92d82c00becac2\", \"skia_unittests\": \"e89318b481a9f4a87c5dab3d03ba9e3f34b98fc5\", \"snapshot_unittests\": \"a1958f713e6210f837bd05ab4e442f78d48301ca\", \"sql_unittests\": \"97de59e3ce802db895e0a04526b11e32bf576495\", \"storage_unittests\": \"28a548c116eab5cf0618fdf9bc817e08ac4da879\", \"sync_integration_tests\": \"b6fc6faaaa81a37658b6aea7d18e8c56c7fd5bf4\", \"traffic_annotation_auditor_unittests\": \"fb219bcab1df29fe86fd759d16352a4a90cd323d\", \"ui_base_unittests\": \"29ea69f1ea4dc924449838c6e87495def0ecb1d7\", \"ui_chromeos_unittests\": \"8d35155d741f7470e1d51d1f1aa299ec792fd0e1\", \"ui_touch_selection_unittests\": \"d5e1b857bc6dcd441f4b2e48a8ba82e8e547bdd4\", \"unit_tests\": \"4c5d8f762df1e238e6374b47b9bd696453c63207\", \"url_unittests\": \"1e1e02ea3b428999da594a32296e1c7871979276\", \"usage_time_limit_unittests\": \"f137f5b335a97ac728f1f79fc8d0bc1c4de44b82\", \"views_unittests\": \"75c825d7b2f253b21aba36d5a213e3304cc16b1f\", \"viz_unittests\": \"3b457ff5dc587cb4c78212719b6fce2357d12734\", \"wayland_client_perftests\": \"9a0adf455a1a5e70735109526bdd825a5ba1eae5\", \"wm_unittests\": \"695c33bd4a19296d811ee61d5ca5e8c7a7f05e74\", \"wtf_unittests\": \"29edea258544e330a64122775bf99640ef485d47\"}}, \"swarming\": {\"bot_dimensions\": {\"builderless\": [\"1\"], \"caches\": [\"builder_49b2c59a41dab71fb2bba5f6f8a4efb56888b0948fbb2a1ad8fa728b25e0dc17_v2\", \"builder_94fa3119a28f4651cdd4fc5b981a64c69b26c151212072333439362290628993_v2\", \"builder_97ff76ae173a9ce5e244c07639f8a3e6b9cacff7958414e43d82a5f30a20f3d2_v2\", \"builder_9ccb21220795a1d1972030c6b5d9b4569472a28790e1458f6676edb8e4495e8f_v2\", \"builder_ffe623326ac74578ce917ff55ef6cefa75094e5ad3ad5fe3409a729d79be647a_v2\", \"git\", \"goma_v2\", \"vpython\"], \"cores\": [\"8\"], \"cpu\": [\"x86\", \"x86-64\", \"x86-64-Broadwell_GCE\", \"x86-64-avx2\"], \"gce\": [\"1\"], \"gpu\": [\"none\"], \"id\": [\"luci-chromium-ci-xenial-us-central1-b-8-40-t54q\"], \"image\": [\"chrome-xenial-19112404-1fe53dcaaf2\"], \"inside_docker\": [\"0\"], \"kvm\": [\"1\"], \"locale\": [\"en_US.UTF-8\"], \"machine_type\": [\"n1-standard-8\"], \"os\": [\"Linux\", \"Ubuntu\", \"Ubuntu-16.04\"], \"pool\": [\"luci.chromium.ci\"], \"python\": [\"2.7.12\"], \"server_version\": [\"4679-6acb960\"], \"ssd\": [\"0\"], \"zone\": [\"us\", \"us-central\", \"us-central1\", \"us-central1-b\"]}}, \"ui\": {\"info\": \"1 Test Suite(s) failed.\\n\\n**unit_tests** failed because of:\\n\\n- All/ArcAppModelBuilderTest.IconLoader/1\"}}",
    "service_account": "chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
    "started_ts": "1575306150123912",
    "status": "COMPLETED",
    "status_changed_ts": "1575306151414173",
    "tags": [
      "build_address:luci.chromium.ci/linux-chromeos-rel/31936",
      "builder:linux-chromeos-rel",
      "buildset:commit/git/ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d",
      "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ae6c1bab2b721a8e3fded7b13b7ee6ad7af1d05d",
      "gitiles_ref:refs/heads/master",
      "scheduler_invocation_id:9060595749589745552",
      "scheduler_job_id:chromium/linux-chromeos-rel",
      "swarming_hostname:chromium-swarm.appspot.com",
      "swarming_tag:log_location:logdog://logs.chromium.org/chromium/buildbucket/cr-buildbucket.appspot.com/8895165685884435120/+/annotations",
      "swarming_tag:luci_project:chromium",
      "swarming_tag:os:Linux",
      "swarming_tag:recipe_name:chromium",
      "swarming_tag:recipe_package:infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",
      "swarming_task_id:48e05f061a6a4c10",
      "user_agent:luci-scheduler"
    ],
    "updated_ts": "1575308395403460",
    "url": "https://ci.chromium.org/b/8895165685884435120",
    "utcnow_ts": "1575308396263008"
  }
}
[17:39:56.291] Invocation finished in 40m31.807511987s with status FAILED