[05:48:30.742] New invocation is queued and will start shortly
[05:48:31.890] Starting the invocation (attempt 1)
[05:48:31.952] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[05:48:31.952] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[05:48:31.952] Buildbucket request:
{
"bucket": "luci.chromium.ci",
"client_operation_id": "9077398397501039952",
"parameters_json": "{\"builder_name\":\"CrWinAsan\",\"properties\":{\"branch\":\"refs/heads/master\",\"repository\":\"https://chromium.googlesource.com/chromium/src.git\",\"revision\":\"9fcf57a3cca5c868d763bd8184c66f7421b192da\"}}",
"pubsub_callback": {
"auth_token": "...",
"topic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
},
"tags": [
"builder:CrWinAsan",
"scheduler_invocation_id:9077398397501039952",
"scheduler_job_id:chromium/CrWinAsan",
"user_agent:luci-scheduler",
"buildset:commit/git/9fcf57a3cca5c868d763bd8184c66f7421b192da",
"buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/9fcf57a3cca5c868d763bd8184c66f7421b192da",
"gitiles_ref:refs/heads/master"
]
}
[05:48:32.810] Buildbucket response:
{
"build": {
"bucket": "luci.chromium.ci",
"canary_preference": "PROD",
"created_by": "project:chromium",
"created_ts": "1559281712169437",
"id": "8911968335338623568",
"parameters_json": "{\"builder_name\": \"CrWinAsan\", \"properties\": {\"branch\": \"refs/heads/master\", \"repository\": \"https://chromium.googlesource.com/chromium/src.git\", \"revision\": \"9fcf57a3cca5c868d763bd8184c66f7421b192da\"}}",
"project": "chromium",
"result_details_json": "{\"properties\": {}}",
"service_account": "chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
"status": "SCHEDULED",
"status_changed_ts": "1559281712630950",
"tags": [
"build_address:luci.chromium.ci/CrWinAsan/3221",
"builder:CrWinAsan",
"buildset:commit/git/9fcf57a3cca5c868d763bd8184c66f7421b192da",
"buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/9fcf57a3cca5c868d763bd8184c66f7421b192da",
"gitiles_ref:refs/heads/master",
"scheduler_invocation_id:9077398397501039952",
"scheduler_job_id:chromium/CrWinAsan",
"swarming_hostname:chromium-swarm.appspot.com",
"swarming_tag:log_location:logdog://logs.chromium.org/chromium/buildbucket/cr-buildbucket.appspot.com/8911968335338623568/+/annotations",
"swarming_tag:luci_project:chromium",
"swarming_tag:recipe_name:chromium",
"swarming_tag:recipe_package:infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",
"swarming_task_id:",
"user_agent:luci-scheduler"
],
"updated_ts": "1559281712631041",
"url": "https://ci.chromium.org/b/8911968335338623568",
"utcnow_ts": "1559281712794968"
}
}
[05:48:32.810] Task URL: https://ci.chromium.org/b/8911968335338623568
[05:48:32.810] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:2:0) after 1m0s
[05:49:32.880] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:2:0)
[05:49:32.880] Timer tick, asking Buildbucket for the build status
[05:49:33.049] Build 8911968335338623568: status "SCHEDULED", result "", failure_reason "", cancelation_reason ""
[05:49:33.050] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:3:0) after 1m0s
[05:50:33.098] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:3:0)
[05:50:33.098] Timer tick, asking Buildbucket for the build status
[05:50:34.691] Build 8911968335338623568: status "SCHEDULED", result "", failure_reason "", cancelation_reason ""
[05:50:34.691] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:4:0) after 1m0s
[05:51:34.711] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:4:0)
[05:51:34.711] Timer tick, asking Buildbucket for the build status
[05:51:35.046] Build 8911968335338623568: status "SCHEDULED", result "", failure_reason "", cancelation_reason ""
[05:51:35.046] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:5:0) after 1m0s
[05:52:35.116] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:5:0)
[05:52:35.116] Timer tick, asking Buildbucket for the build status
[05:52:35.353] Build 8911968335338623568: status "SCHEDULED", result "", failure_reason "", cancelation_reason ""
[05:52:35.353] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:6:0) after 1m0s
[05:53:35.394] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:6:0)
[05:53:35.394] Timer tick, asking Buildbucket for the build status
[05:53:35.497] Build 8911968335338623568: status "SCHEDULED", result "", failure_reason "", cancelation_reason ""
[05:53:35.497] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:7:0) after 1m0s
[05:54:10.377] Received PubSub notification, asking Buildbucket for the build status
[05:54:10.500] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[05:54:35.685] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:7:0)
[05:54:35.685] Timer tick, asking Buildbucket for the build status
[05:54:36.201] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[05:54:36.201] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:9:0) after 1m0s
[05:55:36.175] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:9:0)
[05:55:36.175] Timer tick, asking Buildbucket for the build status
[05:55:36.975] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[05:55:36.975] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:10:0) after 1m0s
[05:56:37.108] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:10:0)
[05:56:37.108] Timer tick, asking Buildbucket for the build status
[05:56:37.956] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[05:56:37.957] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:11:0) after 1m0s
[05:57:38.083] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:11:0)
[05:57:38.083] Timer tick, asking Buildbucket for the build status
[05:57:38.371] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[05:57:38.371] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:12:0) after 1m0s
[05:58:38.551] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:12:0)
[05:58:38.551] Timer tick, asking Buildbucket for the build status
[05:58:39.658] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[05:58:39.658] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:13:0) after 1m0s
[05:59:39.843] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:13:0)
[05:59:39.843] Timer tick, asking Buildbucket for the build status
[05:59:39.981] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[05:59:39.981] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:14:0) after 1m0s
[06:00:40.104] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:14:0)
[06:00:40.104] Timer tick, asking Buildbucket for the build status
[06:00:40.376] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:00:40.376] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:15:0) after 1m0s
[06:01:40.471] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:15:0)
[06:01:40.471] Timer tick, asking Buildbucket for the build status
[06:01:40.581] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:01:40.581] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:16:0) after 1m0s
[06:02:40.676] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:16:0)
[06:02:40.676] Timer tick, asking Buildbucket for the build status
[06:02:40.796] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:02:40.796] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:17:0) after 1m0s
[06:03:40.816] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:17:0)
[06:03:40.816] Timer tick, asking Buildbucket for the build status
[06:03:41.488] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:03:41.488] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:18:0) after 1m0s
[06:04:41.573] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:18:0)
[06:04:41.573] Timer tick, asking Buildbucket for the build status
[06:04:42.036] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:04:42.036] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:19:0) after 1m0s
[06:05:42.107] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:19:0)
[06:05:42.107] Timer tick, asking Buildbucket for the build status
[06:05:42.365] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:05:42.365] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:20:0) after 1m0s
[06:06:42.414] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:20:0)
[06:06:42.414] Timer tick, asking Buildbucket for the build status
[06:06:44.858] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:06:44.858] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:21:0) after 1m0s
[06:07:45.034] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:21:0)
[06:07:45.034] Timer tick, asking Buildbucket for the build status
[06:07:45.411] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:07:45.411] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:22:0) after 1m0s
[06:08:45.541] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:22:0)
[06:08:45.541] Timer tick, asking Buildbucket for the build status
[06:08:47.792] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:08:47.792] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:23:0) after 1m0s
[06:09:47.918] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:23:0)
[06:09:47.918] Timer tick, asking Buildbucket for the build status
[06:09:48.049] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:09:48.049] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:24:0) after 1m0s
[06:10:48.095] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:24:0)
[06:10:48.095] Timer tick, asking Buildbucket for the build status
[06:10:48.909] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:10:48.909] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:25:0) after 1m0s
[06:11:48.973] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:25:0)
[06:11:48.973] Timer tick, asking Buildbucket for the build status
[06:11:50.997] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:11:50.997] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:26:0) after 1m0s
[06:12:51.100] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:26:0)
[06:12:51.100] Timer tick, asking Buildbucket for the build status
[06:12:51.485] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:12:51.485] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:27:0) after 1m0s
[06:13:52.017] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:27:0)
[06:13:52.017] Timer tick, asking Buildbucket for the build status
[06:13:52.305] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:13:52.305] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:28:0) after 1m0s
[06:14:52.329] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:28:0)
[06:14:52.329] Timer tick, asking Buildbucket for the build status
[06:14:52.511] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:14:52.511] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:29:0) after 1m0s
[06:15:52.532] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:29:0)
[06:15:52.532] Timer tick, asking Buildbucket for the build status
[06:15:54.001] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:15:54.001] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:30:0) after 1m0s
[06:16:54.124] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:30:0)
[06:16:54.124] Timer tick, asking Buildbucket for the build status
[06:16:54.290] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:16:54.290] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:31:0) after 1m0s
[06:17:54.305] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:31:0)
[06:17:54.305] Timer tick, asking Buildbucket for the build status
[06:17:54.825] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:17:54.825] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:32:0) after 1m0s
[06:18:54.838] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:32:0)
[06:18:54.838] Timer tick, asking Buildbucket for the build status
[06:18:55.074] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:18:55.074] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:33:0) after 1m0s
[06:19:55.094] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:33:0)
[06:19:55.094] Timer tick, asking Buildbucket for the build status
[06:19:55.234] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:19:55.234] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:34:0) after 1m0s
[06:20:55.271] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:34:0)
[06:20:55.271] Timer tick, asking Buildbucket for the build status
[06:20:55.938] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:20:55.938] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:35:0) after 1m0s
[06:21:55.964] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:35:0)
[06:21:55.964] Timer tick, asking Buildbucket for the build status
[06:21:56.111] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:21:56.111] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:36:0) after 1m0s
[06:22:56.093] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:36:0)
[06:22:56.093] Timer tick, asking Buildbucket for the build status
[06:22:56.205] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:22:56.205] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:37:0) after 1m0s
[06:23:56.227] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:37:0)
[06:23:56.227] Timer tick, asking Buildbucket for the build status
[06:23:56.449] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:23:56.449] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:38:0) after 1m0s
[06:24:56.467] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:38:0)
[06:24:56.467] Timer tick, asking Buildbucket for the build status
[06:24:56.731] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:24:56.731] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:39:0) after 1m0s
[06:25:56.779] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:39:0)
[06:25:56.779] Timer tick, asking Buildbucket for the build status
[06:25:57.252] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:25:57.252] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:40:0) after 1m0s
[06:26:57.277] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:40:0)
[06:26:57.277] Timer tick, asking Buildbucket for the build status
[06:26:57.692] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:26:57.692] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:41:0) after 1m0s
[06:27:57.716] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:41:0)
[06:27:57.716] Timer tick, asking Buildbucket for the build status
[06:27:57.958] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:27:57.958] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:42:0) after 1m0s
[06:28:57.988] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:42:0)
[06:28:57.988] Timer tick, asking Buildbucket for the build status
[06:28:58.116] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:28:58.116] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:43:0) after 1m0s
[06:29:58.140] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:43:0)
[06:29:58.140] Timer tick, asking Buildbucket for the build status
[06:29:58.514] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:29:58.514] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:44:0) after 1m0s
[06:30:58.543] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:44:0)
[06:30:58.543] Timer tick, asking Buildbucket for the build status
[06:30:58.665] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:30:58.665] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:45:0) after 1m0s
[06:31:58.698] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:45:0)
[06:31:58.698] Timer tick, asking Buildbucket for the build status
[06:31:59.066] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:31:59.066] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:46:0) after 1m0s
[06:32:59.076] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:46:0)
[06:32:59.076] Timer tick, asking Buildbucket for the build status
[06:33:00.515] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:33:00.515] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:47:0) after 1m0s
[06:34:01.032] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:47:0)
[06:34:01.032] Timer tick, asking Buildbucket for the build status
[06:34:01.159] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:34:01.159] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:48:0) after 1m0s
[06:35:01.293] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:48:0)
[06:35:01.293] Timer tick, asking Buildbucket for the build status
[06:35:01.485] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:35:01.485] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:49:0) after 1m0s
[06:36:01.594] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:49:0)
[06:36:01.594] Timer tick, asking Buildbucket for the build status
[06:36:04.077] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:36:04.077] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:50:0) after 1m0s
[06:37:04.345] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:50:0)
[06:37:04.345] Timer tick, asking Buildbucket for the build status
[06:37:05.125] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:37:05.125] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:51:0) after 1m0s
[06:38:05.395] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:51:0)
[06:38:05.395] Timer tick, asking Buildbucket for the build status
[06:38:06.083] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:38:06.083] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:52:0) after 1m0s
[06:39:06.146] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:52:0)
[06:39:06.146] Timer tick, asking Buildbucket for the build status
[06:39:06.721] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:39:06.721] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:53:0) after 1m0s
[06:40:06.758] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:53:0)
[06:40:06.758] Timer tick, asking Buildbucket for the build status
[06:40:07.700] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:40:07.700] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:54:0) after 1m0s
[06:41:07.874] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:54:0)
[06:41:07.874] Timer tick, asking Buildbucket for the build status
[06:41:08.392] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:41:08.392] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:55:0) after 1m0s
[06:42:08.500] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:55:0)
[06:42:08.500] Timer tick, asking Buildbucket for the build status
[06:42:08.701] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:42:08.701] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:56:0) after 1m0s
[06:43:08.758] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:56:0)
[06:43:08.758] Timer tick, asking Buildbucket for the build status
[06:43:09.532] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:43:09.532] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:57:0) after 1m0s
[06:44:09.556] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:57:0)
[06:44:09.556] Timer tick, asking Buildbucket for the build status
[06:44:10.562] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:44:10.562] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:58:0) after 1m0s
[06:45:10.642] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:58:0)
[06:45:10.642] Timer tick, asking Buildbucket for the build status
[06:45:10.803] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:45:10.803] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:59:0) after 1m0s
[06:46:10.827] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:59:0)
[06:46:10.827] Timer tick, asking Buildbucket for the build status
[06:46:11.339] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:46:11.339] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:60:0) after 1m0s
[06:47:11.437] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:60:0)
[06:47:11.437] Timer tick, asking Buildbucket for the build status
[06:47:11.997] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:47:11.997] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:61:0) after 1m0s
[06:48:12.018] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:61:0)
[06:48:12.018] Timer tick, asking Buildbucket for the build status
[06:48:12.577] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:48:12.577] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:62:0) after 1m0s
[06:49:12.732] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:62:0)
[06:49:12.732] Timer tick, asking Buildbucket for the build status
[06:49:12.912] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:49:12.912] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:63:0) after 1m0s
[06:50:12.954] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:63:0)
[06:50:12.954] Timer tick, asking Buildbucket for the build status
[06:50:13.285] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:50:13.285] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:64:0) after 1m0s
[06:51:13.306] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:64:0)
[06:51:13.306] Timer tick, asking Buildbucket for the build status
[06:51:15.158] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:51:15.158] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:65:0) after 1m0s
[06:52:15.298] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:65:0)
[06:52:15.298] Timer tick, asking Buildbucket for the build status
[06:52:15.494] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:52:15.494] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:66:0) after 1m0s
[06:53:15.650] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:66:0)
[06:53:15.650] Timer tick, asking Buildbucket for the build status
[06:53:15.884] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:53:15.884] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:67:0) after 1m0s
[06:54:16.567] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:67:0)
[06:54:16.567] Timer tick, asking Buildbucket for the build status
[06:54:16.726] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:54:16.726] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:68:0) after 1m0s
[06:55:16.816] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:68:0)
[06:55:16.816] Timer tick, asking Buildbucket for the build status
[06:55:16.967] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:55:16.967] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:69:0) after 1m0s
[06:56:17.028] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:69:0)
[06:56:17.028] Timer tick, asking Buildbucket for the build status
[06:56:17.149] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:56:17.149] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:70:0) after 1m0s
[06:57:17.140] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:70:0)
[06:57:17.140] Timer tick, asking Buildbucket for the build status
[06:57:17.249] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:57:17.249] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:71:0) after 1m0s
[06:58:17.271] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:71:0)
[06:58:17.271] Timer tick, asking Buildbucket for the build status
[06:58:17.374] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:58:17.374] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:72:0) after 1m0s
[06:59:17.396] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:72:0)
[06:59:17.396] Timer tick, asking Buildbucket for the build status
[06:59:17.770] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[06:59:17.770] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:73:0) after 1m0s
[07:00:17.840] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:73:0)
[07:00:17.840] Timer tick, asking Buildbucket for the build status
[07:00:19.005] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:00:19.005] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:74:0) after 1m0s
[07:01:19.128] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:74:0)
[07:01:19.128] Timer tick, asking Buildbucket for the build status
[07:01:19.472] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:01:19.472] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:75:0) after 1m0s
[07:02:19.546] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:75:0)
[07:02:19.546] Timer tick, asking Buildbucket for the build status
[07:02:22.396] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:02:22.396] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:76:0) after 1m0s
[07:03:22.426] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:76:0)
[07:03:22.426] Timer tick, asking Buildbucket for the build status
[07:03:24.523] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:03:24.523] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:77:0) after 1m0s
[07:04:24.669] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:77:0)
[07:04:24.669] Timer tick, asking Buildbucket for the build status
[07:04:24.751] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:04:24.751] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:78:0) after 1m0s
[07:05:24.777] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:78:0)
[07:05:24.777] Timer tick, asking Buildbucket for the build status
[07:05:25.014] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:05:25.014] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:79:0) after 1m0s
[07:06:25.037] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:79:0)
[07:06:25.037] Timer tick, asking Buildbucket for the build status
[07:06:25.716] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:06:25.716] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:80:0) after 1m0s
[07:07:25.893] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:80:0)
[07:07:25.893] Timer tick, asking Buildbucket for the build status
[07:07:26.008] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:07:26.008] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:81:0) after 1m0s
[07:08:26.210] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:81:0)
[07:08:26.210] Timer tick, asking Buildbucket for the build status
[07:08:26.811] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:08:26.811] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:82:0) after 1m0s
[07:09:26.836] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:82:0)
[07:09:26.836] Timer tick, asking Buildbucket for the build status
[07:09:31.200] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:09:31.201] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:83:0) after 1m0s
[07:10:31.348] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:83:0)
[07:10:31.348] Timer tick, asking Buildbucket for the build status
[07:10:31.545] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:10:31.545] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:84:0) after 1m0s
[07:11:31.600] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:84:0)
[07:11:31.600] Timer tick, asking Buildbucket for the build status
[07:11:33.760] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:11:33.760] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:85:0) after 1m0s
[07:12:33.857] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:85:0)
[07:12:33.857] Timer tick, asking Buildbucket for the build status
[07:12:34.000] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:12:34.000] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:86:0) after 1m0s
[07:13:34.150] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:86:0)
[07:13:34.150] Timer tick, asking Buildbucket for the build status
[07:13:34.478] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:13:34.478] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:87:0) after 1m0s
[07:14:34.503] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:87:0)
[07:14:34.503] Timer tick, asking Buildbucket for the build status
[07:14:34.709] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:14:34.709] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:88:0) after 1m0s
[07:15:34.826] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:88:0)
[07:15:34.826] Timer tick, asking Buildbucket for the build status
[07:15:34.947] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:15:34.947] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:89:0) after 1m0s
[07:16:34.968] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:89:0)
[07:16:34.968] Timer tick, asking Buildbucket for the build status
[07:16:35.168] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:16:35.168] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:90:0) after 1m0s
[07:17:35.320] Handling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:90:0)
[07:17:35.320] Timer tick, asking Buildbucket for the build status
[07:17:35.744] Build 8911968335338623568: status "STARTED", result "", failure_reason "", cancelation_reason ""
[07:17:35.744] Scheduling timer "check-buildbucket-build-status" (chromium/CrWinAsan:9077398397501039952:91:0) after 1m0s
[07:18:12.416] Received PubSub notification, asking Buildbucket for the build status
[07:18:12.518] Build 8911968335338623568: status "COMPLETED", result "FAILURE", failure_reason "BUILD_FAILURE", cancelation_reason ""
[07:18:12.518] Buildbucket build:
{
"build": {
"bucket": "luci.chromium.ci",
"canary_preference": "PROD",
"completed_ts": "1559287091915081",
"created_by": "project:chromium",
"created_ts": "1559281712169437",
"failure_reason": "BUILD_FAILURE",
"id": "8911968335338623568",
"parameters_json": "{\"builder_name\": \"CrWinAsan\", \"properties\": {\"branch\": \"refs/heads/master\", \"repository\": \"https://chromium.googlesource.com/chromium/src.git\", \"revision\": \"9fcf57a3cca5c868d763bd8184c66f7421b192da\"}}",
"project": "chromium",
"result": "FAILURE",
"result_details_json": "{\"error\": {\"message\": \"Step('compile') (retcode: 1)\"}, \"properties\": {\"$recipe_engine/path\": {\"cache_dir\": \"C:\\\\b\\\\s\\\\w\\\\ir\\\\cache\", \"temp_dir\": \"C:\\\\b\\\\s\\\\w\\\\ir\\\\tmp\\\\rt\"}, \"$recipe_engine/runtime\": {\"is_experimental\": false, \"is_luci\": true}, \"bot_id\": \"luci-chromium-ci-win10-32-3-t8m2\", \"branch\": \"refs/heads/master\", \"buildername\": \"CrWinAsan\", \"buildnumber\": 3221, \"got_angle_revision\": \"0a56f0e3d22dad426f0f97e499d70c063983f4b7\", \"got_clang_revision\": \"362195-0d63cef1-0\", \"got_dawn_revision\": \"26d3cf08c209c662a6e2298c301272e2eb8246e4\", \"got_nacl_revision\": \"0ddc033406886a709b901e0c312872529f9705e8\", \"got_revision\": \"9fcf57a3cca5c868d763bd8184c66f7421b192da\", \"got_revision_cp\": \"refs/heads/master@{#665064}\", \"got_swarming_client_revision\": \"779c4f0f8488c64587b75dbb001d18c3c0c4cda9\", \"got_v8_revision\": \"2cb25737c9eb7893ac752674e75d400b4419f67c\", \"got_v8_revision_cp\": \"refs/heads/7.6.303@{#1}\", \"got_webrtc_revision\": \"eb22227c5baf8c35a6222e363b1305d6d529f22b\", \"got_webrtc_revision_cp\": \"refs/heads/master@{#28117}\", \"mastername\": \"chromium.clang\", \"path_config\": \"generic\", \"perf_dashboard_machine_group\": \"ChromiumClang\", \"recipe\": \"chromium\", \"repository\": \"https://chromium.googlesource.com/chromium/src.git\", \"revision\": \"9fcf57a3cca5c868d763bd8184c66f7421b192da\"}, \"swarming\": {\"bot_dimensions\": {\"builderless\": [\"1\"], \"caches\": [\"builder_91c730c3e6ead475130e2ae78bc3f03e09cb9dcb91c3469d4450b29020110401_v2\", \"git\", \"vpython\"], \"cores\": [\"32\"], \"cpu\": [\"x86\", \"x86-64\", \"x86-64-Broadwell_GCE\"], \"gce\": [\"1\"], \"gpu\": [\"none\"], \"id\": [\"luci-chromium-ci-win10-32-3-t8m2\"], \"image\": [\"chrome-win10-1703-2018-04-20-61acaf8983f\"], \"integrity\": [\"high\"], \"locale\": [\"en_US.cp1252\"], \"machine_type\": [\"n1-standard-32\"], \"os\": [\"Windows\", \"Windows-10\", \"Windows-10-15063\"], \"pool\": [\"luci.chromium.ci\"], \"python\": [\"2.7.13\"], \"server_version\": [\"4316-effbcfc\"], \"zone\": [\"us\", \"us-west\", \"us-west1\", \"us-west1-b\"]}}, \"ui\": {\"info\": \"Step('compile') (retcode: 1)\"}}",
"service_account": "chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
"started_ts": "1559282048850312",
"status": "COMPLETED",
"status_changed_ts": "1559282049153376",
"tags": [
"build_address:luci.chromium.ci/CrWinAsan/3221",
"builder:CrWinAsan",
"buildset:commit/git/9fcf57a3cca5c868d763bd8184c66f7421b192da",
"buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/9fcf57a3cca5c868d763bd8184c66f7421b192da",
"gitiles_ref:refs/heads/master",
"scheduler_invocation_id:9077398397501039952",
"scheduler_job_id:chromium/CrWinAsan",
"swarming_hostname:chromium-swarm.appspot.com",
"swarming_tag:log_location:logdog://logs.chromium.org/chromium/buildbucket/cr-buildbucket.appspot.com/8911968335338623568/+/annotations",
"swarming_tag:luci_project:chromium",
"swarming_tag:os:Windows",
"swarming_tag:recipe_name:chromium",
"swarming_tag:recipe_package:infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",
"swarming_task_id:452540474136b910",
"user_agent:luci-scheduler"
],
"updated_ts": "1559287091979440",
"url": "https://ci.chromium.org/b/8911968335338623568",
"utcnow_ts": "1559287092508338"
}
}
[07:18:12.518] Invocation finished in 1h29m41.793620544s with status FAILED