[01:08:30.464] New invocation is queued and will start shortly
[01:08:31.555] Starting the invocation (attempt 1)
[01:08:31.604] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[01:08:31.604] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[01:08:31.604] Buildbucket request:
{
"bucket": "luci.chromium.ci",
"client_operation_id": "9078231386566329168",
"parameters_json": "{\"builder_name\":\"Site Isolation Android\",\"properties\":{\"branch\":\"refs/heads/master\",\"repository\":\"https://chromium.googlesource.com/chromium/src.git\",\"revision\":\"43960d6361f9fbde7a225094f002a33a35e24464\"}}",
"pubsub_callback": {
"auth_token": "...",
"topic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
},
"tags": [
"builder:Site Isolation Android",
"scheduler_invocation_id:9078231386566329168",
"scheduler_job_id:chromium/Site Isolation Android",
"user_agent:luci-scheduler",
"buildset:commit/git/43960d6361f9fbde7a225094f002a33a35e24464",
"buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/43960d6361f9fbde7a225094f002a33a35e24464",
"gitiles_ref:refs/heads/master"
]
}
[01:08:34.774] Buildbucket response:
{
"build": {
"bucket": "luci.chromium.ci",
"canary_preference": "AUTO",
"created_by": "project:chromium",
"created_ts": "1558487312924378",
"id": "8912801323321229232",
"parameters_json": "{\"builder_name\": \"Site Isolation Android\", \"properties\": {\"branch\": \"refs/heads/master\", \"repository\": \"https://chromium.googlesource.com/chromium/src.git\", \"revision\": \"43960d6361f9fbde7a225094f002a33a35e24464\"}}",
"project": "chromium",
"result_details_json": "{\"properties\": {}}",
"service_account": "chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
"status": "SCHEDULED",
"status_changed_ts": "1558487314324278",
"tags": [
"build_address:luci.chromium.ci/Site Isolation Android/34173",
"builder:Site Isolation Android",
"buildset:commit/git/43960d6361f9fbde7a225094f002a33a35e24464",
"buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/43960d6361f9fbde7a225094f002a33a35e24464",
"gitiles_ref:refs/heads/master",
"scheduler_invocation_id:9078231386566329168",
"scheduler_job_id:chromium/Site Isolation Android",
"swarming_hostname:chromium-swarm.appspot.com",
"swarming_tag:log_location:logdog://logs.chromium.org/chromium/buildbucket/cr-buildbucket.appspot.com/8912801323321229232/+/annotations",
"swarming_tag:luci_project:chromium",
"swarming_tag:recipe_name:chromium",
"swarming_tag:recipe_package:infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",
"swarming_task_id:",
"user_agent:luci-scheduler"
],
"updated_ts": "1558487314324368",
"url": "https://ci.chromium.org/b/8912801323321229232",
"utcnow_ts": "1558487314734238"
}
}
[01:08:34.774] Task URL: https://ci.chromium.org/b/8912801323321229232
[01:08:34.774] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:2:0) after 1m0s
[01:08:43.731] Received PubSub notification, asking Buildbucket for the build status
[01:08:44.508] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:09:34.807] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:2:0)
[01:09:34.807] Timer tick, asking Buildbucket for the build status
[01:09:34.997] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:09:34.997] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:4:0) after 1m0s
[01:10:35.038] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:4:0)
[01:10:35.038] Timer tick, asking Buildbucket for the build status
[01:10:36.920] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:10:36.920] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:5:0) after 1m0s
[01:11:36.944] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:5:0)
[01:11:36.944] Timer tick, asking Buildbucket for the build status
[01:11:38.460] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:11:38.460] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:6:0) after 1m0s
[01:12:38.484] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:6:0)
[01:12:38.484] Timer tick, asking Buildbucket for the build status
[01:12:38.720] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:12:38.720] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:7:0) after 1m0s
[01:13:38.739] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:7:0)
[01:13:38.739] Timer tick, asking Buildbucket for the build status
[01:13:38.878] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:13:38.878] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:8:0) after 1m0s
[01:14:38.889] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:8:0)
[01:14:38.889] Timer tick, asking Buildbucket for the build status
[01:14:39.594] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:14:39.594] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:9:0) after 1m0s
[01:15:39.615] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:9:0)
[01:15:39.615] Timer tick, asking Buildbucket for the build status
[01:15:40.762] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:15:40.762] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:10:0) after 1m0s
[01:16:40.769] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:10:0)
[01:16:40.769] Timer tick, asking Buildbucket for the build status
[01:16:40.943] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:16:40.943] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:11:0) after 1m0s
[01:17:41.036] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:11:0)
[01:17:41.036] Timer tick, asking Buildbucket for the build status
[01:17:41.193] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:17:41.193] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:12:0) after 1m0s
[01:18:41.217] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:12:0)
[01:18:41.217] Timer tick, asking Buildbucket for the build status
[01:18:41.430] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:18:41.430] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:13:0) after 1m0s
[01:19:41.454] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:13:0)
[01:19:41.454] Timer tick, asking Buildbucket for the build status
[01:19:41.605] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:19:41.605] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:14:0) after 1m0s
[01:20:41.973] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:14:0)
[01:20:41.973] Timer tick, asking Buildbucket for the build status
[01:20:42.860] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:20:42.860] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:15:0) after 1m0s
[01:21:42.884] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:15:0)
[01:21:42.884] Timer tick, asking Buildbucket for the build status
[01:21:43.440] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:21:43.440] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:16:0) after 1m0s
[01:22:43.440] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:16:0)
[01:22:43.440] Timer tick, asking Buildbucket for the build status
[01:22:44.076] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:22:44.076] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:17:0) after 1m0s
[01:23:44.108] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:17:0)
[01:23:44.108] Timer tick, asking Buildbucket for the build status
[01:23:44.218] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:23:44.218] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:18:0) after 1m0s
[01:24:44.253] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:18:0)
[01:24:44.253] Timer tick, asking Buildbucket for the build status
[01:24:44.801] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:24:44.801] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:19:0) after 1m0s
[01:25:44.820] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:19:0)
[01:25:44.820] Timer tick, asking Buildbucket for the build status
[01:25:45.424] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:25:45.424] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:20:0) after 1m0s
[01:26:45.448] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:20:0)
[01:26:45.448] Timer tick, asking Buildbucket for the build status
[01:26:46.323] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:26:46.323] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:21:0) after 1m0s
[01:27:46.379] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:21:0)
[01:27:46.380] Timer tick, asking Buildbucket for the build status
[01:27:46.531] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:27:46.531] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:22:0) after 1m0s
[01:28:46.699] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:22:0)
[01:28:46.699] Timer tick, asking Buildbucket for the build status
[01:28:48.623] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:28:48.623] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:23:0) after 1m0s
[01:29:48.645] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:23:0)
[01:29:48.645] Timer tick, asking Buildbucket for the build status
[01:29:49.754] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:29:49.754] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:24:0) after 1m0s
[01:30:49.774] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:24:0)
[01:30:49.774] Timer tick, asking Buildbucket for the build status
[01:30:50.753] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:30:50.753] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:25:0) after 1m0s
[01:31:50.798] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:25:0)
[01:31:50.798] Timer tick, asking Buildbucket for the build status
[01:31:50.910] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:31:50.910] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:26:0) after 1m0s
[01:32:50.939] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:26:0)
[01:32:50.939] Timer tick, asking Buildbucket for the build status
[01:32:51.565] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:32:51.565] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:27:0) after 1m0s
[01:33:51.585] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:27:0)
[01:33:51.585] Timer tick, asking Buildbucket for the build status
[01:33:51.772] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:33:51.772] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:28:0) after 1m0s
[01:34:51.794] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:28:0)
[01:34:51.794] Timer tick, asking Buildbucket for the build status
[01:34:51.915] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:34:51.915] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:29:0) after 1m0s
[01:35:51.931] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:29:0)
[01:35:51.931] Timer tick, asking Buildbucket for the build status
[01:35:52.284] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:35:52.284] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:30:0) after 1m0s
[01:36:52.306] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:30:0)
[01:36:52.306] Timer tick, asking Buildbucket for the build status
[01:36:52.711] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:36:52.711] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:31:0) after 1m0s
[01:37:52.771] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:31:0)
[01:37:52.771] Timer tick, asking Buildbucket for the build status
[01:37:54.834] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:37:54.834] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:32:0) after 1m0s
[01:38:54.959] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:32:0)
[01:38:54.959] Timer tick, asking Buildbucket for the build status
[01:38:55.816] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:38:55.816] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:33:0) after 1m0s
[01:39:56.009] Handling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:33:0)
[01:39:56.009] Timer tick, asking Buildbucket for the build status
[01:39:58.473] Build 8912801323321229232: status "STARTED", result "", failure_reason "", cancelation_reason ""
[01:39:58.473] Scheduling timer "check-buildbucket-build-status" (chromium/Site Isolation Android:9078231386566329168:34:0) after 1m0s
[01:40:00.365] Received PubSub notification, asking Buildbucket for the build status
[01:40:00.474] Build 8912801323321229232: status "COMPLETED", result "FAILURE", failure_reason "BUILD_FAILURE", cancelation_reason ""
[01:40:00.474] Buildbucket build:
{
"build": {
"bucket": "luci.chromium.ci",
"canary_preference": "AUTO",
"completed_ts": "1558489198015362",
"created_by": "project:chromium",
"created_ts": "1558487312924378",
"failure_reason": "BUILD_FAILURE",
"id": "8912801323321229232",
"parameters_json": "{\"builder_name\": \"Site Isolation Android\", \"properties\": {\"branch\": \"refs/heads/master\", \"repository\": \"https://chromium.googlesource.com/chromium/src.git\", \"revision\": \"43960d6361f9fbde7a225094f002a33a35e24464\"}}",
"project": "chromium",
"result": "FAILURE",
"result_details_json": "{\"error\": {\"message\": \"3 tests failed: ['site_per_process_content_browsertests on Android device Nexus 5X', 'site_per_process_components_unittests on Android device Nexus 5X', 'site_per_process_content_unittests on Android device Nexus 5X']\"}, \"properties\": {\"$recipe_engine/path\": {\"cache_dir\": \"/b/swarming/w/ir/cache\", \"temp_dir\": \"/b/swarming/w/ir/tmp/rt\"}, \"$recipe_engine/runtime\": {\"is_experimental\": false, \"is_luci\": true}, \"bot_id\": \"swarm2384-c4\", \"branch\": \"refs/heads/master\", \"buildername\": \"Site Isolation Android\", \"buildnumber\": 34173, \"got_angle_revision\": \"fb8e1b25ad7165591b9df6a1190316610bd6476b\", \"got_dawn_revision\": \"c0c7e2f85bce063edf595ac014c040f0c7d04280\", \"got_nacl_revision\": \"4744a43b6fc4aaa5230541efad113b3cd86ea0f8\", \"got_revision\": \"43960d6361f9fbde7a225094f002a33a35e24464\", \"got_revision_cp\": \"refs/heads/master@{#662015}\", \"got_swarming_client_revision\": \"1b65f4e862045f3ba430a4cbd0643f5b1b66c230\", \"got_v8_revision\": \"6ba8b68b85689114be67b049e98820cad056c75d\", \"got_v8_revision_cp\": \"refs/heads/7.6.246@{#1}\", \"got_webrtc_revision\": \"4f08faae82ccb0a252e83aec2baa9598c9f1716a\", \"got_webrtc_revision_cp\": \"refs/heads/master@{#28016}\", \"isolate_server\": \"https://isolateserver.appspot.com\", \"mastername\": \"chromium.fyi\", \"path_config\": \"generic\", \"recipe\": \"chromium\", \"repository\": \"https://chromium.googlesource.com/chromium/src.git\", \"revision\": \"43960d6361f9fbde7a225094f002a33a35e24464\", \"swarm_hashes\": {\"components_browsertests\": \"a07d9b60deffde7a9a54ce8e228e80c63aea84ed\", \"components_unittests\": \"2787ca1e5bae436e6c1f0892f83d7d6f3fd85e16\", \"content_browsertests\": \"7f4b5a0339abe2001bd128c1e2977251e4eac396\", \"content_unittests\": \"9e134f2fc9e3576e0a511ef06b7c1d663bd251b2\", \"unit_tests\": \"ad994b5d8c51d78c350663c048456ffd1331f4b5\"}, \"swarm_hashes_refs/heads/master(at){#662015}_without_patch\": {\"components_browsertests\": \"a07d9b60deffde7a9a54ce8e228e80c63aea84ed\", \"components_unittests\": \"2787ca1e5bae436e6c1f0892f83d7d6f3fd85e16\", \"content_browsertests\": \"7f4b5a0339abe2001bd128c1e2977251e4eac396\", \"content_unittests\": \"9e134f2fc9e3576e0a511ef06b7c1d663bd251b2\", \"unit_tests\": \"ad994b5d8c51d78c350663c048456ffd1331f4b5\"}}, \"swarming\": {\"bot_dimensions\": {\"builder\": [\"Site Isolation Android\"], \"caches\": [\"builder_e961d9d7e9bca679490a4aaa8599bd975f9b7e0b713a988ed6895a059d04d2ac_v2\", \"git\", \"goma_v2\", \"vpython\"], \"cores\": [\"8\"], \"cpu\": [\"x86\", \"x86-64\", \"x86-64-Ivy_Bridge_GCE\"], \"gce\": [\"1\"], \"gpu\": [\"none\"], \"id\": [\"swarm2384-c4\"], \"inside_docker\": [\"0\"], \"kvm\": [\"0\"], \"machine_type\": [\"n1-standard-8\"], \"os\": [\"Linux\", \"Ubuntu\", \"Ubuntu-14.04\"], \"pool\": [\"luci.chromium.ci\"], \"python\": [\"2.7.6\"], \"server_version\": [\"4303-bae210e\"], \"zone\": [\"us\", \"us-central\", \"us-central1\", \"us-central1-f\"]}}, \"ui\": {\"info\": \"3 tests failed: ['site_per_process_content_browsertests on Android device Nexus 5X', 'site_per_process_components_unittests on Android device Nexus 5X', 'site_per_process_content_unittests on Android device Nexus 5X']\"}}",
"service_account": "chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
"started_ts": "1558487322303885",
"status": "COMPLETED",
"status_changed_ts": "1558487322695955",
"tags": [
"build_address:luci.chromium.ci/Site Isolation Android/34173",
"builder:Site Isolation Android",
"buildset:commit/git/43960d6361f9fbde7a225094f002a33a35e24464",
"buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/43960d6361f9fbde7a225094f002a33a35e24464",
"gitiles_ref:refs/heads/master",
"scheduler_invocation_id:9078231386566329168",
"scheduler_job_id:chromium/Site Isolation Android",
"swarming_hostname:chromium-swarm.appspot.com",
"swarming_tag:log_location:logdog://logs.chromium.org/chromium/buildbucket/cr-buildbucket.appspot.com/8912801323321229232/+/annotations",
"swarming_tag:luci_project:chromium",
"swarming_tag:os:Linux",
"swarming_tag:recipe_name:chromium",
"swarming_tag:recipe_package:infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",
"swarming_task_id:44f5e6bfd9248110",
"user_agent:luci-scheduler"
],
"updated_ts": "1558489198070027",
"url": "https://ci.chromium.org/b/8912801323321229232",
"utcnow_ts": "1558489200465736"
}
}
[01:40:00.474] Invocation finished in 31m30.02843021s with status FAILED