[10:44:43.374] New invocation is queued and will start shortly
[10:44:43.374] Triggered by user:fuchsia-prod-builder@fuchsia-infra.iam.gserviceaccount.com
[10:44:44.644] Starting the invocation (attempt 1)
[10:44:44.685] Popped buildset tag "buildset:commit/gitiles/fuchsia.googlesource.com/integration/+/37820bd8b4247a0ab74601b360f66ad3b9146829"
[10:44:44.685] Reconstructed gitiles commit from tags
[10:44:44.685] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[10:44:44.686] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[10:44:44.686] Buildbucket request:
{
"requestId": "8999013385609159344",
"builder": {
"project": "fuchsia",
"bucket": "prod",
"builder": "clang-ml-training"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"triggers": [
{
"buildbucket": {
"properties": {
"clang_cas_digest": "ab57d4c3a810442ba2a03ce3e076f2b0390e190f510d0fd121b7f35a529662cc/399",
"corpus_cas_digest": "a1ef92c476507802135c59a21b07c538af08f9723536ce440ab1dabb9cfb3056/247",
"fint_params_paths": {
"arm64": "integration/infra/config/generated/fuchsia/fint_params/prod/clang-ml-training-arm64.textproto",
"x64": "integration/infra/config/generated/fuchsia/fint_params/prod/clang-ml-training-x64.textproto"
},
"fuchsia_git_revision": "3f6d0e8d18fcaab85aaa195bef7d7f57e19b1ec6",
"iterations": 2000,
"manifest": "flower",
"model_cas_digest": "0b19aa0cd848ec77fe1273fa21f03d3ae3510e9f04a64009cc7f3d48694dfd16/4656",
"remote": "https://fuchsia.googlesource.com/integration",
"step_iterations": 1000,
"total_iterations": 3000,
"warmstart_cas_digest": "23497a9be3a67384589dd5a480a1f8ee11bc5526054c9623989d0ac79744354b/2401"
},
"tags": [
"buildset:commit/gitiles/fuchsia.googlesource.com/integration/+/37820bd8b4247a0ab74601b360f66ad3b9146829",
"gitiles_ref:refs/heads/main",
"parent_buildername:clang-ml-training",
"user_agent:recipe"
]
},
"id": "345eb1ea-3ad1-4078-8316-c4a0ee16e3b9",
"title": "clang-ml-training/0"
}
]
},
"clang_cas_digest": "ab57d4c3a810442ba2a03ce3e076f2b0390e190f510d0fd121b7f35a529662cc/399",
"corpus_cas_digest": "a1ef92c476507802135c59a21b07c538af08f9723536ce440ab1dabb9cfb3056/247",
"fint_params_paths": {
"arm64": "integration/infra/config/generated/fuchsia/fint_params/prod/clang-ml-training-arm64.textproto",
"x64": "integration/infra/config/generated/fuchsia/fint_params/prod/clang-ml-training-x64.textproto"
},
"fuchsia_git_revision": "3f6d0e8d18fcaab85aaa195bef7d7f57e19b1ec6",
"iterations": 2000,
"manifest": "flower",
"model_cas_digest": "0b19aa0cd848ec77fe1273fa21f03d3ae3510e9f04a64009cc7f3d48694dfd16/4656",
"remote": "https://fuchsia.googlesource.com/integration",
"step_iterations": 1000,
"total_iterations": 3000,
"warmstart_cas_digest": "23497a9be3a67384589dd5a480a1f8ee11bc5526054c9623989d0ac79744354b/2401"
},
"gitilesCommit": {
"host": "fuchsia.googlesource.com",
"project": "integration",
"id": "37820bd8b4247a0ab74601b360f66ad3b9146829",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8999013385609159344"
},
{
"key": "scheduler_job_id",
"value": "fuchsia/prod-clang-ml-training"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "clang-ml-training"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[10:44:45.151] Scheduled build:
{
"id": "8833583323493503521",
"builder": {
"project": "fuchsia",
"bucket": "prod",
"builder": "clang-ml-training"
},
"createdBy": "project:fuchsia",
"createTime": "2021-10-12T10:44:44.757222449Z",
"updateTime": "2021-10-12T10:44:44.948905913Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "fuchsia.googlesource.com",
"project": "integration",
"id": "37820bd8b4247a0ab74601b360f66ad3b9146829",
"ref": "refs/heads/main"
}
}
}
[10:44:45.151] Task URL: https://cr-buildbucket.appspot.com/build/8833583323493503521
[10:44:45.151] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:2:0) after 8m10s
[10:44:50.706] Received PubSub notification, asking Buildbucket for the build status
[10:44:50.756] Build status: STARTED
[10:52:55.359] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:2:0)
[10:52:55.403] Build status: STARTED
[10:52:55.403] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:4:0) after 2m13s
[10:55:08.633] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:4:0)
[10:55:08.658] Build status: STARTED
[10:55:08.658] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:5:0) after 5m20s
[11:00:28.680] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:5:0)
[11:00:28.713] Build status: STARTED
[11:00:28.713] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:6:0) after 9m41s
[11:10:09.946] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:6:0)
[11:10:09.975] Build status: STARTED
[11:10:09.975] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:7:0) after 8m48s
[11:18:58.048] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:7:0)
[11:18:58.089] Build status: STARTED
[11:18:58.089] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:8:0) after 4m24s
[11:23:22.107] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:8:0)
[11:23:22.131] Build status: STARTED
[11:23:22.131] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:9:0) after 9m44s
[11:33:06.294] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:9:0)
[11:33:06.324] Build status: STARTED
[11:33:06.324] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:10:0) after 7m47s
[11:40:53.357] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:10:0)
[11:40:53.496] Build status: STARTED
[11:40:53.496] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:11:0) after 4m29s
[11:45:22.525] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:11:0)
[11:45:22.557] Build status: STARTED
[11:45:22.557] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:12:0) after 9m42s
[11:55:04.792] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:12:0)
[11:55:04.823] Build status: STARTED
[11:55:04.823] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:13:0) after 9m13s
[12:04:17.840] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:13:0)
[12:04:17.873] Build status: STARTED
[12:04:17.873] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:14:0) after 1m21s
[12:05:38.842] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:14:0)
[12:05:38.900] Build status: STARTED
[12:05:38.900] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:15:0) after 4m2s
[12:09:41.045] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:15:0)
[12:09:41.101] Build status: STARTED
[12:09:41.101] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:16:0) after 7m32s
[12:17:13.118] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:16:0)
[12:17:13.148] Build status: STARTED
[12:17:13.148] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:17:0) after 9m40s
[12:26:53.376] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:17:0)
[12:26:53.403] Build status: STARTED
[12:26:53.403] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:18:0) after 7m17s
[12:34:10.603] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:18:0)
[12:34:10.632] Build status: STARTED
[12:34:10.632] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:19:0) after 8m24s
[12:42:34.653] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:19:0)
[12:42:34.688] Build status: STARTED
[12:42:34.688] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:20:0) after 6m29s
[12:49:04.040] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:20:0)
[12:49:04.066] Build status: STARTED
[12:49:04.066] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:21:0) after 9m38s
[12:58:42.156] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:21:0)
[12:58:42.269] Build status: STARTED
[12:58:42.269] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:22:0) after 2m4s
[13:00:46.274] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:22:0)
[13:00:46.309] Build status: STARTED
[13:00:46.309] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:23:0) after 5m38s
[13:06:24.472] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:23:0)
[13:06:24.519] Build status: STARTED
[13:06:24.519] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:24:0) after 3m16s
[13:09:40.588] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:24:0)
[13:09:40.646] Build status: STARTED
[13:09:40.646] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:25:0) after 6m12s
[13:15:52.905] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:25:0)
[13:15:53.067] Build status: STARTED
[13:15:53.067] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:26:0) after 5m44s
[13:21:37.103] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:26:0)
[13:21:37.138] Build status: STARTED
[13:21:37.138] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:27:0) after 7m15s
[13:28:52.162] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:27:0)
[13:28:52.201] Build status: STARTED
[13:28:52.201] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:28:0) after 3m38s
[13:32:30.295] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:28:0)
[13:32:30.337] Build status: STARTED
[13:32:30.337] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:29:0) after 6m39s
[13:39:09.356] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:29:0)
[13:39:09.384] Build status: STARTED
[13:39:09.384] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:30:0) after 2m6s
[13:41:15.719] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:30:0)
[13:41:15.750] Build status: STARTED
[13:41:15.750] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:31:0) after 2m59s
[13:44:14.917] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:31:0)
[13:44:14.946] Build status: STARTED
[13:44:14.946] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:32:0) after 7m46s
[13:52:01.215] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:32:0)
[13:52:01.243] Build status: STARTED
[13:52:01.243] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:33:0) after 8m47s
[14:00:48.261] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:33:0)
[14:00:48.319] Build status: STARTED
[14:00:48.319] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:34:0) after 8m33s
[14:09:21.337] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:34:0)
[14:09:21.375] Build status: STARTED
[14:09:21.375] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:35:0) after 4m8s
[14:13:29.735] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:35:0)
[14:13:29.780] Build status: STARTED
[14:13:29.780] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:36:0) after 3m48s
[14:17:17.969] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:36:0)
[14:17:18.011] Build status: STARTED
[14:17:18.011] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:37:0) after 6m12s
[14:23:30.453] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:37:0)
[14:23:30.497] Build status: STARTED
[14:23:30.497] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:38:0) after 3m59s
[14:27:30.209] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:38:0)
[14:27:30.358] Build status: STARTED
[14:27:30.358] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:39:0) after 6m58s
[14:34:28.510] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:39:0)
[14:34:28.556] Build status: STARTED
[14:34:28.556] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:40:0) after 5m9s
[14:39:37.580] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:40:0)
[14:39:37.612] Build status: STARTED
[14:39:37.612] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:41:0) after 8m24s
[14:48:01.656] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:41:0)
[14:48:01.699] Build status: STARTED
[14:48:01.699] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:42:0) after 5m29s
[14:53:30.846] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:42:0)
[14:53:30.872] Build status: STARTED
[14:53:30.872] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:43:0) after 3m23s
[14:56:53.895] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:43:0)
[14:56:54.038] Build status: STARTED
[14:56:54.038] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:44:0) after 4m59s
[15:01:53.110] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:44:0)
[15:01:53.136] Build status: STARTED
[15:01:53.136] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:45:0) after 2m5s
[15:03:58.631] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:45:0)
[15:03:58.662] Build status: STARTED
[15:03:58.662] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:46:0) after 6m25s
[15:10:23.871] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:46:0)
[15:10:23.903] Build status: STARTED
[15:10:23.903] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:47:0) after 8m46s
[15:19:10.178] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:47:0)
[15:19:10.380] Build status: STARTED
[15:19:10.380] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:48:0) after 8m6s
[15:27:17.153] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:48:0)
[15:27:17.189] Build status: STARTED
[15:27:17.189] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:49:0) after 2m40s
[15:29:57.307] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:49:0)
[15:29:57.334] Build status: STARTED
[15:29:57.334] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:50:0) after 4m7s
[15:34:04.844] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:50:0)
[15:34:04.877] Build status: STARTED
[15:34:04.877] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:51:0) after 3m24s
[15:37:29.425] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:51:0)
[15:37:29.552] Build status: STARTED
[15:37:29.552] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:52:0) after 8m27s
[15:45:56.579] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:52:0)
[15:45:56.653] Build status: STARTED
[15:45:56.653] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:53:0) after 4m58s
[15:50:54.673] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:53:0)
[15:50:54.707] Build status: STARTED
[15:50:54.707] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:54:0) after 9m55s
[16:00:49.742] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:54:0)
[16:00:49.772] Build status: STARTED
[16:00:49.772] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:55:0) after 9m28s
[16:10:17.919] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:55:0)
[16:10:17.958] Build status: STARTED
[16:10:17.958] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:56:0) after 3m11s
[16:13:29.359] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:56:0)
[16:13:29.394] Build status: STARTED
[16:13:29.394] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:57:0) after 1m53s
[16:15:22.581] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:57:0)
[16:15:22.611] Build status: STARTED
[16:15:22.611] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:58:0) after 1m6s
[16:16:28.694] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:58:0)
[16:16:28.776] Build status: STARTED
[16:16:28.776] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:59:0) after 6m57s
[16:23:25.782] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:59:0)
[16:23:26.837] Build status: STARTED
[16:23:26.837] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:60:0) after 1m32s
[16:24:59.030] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:60:0)
[16:24:59.053] Build status: STARTED
[16:24:59.054] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:61:0) after 8m17s
[16:33:16.361] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:61:0)
[16:33:16.404] Build status: STARTED
[16:33:16.404] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:62:0) after 1m34s
[16:34:50.571] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:62:0)
[16:34:50.600] Build status: STARTED
[16:34:50.600] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:63:0) after 5m21s
[16:40:11.976] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:63:0)
[16:40:12.030] Build status: STARTED
[16:40:12.030] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:64:0) after 1m44s
[16:41:56.117] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:64:0)
[16:41:56.151] Build status: STARTED
[16:41:56.151] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:65:0) after 8m23s
[16:50:19.272] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:65:0)
[16:50:19.299] Build status: STARTED
[16:50:19.299] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:66:0) after 4m30s
[16:54:49.594] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:66:0)
[16:54:49.628] Build status: STARTED
[16:54:49.628] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:67:0) after 8m5s
[17:02:54.989] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:67:0)
[17:02:55.014] Build status: STARTED
[17:02:55.014] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:68:0) after 3m37s
[17:06:32.177] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:68:0)
[17:06:32.248] Build status: STARTED
[17:06:32.248] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:69:0) after 3m9s
[17:09:41.547] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:69:0)
[17:09:41.583] Build status: STARTED
[17:09:41.583] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:70:0) after 3m40s
[17:13:21.852] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:70:0)
[17:13:21.883] Build status: STARTED
[17:13:21.883] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:71:0) after 7m44s
[17:21:05.966] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:71:0)
[17:21:05.994] Build status: STARTED
[17:21:05.994] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:72:0) after 8m32s
[17:29:38.013] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:72:0)
[17:29:38.040] Build status: STARTED
[17:29:38.040] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:73:0) after 7m11s
[17:36:49.503] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:73:0)
[17:36:49.531] Build status: STARTED
[17:36:49.531] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:74:0) after 1m19s
[17:38:08.832] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:74:0)
[17:38:08.881] Build status: STARTED
[17:38:08.881] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:75:0) after 1m16s
[17:39:24.903] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:75:0)
[17:39:25.012] Build status: STARTED
[17:39:25.012] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:76:0) after 8m56s
[17:48:21.143] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:76:0)
[17:48:21.187] Build status: STARTED
[17:48:21.187] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:77:0) after 7m32s
[17:55:53.672] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:77:0)
[17:55:53.718] Build status: STARTED
[17:55:53.718] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:78:0) after 8m34s
[18:04:28.670] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:78:0)
[18:04:28.699] Build status: STARTED
[18:04:28.699] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:79:0) after 1m8s
[18:05:36.755] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:79:0)
[18:05:36.782] Build status: STARTED
[18:05:36.782] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:80:0) after 7m32s
[18:13:08.868] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:80:0)
[18:13:08.897] Build status: STARTED
[18:13:08.898] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:81:0) after 5m46s
[18:18:55.030] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:81:0)
[18:18:55.058] Build status: STARTED
[18:18:55.058] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:82:0) after 2m10s
[18:21:05.129] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:82:0)
[18:21:05.156] Build status: STARTED
[18:21:05.156] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:83:0) after 8m37s
[18:29:42.226] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:83:0)
[18:29:42.267] Build status: STARTED
[18:29:42.267] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:84:0) after 5m56s
[18:35:38.293] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:84:0)
[18:35:38.367] Build status: STARTED
[18:35:38.367] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:85:0) after 2m4s
[18:37:42.466] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:85:0)
[18:37:42.491] Build status: STARTED
[18:37:42.491] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:86:0) after 6m37s
[18:44:19.502] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:86:0)
[18:44:19.531] Build status: STARTED
[18:44:19.531] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:87:0) after 3m22s
[18:47:41.643] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:87:0)
[18:47:41.672] Build status: STARTED
[18:47:41.672] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:88:0) after 8m17s
[18:55:58.740] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:88:0)
[18:55:58.768] Build status: STARTED
[18:55:58.768] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:89:0) after 8m34s
[19:04:32.832] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:89:0)
[19:04:32.891] Build status: STARTED
[19:04:32.891] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:90:0) after 3m48s
[19:08:20.914] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:90:0)
[19:08:20.976] Build status: STARTED
[19:08:20.976] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:91:0) after 5m38s
[19:13:58.996] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:91:0)
[19:13:59.028] Build status: STARTED
[19:13:59.028] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:92:0) after 5m41s
[19:19:40.041] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:92:0)
[19:19:40.076] Build status: STARTED
[19:19:40.076] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:93:0) after 7m25s
[19:27:05.058] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:93:0)
[19:27:05.100] Build status: STARTED
[19:27:05.100] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:94:0) after 2m53s
[19:29:58.198] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:94:0)
[19:29:58.241] Build status: STARTED
[19:29:58.241] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:95:0) after 1m21s
[19:31:19.329] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:95:0)
[19:31:19.359] Build status: STARTED
[19:31:19.359] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:96:0) after 7m1s
[19:38:20.392] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:96:0)
[19:38:20.454] Build status: STARTED
[19:38:20.454] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:97:0) after 4m34s
[19:42:54.523] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:97:0)
[19:42:54.551] Build status: STARTED
[19:42:54.551] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:98:0) after 6m56s
[19:49:50.590] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:98:0)
[19:49:50.621] Build status: STARTED
[19:49:50.621] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:99:0) after 9m29s
[19:59:19.962] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:99:0)
[19:59:19.994] Build status: STARTED
[19:59:19.994] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:100:0) after 4m15s
[20:03:35.098] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:100:0)
[20:03:35.123] Build status: STARTED
[20:03:35.123] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:101:0) after 5m44s
[20:09:19.144] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:101:0)
[20:09:19.175] Build status: STARTED
[20:09:19.175] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:102:0) after 2m41s
[20:12:00.194] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:102:0)
[20:12:00.227] Build status: STARTED
[20:12:00.227] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:103:0) after 6m25s
[20:18:25.252] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:103:0)
[20:18:25.299] Build status: STARTED
[20:18:25.299] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:104:0) after 5m51s
[20:24:16.279] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:104:0)
[20:24:16.320] Build status: STARTED
[20:24:16.320] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:105:0) after 8m4s
[20:32:20.361] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:105:0)
[20:32:20.403] Build status: STARTED
[20:32:20.403] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:106:0) after 8m42s
[20:41:02.431] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:106:0)
[20:41:02.480] Build status: STARTED
[20:41:02.480] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:107:0) after 6m15s
[20:47:17.508] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:107:0)
[20:47:17.537] Build status: STARTED
[20:47:17.537] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:108:0) after 1m51s
[20:49:08.752] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:108:0)
[20:49:08.788] Build status: STARTED
[20:49:08.788] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:109:0) after 2m55s
[20:52:03.824] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:109:0)
[20:52:03.856] Build status: STARTED
[20:52:03.856] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:110:0) after 1m21s
[20:53:24.883] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:110:0)
[20:53:24.974] Build status: STARTED
[20:53:24.974] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:111:0) after 6m1s
[20:59:26.084] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:111:0)
[20:59:26.110] Build status: STARTED
[20:59:26.111] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:112:0) after 9m18s
[21:08:44.149] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:112:0)
[21:08:44.182] Build status: STARTED
[21:08:44.182] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:113:0) after 6m57s
[21:15:41.202] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:113:0)
[21:15:41.226] Build status: STARTED
[21:15:41.226] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:114:0) after 8m23s
[21:24:04.245] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:114:0)
[21:24:04.278] Build status: STARTED
[21:24:04.278] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:115:0) after 4m53s
[21:28:57.297] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:115:0)
[21:28:57.344] Build status: STARTED
[21:28:57.345] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:116:0) after 6m8s
[21:35:05.608] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:116:0)
[21:35:05.633] Build status: STARTED
[21:35:05.633] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:117:0) after 6m49s
[21:41:54.740] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:117:0)
[21:41:54.770] Build status: STARTED
[21:41:54.770] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:118:0) after 1m32s
[21:43:26.880] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:118:0)
[21:43:26.920] Build status: STARTED
[21:43:26.920] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:119:0) after 4m24s
[21:47:51.162] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:119:0)
[21:47:51.194] Build status: STARTED
[21:47:51.194] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:120:0) after 1m58s
[21:49:49.491] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:120:0)
[21:49:49.520] Build status: STARTED
[21:49:49.520] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:121:0) after 1m53s
[21:51:42.916] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:121:0)
[21:51:42.941] Build status: STARTED
[21:51:42.941] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:122:0) after 4m13s
[21:55:55.963] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:122:0)
[21:55:55.987] Build status: STARTED
[21:55:55.987] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:123:0) after 5m54s
[22:01:50.008] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:123:0)
[22:01:50.042] Build status: STARTED
[22:01:50.042] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:124:0) after 7m48s
[22:09:38.068] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:124:0)
[22:09:38.107] Build status: STARTED
[22:09:38.107] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:125:0) after 5m20s
[22:14:58.178] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:125:0)
[22:14:58.214] Build status: STARTED
[22:14:58.214] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:126:0) after 3m30s
[22:18:28.545] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:126:0)
[22:18:28.577] Build status: STARTED
[22:18:28.577] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:127:0) after 5m59s
[22:24:27.997] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:127:0)
[22:24:28.049] Build status: STARTED
[22:24:28.049] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:128:0) after 8m56s
[22:33:24.394] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:128:0)
[22:33:24.425] Build status: STARTED
[22:33:24.425] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:129:0) after 4m7s
[22:37:31.603] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:129:0)
[22:37:31.629] Build status: STARTED
[22:37:31.629] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:130:0) after 7m56s
[22:45:27.682] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:130:0)
[22:45:27.715] Build status: STARTED
[22:45:27.715] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:131:0) after 5m8s
[22:50:36.000] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:131:0)
[22:50:36.041] Build status: STARTED
[22:50:36.041] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:132:0) after 6m4s
[22:56:40.068] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:132:0)
[22:56:40.096] Build status: STARTED
[22:56:40.096] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:133:0) after 4m3s
[23:00:43.273] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:133:0)
[23:00:43.318] Build status: STARTED
[23:00:43.318] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:134:0) after 4m8s
[23:04:51.377] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:134:0)
[23:04:51.509] Build status: STARTED
[23:04:51.510] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:135:0) after 4m32s
[23:09:23.749] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:135:0)
[23:09:23.804] Build status: STARTED
[23:09:23.804] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:136:0) after 8m4s
[23:17:27.824] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:136:0)
[23:17:27.853] Build status: STARTED
[23:17:27.853] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:137:0) after 8m27s
[23:25:55.071] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:137:0)
[23:25:55.098] Build status: STARTED
[23:25:55.098] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:138:0) after 6m16s
[23:32:11.525] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:138:0)
[23:32:11.565] Build status: STARTED
[23:32:11.565] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:139:0) after 5m29s
[23:37:40.736] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:139:0)
[23:37:40.833] Build status: STARTED
[23:37:40.833] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:140:0) after 1m24s
[23:39:05.061] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:140:0)
[23:39:05.096] Build status: STARTED
[23:39:05.096] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:141:0) after 8m29s
[23:47:34.327] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:141:0)
[23:47:34.355] Build status: STARTED
[23:47:34.355] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:142:0) after 5m14s
[23:52:48.429] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:142:0)
[23:52:48.475] Build status: STARTED
[23:52:48.475] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:143:0) after 5m7s
[23:57:55.748] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:143:0)
[23:57:55.780] Build status: STARTED
[23:57:55.780] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:144:0) after 4m4s
[00:01:59.802] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:144:0)
[00:01:59.835] Build status: STARTED
[00:01:59.835] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:145:0) after 8m19s
[00:10:19.143] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:145:0)
[00:10:19.205] Build status: STARTED
[00:10:19.205] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:146:0) after 8m48s
[00:19:07.306] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:146:0)
[00:19:07.336] Build status: STARTED
[00:19:07.336] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:147:0) after 2m59s
[00:22:06.927] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:147:0)
[00:22:07.101] Build status: STARTED
[00:22:07.101] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:148:0) after 8m21s
[00:30:28.138] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:148:0)
[00:30:28.175] Build status: STARTED
[00:30:28.175] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:149:0) after 5m13s
[00:35:41.564] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:149:0)
[00:35:41.593] Build status: STARTED
[00:35:41.593] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:150:0) after 7m45s
[00:43:26.763] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:150:0)
[00:43:26.795] Build status: STARTED
[00:43:26.795] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:151:0) after 5m49s
[00:49:15.817] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:151:0)
[00:49:15.850] Build status: STARTED
[00:49:15.850] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:152:0) after 8m54s
[00:58:10.125] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:152:0)
[00:58:10.177] Build status: STARTED
[00:58:10.177] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:153:0) after 4m10s
[01:02:21.126] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:153:0)
[01:02:21.158] Build status: STARTED
[01:02:21.158] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:154:0) after 6m57s
[01:09:18.173] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:154:0)
[01:09:18.207] Build status: STARTED
[01:09:18.207] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:155:0) after 4m10s
[01:13:28.242] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:155:0)
[01:13:28.275] Build status: STARTED
[01:13:28.275] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:156:0) after 1m6s
[01:14:34.294] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:156:0)
[01:14:34.323] Build status: STARTED
[01:14:34.323] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:157:0) after 2m3s
[01:16:37.337] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:157:0)
[01:16:37.388] Build status: STARTED
[01:16:37.388] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:158:0) after 5m56s
[01:22:33.444] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:158:0)
[01:22:33.504] Build status: STARTED
[01:22:33.504] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:159:0) after 7m35s
[01:30:08.727] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:159:0)
[01:30:08.756] Build status: STARTED
[01:30:08.756] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:160:0) after 4m48s
[01:34:56.876] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:160:0)
[01:34:56.908] Build status: STARTED
[01:34:56.908] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:161:0) after 7m21s
[01:42:17.929] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:161:0)
[01:42:17.958] Build status: STARTED
[01:42:17.958] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:162:0) after 7m9s
[01:49:27.033] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:162:0)
[01:49:27.088] Build status: STARTED
[01:49:27.088] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:163:0) after 3m19s
[01:52:46.114] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:163:0)
[01:52:46.154] Build status: STARTED
[01:52:46.154] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:164:0) after 6m23s
[01:59:09.545] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:164:0)
[01:59:09.574] Build status: STARTED
[01:59:09.574] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:165:0) after 8m28s
[02:07:37.923] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:165:0)
[02:07:37.953] Build status: STARTED
[02:07:37.953] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:166:0) after 6m20s
[02:13:58.066] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:166:0)
[02:13:58.098] Build status: STARTED
[02:13:58.098] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:167:0) after 3m19s
[02:17:17.277] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:167:0)
[02:17:17.339] Build status: STARTED
[02:17:17.339] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:168:0) after 3m49s
[02:21:06.368] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:168:0)
[02:21:06.397] Build status: STARTED
[02:21:06.397] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:169:0) after 3m20s
[02:24:26.660] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:169:0)
[02:24:26.688] Build status: STARTED
[02:24:26.688] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:170:0) after 4m39s
[02:29:06.081] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:170:0)
[02:29:06.113] Build status: STARTED
[02:29:06.113] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:171:0) after 7m28s
[02:36:34.136] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:171:0)
[02:36:34.168] Build status: STARTED
[02:36:34.168] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:172:0) after 9m28s
[02:46:02.250] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:172:0)
[02:46:02.284] Build status: STARTED
[02:46:02.284] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:173:0) after 7m39s
[02:53:41.323] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:173:0)
[02:53:41.375] Build status: STARTED
[02:53:41.375] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:174:0) after 5m3s
[02:58:44.393] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:174:0)
[02:58:44.424] Build status: STARTED
[02:58:44.424] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:175:0) after 9m42s
[03:08:26.443] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:175:0)
[03:08:26.476] Build status: STARTED
[03:08:26.476] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:176:0) after 2m27s
[03:10:53.497] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:176:0)
[03:10:53.523] Build status: STARTED
[03:10:53.523] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:177:0) after 3m50s
[03:14:43.591] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:177:0)
[03:14:43.620] Build status: STARTED
[03:14:43.620] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:178:0) after 7m51s
[03:22:34.661] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:178:0)
[03:22:34.718] Build status: STARTED
[03:22:34.719] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:179:0) after 3m51s
[03:26:26.028] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:179:0)
[03:26:26.054] Build status: STARTED
[03:26:26.054] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:180:0) after 9m13s
[03:35:39.169] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:180:0)
[03:35:39.202] Build status: STARTED
[03:35:39.202] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:181:0) after 9m41s
[03:45:20.315] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:181:0)
[03:45:20.355] Build status: STARTED
[03:45:20.355] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:182:0) after 8m9s
[03:53:29.620] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:182:0)
[03:53:29.646] Build status: STARTED
[03:53:29.646] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:183:0) after 1m49s
[03:55:18.765] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:183:0)
[03:55:18.796] Build status: STARTED
[03:55:18.796] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:184:0) after 9m59s
[04:05:17.819] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:184:0)
[04:05:17.860] Build status: STARTED
[04:05:17.860] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999013385609159344:185:0) after 7m41s
[04:11:22.979] Received PubSub notification, asking Buildbucket for the build status
[04:11:23.007] Build:
{
"id": "8833583323493503521",
"builder": {
"project": "fuchsia",
"bucket": "prod",
"builder": "clang-ml-training"
},
"createdBy": "project:fuchsia",
"createTime": "2021-10-12T10:44:44.757222449Z",
"startTime": "2021-10-12T10:44:49.587743Z",
"endTime": "2021-10-13T04:11:19.473704306Z",
"updateTime": "2021-10-13T04:11:19.474062512Z",
"status": "SUCCESS",
"input": {
"gitilesCommit": {
"host": "fuchsia.googlesource.com",
"project": "integration",
"id": "37820bd8b4247a0ab74601b360f66ad3b9146829",
"ref": "refs/heads/main"
}
}
}
[04:11:23.007] Invocation finished in 17h26m39.650739946s with status SUCCEEDED