[01:13:33.162] New invocation is queued and will start shortly
[01:13:33.162] Triggered by user:fuchsia-prod-builder@fuchsia-infra.iam.gserviceaccount.com
[01:13:34.305] Starting the invocation (attempt 1)
[01:13:34.339] Popped buildset tag "buildset:commit/gitiles/fuchsia.googlesource.com/integration/+/37820bd8b4247a0ab74601b360f66ad3b9146829"
[01:13:34.339] Reconstructed gitiles commit from tags
[01:13:34.339] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[01:13:34.339] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[01:13:34.339] Buildbucket request:
{
"requestId": "8999139917494565232",
"builder": {
"project": "fuchsia",
"bucket": "prod",
"builder": "clang-ml-training"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"triggers": [
{
"buildbucket": {
"properties": {
"clang_cas_digest": "ab57d4c3a810442ba2a03ce3e076f2b0390e190f510d0fd121b7f35a529662cc/399",
"corpus_cas_digest": "a1ef92c476507802135c59a21b07c538af08f9723536ce440ab1dabb9cfb3056/247",
"fint_params_paths": {
"arm64": "integration/infra/config/generated/fuchsia/fint_params/prod/clang-ml-training-arm64.textproto",
"x64": "integration/infra/config/generated/fuchsia/fint_params/prod/clang-ml-training-x64.textproto"
},
"fuchsia_git_revision": "3f6d0e8d18fcaab85aaa195bef7d7f57e19b1ec6",
"manifest": "flower",
"remote": "https://fuchsia.googlesource.com/integration",
"warmstart_cas_digest": "23497a9be3a67384589dd5a480a1f8ee11bc5526054c9623989d0ac79744354b/2401"
},
"tags": [
"buildset:commit/gitiles/fuchsia.googlesource.com/integration/+/37820bd8b4247a0ab74601b360f66ad3b9146829",
"gitiles_ref:refs/heads/main",
"parent_buildername:clang-ml-training",
"user_agent:recipe"
]
},
"id": "58a08e06-c48d-4ec4-a398-d1413360bb37",
"title": "clang-ml-training/0"
}
]
},
"clang_cas_digest": "ab57d4c3a810442ba2a03ce3e076f2b0390e190f510d0fd121b7f35a529662cc/399",
"corpus_cas_digest": "a1ef92c476507802135c59a21b07c538af08f9723536ce440ab1dabb9cfb3056/247",
"fint_params_paths": {
"arm64": "integration/infra/config/generated/fuchsia/fint_params/prod/clang-ml-training-arm64.textproto",
"x64": "integration/infra/config/generated/fuchsia/fint_params/prod/clang-ml-training-x64.textproto"
},
"fuchsia_git_revision": "3f6d0e8d18fcaab85aaa195bef7d7f57e19b1ec6",
"manifest": "flower",
"remote": "https://fuchsia.googlesource.com/integration",
"warmstart_cas_digest": "23497a9be3a67384589dd5a480a1f8ee11bc5526054c9623989d0ac79744354b/2401"
},
"gitilesCommit": {
"host": "fuchsia.googlesource.com",
"project": "integration",
"id": "37820bd8b4247a0ab74601b360f66ad3b9146829",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8999139917494565232"
},
{
"key": "scheduler_job_id",
"value": "fuchsia/prod-clang-ml-training"
},
{
"key": "user_agent",
"value": "luci-scheduler"
},
{
"key": "parent_buildername",
"value": "clang-ml-training"
},
{
"key": "user_agent",
"value": "recipe"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[01:13:34.808] Scheduled build:
{
"id": "8833709855505349009",
"builder": {
"project": "fuchsia",
"bucket": "prod",
"builder": "clang-ml-training"
},
"createdBy": "project:fuchsia",
"createTime": "2021-10-11T01:13:34.427286198Z",
"updateTime": "2021-10-11T01:13:34.640134515Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "fuchsia.googlesource.com",
"project": "integration",
"id": "37820bd8b4247a0ab74601b360f66ad3b9146829",
"ref": "refs/heads/main"
}
},
"canary": true
}
[01:13:34.808] Task URL: https://cr-buildbucket.appspot.com/build/8833709855505349009
[01:13:34.809] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:2:0) after 2m52s
[01:15:02.189] Received PubSub notification, asking Buildbucket for the build status
[01:15:02.217] Build status: STARTED
[01:16:26.926] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:2:0)
[01:16:26.976] Build status: STARTED
[01:16:26.976] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:4:0) after 5m9s
[01:21:35.992] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:4:0)
[01:21:36.048] Build status: STARTED
[01:21:36.048] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:5:0) after 9m29s
[01:31:05.339] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:5:0)
[01:31:05.396] Build status: STARTED
[01:31:05.396] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:6:0) after 6m58s
[01:38:03.458] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:6:0)
[01:38:03.489] Build status: STARTED
[01:38:03.489] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:7:0) after 8m45s
[01:46:48.514] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:7:0)
[01:46:48.548] Build status: STARTED
[01:46:48.548] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:8:0) after 2m15s
[01:49:03.564] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:8:0)
[01:49:03.598] Build status: STARTED
[01:49:03.598] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:9:0) after 4m10s
[01:53:13.615] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:9:0)
[01:53:13.647] Build status: STARTED
[01:53:13.647] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:10:0) after 7m13s
[02:00:26.675] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:10:0)
[02:00:26.708] Build status: STARTED
[02:00:26.708] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:11:0) after 6m25s
[02:06:51.976] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:11:0)
[02:06:52.011] Build status: STARTED
[02:06:52.011] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:12:0) after 3m59s
[02:10:51.028] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:12:0)
[02:10:51.071] Build status: STARTED
[02:10:51.071] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:13:0) after 7m43s
[02:18:34.086] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:13:0)
[02:18:34.122] Build status: STARTED
[02:18:34.122] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:14:0) after 4m27s
[02:23:01.350] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:14:0)
[02:23:01.472] Build status: STARTED
[02:23:01.472] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:15:0) after 6m38s
[02:29:39.492] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:15:0)
[02:29:39.570] Build status: STARTED
[02:29:39.570] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:16:0) after 7m17s
[02:36:56.699] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:16:0)
[02:36:56.729] Build status: STARTED
[02:36:56.729] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:17:0) after 3m54s
[02:40:50.788] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:17:0)
[02:40:50.819] Build status: STARTED
[02:40:50.819] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:18:0) after 5m41s
[02:46:31.902] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:18:0)
[02:46:31.934] Build status: STARTED
[02:46:31.934] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:19:0) after 1m54s
[02:48:26.091] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:19:0)
[02:48:26.117] Build status: STARTED
[02:48:26.117] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:20:0) after 2m55s
[02:51:21.262] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:20:0)
[02:51:21.292] Build status: STARTED
[02:51:21.292] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:21:0) after 6m40s
[02:58:01.490] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:21:0)
[02:58:01.545] Build status: STARTED
[02:58:01.545] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:22:0) after 6m41s
[03:04:42.592] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:22:0)
[03:04:42.631] Build status: STARTED
[03:04:42.631] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:23:0) after 4m39s
[03:09:21.649] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:23:0)
[03:09:21.689] Build status: STARTED
[03:09:21.689] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:24:0) after 4m29s
[03:13:50.724] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:24:0)
[03:13:50.762] Build status: STARTED
[03:13:50.762] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:25:0) after 2m54s
[03:16:44.780] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:25:0)
[03:16:44.806] Build status: STARTED
[03:16:44.807] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:26:0) after 2m3s
[03:18:47.831] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:26:0)
[03:18:47.859] Build status: STARTED
[03:18:47.859] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:27:0) after 5m52s
[03:24:39.916] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:27:0)
[03:24:39.942] Build status: STARTED
[03:24:39.942] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:28:0) after 3m24s
[03:28:04.063] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:28:0)
[03:28:04.095] Build status: STARTED
[03:28:04.095] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:29:0) after 3m12s
[03:31:16.114] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:29:0)
[03:31:16.139] Build status: STARTED
[03:31:16.139] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:30:0) after 9m27s
[03:40:43.260] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:30:0)
[03:40:43.288] Build status: STARTED
[03:40:43.288] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:31:0) after 9m2s
[03:49:45.355] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:31:0)
[03:49:45.446] Build status: STARTED
[03:49:45.447] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:32:0) after 9m37s
[03:59:22.468] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:32:0)
[03:59:22.501] Build status: STARTED
[03:59:22.501] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:33:0) after 5m29s
[04:04:51.521] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:33:0)
[04:04:51.553] Build status: STARTED
[04:04:51.553] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:34:0) after 2m28s
[04:07:19.600] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:34:0)
[04:07:19.629] Build status: STARTED
[04:07:19.629] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:35:0) after 2m57s
[04:10:16.661] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:35:0)
[04:10:16.701] Build status: STARTED
[04:10:16.701] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:36:0) after 2m43s
[04:12:59.824] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:36:0)
[04:12:59.851] Build status: STARTED
[04:12:59.851] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:37:0) after 4m48s
[04:17:47.878] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:37:0)
[04:17:47.907] Build status: STARTED
[04:17:47.907] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:38:0) after 2m20s
[04:20:07.927] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:38:0)
[04:20:07.953] Build status: STARTED
[04:20:07.953] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:39:0) after 2m31s
[04:22:39.032] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:39:0)
[04:22:39.104] Build status: STARTED
[04:22:39.104] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:40:0) after 3m34s
[04:26:13.165] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:40:0)
[04:26:13.199] Build status: STARTED
[04:26:13.199] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:41:0) after 8m20s
[04:34:33.462] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:41:0)
[04:34:33.508] Build status: STARTED
[04:34:33.508] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:42:0) after 1m16s
[04:35:49.637] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:42:0)
[04:35:49.664] Build status: STARTED
[04:35:49.664] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:43:0) after 4m19s
[04:40:08.700] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:43:0)
[04:40:08.795] Build status: STARTED
[04:40:08.795] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:44:0) after 4m48s
[04:44:57.105] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:44:0)
[04:44:57.131] Build status: STARTED
[04:44:57.131] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:45:0) after 8m47s
[04:53:44.256] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:45:0)
[04:53:44.313] Build status: STARTED
[04:53:44.313] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:46:0) after 2m10s
[04:55:54.330] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:46:0)
[04:55:54.355] Build status: STARTED
[04:55:54.355] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:47:0) after 5m46s
[05:01:40.376] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:47:0)
[05:01:40.403] Build status: STARTED
[05:01:40.403] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:48:0) after 5m30s
[05:07:10.425] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:48:0)
[05:07:10.450] Build status: STARTED
[05:07:10.450] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:49:0) after 8m17s
[05:15:27.443] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:49:0)
[05:15:27.475] Build status: STARTED
[05:15:27.475] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:50:0) after 5m20s
[05:20:47.494] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:50:0)
[05:20:47.529] Build status: STARTED
[05:20:47.529] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:51:0) after 2m51s
[05:23:38.557] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:51:0)
[05:23:38.587] Build status: STARTED
[05:23:38.587] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:52:0) after 2m26s
[05:26:04.628] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:52:0)
[05:26:04.663] Build status: STARTED
[05:26:04.663] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:53:0) after 5m59s
[05:32:03.689] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:53:0)
[05:32:03.740] Build status: STARTED
[05:32:03.740] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:54:0) after 6m4s
[05:38:07.766] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:54:0)
[05:38:07.802] Build status: STARTED
[05:38:07.802] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:55:0) after 3m23s
[05:41:30.885] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:55:0)
[05:41:30.914] Build status: STARTED
[05:41:30.914] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:56:0) after 5m42s
[05:47:13.114] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:56:0)
[05:47:13.164] Build status: STARTED
[05:47:13.164] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:57:0) after 1m54s
[05:49:07.203] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:57:0)
[05:49:07.415] Build status: STARTED
[05:49:07.415] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:58:0) after 4m45s
[05:53:52.536] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:58:0)
[05:53:52.561] Build status: STARTED
[05:53:52.561] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:59:0) after 9m21s
[06:03:13.633] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:59:0)
[06:03:13.664] Build status: STARTED
[06:03:13.664] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:60:0) after 7m1s
[06:10:14.859] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:60:0)
[06:10:14.893] Build status: STARTED
[06:10:14.893] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:61:0) after 7m53s
[06:18:07.921] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:61:0)
[06:18:07.970] Build status: STARTED
[06:18:07.970] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:62:0) after 1m36s
[06:19:44.017] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:62:0)
[06:19:44.054] Build status: STARTED
[06:19:44.054] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:63:0) after 3m54s
[06:23:38.063] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:63:0)
[06:23:38.087] Build status: STARTED
[06:23:38.087] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:64:0) after 8m42s
[06:32:20.124] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:64:0)
[06:32:20.150] Build status: STARTED
[06:32:20.150] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:65:0) after 2m27s
[06:34:47.172] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:65:0)
[06:34:47.204] Build status: STARTED
[06:34:47.204] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:66:0) after 1m59s
[06:36:46.221] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:66:0)
[06:36:46.249] Build status: STARTED
[06:36:46.249] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:67:0) after 3m12s
[06:39:58.378] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:67:0)
[06:39:58.410] Build status: STARTED
[06:39:58.410] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:68:0) after 6m56s
[06:46:54.579] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:68:0)
[06:46:54.691] Build status: STARTED
[06:46:54.691] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:69:0) after 1m30s
[06:48:24.711] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:69:0)
[06:48:24.740] Build status: STARTED
[06:48:24.740] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:70:0) after 9m30s
[06:57:54.758] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:70:0)
[06:57:54.790] Build status: STARTED
[06:57:54.790] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:71:0) after 8m12s
[07:06:06.908] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:71:0)
[07:06:06.944] Build status: STARTED
[07:06:06.944] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:72:0) after 4m15s
[07:10:21.970] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:72:0)
[07:10:22.058] Build status: STARTED
[07:10:22.058] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:73:0) after 2m15s
[07:12:37.096] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:73:0)
[07:12:37.173] Build status: STARTED
[07:12:37.173] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:74:0) after 8m51s
[07:21:28.185] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:74:0)
[07:21:28.214] Build status: STARTED
[07:21:28.214] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:75:0) after 8m30s
[07:29:58.333] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:75:0)
[07:29:58.363] Build status: STARTED
[07:29:58.363] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:76:0) after 7m32s
[07:37:30.387] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:76:0)
[07:37:30.440] Build status: STARTED
[07:37:30.440] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:77:0) after 5m2s
[07:42:32.459] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:77:0)
[07:42:32.483] Build status: STARTED
[07:42:32.483] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:78:0) after 8m19s
[07:50:51.530] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:78:0)
[07:50:51.558] Build status: STARTED
[07:50:51.558] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:79:0) after 5m26s
[07:56:17.585] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:79:0)
[07:56:17.615] Build status: STARTED
[07:56:17.615] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:80:0) after 5m11s
[08:01:28.644] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:80:0)
[08:01:28.686] Build status: STARTED
[08:01:28.686] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:81:0) after 5m30s
[08:06:58.708] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:81:0)
[08:06:58.754] Build status: STARTED
[08:06:58.754] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:82:0) after 3m58s
[08:10:56.927] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:82:0)
[08:10:56.978] Build status: STARTED
[08:10:56.978] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:83:0) after 4m54s
[08:15:50.977] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:83:0)
[08:15:51.011] Build status: STARTED
[08:15:51.011] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:84:0) after 8m35s
[08:24:26.050] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:84:0)
[08:24:26.085] Build status: STARTED
[08:24:26.085] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:85:0) after 7m45s
[08:32:11.109] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:85:0)
[08:32:11.153] Build status: STARTED
[08:32:11.153] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:86:0) after 4m16s
[08:36:27.163] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:86:0)
[08:36:27.192] Build status: STARTED
[08:36:27.192] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:87:0) after 8m6s
[08:44:33.262] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:87:0)
[08:44:33.295] Build status: STARTED
[08:44:33.295] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:88:0) after 8m3s
[08:52:36.645] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:88:0)
[08:52:36.675] Build status: STARTED
[08:52:36.675] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:89:0) after 4m6s
[08:56:42.976] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:89:0)
[08:56:43.028] Build status: STARTED
[08:56:43.028] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:90:0) after 6m30s
[09:03:13.225] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:90:0)
[09:03:13.252] Build status: STARTED
[09:03:13.252] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:91:0) after 6m3s
[09:09:16.422] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:91:0)
[09:09:16.446] Build status: STARTED
[09:09:16.446] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:92:0) after 8m3s
[09:17:19.461] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:92:0)
[09:17:19.521] Build status: STARTED
[09:17:19.521] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:93:0) after 5m14s
[09:22:33.539] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:93:0)
[09:22:33.572] Build status: STARTED
[09:22:33.572] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:94:0) after 7m23s
[09:29:56.653] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:94:0)
[09:29:56.687] Build status: STARTED
[09:29:56.687] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:95:0) after 6m10s
[09:36:06.755] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:95:0)
[09:36:06.804] Build status: STARTED
[09:36:06.804] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:96:0) after 8m41s
[09:44:47.950] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:96:0)
[09:44:47.981] Build status: STARTED
[09:44:47.981] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:97:0) after 5m39s
[09:50:27.095] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:97:0)
[09:50:27.125] Build status: STARTED
[09:50:27.125] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:98:0) after 7m40s
[09:58:07.175] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:98:0)
[09:58:07.208] Build status: STARTED
[09:58:07.208] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:99:0) after 3m59s
[10:02:06.338] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:99:0)
[10:02:06.375] Build status: STARTED
[10:02:06.375] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:100:0) after 4m44s
[10:06:50.400] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:100:0)
[10:06:50.426] Build status: STARTED
[10:06:50.426] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:101:0) after 2m57s
[10:09:47.555] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:101:0)
[10:09:47.587] Build status: STARTED
[10:09:47.587] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:102:0) after 2m45s
[10:12:32.724] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:102:0)
[10:12:32.754] Build status: STARTED
[10:12:32.754] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:103:0) after 5m32s
[10:18:04.826] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:103:0)
[10:18:04.915] Build status: STARTED
[10:18:04.915] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:104:0) after 8m1s
[10:26:05.938] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:104:0)
[10:26:05.977] Build status: STARTED
[10:26:05.977] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:105:0) after 4m9s
[10:30:15.018] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:105:0)
[10:30:15.046] Build status: STARTED
[10:30:15.046] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:106:0) after 2m28s
[10:32:43.069] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:106:0)
[10:32:43.118] Build status: STARTED
[10:32:43.118] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:107:0) after 7m14s
[10:39:57.176] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:107:0)
[10:39:57.204] Build status: STARTED
[10:39:57.204] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:108:0) after 9m55s
[10:49:52.350] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:108:0)
[10:49:52.390] Build status: STARTED
[10:49:52.391] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:109:0) after 1m16s
[10:51:08.369] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:109:0)
[10:51:08.397] Build status: STARTED
[10:51:08.397] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:110:0) after 2m15s
[10:53:23.428] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:110:0)
[10:53:23.487] Build status: STARTED
[10:53:23.487] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:111:0) after 3m46s
[10:57:09.509] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:111:0)
[10:57:09.537] Build status: STARTED
[10:57:09.537] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:112:0) after 9m59s
[11:07:08.555] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:112:0)
[11:07:08.594] Build status: STARTED
[11:07:08.594] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:113:0) after 4m25s
[11:11:33.611] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:113:0)
[11:11:33.640] Build status: STARTED
[11:11:33.640] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:114:0) after 6m22s
[11:17:55.872] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:114:0)
[11:17:55.906] Build status: STARTED
[11:17:55.906] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:115:0) after 6m55s
[11:24:50.928] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:115:0)
[11:24:50.975] Build status: STARTED
[11:24:50.975] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:116:0) after 5m37s
[11:30:27.994] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:116:0)
[11:30:28.026] Build status: STARTED
[11:30:28.026] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:117:0) after 3m55s
[11:34:23.143] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:117:0)
[11:34:23.265] Build status: STARTED
[11:34:23.265] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:118:0) after 5m27s
[11:39:50.285] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:118:0)
[11:39:50.369] Build status: STARTED
[11:39:50.369] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:119:0) after 7m0s
[11:46:50.564] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:119:0)
[11:46:50.657] Build status: STARTED
[11:46:50.658] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:120:0) after 2m49s
[11:49:39.849] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:120:0)
[11:49:39.888] Build status: STARTED
[11:49:39.888] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:121:0) after 5m38s
[11:55:18.020] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:121:0)
[11:55:18.168] Build status: STARTED
[11:55:18.168] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:122:0) after 1m37s
[11:56:55.406] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:122:0)
[11:56:55.434] Build status: STARTED
[11:56:55.434] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:123:0) after 6m3s
[12:02:58.544] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:123:0)
[12:02:58.585] Build status: STARTED
[12:02:58.585] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:124:0) after 3m42s
[12:06:40.724] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:124:0)
[12:06:40.754] Build status: STARTED
[12:06:40.754] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:125:0) after 8m41s
[12:15:21.775] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:125:0)
[12:15:21.817] Build status: STARTED
[12:15:21.817] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:126:0) after 6m27s
[12:21:48.839] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:126:0)
[12:21:48.982] Build status: STARTED
[12:21:48.982] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:127:0) after 3m27s
[12:25:16.146] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:127:0)
[12:25:16.175] Build status: STARTED
[12:25:16.175] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:128:0) after 2m30s
[12:27:46.197] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:128:0)
[12:27:46.230] Build status: STARTED
[12:27:46.230] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:129:0) after 6m6s
[12:33:52.267] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:129:0)
[12:33:52.298] Build status: STARTED
[12:33:52.298] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:130:0) after 8m54s
[12:42:46.320] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:130:0)
[12:42:46.380] Build status: STARTED
[12:42:46.380] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:131:0) after 4m21s
[12:47:07.399] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:131:0)
[12:47:07.436] Build status: STARTED
[12:47:07.436] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:132:0) after 1m41s
[12:48:48.677] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:132:0)
[12:48:48.715] Build status: STARTED
[12:48:48.715] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:133:0) after 9m0s
[12:57:49.368] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:133:0)
[12:57:49.406] Build status: STARTED
[12:57:49.406] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:134:0) after 9m10s
[13:06:59.438] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:134:0)
[13:07:01.285] Build status: STARTED
[13:07:01.285] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:135:0) after 3m43s
[13:10:44.316] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:135:0)
[13:10:44.360] Build status: STARTED
[13:10:44.360] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:136:0) after 2m58s
[13:13:42.344] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:136:0)
[13:13:42.373] Build status: STARTED
[13:13:42.373] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:137:0) after 1m53s
[13:15:35.407] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:137:0)
[13:15:35.452] Build status: STARTED
[13:15:35.452] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:138:0) after 7m54s
[13:23:29.550] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:138:0)
[13:23:29.581] Build status: STARTED
[13:23:29.581] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:139:0) after 8m6s
[13:31:35.609] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:139:0)
[13:31:35.637] Build status: STARTED
[13:31:35.637] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:140:0) after 4m21s
[13:35:56.662] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:140:0)
[13:35:56.691] Build status: STARTED
[13:35:56.691] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:141:0) after 4m2s
[13:39:58.849] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:141:0)
[13:39:58.886] Build status: STARTED
[13:39:58.887] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:142:0) after 2m12s
[13:42:10.913] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:142:0)
[13:42:10.942] Build status: STARTED
[13:42:10.942] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:143:0) after 1m10s
[13:43:21.027] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:143:0)
[13:43:21.064] Build status: STARTED
[13:43:21.064] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:144:0) after 1m14s
[13:44:35.188] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:144:0)
[13:44:35.215] Build status: STARTED
[13:44:35.215] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:145:0) after 1m55s
[13:46:30.314] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:145:0)
[13:46:30.342] Build status: STARTED
[13:46:30.342] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:146:0) after 5m57s
[13:52:27.362] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:146:0)
[13:52:27.390] Build status: STARTED
[13:52:27.390] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:147:0) after 4m10s
[13:56:37.809] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:147:0)
[13:56:37.837] Build status: STARTED
[13:56:37.837] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:148:0) after 1m54s
[13:58:31.875] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:148:0)
[13:58:31.917] Build status: STARTED
[13:58:31.917] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:149:0) after 4m42s
[14:03:13.937] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:149:0)
[14:03:13.969] Build status: STARTED
[14:03:13.970] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:150:0) after 1m35s
[14:04:49.038] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:150:0)
[14:04:49.079] Build status: STARTED
[14:04:49.079] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:151:0) after 6m14s
[14:11:03.198] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:151:0)
[14:11:03.228] Build status: STARTED
[14:11:03.228] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:152:0) after 4m53s
[14:15:56.252] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:152:0)
[14:15:56.284] Build status: STARTED
[14:15:56.284] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:153:0) after 1m29s
[14:17:25.320] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:153:0)
[14:17:25.420] Build status: STARTED
[14:17:25.420] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:154:0) after 4m13s
[14:21:38.444] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:154:0)
[14:21:38.487] Build status: STARTED
[14:21:38.487] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:155:0) after 1m7s
[14:22:45.512] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:155:0)
[14:22:45.544] Build status: STARTED
[14:22:45.544] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:156:0) after 3m36s
[14:26:21.567] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:156:0)
[14:26:21.594] Build status: STARTED
[14:26:21.594] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:157:0) after 9m53s
[14:36:14.616] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:157:0)
[14:36:14.694] Build status: STARTED
[14:36:14.694] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:158:0) after 3m23s
[14:39:37.711] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:158:0)
[14:39:37.740] Build status: STARTED
[14:39:37.740] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:159:0) after 9m49s
[14:49:26.753] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:159:0)
[14:49:26.782] Build status: STARTED
[14:49:26.782] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:160:0) after 5m17s
[14:54:43.861] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:160:0)
[14:54:43.902] Build status: STARTED
[14:54:43.902] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:161:0) after 8m51s
[15:03:34.985] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:161:0)
[15:03:35.014] Build status: STARTED
[15:03:35.014] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:162:0) after 4m42s
[15:08:17.303] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:162:0)
[15:08:17.359] Build status: STARTED
[15:08:17.359] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:163:0) after 3m0s
[15:11:17.389] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:163:0)
[15:11:17.418] Build status: STARTED
[15:11:17.418] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:164:0) after 4m19s
[15:15:36.480] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:164:0)
[15:15:36.515] Build status: STARTED
[15:15:36.515] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:165:0) after 6m36s
[15:22:12.541] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:165:0)
[15:22:12.569] Build status: STARTED
[15:22:12.569] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:166:0) after 5m53s
[15:28:05.698] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:166:0)
[15:28:05.730] Build status: STARTED
[15:28:05.730] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:167:0) after 4m37s
[15:32:42.738] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:167:0)
[15:32:42.784] Build status: STARTED
[15:32:42.784] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:168:0) after 8m38s
[15:41:20.908] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:168:0)
[15:41:20.934] Build status: STARTED
[15:41:20.934] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:169:0) after 9m5s
[15:50:25.956] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:169:0)
[15:50:25.984] Build status: STARTED
[15:50:25.984] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:170:0) after 2m50s
[15:53:16.254] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:170:0)
[15:53:16.282] Build status: STARTED
[15:53:16.282] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:171:0) after 6m14s
[15:59:30.405] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:171:0)
[15:59:30.445] Build status: STARTED
[15:59:30.445] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:172:0) after 9m50s
[16:09:20.469] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:172:0)
[16:09:20.506] Build status: STARTED
[16:09:20.506] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:173:0) after 9m14s
[16:18:34.526] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:173:0)
[16:18:34.802] Build status: STARTED
[16:18:34.802] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:174:0) after 4m3s
[16:22:37.873] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:174:0)
[16:22:37.926] Build status: STARTED
[16:22:37.926] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:175:0) after 4m27s
[16:27:05.073] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:175:0)
[16:27:05.205] Build status: STARTED
[16:27:05.205] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:176:0) after 4m56s
[16:32:01.352] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:176:0)
[16:32:01.385] Build status: STARTED
[16:32:01.385] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:177:0) after 2m55s
[16:34:56.457] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:177:0)
[16:34:56.494] Build status: STARTED
[16:34:56.494] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:178:0) after 4m23s
[16:39:19.730] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:178:0)
[16:39:19.762] Build status: STARTED
[16:39:19.762] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:179:0) after 5m46s
[16:45:05.786] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:179:0)
[16:45:05.815] Build status: STARTED
[16:45:05.815] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:180:0) after 6m31s
[16:51:37.034] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:180:0)
[16:51:37.076] Build status: STARTED
[16:51:37.076] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:181:0) after 5m5s
[16:56:42.281] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:181:0)
[16:56:42.311] Build status: STARTED
[16:56:42.311] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:182:0) after 8m1s
[17:04:43.320] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:182:0)
[17:04:43.349] Build status: STARTED
[17:04:43.349] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:183:0) after 5m16s
[17:09:59.370] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:183:0)
[17:09:59.400] Build status: STARTED
[17:09:59.400] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:184:0) after 4m20s
[17:14:19.764] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:184:0)
[17:14:19.798] Build status: STARTED
[17:14:19.798] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:185:0) after 4m38s
[17:18:57.899] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:185:0)
[17:18:58.018] Build status: STARTED
[17:18:58.018] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:186:0) after 2m9s
[17:21:07.069] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:186:0)
[17:21:07.109] Build status: STARTED
[17:21:07.109] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:187:0) after 1m59s
[17:23:06.134] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:187:0)
[17:23:06.166] Build status: STARTED
[17:23:06.166] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:188:0) after 4m29s
[17:27:35.189] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:188:0)
[17:27:35.215] Build status: STARTED
[17:27:35.215] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:189:0) after 9m58s
[17:37:33.812] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:189:0)
[17:37:33.839] Build status: STARTED
[17:37:33.839] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:190:0) after 9m27s
[17:47:00.869] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:190:0)
[17:47:00.897] Build status: STARTED
[17:47:00.897] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:191:0) after 1m56s
[17:48:57.204] Handling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:191:0)
[17:48:57.238] Build status: STARTED
[17:48:57.238] Scheduling timer "check-buildbucket-build-status" (fuchsia/prod-clang-ml-training:8999139917494565232:192:0) after 9m18s
[17:55:39.503] Received PubSub notification, asking Buildbucket for the build status
[17:55:39.530] Build:
{
"id": "8833709855505349009",
"builder": {
"project": "fuchsia",
"bucket": "prod",
"builder": "clang-ml-training"
},
"createdBy": "project:fuchsia",
"createTime": "2021-10-11T01:13:34.427286198Z",
"startTime": "2021-10-11T01:15:01.503541Z",
"endTime": "2021-10-11T17:55:39.297966260Z",
"updateTime": "2021-10-11T17:55:39.298526538Z",
"status": "SUCCESS",
"input": {
"gitilesCommit": {
"host": "fuchsia.googlesource.com",
"project": "integration",
"id": "37820bd8b4247a0ab74601b360f66ad3b9146829",
"ref": "refs/heads/main"
}
},
"canary": true
}
[17:55:39.530] Invocation finished in 16h42m6.382516536s with status SUCCEEDED