Status: FAILED
Triggered by: -
Duration: 12 hours
Actions:

Definition (rev 7e7ca5cd047c45b3be7281cd6c03a9eb5b0edf34)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "reclient"
  builder: "Windows Cross deterministic"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "89d961537da4dff8bef27e51ad0913b105bc87b9"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/89d961537da4dff8bef27e51ad0913b105bc87b9
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[02:54:22.968] New invocation is queued and will start shortly
[02:54:24.149] Starting the invocation (attempt 1)
[02:54:24.178] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/89d961537da4dff8bef27e51ad0913b105bc87b9"
[02:54:24.178] Popped gitiles commit info from properties and tags
[02:54:24.178] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[02:54:24.178] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[02:54:24.178] Buildbucket request:
{
  "requestId": "8910076755836825696",
  "builder": {
    "project": "chromium",
    "bucket": "reclient",
    "builder": "Windows Cross deterministic"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "89d961537da4dff8bef27e51ad0913b105bc87b9",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8910076755836825696"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/Windows Cross deterministic"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[02:54:24.806] Scheduled build:
{
  "id": "8744646693848212609",
  "builder": {
    "project": "chromium",
    "bucket": "reclient",
    "builder": "Windows Cross deterministic"
  },
  "number": 2390,
  "createdBy": "project:chromium",
  "createTime": "2024-06-20T02:54:24.238424389Z",
  "updateTime": "2024-06-20T02:54:24.238424389Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "89d961537da4dff8bef27e51ad0913b105bc87b9",
      "ref": "refs/heads/main"
    }
  }
}
[02:54:24.806] Task URL: https://cr-buildbucket.appspot.com/build/8744646693848212609
[02:54:24.806] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:2:0) after 3m45s
[02:54:25.701] Received PubSub notification, asking Buildbucket for the build status
[02:54:25.719] Build status: SCHEDULED
[02:58:09.925] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:2:0)
[02:58:09.952] Build status: SCHEDULED
[02:58:09.952] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:4:0) after 6m3s
[03:04:13.034] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:4:0)
[03:04:13.060] Build status: SCHEDULED
[03:04:13.060] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:5:0) after 7m25s
[03:11:38.103] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:5:0)
[03:11:38.124] Build status: SCHEDULED
[03:11:38.124] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:6:0) after 1m1s
[03:12:39.146] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:6:0)
[03:12:39.161] Build status: SCHEDULED
[03:12:39.161] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:7:0) after 6m50s
[03:19:29.224] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:7:0)
[03:19:29.243] Build status: SCHEDULED
[03:19:29.243] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:8:0) after 6m1s
[03:25:30.413] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:8:0)
[03:25:30.466] Build status: SCHEDULED
[03:25:30.466] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:9:0) after 5m48s
[03:31:18.459] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:9:0)
[03:31:18.486] Build status: SCHEDULED
[03:31:18.486] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:10:0) after 9m46s
[03:41:04.466] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:10:0)
[03:41:04.488] Build status: SCHEDULED
[03:41:04.488] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:11:0) after 5m50s
[03:46:54.641] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:11:0)
[03:46:54.665] Build status: SCHEDULED
[03:46:54.665] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:12:0) after 2m28s
[03:49:22.915] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:12:0)
[03:49:22.931] Build status: SCHEDULED
[03:49:22.931] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:13:0) after 1m1s
[03:50:24.010] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:13:0)
[03:50:24.043] Build status: SCHEDULED
[03:50:24.043] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:14:0) after 9m0s
[03:52:51.083] Received PubSub notification, asking Buildbucket for the build status
[03:52:51.105] Build status: STARTED
[03:59:24.116] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:14:0)
[03:59:24.134] Build status: STARTED
[03:59:24.134] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:16:0) after 1m2s
[04:00:26.140] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:16:0)
[04:00:26.168] Build status: STARTED
[04:00:26.168] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:17:0) after 8m4s
[04:08:30.331] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:17:0)
[04:08:30.350] Build status: STARTED
[04:08:30.351] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:18:0) after 8m22s
[04:16:52.755] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:18:0)
[04:16:52.793] Build status: STARTED
[04:16:52.793] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:19:0) after 9m13s
[04:26:05.853] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:19:0)
[04:26:05.874] Build status: STARTED
[04:26:05.874] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:20:0) after 9m35s
[04:35:40.900] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:20:0)
[04:35:40.934] Build status: STARTED
[04:35:40.934] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:21:0) after 9m48s
[04:45:29.072] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:21:0)
[04:45:29.096] Build status: STARTED
[04:45:29.096] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:22:0) after 5m0s
[04:50:29.079] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:22:0)
[04:50:29.116] Build status: STARTED
[04:50:29.116] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:23:0) after 5m36s
[04:56:05.143] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:23:0)
[04:56:05.279] Build status: STARTED
[04:56:05.279] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:24:0) after 5m6s
[05:01:11.514] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:24:0)
[05:01:11.638] Build status: STARTED
[05:01:11.638] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:25:0) after 2m23s
[05:03:34.742] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:25:0)
[05:03:34.768] Build status: STARTED
[05:03:34.768] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:26:0) after 4m31s
[05:08:05.797] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:26:0)
[05:08:05.818] Build status: STARTED
[05:08:05.818] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:27:0) after 8m13s
[05:16:18.930] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:27:0)
[05:16:18.953] Build status: STARTED
[05:16:18.953] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:28:0) after 5m16s
[05:21:34.977] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:28:0)
[05:21:35.061] Build status: STARTED
[05:21:35.061] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:29:0) after 2m28s
[05:24:03.144] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:29:0)
[05:24:03.169] Build status: STARTED
[05:24:03.169] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:30:0) after 8m46s
[05:32:49.188] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:30:0)
[05:32:49.207] Build status: STARTED
[05:32:49.207] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:31:0) after 2m32s
[05:35:21.224] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:31:0)
[05:35:21.243] Build status: STARTED
[05:35:21.243] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:32:0) after 2m13s
[05:37:34.426] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:32:0)
[05:37:34.469] Build status: STARTED
[05:37:34.470] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:33:0) after 7m0s
[05:44:34.502] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:33:0)
[05:44:34.519] Build status: STARTED
[05:44:34.519] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:34:0) after 6m9s
[05:50:43.562] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:34:0)
[05:50:43.578] Build status: STARTED
[05:50:43.578] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:35:0) after 3m23s
[05:54:06.695] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:35:0)
[05:54:06.711] Build status: STARTED
[05:54:06.711] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:36:0) after 6m49s
[06:00:55.786] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:36:0)
[06:00:55.819] Build status: STARTED
[06:00:55.819] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:37:0) after 5m20s
[06:06:16.290] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:37:0)
[06:06:16.314] Build status: STARTED
[06:06:16.314] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:38:0) after 8m29s
[06:14:45.374] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:38:0)
[06:14:45.402] Build status: STARTED
[06:14:45.402] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:39:0) after 5m43s
[06:20:28.500] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:39:0)
[06:20:28.522] Build status: STARTED
[06:20:28.522] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:40:0) after 7m48s
[06:28:16.569] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:40:0)
[06:28:16.629] Build status: STARTED
[06:28:16.629] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:41:0) after 5m35s
[06:33:51.676] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:41:0)
[06:33:51.823] Build status: STARTED
[06:33:51.823] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:42:0) after 7m53s
[06:41:45.024] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:42:0)
[06:41:45.044] Build status: STARTED
[06:41:45.044] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:43:0) after 5m3s
[06:46:48.068] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:43:0)
[06:46:48.086] Build status: STARTED
[06:46:48.086] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:44:0) after 5m23s
[06:52:11.106] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:44:0)
[06:52:11.125] Build status: STARTED
[06:52:11.125] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:45:0) after 8m50s
[07:01:01.218] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:45:0)
[07:01:01.253] Build status: STARTED
[07:01:01.253] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:46:0) after 1m0s
[07:02:01.326] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:46:0)
[07:02:01.389] Build status: STARTED
[07:02:01.389] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:47:0) after 1m43s
[07:03:44.534] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:47:0)
[07:03:44.566] Build status: STARTED
[07:03:44.566] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:48:0) after 9m37s
[07:13:21.594] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:48:0)
[07:13:21.632] Build status: STARTED
[07:13:21.632] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:49:0) after 3m48s
[07:17:09.868] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:49:0)
[07:17:09.902] Build status: STARTED
[07:17:09.902] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:50:0) after 8m7s
[07:25:16.943] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:50:0)
[07:25:16.959] Build status: STARTED
[07:25:16.959] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:51:0) after 8m50s
[07:34:07.167] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:51:0)
[07:34:07.232] Build status: STARTED
[07:34:07.232] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:52:0) after 8m23s
[07:42:30.255] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:52:0)
[07:42:30.287] Build status: STARTED
[07:42:30.287] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:53:0) after 1m13s
[07:43:43.391] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:53:0)
[07:43:43.416] Build status: STARTED
[07:43:43.416] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:54:0) after 6m1s
[07:49:44.446] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:54:0)
[07:49:44.474] Build status: STARTED
[07:49:44.474] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:55:0) after 1m0s
[07:50:44.735] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:55:0)
[07:50:44.827] Build status: STARTED
[07:50:44.827] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:56:0) after 7m8s
[07:57:53.177] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:56:0)
[07:57:53.194] Build status: STARTED
[07:57:53.194] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:57:0) after 3m10s
[08:01:03.243] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:57:0)
[08:01:03.298] Build status: STARTED
[08:01:03.298] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:58:0) after 9m29s
[08:10:32.419] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:58:0)
[08:10:33.764] Build status: STARTED
[08:10:33.764] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:59:0) after 2m59s
[08:13:32.858] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:59:0)
[08:13:32.881] Build status: STARTED
[08:13:32.881] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:60:0) after 5m50s
[08:19:23.049] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:60:0)
[08:19:23.090] Build status: STARTED
[08:19:23.090] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:61:0) after 9m57s
[08:29:20.092] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:61:0)
[08:29:20.113] Build status: STARTED
[08:29:20.113] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:62:0) after 5m50s
[08:35:10.234] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:62:0)
[08:35:10.296] Build status: STARTED
[08:35:10.296] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:63:0) after 8m9s
[08:43:19.317] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:63:0)
[08:43:19.381] Build status: STARTED
[08:43:19.381] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:64:0) after 4m21s
[08:47:40.545] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:64:0)
[08:47:40.563] Build status: STARTED
[08:47:40.563] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:65:0) after 7m32s
[08:55:12.797] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:65:0)
[08:55:12.815] Build status: STARTED
[08:55:12.815] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:66:0) after 6m16s
[09:01:29.130] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:66:0)
[09:01:29.159] Build status: STARTED
[09:01:29.159] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:67:0) after 5m36s
[09:07:05.635] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:67:0)
[09:07:05.673] Build status: STARTED
[09:07:05.674] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:68:0) after 6m16s
[09:13:22.021] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:68:0)
[09:13:22.048] Build status: STARTED
[09:13:22.048] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:69:0) after 1m43s
[09:15:05.185] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:69:0)
[09:15:05.208] Build status: STARTED
[09:15:05.208] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:70:0) after 9m51s
[09:24:56.235] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:70:0)
[09:24:56.252] Build status: STARTED
[09:24:56.252] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:71:0) after 1m54s
[09:26:50.326] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:71:0)
[09:26:50.423] Build status: STARTED
[09:26:50.423] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:72:0) after 7m8s
[09:33:58.607] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:72:0)
[09:33:58.632] Build status: STARTED
[09:33:58.632] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:73:0) after 3m46s
[09:37:44.663] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:73:0)
[09:37:44.694] Build status: STARTED
[09:37:44.694] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:74:0) after 4m12s
[09:41:56.887] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:74:0)
[09:41:56.906] Build status: STARTED
[09:41:56.906] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:75:0) after 2m31s
[09:44:27.991] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:75:0)
[09:44:28.059] Build status: STARTED
[09:44:28.059] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:76:0) after 6m41s
[09:51:09.182] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:76:0)
[09:51:09.202] Build status: STARTED
[09:51:09.202] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:77:0) after 1m48s
[09:52:57.197] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:77:0)
[09:52:57.238] Build status: STARTED
[09:52:57.238] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:78:0) after 9m56s
[10:02:53.368] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:78:0)
[10:02:53.395] Build status: STARTED
[10:02:53.395] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:79:0) after 8m29s
[10:11:22.417] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:79:0)
[10:11:22.434] Build status: STARTED
[10:11:22.434] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:80:0) after 6m0s
[10:17:22.695] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:80:0)
[10:17:22.730] Build status: STARTED
[10:17:22.730] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:81:0) after 6m8s
[10:23:30.999] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:81:0)
[10:23:31.035] Build status: STARTED
[10:23:31.035] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:82:0) after 3m45s
[10:27:16.404] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:82:0)
[10:27:16.425] Build status: STARTED
[10:27:16.425] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:83:0) after 6m3s
[10:33:19.449] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:83:0)
[10:33:19.467] Build status: STARTED
[10:33:19.467] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:84:0) after 8m51s
[10:42:10.505] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:84:0)
[10:42:10.607] Build status: STARTED
[10:42:10.607] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:85:0) after 9m24s
[10:51:34.748] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:85:0)
[10:51:34.770] Build status: STARTED
[10:51:34.770] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:86:0) after 2m59s
[10:54:33.910] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:86:0)
[10:54:33.931] Build status: STARTED
[10:54:33.932] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:87:0) after 2m43s
[10:57:17.070] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:87:0)
[10:57:17.088] Build status: STARTED
[10:57:17.088] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:88:0) after 9m31s
[11:06:48.191] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:88:0)
[11:06:48.209] Build status: STARTED
[11:06:48.209] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:89:0) after 1m4s
[11:07:52.240] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:89:0)
[11:07:52.262] Build status: STARTED
[11:07:52.262] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:90:0) after 1m2s
[11:08:54.311] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:90:0)
[11:08:54.374] Build status: STARTED
[11:08:54.374] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:91:0) after 6m17s
[11:15:11.696] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:91:0)
[11:15:11.716] Build status: STARTED
[11:15:11.716] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:92:0) after 8m28s
[11:23:39.886] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:92:0)
[11:23:39.918] Build status: STARTED
[11:23:39.918] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:93:0) after 2m11s
[11:25:51.004] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:93:0)
[11:25:51.025] Build status: STARTED
[11:25:51.025] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:94:0) after 7m41s
[11:33:32.307] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:94:0)
[11:33:32.327] Build status: STARTED
[11:33:32.327] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:95:0) after 5m55s
[11:39:27.411] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:95:0)
[11:39:27.429] Build status: STARTED
[11:39:27.429] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:96:0) after 5m50s
[11:45:17.723] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:96:0)
[11:45:17.762] Build status: STARTED
[11:45:17.762] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:97:0) after 1m41s
[11:46:58.789] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:97:0)
[11:46:58.812] Build status: STARTED
[11:46:58.812] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:98:0) after 6m43s
[11:53:41.865] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:98:0)
[11:53:41.919] Build status: STARTED
[11:53:41.919] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:99:0) after 6m44s
[12:00:26.100] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:99:0)
[12:00:26.124] Build status: STARTED
[12:00:26.124] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:100:0) after 1m13s
[12:01:39.157] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:100:0)
[12:01:39.185] Build status: STARTED
[12:01:39.185] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:101:0) after 8m29s
[12:10:08.558] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:101:0)
[12:10:08.626] Build status: STARTED
[12:10:08.626] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:102:0) after 8m35s
[12:18:43.810] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:102:0)
[12:18:43.832] Build status: STARTED
[12:18:43.832] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:103:0) after 5m55s
[12:24:38.991] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:103:0)
[12:24:39.015] Build status: STARTED
[12:24:39.015] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:104:0) after 6m59s
[12:31:38.026] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:104:0)
[12:31:38.047] Build status: STARTED
[12:31:38.047] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:105:0) after 9m32s
[12:41:10.120] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:105:0)
[12:41:10.136] Build status: STARTED
[12:41:10.136] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:106:0) after 6m48s
[12:47:58.153] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:106:0)
[12:47:58.183] Build status: STARTED
[12:47:58.183] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:107:0) after 6m37s
[12:54:35.843] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:107:0)
[12:54:36.008] Build status: STARTED
[12:54:36.008] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:108:0) after 6m8s
[13:00:44.012] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:108:0)
[13:00:44.101] Build status: STARTED
[13:00:44.101] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:109:0) after 3m6s
[13:03:50.453] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:109:0)
[13:03:50.471] Build status: STARTED
[13:03:50.471] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:110:0) after 4m40s
[13:08:30.626] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:110:0)
[13:08:30.828] Build status: STARTED
[13:08:30.828] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:111:0) after 9m13s
[13:17:43.864] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:111:0)
[13:17:43.887] Build status: STARTED
[13:17:43.888] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:112:0) after 7m27s
[13:25:10.925] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:112:0)
[13:25:10.948] Build status: STARTED
[13:25:10.948] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:113:0) after 1m43s
[13:26:53.984] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:113:0)
[13:26:54.009] Build status: STARTED
[13:26:54.009] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:114:0) after 2m42s
[13:29:36.137] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:114:0)
[13:29:36.174] Build status: STARTED
[13:29:36.174] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:115:0) after 2m26s
[13:32:02.212] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:115:0)
[13:32:02.232] Build status: STARTED
[13:32:02.233] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:116:0) after 9m27s
[13:41:29.291] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:116:0)
[13:41:29.309] Build status: STARTED
[13:41:29.309] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:117:0) after 2m13s
[13:43:42.358] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:117:0)
[13:43:42.378] Build status: STARTED
[13:43:42.378] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:118:0) after 7m52s
[13:51:34.400] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:118:0)
[13:51:34.421] Build status: STARTED
[13:51:34.421] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:119:0) after 4m8s
[13:55:42.492] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:119:0)
[13:55:42.517] Build status: STARTED
[13:55:42.517] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:120:0) after 3m30s
[13:59:12.609] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:120:0)
[13:59:12.628] Build status: STARTED
[13:59:12.628] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:121:0) after 9m27s
[14:08:39.651] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:121:0)
[14:08:39.672] Build status: STARTED
[14:08:39.672] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:122:0) after 4m25s
[14:13:04.655] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:122:0)
[14:13:04.680] Build status: STARTED
[14:13:04.680] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:123:0) after 8m29s
[14:21:33.712] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:123:0)
[14:21:33.759] Build status: STARTED
[14:21:33.759] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:124:0) after 9m31s
[14:31:04.883] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:124:0)
[14:31:05.110] Build status: STARTED
[14:31:05.110] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:125:0) after 2m59s
[14:34:04.142] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:125:0)
[14:34:04.176] Build status: STARTED
[14:34:04.176] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:126:0) after 4m20s
[14:38:24.228] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:126:0)
[14:38:24.256] Build status: STARTED
[14:38:24.256] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:127:0) after 1m1s
[14:39:25.285] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:127:0)
[14:39:25.305] Build status: STARTED
[14:39:25.305] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:128:0) after 3m16s
[14:42:41.321] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:128:0)
[14:42:41.365] Build status: STARTED
[14:42:41.365] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:129:0) after 5m50s
[14:48:31.387] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:129:0)
[14:48:31.411] Build status: STARTED
[14:48:31.411] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:130:0) after 8m8s
[14:56:39.434] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:130:0)
[14:56:39.458] Build status: STARTED
[14:56:39.458] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:131:0) after 2m15s
[14:58:54.488] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:131:0)
[14:58:54.517] Build status: STARTED
[14:58:54.517] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:132:0) after 8m15s
[15:07:09.605] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:132:0)
[15:07:09.628] Build status: STARTED
[15:07:09.628] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:133:0) after 2m3s
[15:09:12.666] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:133:0)
[15:09:12.696] Build status: STARTED
[15:09:12.696] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:134:0) after 2m9s
[15:11:21.684] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:134:0)
[15:11:21.714] Build status: STARTED
[15:11:21.714] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:135:0) after 9m6s
[15:20:27.694] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:135:0)
[15:20:27.743] Build status: STARTED
[15:20:27.743] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:136:0) after 4m37s
[15:25:04.780] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:136:0)
[15:25:04.823] Build status: STARTED
[15:25:04.823] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:137:0) after 8m17s
[15:33:21.908] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:137:0)
[15:33:21.926] Build status: STARTED
[15:33:21.926] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:138:0) after 4m59s
[15:38:20.974] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:138:0)
[15:38:20.993] Build status: STARTED
[15:38:20.993] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8910076755836825696:139:0) after 7m27s
[15:38:50.784] Received PubSub notification, asking Buildbucket for the build status
[15:38:50.808] Build:
{
  "id": "8744646693848212609",
  "builder": {
    "project": "chromium",
    "bucket": "reclient",
    "builder": "Windows Cross deterministic"
  },
  "number": 2390,
  "createdBy": "project:chromium",
  "createTime": "2024-06-20T02:54:24.238424389Z",
  "startTime": "2024-06-20T03:52:50.895923047Z",
  "endTime": "2024-06-20T15:38:50.462389470Z",
  "updateTime": "2024-06-20T15:38:50.462389470Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "89d961537da4dff8bef27e51ad0913b105bc87b9",
      "ref": "refs/heads/main"
    }
  }
}
[15:38:50.808] Invocation finished in 12h44m27.849564984s with status FAILED