Status: FAILED
Triggered by: -
Duration: 12 hours
Actions:

Definition (rev 7e7ca5cd047c45b3be7281cd6c03a9eb5b0edf34)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "reclient"
  builder: "Windows Cross deterministic"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "56415bb7ad5750f77b6ea173dd7da2c4b05bdfe4"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/56415bb7ad5750f77b6ea173dd7da2c4b05bdfe4
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[23:20:41.586] New invocation is queued and will start shortly
[23:20:42.686] Starting the invocation (attempt 1)
[23:20:42.720] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/56415bb7ad5750f77b6ea173dd7da2c4b05bdfe4"
[23:20:42.721] Popped gitiles commit info from properties and tags
[23:20:42.721] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[23:20:42.721] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[23:20:42.721] Buildbucket request:
{
  "requestId": "8906738112278401888",
  "builder": {
    "project": "chromium",
    "bucket": "reclient",
    "builder": "Windows Cross deterministic"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "56415bb7ad5750f77b6ea173dd7da2c4b05bdfe4",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8906738112278401888"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/Windows Cross deterministic"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[23:20:43.496] Scheduled build:
{
  "id": "8741308050344685249",
  "builder": {
    "project": "chromium",
    "bucket": "reclient",
    "builder": "Windows Cross deterministic"
  },
  "number": 2471,
  "createdBy": "project:chromium",
  "createTime": "2024-07-26T23:20:42.799311056Z",
  "updateTime": "2024-07-26T23:20:42.799311056Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "56415bb7ad5750f77b6ea173dd7da2c4b05bdfe4",
      "ref": "refs/heads/main"
    }
  }
}
[23:20:43.496] Task URL: https://cr-buildbucket.appspot.com/build/8741308050344685249
[23:20:43.496] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:2:0) after 2m18s
[23:20:44.360] Received PubSub notification, asking Buildbucket for the build status
[23:20:44.387] Build status: SCHEDULED
[23:23:01.521] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:2:0)
[23:23:01.540] Build status: SCHEDULED
[23:23:01.540] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:4:0) after 9m45s
[23:32:05.308] Received PubSub notification, asking Buildbucket for the build status
[23:32:05.329] Build status: STARTED
[23:32:46.586] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:4:0)
[23:32:46.605] Build status: STARTED
[23:32:46.605] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:6:0) after 4m16s
[23:37:02.638] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:6:0)
[23:37:02.664] Build status: STARTED
[23:37:02.664] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:7:0) after 8m50s
[23:45:52.741] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:7:0)
[23:45:52.799] Build status: STARTED
[23:45:52.799] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:8:0) after 4m0s
[23:49:52.815] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:8:0)
[23:49:52.846] Build status: STARTED
[23:49:52.846] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:9:0) after 5m34s
[23:55:26.875] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:9:0)
[23:55:26.912] Build status: STARTED
[23:55:26.913] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:10:0) after 4m16s
[23:59:42.937] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:10:0)
[23:59:42.958] Build status: STARTED
[23:59:42.958] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:11:0) after 7m37s
[00:07:19.984] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:11:0)
[00:07:20.062] Build status: STARTED
[00:07:20.062] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:12:0) after 5m10s
[00:12:30.097] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:12:0)
[00:12:30.118] Build status: STARTED
[00:12:30.118] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:13:0) after 5m42s
[00:18:12.156] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:13:0)
[00:18:12.296] Build status: STARTED
[00:18:12.296] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:14:0) after 8m41s
[00:26:53.325] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:14:0)
[00:26:53.375] Build status: STARTED
[00:26:53.375] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:15:0) after 1m55s
[00:28:48.357] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:15:0)
[00:28:48.377] Build status: STARTED
[00:28:48.377] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:16:0) after 5m50s
[00:34:38.406] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:16:0)
[00:34:38.442] Build status: STARTED
[00:34:38.442] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:17:0) after 4m22s
[00:39:00.478] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:17:0)
[00:39:00.499] Build status: STARTED
[00:39:00.499] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:18:0) after 7m28s
[00:46:28.577] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:18:0)
[00:46:28.597] Build status: STARTED
[00:46:28.597] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:19:0) after 4m47s
[00:51:15.627] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:19:0)
[00:51:15.648] Build status: STARTED
[00:51:15.648] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:20:0) after 9m43s
[01:00:58.681] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:20:0)
[01:00:58.696] Build status: STARTED
[01:00:58.696] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:21:0) after 9m38s
[01:10:36.742] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:21:0)
[01:10:36.764] Build status: STARTED
[01:10:36.765] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:22:0) after 2m45s
[01:13:21.795] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:22:0)
[01:13:21.817] Build status: STARTED
[01:13:21.817] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:23:0) after 2m27s
[01:15:48.787] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:23:0)
[01:15:48.843] Build status: STARTED
[01:15:48.843] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:24:0) after 1m9s
[01:16:57.924] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:24:0)
[01:16:57.944] Build status: STARTED
[01:16:57.944] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:25:0) after 9m37s
[01:26:34.972] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:25:0)
[01:26:34.989] Build status: STARTED
[01:26:34.989] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:26:0) after 9m52s
[01:36:27.017] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:26:0)
[01:36:27.039] Build status: STARTED
[01:36:27.039] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:27:0) after 7m3s
[01:43:30.115] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:27:0)
[01:43:30.147] Build status: STARTED
[01:43:30.148] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:28:0) after 1m32s
[01:45:02.186] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:28:0)
[01:45:02.206] Build status: STARTED
[01:45:02.206] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:29:0) after 7m53s
[01:52:55.299] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:29:0)
[01:52:55.317] Build status: STARTED
[01:52:55.317] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:30:0) after 2m31s
[01:55:26.367] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:30:0)
[01:55:26.417] Build status: STARTED
[01:55:26.417] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:31:0) after 5m56s
[02:01:22.454] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:31:0)
[02:01:22.509] Build status: STARTED
[02:01:22.509] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:32:0) after 2m9s
[02:03:31.538] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:32:0)
[02:03:31.555] Build status: STARTED
[02:03:31.555] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:33:0) after 9m20s
[02:12:52.435] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:33:0)
[02:12:52.480] Build status: STARTED
[02:12:52.480] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:34:0) after 2m58s
[02:15:50.624] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:34:0)
[02:15:50.695] Build status: STARTED
[02:15:50.695] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:35:0) after 6m40s
[02:22:31.190] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:35:0)
[02:22:31.240] Build status: STARTED
[02:22:31.240] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:36:0) after 3m8s
[02:25:39.416] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:36:0)
[02:25:39.438] Build status: STARTED
[02:25:39.438] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:37:0) after 2m28s
[02:28:07.602] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:37:0)
[02:28:07.622] Build status: STARTED
[02:28:07.622] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:38:0) after 2m6s
[02:30:13.788] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:38:0)
[02:30:13.810] Build status: STARTED
[02:30:13.810] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:39:0) after 3m44s
[02:33:57.921] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:39:0)
[02:33:57.939] Build status: STARTED
[02:33:57.939] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:40:0) after 2m13s
[02:36:11.003] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:40:0)
[02:36:11.133] Build status: STARTED
[02:36:11.133] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:41:0) after 4m27s
[02:40:38.820] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:41:0)
[02:40:38.839] Build status: STARTED
[02:40:38.839] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:42:0) after 7m22s
[02:48:00.976] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:42:0)
[02:48:01.001] Build status: STARTED
[02:48:01.001] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:43:0) after 5m1s
[02:53:02.084] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:43:0)
[02:53:02.150] Build status: STARTED
[02:53:02.150] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:44:0) after 3m7s
[02:56:09.163] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:44:0)
[02:56:09.181] Build status: STARTED
[02:56:09.181] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:45:0) after 6m30s
[03:02:39.277] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:45:0)
[03:02:39.330] Build status: STARTED
[03:02:39.330] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:46:0) after 7m42s
[03:10:21.385] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:46:0)
[03:10:21.460] Build status: STARTED
[03:10:21.460] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:47:0) after 2m11s
[03:12:32.689] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:47:0)
[03:12:32.719] Build status: STARTED
[03:12:32.719] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:48:0) after 9m7s
[03:21:40.017] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:48:0)
[03:21:40.036] Build status: STARTED
[03:21:40.036] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:49:0) after 2m22s
[03:24:02.167] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:49:0)
[03:24:02.186] Build status: STARTED
[03:24:02.186] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:50:0) after 3m8s
[03:27:10.661] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:50:0)
[03:27:10.687] Build status: STARTED
[03:27:10.687] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:51:0) after 1m5s
[03:28:15.813] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:51:0)
[03:28:15.832] Build status: STARTED
[03:28:15.832] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:52:0) after 8m53s
[03:37:08.912] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:52:0)
[03:37:08.935] Build status: STARTED
[03:37:08.935] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:53:0) after 4m35s
[03:41:44.099] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:53:0)
[03:41:44.128] Build status: STARTED
[03:41:44.128] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:54:0) after 2m14s
[03:43:58.153] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:54:0)
[03:43:58.232] Build status: STARTED
[03:43:58.232] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:55:0) after 7m56s
[03:51:54.278] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:55:0)
[03:51:54.295] Build status: STARTED
[03:51:54.296] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:56:0) after 6m56s
[03:58:50.409] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:56:0)
[03:58:50.447] Build status: STARTED
[03:58:50.447] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:57:0) after 3m18s
[04:02:08.481] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:57:0)
[04:02:08.504] Build status: STARTED
[04:02:08.504] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:58:0) after 2m40s
[04:04:48.743] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:58:0)
[04:04:48.767] Build status: STARTED
[04:04:48.767] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:59:0) after 7m11s
[04:12:00.087] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:59:0)
[04:12:00.107] Build status: STARTED
[04:12:00.107] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:60:0) after 2m13s
[04:14:13.422] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:60:0)
[04:14:13.448] Build status: STARTED
[04:14:13.449] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:61:0) after 5m25s
[04:19:38.619] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:61:0)
[04:19:38.641] Build status: STARTED
[04:19:38.641] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:62:0) after 3m23s
[04:23:01.736] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:62:0)
[04:23:01.757] Build status: STARTED
[04:23:01.757] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:63:0) after 6m34s
[04:29:36.056] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:63:0)
[04:29:36.168] Build status: STARTED
[04:29:36.168] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:64:0) after 4m59s
[04:34:35.202] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:64:0)
[04:34:35.227] Build status: STARTED
[04:34:35.227] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:65:0) after 5m38s
[04:40:13.254] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:65:0)
[04:40:13.272] Build status: STARTED
[04:40:13.272] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:66:0) after 5m4s
[04:45:17.385] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:66:0)
[04:45:17.485] Build status: STARTED
[04:45:17.485] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:67:0) after 2m43s
[04:48:00.535] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:67:0)
[04:48:00.589] Build status: STARTED
[04:48:00.589] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:68:0) after 8m32s
[04:56:32.679] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:68:0)
[04:56:32.696] Build status: STARTED
[04:56:32.696] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:69:0) after 5m40s
[05:02:12.761] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:69:0)
[05:02:12.781] Build status: STARTED
[05:02:12.781] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:70:0) after 7m38s
[05:09:50.944] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:70:0)
[05:09:50.963] Build status: STARTED
[05:09:50.963] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:71:0) after 5m43s
[05:15:34.008] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:71:0)
[05:15:34.031] Build status: STARTED
[05:15:34.031] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:72:0) after 4m2s
[05:19:36.169] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:72:0)
[05:19:36.186] Build status: STARTED
[05:19:36.186] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:73:0) after 4m34s
[05:24:10.232] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:73:0)
[05:24:10.287] Build status: STARTED
[05:24:10.287] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:74:0) after 9m6s
[05:33:16.485] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:74:0)
[05:33:16.514] Build status: STARTED
[05:33:16.514] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:75:0) after 5m25s
[05:38:41.861] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:75:0)
[05:38:41.916] Build status: STARTED
[05:38:41.916] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:76:0) after 7m47s
[05:46:28.963] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:76:0)
[05:46:29.042] Build status: STARTED
[05:46:29.042] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:77:0) after 1m28s
[05:47:57.175] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:77:0)
[05:47:57.193] Build status: STARTED
[05:47:57.193] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:78:0) after 4m49s
[05:52:46.467] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:78:0)
[05:52:46.485] Build status: STARTED
[05:52:46.485] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:79:0) after 9m6s
[06:01:52.955] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:79:0)
[06:01:53.022] Build status: STARTED
[06:01:53.022] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:80:0) after 7m35s
[06:09:28.326] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:80:0)
[06:09:28.346] Build status: STARTED
[06:09:28.346] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:81:0) after 7m49s
[06:17:17.491] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:81:0)
[06:17:17.508] Build status: STARTED
[06:17:17.508] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:82:0) after 4m12s
[06:21:29.804] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:82:0)
[06:21:29.863] Build status: STARTED
[06:21:29.863] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:83:0) after 4m15s
[06:25:44.868] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:83:0)
[06:25:44.904] Build status: STARTED
[06:25:44.904] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:84:0) after 4m6s
[06:29:51.182] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:84:0)
[06:29:51.204] Build status: STARTED
[06:29:51.204] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:85:0) after 8m56s
[06:38:47.214] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:85:0)
[06:38:47.231] Build status: STARTED
[06:38:47.231] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:86:0) after 4m13s
[06:43:00.265] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:86:0)
[06:43:00.299] Build status: STARTED
[06:43:00.299] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:87:0) after 4m10s
[06:47:10.481] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:87:0)
[06:47:10.502] Build status: STARTED
[06:47:10.502] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:88:0) after 3m32s
[06:50:42.607] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:88:0)
[06:50:42.645] Build status: STARTED
[06:50:42.645] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:89:0) after 5m30s
[06:56:12.798] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:89:0)
[06:56:12.855] Build status: STARTED
[06:56:12.855] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:90:0) after 7m11s
[07:03:23.936] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:90:0)
[07:03:23.964] Build status: STARTED
[07:03:23.964] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:91:0) after 8m10s
[07:11:34.183] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:91:0)
[07:11:34.209] Build status: STARTED
[07:11:34.209] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:92:0) after 7m35s
[07:19:09.471] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:92:0)
[07:19:09.505] Build status: STARTED
[07:19:09.505] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:93:0) after 1m4s
[07:20:13.531] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:93:0)
[07:20:13.549] Build status: STARTED
[07:20:13.549] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:94:0) after 5m50s
[07:26:03.603] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:94:0)
[07:26:03.621] Build status: STARTED
[07:26:03.621] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:95:0) after 2m4s
[07:28:07.789] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:95:0)
[07:28:07.843] Build status: STARTED
[07:28:07.843] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:96:0) after 9m15s
[07:37:23.195] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:96:0)
[07:37:23.238] Build status: STARTED
[07:37:23.238] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:97:0) after 1m43s
[07:39:06.313] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:97:0)
[07:39:06.348] Build status: STARTED
[07:39:06.348] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:98:0) after 1m57s
[07:41:03.388] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:98:0)
[07:41:03.410] Build status: STARTED
[07:41:03.410] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:99:0) after 8m48s
[07:49:51.682] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:99:0)
[07:49:51.704] Build status: STARTED
[07:49:51.704] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:100:0) after 6m44s
[07:56:35.954] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:100:0)
[07:56:35.973] Build status: STARTED
[07:56:35.973] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:101:0) after 7m11s
[08:03:47.177] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:101:0)
[08:03:47.196] Build status: STARTED
[08:03:47.196] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:102:0) after 4m5s
[08:07:52.439] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:102:0)
[08:07:52.461] Build status: STARTED
[08:07:52.461] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:103:0) after 8m53s
[08:16:45.574] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:103:0)
[08:16:45.597] Build status: STARTED
[08:16:45.597] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:104:0) after 8m3s
[08:24:48.627] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:104:0)
[08:24:48.663] Build status: STARTED
[08:24:48.663] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:105:0) after 6m56s
[08:31:44.678] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:105:0)
[08:31:44.700] Build status: STARTED
[08:31:44.700] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:106:0) after 7m15s
[08:38:59.740] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:106:0)
[08:38:59.761] Build status: STARTED
[08:38:59.761] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:107:0) after 3m18s
[08:42:17.798] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:107:0)
[08:42:17.824] Build status: STARTED
[08:42:17.824] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:108:0) after 3m58s
[08:46:15.905] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:108:0)
[08:46:15.929] Build status: STARTED
[08:46:15.929] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:109:0) after 6m6s
[08:52:22.275] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:109:0)
[08:52:22.321] Build status: STARTED
[08:52:22.321] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:110:0) after 5m50s
[08:58:12.353] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:110:0)
[08:58:12.386] Build status: STARTED
[08:58:12.386] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:111:0) after 9m12s
[09:07:24.521] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:111:0)
[09:07:24.551] Build status: STARTED
[09:07:24.551] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:112:0) after 6m38s
[09:14:02.752] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:112:0)
[09:14:02.811] Build status: STARTED
[09:14:02.811] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:113:0) after 4m51s
[09:18:53.865] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:113:0)
[09:18:53.886] Build status: STARTED
[09:18:53.886] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:114:0) after 1m57s
[09:20:50.940] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:114:0)
[09:20:50.996] Build status: STARTED
[09:20:50.996] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:115:0) after 5m19s
[09:26:10.067] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:115:0)
[09:26:10.088] Build status: STARTED
[09:26:10.088] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:116:0) after 1m23s
[09:27:33.135] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:116:0)
[09:27:33.155] Build status: STARTED
[09:27:33.155] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:117:0) after 3m51s
[09:31:24.180] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:117:0)
[09:31:24.219] Build status: STARTED
[09:31:24.219] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:118:0) after 2m26s
[09:33:50.249] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:118:0)
[09:33:50.267] Build status: STARTED
[09:33:50.267] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:119:0) after 5m30s
[09:39:20.319] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:119:0)
[09:39:20.380] Build status: STARTED
[09:39:20.380] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:120:0) after 8m49s
[09:48:09.427] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:120:0)
[09:48:09.448] Build status: STARTED
[09:48:09.448] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:121:0) after 8m53s
[09:57:02.495] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:121:0)
[09:57:02.561] Build status: STARTED
[09:57:02.561] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:122:0) after 1m7s
[09:58:09.602] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:122:0)
[09:58:09.622] Build status: STARTED
[09:58:09.622] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:123:0) after 9m21s
[10:07:30.634] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:123:0)
[10:07:30.681] Build status: STARTED
[10:07:30.681] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:124:0) after 6m15s
[10:13:45.776] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:124:0)
[10:13:45.796] Build status: STARTED
[10:13:45.796] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:125:0) after 7m4s
[10:20:49.923] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:125:0)
[10:20:49.943] Build status: STARTED
[10:20:49.943] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:126:0) after 9m55s
[10:30:45.082] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:126:0)
[10:30:45.100] Build status: STARTED
[10:30:45.100] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:127:0) after 1m24s
[10:32:09.229] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:127:0)
[10:32:09.249] Build status: STARTED
[10:32:09.249] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:128:0) after 6m14s
[10:38:23.457] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:128:0)
[10:38:23.476] Build status: STARTED
[10:38:23.476] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:129:0) after 9m11s
[10:47:34.472] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:129:0)
[10:47:34.492] Build status: STARTED
[10:47:34.492] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:130:0) after 7m3s
[10:54:37.492] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:130:0)
[10:54:37.508] Build status: STARTED
[10:54:37.508] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:131:0) after 9m47s
[11:04:24.689] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:131:0)
[11:04:24.715] Build status: STARTED
[11:04:24.715] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:132:0) after 6m51s
[11:11:15.748] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:132:0)
[11:11:15.769] Build status: STARTED
[11:11:15.769] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:133:0) after 9m59s
[11:21:14.816] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:133:0)
[11:21:14.836] Build status: STARTED
[11:21:14.836] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:134:0) after 6m42s
[11:27:56.865] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:134:0)
[11:27:56.894] Build status: STARTED
[11:27:56.894] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:135:0) after 1m55s
[11:29:51.920] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:135:0)
[11:29:51.939] Build status: STARTED
[11:29:51.939] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:136:0) after 2m10s
[11:32:02.076] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:136:0)
[11:32:02.209] Build status: STARTED
[11:32:02.209] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:137:0) after 9m41s
[11:41:43.449] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:137:0)
[11:41:43.543] Build status: STARTED
[11:41:43.543] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:138:0) after 8m10s
[11:49:53.673] Handling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:138:0)
[11:49:53.692] Build status: STARTED
[11:49:53.692] Scheduling timer "check-buildbucket-build-status" (chromium/Windows Cross deterministic:8906738112278401888:139:0) after 7m26s
[11:54:35.777] Received PubSub notification, asking Buildbucket for the build status
[11:54:35.798] Build:
{
  "id": "8741308050344685249",
  "builder": {
    "project": "chromium",
    "bucket": "reclient",
    "builder": "Windows Cross deterministic"
  },
  "number": 2471,
  "createdBy": "project:chromium",
  "createTime": "2024-07-26T23:20:42.799311056Z",
  "startTime": "2024-07-26T23:31:41.611829458Z",
  "endTime": "2024-07-27T11:54:35.530571853Z",
  "updateTime": "2024-07-27T11:54:35.530571853Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "56415bb7ad5750f77b6ea173dd7da2c4b05bdfe4",
      "ref": "refs/heads/main"
    }
  }
}
[11:54:35.798] Invocation finished in 12h33m54.227125584s with status FAILED