[21:21:28.677] New invocation is queued and will start shortly
[21:21:29.795] Starting the invocation (attempt 1)
[21:21:29.818] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/f0fd8115bc3287fd318a59699b134cfdc003cda7"
[21:21:29.818] Popped gitiles commit info from properties and tags
[21:21:29.818] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[21:21:29.818] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[21:21:29.819] Buildbucket request:
{
"requestId": "8881016074188901440",
"builder": {
"project": "v8",
"bucket": "ci",
"builder": "Linux V8 API Stability"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8881016074188901440",
"job": "v8/Linux V8 API Stability",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "5697d41da2c74b55bee97a1984402e6e7b1bc99e"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@5697d41da2c74b55bee97a1984402e6e7b1bc99e",
"title": "5697d41da2c74b55bee97a1984402e6e7b1bc99e",
"url": "https://chromium.googlesource.com/chromium/src/+/5697d41da2c74b55bee97a1984402e6e7b1bc99e"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "6684e58a692d62f297f6b95d1782a152eb6e6aee"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@6684e58a692d62f297f6b95d1782a152eb6e6aee",
"title": "6684e58a692d62f297f6b95d1782a152eb6e6aee",
"url": "https://chromium.googlesource.com/chromium/src/+/6684e58a692d62f297f6b95d1782a152eb6e6aee"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "78a1a291695f5d17c03c686f47d5756fe6e79034"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@78a1a291695f5d17c03c686f47d5756fe6e79034",
"title": "78a1a291695f5d17c03c686f47d5756fe6e79034",
"url": "https://chromium.googlesource.com/chromium/src/+/78a1a291695f5d17c03c686f47d5756fe6e79034"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "316c191d4997b8dd293368aa104a441b776fd4a4"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@316c191d4997b8dd293368aa104a441b776fd4a4",
"title": "316c191d4997b8dd293368aa104a441b776fd4a4",
"url": "https://chromium.googlesource.com/chromium/src/+/316c191d4997b8dd293368aa104a441b776fd4a4"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "ab4cb71a26214771ad1fb1b89a43918d6d90cfba"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@ab4cb71a26214771ad1fb1b89a43918d6d90cfba",
"title": "ab4cb71a26214771ad1fb1b89a43918d6d90cfba",
"url": "https://chromium.googlesource.com/chromium/src/+/ab4cb71a26214771ad1fb1b89a43918d6d90cfba"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "f96385c039287dab860378f679e29563f9be0c0f"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@f96385c039287dab860378f679e29563f9be0c0f",
"title": "f96385c039287dab860378f679e29563f9be0c0f",
"url": "https://chromium.googlesource.com/chromium/src/+/f96385c039287dab860378f679e29563f9be0c0f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "70e227012a77e58b1ad4e6cdf312cbdb41bbf7d4"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@70e227012a77e58b1ad4e6cdf312cbdb41bbf7d4",
"title": "70e227012a77e58b1ad4e6cdf312cbdb41bbf7d4",
"url": "https://chromium.googlesource.com/chromium/src/+/70e227012a77e58b1ad4e6cdf312cbdb41bbf7d4"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "c59eeff40f493459618031de68c3abdc932d1d8d"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@c59eeff40f493459618031de68c3abdc932d1d8d",
"title": "c59eeff40f493459618031de68c3abdc932d1d8d",
"url": "https://chromium.googlesource.com/chromium/src/+/c59eeff40f493459618031de68c3abdc932d1d8d"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "eaccc8fd8dc0e28a8f168b0c53d1682d25133ab3"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@eaccc8fd8dc0e28a8f168b0c53d1682d25133ab3",
"title": "eaccc8fd8dc0e28a8f168b0c53d1682d25133ab3",
"url": "https://chromium.googlesource.com/chromium/src/+/eaccc8fd8dc0e28a8f168b0c53d1682d25133ab3"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "d1ed30295c1d4410d7ddd2168d4ee039f68c05a1"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@d1ed30295c1d4410d7ddd2168d4ee039f68c05a1",
"title": "d1ed30295c1d4410d7ddd2168d4ee039f68c05a1",
"url": "https://chromium.googlesource.com/chromium/src/+/d1ed30295c1d4410d7ddd2168d4ee039f68c05a1"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "3f0ac3706fb37e5b553edce092581ea5f52b44e9"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@3f0ac3706fb37e5b553edce092581ea5f52b44e9",
"title": "3f0ac3706fb37e5b553edce092581ea5f52b44e9",
"url": "https://chromium.googlesource.com/chromium/src/+/3f0ac3706fb37e5b553edce092581ea5f52b44e9"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "aa8fb1e131af6075a47eda1cf426d11582836b19"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@aa8fb1e131af6075a47eda1cf426d11582836b19",
"title": "aa8fb1e131af6075a47eda1cf426d11582836b19",
"url": "https://chromium.googlesource.com/chromium/src/+/aa8fb1e131af6075a47eda1cf426d11582836b19"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "2d3c9cbf35d902e2b83311561030c5cd568e419b"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@2d3c9cbf35d902e2b83311561030c5cd568e419b",
"title": "2d3c9cbf35d902e2b83311561030c5cd568e419b",
"url": "https://chromium.googlesource.com/chromium/src/+/2d3c9cbf35d902e2b83311561030c5cd568e419b"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "a318f12017337a3a228571b992c096153bf0a5f5"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@a318f12017337a3a228571b992c096153bf0a5f5",
"title": "a318f12017337a3a228571b992c096153bf0a5f5",
"url": "https://chromium.googlesource.com/chromium/src/+/a318f12017337a3a228571b992c096153bf0a5f5"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "1d981ba545a81ad0fbbafe0384847dd5f8116b1f"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@1d981ba545a81ad0fbbafe0384847dd5f8116b1f",
"title": "1d981ba545a81ad0fbbafe0384847dd5f8116b1f",
"url": "https://chromium.googlesource.com/chromium/src/+/1d981ba545a81ad0fbbafe0384847dd5f8116b1f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "ddae644ffc50d7b8ea90deaf9f51931670c15c5f"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@ddae644ffc50d7b8ea90deaf9f51931670c15c5f",
"title": "ddae644ffc50d7b8ea90deaf9f51931670c15c5f",
"url": "https://chromium.googlesource.com/chromium/src/+/ddae644ffc50d7b8ea90deaf9f51931670c15c5f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "9a5c17cb2a8ecc8835d8d98a9b3ac0fd3092c6c8"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@9a5c17cb2a8ecc8835d8d98a9b3ac0fd3092c6c8",
"title": "9a5c17cb2a8ecc8835d8d98a9b3ac0fd3092c6c8",
"url": "https://chromium.googlesource.com/chromium/src/+/9a5c17cb2a8ecc8835d8d98a9b3ac0fd3092c6c8"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "f8d2ac02032d313e079d1ce84a52334588578d87"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@f8d2ac02032d313e079d1ce84a52334588578d87",
"title": "f8d2ac02032d313e079d1ce84a52334588578d87",
"url": "https://chromium.googlesource.com/chromium/src/+/f8d2ac02032d313e079d1ce84a52334588578d87"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "d637dbeb7e5de40a7f22306a1e0399effab8b3a8"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@d637dbeb7e5de40a7f22306a1e0399effab8b3a8",
"title": "d637dbeb7e5de40a7f22306a1e0399effab8b3a8",
"url": "https://chromium.googlesource.com/chromium/src/+/d637dbeb7e5de40a7f22306a1e0399effab8b3a8"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "4ba11f910ffd01224f1ad253fd899b3f0f32af77"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@4ba11f910ffd01224f1ad253fd899b3f0f32af77",
"title": "4ba11f910ffd01224f1ad253fd899b3f0f32af77",
"url": "https://chromium.googlesource.com/chromium/src/+/4ba11f910ffd01224f1ad253fd899b3f0f32af77"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "d692e77347323a9b737870925efd0684963f7225"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@d692e77347323a9b737870925efd0684963f7225",
"title": "d692e77347323a9b737870925efd0684963f7225",
"url": "https://chromium.googlesource.com/chromium/src/+/d692e77347323a9b737870925efd0684963f7225"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "818bdc4bcec3b850488b067bb3def7790753c5c0"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@818bdc4bcec3b850488b067bb3def7790753c5c0",
"title": "818bdc4bcec3b850488b067bb3def7790753c5c0",
"url": "https://chromium.googlesource.com/chromium/src/+/818bdc4bcec3b850488b067bb3def7790753c5c0"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "b9b65a7d43a92feb980244123420ebb7e00ceb5f"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@b9b65a7d43a92feb980244123420ebb7e00ceb5f",
"title": "b9b65a7d43a92feb980244123420ebb7e00ceb5f",
"url": "https://chromium.googlesource.com/chromium/src/+/b9b65a7d43a92feb980244123420ebb7e00ceb5f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "5be8c1d0c8c1c7582d419282a21e52e34a9fff86"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@5be8c1d0c8c1c7582d419282a21e52e34a9fff86",
"title": "5be8c1d0c8c1c7582d419282a21e52e34a9fff86",
"url": "https://chromium.googlesource.com/chromium/src/+/5be8c1d0c8c1c7582d419282a21e52e34a9fff86"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "cd4c262cd5c49be15b466b62b33bec42b2359c08"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@cd4c262cd5c49be15b466b62b33bec42b2359c08",
"title": "cd4c262cd5c49be15b466b62b33bec42b2359c08",
"url": "https://chromium.googlesource.com/chromium/src/+/cd4c262cd5c49be15b466b62b33bec42b2359c08"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "ba185417f8f00e96c06bcf8789b32fdb8e38dab2"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@ba185417f8f00e96c06bcf8789b32fdb8e38dab2",
"title": "ba185417f8f00e96c06bcf8789b32fdb8e38dab2",
"url": "https://chromium.googlesource.com/chromium/src/+/ba185417f8f00e96c06bcf8789b32fdb8e38dab2"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "0d66293985de1518298e3cbbf9016d9c9d8a6548"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@0d66293985de1518298e3cbbf9016d9c9d8a6548",
"title": "0d66293985de1518298e3cbbf9016d9c9d8a6548",
"url": "https://chromium.googlesource.com/chromium/src/+/0d66293985de1518298e3cbbf9016d9c9d8a6548"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "da99b406840dfb988aed33e0d050a6add175bfb4"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@da99b406840dfb988aed33e0d050a6add175bfb4",
"title": "da99b406840dfb988aed33e0d050a6add175bfb4",
"url": "https://chromium.googlesource.com/chromium/src/+/da99b406840dfb988aed33e0d050a6add175bfb4"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "e57fb9d3a03c8bb9ef840fb45f59d672d47c2912"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@e57fb9d3a03c8bb9ef840fb45f59d672d47c2912",
"title": "e57fb9d3a03c8bb9ef840fb45f59d672d47c2912",
"url": "https://chromium.googlesource.com/chromium/src/+/e57fb9d3a03c8bb9ef840fb45f59d672d47c2912"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "4e7a681d1d298550d076cc72181a4b55cf7c463f"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@4e7a681d1d298550d076cc72181a4b55cf7c463f",
"title": "4e7a681d1d298550d076cc72181a4b55cf7c463f",
"url": "https://chromium.googlesource.com/chromium/src/+/4e7a681d1d298550d076cc72181a4b55cf7c463f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "3c693f1a5e2ba0ccf61fa3b79626c52bfe7a2f0a"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@3c693f1a5e2ba0ccf61fa3b79626c52bfe7a2f0a",
"title": "3c693f1a5e2ba0ccf61fa3b79626c52bfe7a2f0a",
"url": "https://chromium.googlesource.com/chromium/src/+/3c693f1a5e2ba0ccf61fa3b79626c52bfe7a2f0a"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://chromium.googlesource.com/chromium/src",
"revision": "f0fd8115bc3287fd318a59699b134cfdc003cda7"
},
"id": "https://chromium.googlesource.com/chromium/src/+/refs/heads/main@f0fd8115bc3287fd318a59699b134cfdc003cda7",
"title": "f0fd8115bc3287fd318a59699b134cfdc003cda7",
"url": "https://chromium.googlesource.com/chromium/src/+/f0fd8115bc3287fd318a59699b134cfdc003cda7"
}
]
}
},
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "f0fd8115bc3287fd318a59699b134cfdc003cda7",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8881016074188901440"
},
{
"key": "scheduler_job_id",
"value": "v8/Linux V8 API Stability"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[21:21:30.725] Scheduled build:
{
"id": "8715586012250365825",
"builder": {
"project": "v8",
"bucket": "ci",
"builder": "Linux V8 API Stability"
},
"number": 169791,
"createdBy": "project:v8",
"createTime": "2025-05-06T21:21:29.895013702Z",
"updateTime": "2025-05-06T21:21:29.895013702Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "f0fd8115bc3287fd318a59699b134cfdc003cda7",
"ref": "refs/heads/main"
}
}
}
[21:21:30.725] Task URL: https://cr-buildbucket.appspot.com/build/8715586012250365825
[21:21:30.725] Scheduling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:2:0) after 3m8s
[21:21:31.514] Received PubSub notification, asking Buildbucket for the build status
[21:21:31.574] Build status: SCHEDULED
[21:21:37.430] Received PubSub notification, asking Buildbucket for the build status
[21:21:37.455] Build status: STARTED
[21:24:38.780] Handling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:2:0)
[21:24:38.816] Build status: STARTED
[21:24:38.816] Scheduling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:5:0) after 2m38s
[21:27:16.860] Handling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:5:0)
[21:27:16.889] Build status: STARTED
[21:27:16.889] Scheduling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:6:0) after 8m38s
[21:35:54.958] Handling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:6:0)
[21:35:55.012] Build status: STARTED
[21:35:55.012] Scheduling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:7:0) after 6m19s
[21:42:14.035] Handling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:7:0)
[21:42:14.068] Build status: STARTED
[21:42:14.068] Scheduling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:8:0) after 3m56s
[21:46:10.086] Handling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:8:0)
[21:46:10.112] Build status: STARTED
[21:46:10.112] Scheduling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:9:0) after 8m45s
[21:54:55.185] Handling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:9:0)
[21:54:55.201] Build status: STARTED
[21:54:55.201] Scheduling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:10:0) after 2m44s
[21:57:39.225] Handling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:10:0)
[21:57:39.242] Build status: STARTED
[21:57:39.242] Scheduling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:11:0) after 2m2s
[21:59:41.267] Handling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:11:0)
[21:59:41.293] Build status: STARTED
[21:59:41.293] Scheduling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:12:0) after 3m35s
[22:03:16.310] Handling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:12:0)
[22:03:16.403] Build status: STARTED
[22:03:16.403] Scheduling timer "check-buildbucket-build-status" (v8/Linux V8 API Stability:8881016074188901440:13:0) after 7m10s
[22:03:30.937] Received PubSub notification, asking Buildbucket for the build status
[22:03:30.961] Build:
{
"id": "8715586012250365825",
"builder": {
"project": "v8",
"bucket": "ci",
"builder": "Linux V8 API Stability"
},
"number": 169791,
"createdBy": "project:v8",
"createTime": "2025-05-06T21:21:29.895013702Z",
"startTime": "2025-05-06T21:21:37.241248292Z",
"endTime": "2025-05-06T22:03:29.775970291Z",
"updateTime": "2025-05-06T22:03:29.775970291Z",
"status": "INFRA_FAILURE",
"input": {
"gitilesCommit": {
"host": "chromium.googlesource.com",
"project": "chromium/src",
"id": "f0fd8115bc3287fd318a59699b134cfdc003cda7",
"ref": "refs/heads/main"
}
}
}
[22:03:30.961] Invocation finished in 42m2.29880439s with status FAILED