[23:03:02.948] New invocation is queued and will start shortly
[23:03:04.597] Starting the invocation (attempt 1)
[23:03:04.627] Popped buildset tag "buildset:commit/gitiles/fuchsia.googlesource.com/third_party/rust/+/72fdf913c53dd0e75313ba83e4aa80df3f6e2871"
[23:03:04.627] Popped gitiles commit info from properties and tags
[23:03:04.628] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[23:03:04.630] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[23:03:04.631] Buildbucket request:
{
"requestId": "8911359667619614832",
"builder": {
"project": "fuchsia",
"bucket": "toolchain.ci",
"builder": "rust_test-x64-linux"
},
"properties": {
"$recipe_engine/scheduler": {
"hostname": "luci-scheduler.appspot.com",
"invocation": "8911359667619614832",
"job": "fuchsia/rust_test-x64-linux-49def413",
"triggers": [
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "822c3203c65a16bbe14c1b92894ad7fd72ebb432"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@822c3203c65a16bbe14c1b92894ad7fd72ebb432",
"title": "822c3203c65a16bbe14c1b92894ad7fd72ebb432",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/822c3203c65a16bbe14c1b92894ad7fd72ebb432"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "f0273664ce275afe78a25b75f001b77d09186113"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@f0273664ce275afe78a25b75f001b77d09186113",
"title": "f0273664ce275afe78a25b75f001b77d09186113",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/f0273664ce275afe78a25b75f001b77d09186113"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "7cd732f990ec341efdac53f6519a44b72e07861f"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@7cd732f990ec341efdac53f6519a44b72e07861f",
"title": "7cd732f990ec341efdac53f6519a44b72e07861f",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/7cd732f990ec341efdac53f6519a44b72e07861f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "c3de4b3aad6a2be86d4711086267e21660bf2b23"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@c3de4b3aad6a2be86d4711086267e21660bf2b23",
"title": "c3de4b3aad6a2be86d4711086267e21660bf2b23",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/c3de4b3aad6a2be86d4711086267e21660bf2b23"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "f4b060e452fa00a04f20ad7a0ec140fda48f3b50"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@f4b060e452fa00a04f20ad7a0ec140fda48f3b50",
"title": "f4b060e452fa00a04f20ad7a0ec140fda48f3b50",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/f4b060e452fa00a04f20ad7a0ec140fda48f3b50"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "f67a0eb2b72294f108051381cbc833c295aadad9"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@f67a0eb2b72294f108051381cbc833c295aadad9",
"title": "f67a0eb2b72294f108051381cbc833c295aadad9",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/f67a0eb2b72294f108051381cbc833c295aadad9"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "9b2e41a21800bcd56c5ce3307ab7e484f83d51b3"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@9b2e41a21800bcd56c5ce3307ab7e484f83d51b3",
"title": "9b2e41a21800bcd56c5ce3307ab7e484f83d51b3",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/9b2e41a21800bcd56c5ce3307ab7e484f83d51b3"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "b8c6008fbcb0a1f9f8af1ad7b6b196c950351b67"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@b8c6008fbcb0a1f9f8af1ad7b6b196c950351b67",
"title": "b8c6008fbcb0a1f9f8af1ad7b6b196c950351b67",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/b8c6008fbcb0a1f9f8af1ad7b6b196c950351b67"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "8db363c44b1512b5ba30b10ac3ad915228276375"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@8db363c44b1512b5ba30b10ac3ad915228276375",
"title": "8db363c44b1512b5ba30b10ac3ad915228276375",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/8db363c44b1512b5ba30b10ac3ad915228276375"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "5e8df95dbb445b2891610a9e46ce31c2b615bc09"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@5e8df95dbb445b2891610a9e46ce31c2b615bc09",
"title": "5e8df95dbb445b2891610a9e46ce31c2b615bc09",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/5e8df95dbb445b2891610a9e46ce31c2b615bc09"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "14f9c63759b67ad725dbecb003bcb7752fc92d69"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@14f9c63759b67ad725dbecb003bcb7752fc92d69",
"title": "14f9c63759b67ad725dbecb003bcb7752fc92d69",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/14f9c63759b67ad725dbecb003bcb7752fc92d69"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "7d151fa3b015ad1dbac18933c58db3413a964534"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@7d151fa3b015ad1dbac18933c58db3413a964534",
"title": "7d151fa3b015ad1dbac18933c58db3413a964534",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/7d151fa3b015ad1dbac18933c58db3413a964534"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "7894a1148371e4d0fa665c183615a76f9e2ec656"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@7894a1148371e4d0fa665c183615a76f9e2ec656",
"title": "7894a1148371e4d0fa665c183615a76f9e2ec656",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/7894a1148371e4d0fa665c183615a76f9e2ec656"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "81895065bb541b9e79f402d2446c1c83243b37d0"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@81895065bb541b9e79f402d2446c1c83243b37d0",
"title": "81895065bb541b9e79f402d2446c1c83243b37d0",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/81895065bb541b9e79f402d2446c1c83243b37d0"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "ffb1b2c14809ae5d15a078a9cce5299865d7dd73"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@ffb1b2c14809ae5d15a078a9cce5299865d7dd73",
"title": "ffb1b2c14809ae5d15a078a9cce5299865d7dd73",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/ffb1b2c14809ae5d15a078a9cce5299865d7dd73"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "874670399c43c58a6e21d46662979fb4fb8a0b8d"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@874670399c43c58a6e21d46662979fb4fb8a0b8d",
"title": "874670399c43c58a6e21d46662979fb4fb8a0b8d",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/874670399c43c58a6e21d46662979fb4fb8a0b8d"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "54b2e86db7f522294254ae5e4572feba28e0b929"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@54b2e86db7f522294254ae5e4572feba28e0b929",
"title": "54b2e86db7f522294254ae5e4572feba28e0b929",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/54b2e86db7f522294254ae5e4572feba28e0b929"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "b710404f3b3990af8eebe2e168f6465535dfcc82"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@b710404f3b3990af8eebe2e168f6465535dfcc82",
"title": "b710404f3b3990af8eebe2e168f6465535dfcc82",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/b710404f3b3990af8eebe2e168f6465535dfcc82"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "e704858b0b41b72da9645f9a51e2e55a482fa5c4"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@e704858b0b41b72da9645f9a51e2e55a482fa5c4",
"title": "e704858b0b41b72da9645f9a51e2e55a482fa5c4",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/e704858b0b41b72da9645f9a51e2e55a482fa5c4"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "69a8c139f1d7c97dc8b9bc0086e042fa6c79a389"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@69a8c139f1d7c97dc8b9bc0086e042fa6c79a389",
"title": "69a8c139f1d7c97dc8b9bc0086e042fa6c79a389",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/69a8c139f1d7c97dc8b9bc0086e042fa6c79a389"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "9abf8b105e22dd876834bf5094c77b0f2702f974"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@9abf8b105e22dd876834bf5094c77b0f2702f974",
"title": "9abf8b105e22dd876834bf5094c77b0f2702f974",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/9abf8b105e22dd876834bf5094c77b0f2702f974"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "edba40122a464231cff74a3ac8beecaddaebce1f"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@edba40122a464231cff74a3ac8beecaddaebce1f",
"title": "edba40122a464231cff74a3ac8beecaddaebce1f",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/edba40122a464231cff74a3ac8beecaddaebce1f"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "36cab1260d64d031d91c5b60bda5b4988f643166"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@36cab1260d64d031d91c5b60bda5b4988f643166",
"title": "36cab1260d64d031d91c5b60bda5b4988f643166",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/36cab1260d64d031d91c5b60bda5b4988f643166"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "fcc0b641e9c1a4b8dbdefa6c98b1f9924bc5ba76"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@fcc0b641e9c1a4b8dbdefa6c98b1f9924bc5ba76",
"title": "fcc0b641e9c1a4b8dbdefa6c98b1f9924bc5ba76",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/fcc0b641e9c1a4b8dbdefa6c98b1f9924bc5ba76"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "9c8e46d08aab7b5bf06d2e9c699b2a4de951e534"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@9c8e46d08aab7b5bf06d2e9c699b2a4de951e534",
"title": "9c8e46d08aab7b5bf06d2e9c699b2a4de951e534",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/9c8e46d08aab7b5bf06d2e9c699b2a4de951e534"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "33c02c37129777a985c8a712e59a29862a8f21c9"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@33c02c37129777a985c8a712e59a29862a8f21c9",
"title": "33c02c37129777a985c8a712e59a29862a8f21c9",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/33c02c37129777a985c8a712e59a29862a8f21c9"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "1c17449ae39ee2d3ab5f7a056db4a23d7f0629b4"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@1c17449ae39ee2d3ab5f7a056db4a23d7f0629b4",
"title": "1c17449ae39ee2d3ab5f7a056db4a23d7f0629b4",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/1c17449ae39ee2d3ab5f7a056db4a23d7f0629b4"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "fa58891f992b09dddd42cb90099d54ea04282a9a"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@fa58891f992b09dddd42cb90099d54ea04282a9a",
"title": "fa58891f992b09dddd42cb90099d54ea04282a9a",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/fa58891f992b09dddd42cb90099d54ea04282a9a"
},
{
"gitiles": {
"ref": "refs/heads/main",
"repo": "https://fuchsia.googlesource.com/third_party/rust",
"revision": "72fdf913c53dd0e75313ba83e4aa80df3f6e2871"
},
"id": "https://fuchsia.googlesource.com/third_party/rust/+/refs/heads/main@72fdf913c53dd0e75313ba83e4aa80df3f6e2871",
"title": "72fdf913c53dd0e75313ba83e4aa80df3f6e2871",
"url": "https://fuchsia.googlesource.com/third_party/rust/+/72fdf913c53dd0e75313ba83e4aa80df3f6e2871"
}
]
}
},
"gitilesCommit": {
"host": "fuchsia.googlesource.com",
"project": "third_party/rust",
"id": "72fdf913c53dd0e75313ba83e4aa80df3f6e2871",
"ref": "refs/heads/main"
},
"tags": [
{
"key": "scheduler_invocation_id",
"value": "8911359667619614832"
},
{
"key": "scheduler_job_id",
"value": "fuchsia/rust_test-x64-linux-49def413"
},
{
"key": "user_agent",
"value": "luci-scheduler"
}
],
"notify": {
"pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
}
}
[23:03:05.678] Scheduled build:
{
"id": "8745929604923807633",
"builder": {
"project": "fuchsia",
"bucket": "toolchain.ci",
"builder": "rust_test-x64-linux"
},
"createdBy": "project:fuchsia",
"createTime": "2024-06-05T23:03:04.895864448Z",
"updateTime": "2024-06-05T23:03:04.895864448Z",
"status": "SCHEDULED",
"input": {
"gitilesCommit": {
"host": "fuchsia.googlesource.com",
"project": "third_party/rust",
"id": "72fdf913c53dd0e75313ba83e4aa80df3f6e2871",
"ref": "refs/heads/main"
}
}
}
[23:03:05.678] Task URL: https://cr-buildbucket.appspot.com/build/8745929604923807633
[23:03:05.678] Scheduling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:2:0) after 5m44s
[23:03:06.073] Received PubSub notification, asking Buildbucket for the build status
[23:03:06.092] Build status: SCHEDULED
[23:08:49.681] Handling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:2:0)
[23:08:49.702] Build status: SCHEDULED
[23:08:49.702] Scheduling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:4:0) after 9m43s
[23:18:32.720] Handling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:4:0)
[23:18:32.738] Build status: SCHEDULED
[23:18:32.738] Scheduling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:5:0) after 3m18s
[23:21:50.758] Handling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:5:0)
[23:21:50.777] Build status: SCHEDULED
[23:21:50.777] Scheduling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:6:0) after 4m33s
[23:26:23.808] Handling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:6:0)
[23:26:23.859] Build status: SCHEDULED
[23:26:23.859] Scheduling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:7:0) after 7m57s
[23:32:05.122] Received PubSub notification, asking Buildbucket for the build status
[23:32:05.149] Build status: STARTED
[23:34:20.849] Handling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:7:0)
[23:34:20.868] Build status: STARTED
[23:34:20.868] Scheduling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:9:0) after 7m41s
[23:42:01.885] Handling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:9:0)
[23:42:01.905] Build status: STARTED
[23:42:01.905] Scheduling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:10:0) after 9m4s
[23:51:05.973] Handling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:10:0)
[23:51:06.352] Build status: STARTED
[23:51:06.352] Scheduling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:11:0) after 9m21s
[00:00:27.379] Handling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:11:0)
[00:00:27.416] Build status: STARTED
[00:00:27.416] Scheduling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:12:0) after 1m43s
[00:02:10.517] Handling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:12:0)
[00:02:10.568] Build status: STARTED
[00:02:10.568] Scheduling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:13:0) after 2m48s
[00:04:58.595] Handling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:13:0)
[00:04:58.681] Build status: STARTED
[00:04:58.681] Scheduling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:14:0) after 7m46s
[00:12:44.715] Handling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:14:0)
[00:12:44.735] Build status: STARTED
[00:12:44.735] Scheduling timer "check-buildbucket-build-status" (fuchsia/rust_test-x64-linux-49def413:8911359667619614832:15:0) after 7m42s
[00:13:40.373] Received PubSub notification, asking Buildbucket for the build status
[00:13:40.488] Build:
{
"id": "8745929604923807633",
"builder": {
"project": "fuchsia",
"bucket": "toolchain.ci",
"builder": "rust_test-x64-linux"
},
"createdBy": "project:fuchsia",
"createTime": "2024-06-05T23:03:04.895864448Z",
"startTime": "2024-06-05T23:32:03.915346962Z",
"endTime": "2024-06-06T00:13:39.802163293Z",
"updateTime": "2024-06-06T00:13:39.802163293Z",
"status": "FAILURE",
"input": {
"gitilesCommit": {
"host": "fuchsia.googlesource.com",
"project": "third_party/rust",
"id": "72fdf913c53dd0e75313ba83e4aa80df3f6e2871",
"ref": "refs/heads/main"
}
}
}
[00:13:40.488] Invocation finished in 1h10m37.547954035s with status FAILED