Status: FAILED
Triggered by: chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 15 minutes
Actions:

Definition (rev dbc5bbb4d5c973373f7e6afa20eb56bb1648ac04)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Linux TSan Tests"
>

Incoming triggers

  • Linux TSan Builder/584 (2 weeks ago by chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com)

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98",
        "project": "chromium/src",
        "ref": "refs/branch-heads/7103"
      }
    ]
  },
  "branch": "refs/branch-heads/7103",
  "deps_revision_overrides": {
    "src": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8716311830953752737"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8716311830953752737"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8716311830953752737"
    }
  ],
  "parent_build_id": "8716311830953752737",
  "parent_builder_group": "chromium.memory",
  "parent_buildername": "Linux TSan Builder",
  "parent_got_angle_revision": "ecc378cc61109732d174d6542c41fd523c331b13",
  "parent_got_dawn_revision": "1cffe7ec763900d104e4df62bc96d93f572157cb",
  "parent_got_revision": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98",
  "parent_got_revision_cp": "refs/branch-heads/7103@{#1722}",
  "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
  "parent_got_v8_revision": "b6178615ecae6d84b347cb7a1812cad9afca51f2",
  "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#15}",
  "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
  "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98",
  "swarm_hashes": {
    "absl_hardening_tests": "36d9e88be655cdeb6e08dc992155a96adc145d051f480c140073f9c17e21ccea/488",
    "accessibility_unittests": "e4e91ae83a5fe7eda5fe16c489f0c03affd38ae777e69b160514d0785c0ddb23/488",
    "angle_unittests": "8e68695a447fb8481cf2a4e1c5ae7ed60ab7a0cb611c62aa5d292ab2896916e9/489",
    "app_shell_unittests": "d66cdb80fab6ba9415d28d3ab91691a966809f83a0052623b7670a45404eed6e/649",
    "aura_unittests": "b3d37cfc0583d11df69827371145aec8363c4b95bf23837355a96c4616d446f7/488",
    "base_unittests": "1aca32194dc6197704f0e25ef21b4c14ababa184f92f39c21e5e46c07bc08d00/566",
    "blink_common_unittests": "4aceeeff73c708c7d211ae23c451120d8eb21ca95d2132654439c579e31bb221/566",
    "blink_fuzzer_unittests": "b6e11813ea55e9d3d5f153c682e8c811122ef10188170de15ec01fafd792f794/488",
    "blink_heap_unittests": "ddf6e147521363f8998a8e023309597d8ca13c695d021fde907268d074933e33/488",
    "blink_platform_unittests": "4965b88726925900b43970ea84ab202a9d09f2f70ded17a83f160c0f0839f54e/645",
    "blink_unittests": "a2c5cea4a8341ad9e4afb62f4c5c7b101c15b9597daaa293316e760be0e95ac2/729",
    "boringssl_crypto_tests": "4683f7679c42e32f9d75df888da52eef7f088b966861d5f7863d369e64b2db58/489",
    "boringssl_ssl_tests": "00941761c02c9c2dc7e96f4bb5925a749dbb7d66924d66998a79a7ea7a9f8c76/488",
    "capture_unittests": "9dba2c6bbea5aab77bb240b86c2358621ecc51b2aec0c575e8f732a71b3f24f4/567",
    "cast_unittests": "e55788aa8fe5ef062652b00308cf3a2e2d82d9986f42689ce0f445f49d7852de/488",
    "cc_unittests": "fc3bf0bb542a1bbd90d0035a16a4068cbe38910f5183de1eaa78ec77737b86af/648",
    "chrome_app_unittests": "eedbd24bd6080bc5a8bc4aa627657d596359e9ff6d097b8642f284686ecd08e6/566",
    "chromedriver_unittests": "a1a60b4aead1094f6b50380095ad4047fa0ba55ee48fc29fddf71a88c5071277/645",
    "components_browsertests": "9d6362b27b239006ffb0d8e3008b83f82e8a040de7d2750e08132d5ea0a9933d/732",
    "components_unittests": "54dee7e3bbdd9fe219830ceac36edf56e0d2d92d5012bb729e4ac18d854800e4/732",
    "compositor_unittests": "d60774c7e08988d63e9546927f38fe1e9938906fa7c342eec43c9adf62bfefff/564",
    "content_browsertests": "f92683039e6d9b19c2281408088cd8e1df15372c13c0c51234263456190d6e10/805",
    "content_unittests": "15df00cdf8c33ea53974fe117f874da079b3e4ed2300fc9aefa57f22b3feda04/725",
    "cronet_tests": "2f33d0e0d1a3e515dde139987f6733f3d2ab5485698ee5e8ca53e078349ef9c5/649",
    "cronet_unittests": "9682843cceb83cd83e164e5225d320a5b9a18492bd165611135540b1e6c69e70/565",
    "crypto_unittests": "43b3f1030ee54e35d1855406d2671327158cda5caa60eda3344954a744540f0b/488",
    "dbus_unittests": "f6c9597532bd98b42dce866e3a5f875e5ea747ace778c44daf9f1855608712a4/488",
    "device_unittests": "bb140ece649a976ca7015a18f5ac4f611d08d052e54a5843d0db65e0363199d2/565",
    "display_unittests": "15f1c4803ad0ec03e4bd724c3251dd59a46d014dc216c2127eef175662712091/488",
    "env_chromium_unittests": "27ebf5d2693277ed4b196be8b8fbcf23f6089a4a99b1e071b6e1bb225c604076/488",
    "events_unittests": "c76ccea81c8007e5add069c3f4e0dc65c1228cb6b3092e18307dbf9d6ebb1318/488",
    "extensions_browsertests": "552981fee294a32286e08c36eeac05f1bf89f3f3c6f4c58271d00e17818ee5a7/651",
    "extensions_unittests": "540bb7983e13f12b1061a7dba0c264b8f325d6791ed56d5bc08c919ec9e8988c/730",
    "filesystem_service_unittests": "aac43505adcf186852dccca109c4f8e63d18480412b7a3d1a5727f213a4bcaad/488",
    "fuzzing_unittests": "69c64d4850606623c0b5c7666cf1ea3866da025dac6acecacb1dc9d7e48abac8/488",
    "gcm_unit_tests": "678a4209d5ae18e46235348dc5cd9603448a41525341487291439da2e38311a7/565",
    "gfx_unittests": "a6fa9487005ce4d193620a93fdc3add7a5821da09bd5fc150a439d696e6f2b42/564",
    "gin_unittests": "aaabbd2ed1415f99db7d953f09ec178d08c497b9a18eeacaa0256700ac33c66a/565",
    "gl_tests": "35ffce76b0c48ffbc35e642c5cfe24a2ec8bff39a8c6bedf03edf157e10ce20a/488",
    "gl_unittests": "bf0754d1f6ea27b3c9796d3063e6bf223ee99c123b3d7c485b319ebca6dcc001/488",
    "google_apis_unittests": "80e9be1d35e39a6127b64b9ca72e2f68bc68991fb81f2bf9b3af498f3364f476/650",
    "gpu_unittests": "6cd75c4065c1cd9e7589079066a8240552d6cb9d7d6c68a744f7de8e0bde82bc/565",
    "gwp_asan_unittests": "181aedbcdad5fe4cada2853b056753ba07a375030a1a3f60e32489db9bbb1ee5/489",
    "headless_browsertests": "34ea2280b7c54727272ae59e5471320055a259b7468b5391da6245968c011df0/649",
    "headless_unittests": "acb8f90810e65b2a1f1c220dd01a0c792df4bee9c1d26347f200316a00691b63/488",
    "interactive_ui_tests": "4520c4a13170f2de31dd17330bcabc7df05caa813a26c8c2ef4235de69635763/973",
    "ipc_tests": "ae2b89f73843c615b767bb09fa5f6a7f6837c3ce4f2d6e19289266ee05a99dbf/488",
    "latency_unittests": "a48a766c02915f953d3b93fc1a11a022b328e892c9c7013b585184cf015c1696/488",
    "leveldb_unittests": "809b8ca1f0e16faa02fa0cc40316a37b234b13c5bfc9418cca8b9c33d5f51a20/488",
    "libjingle_xmpp_unittests": "63f2fb4017f6f4a78f1bca7858735cce63cf1117c6c2d1296375a6b04ecefe5d/488",
    "liburlpattern_unittests": "2bf5c9a1352a18263269570cab94d3d1128dd596f370b9235127e194b3e4dbb9/488",
    "media_unittests": "c82ce1cc04da9472b59f2e4af771dae67f5037e46c99c915406e47bc4c9a8879/568",
    "message_center_unittests": "933d16c7c77a37d6b181170b8469e03dffb4b9fb728880b4facc7af302212b92/488",
    "midi_unittests": "2bb347c7e0378f6ad154a3d8f4f893ddacadf739fcdc6f333bc76e823469d9d6/488",
    "mojo_unittests": "ed08a7c303629e4b8f121ffb2a6d4d3c718a60569011fcb4f2649036066d4396/566",
    "native_theme_unittests": "8191f7a2a604ecead9e4b8bcecadf707e19656a13b671aab623cc6e06d1eb796/488",
    "net_unittests": "c463f9818d37aa5d7352b95a5ce57c4689f864d13482a6492f58e6281de8d658/567",
    "openscreen_unittests": "4ddd5ed8dd031c7def0c5efaf5431218f6c2c20ecdf50e5a9ecd0ae189b985a7/489",
    "ozone_unittests": "cfebb380aaa2cef15eb166519c564a6ad9b599f39c776008c83bb1fdc4fdaead/488",
    "ozone_x11_unittests": "79babbcee57c8723c8b6d78f9675d8d48541d021f767e7cd99115f18a4f31785/488",
    "pdf_unittests": "e872ca863fad108694ae4649080f53b5fab204213e545ef2207c071e04742925/565",
    "perfetto_unittests": "c4fe82082c3d32526f932ec61d4de64f269d7b913b0a05143da231ff59c7fdda/488",
    "printing_unittests": "329955784942d9624a09770307bf28f2730f68b8796d61b6af245b6d4941d282/488",
    "pthreadpool_unittests": "62c9f1f9b901a9dfd9443bd59b228b485e40afe16ac9a5f0f9d82e92395658e8/488",
    "remoting_unittests": "3136d7112f2e2da44f25bf02472f24de85921a325432ad1a502d9bf211a4700a/565",
    "sandbox_linux_unittests": "9af1209f6fcba7f3f8d68de178ce43ed956e7417d665af3426933487a536ebb6/488",
    "services_unittests": "70f7b80984ff293ecec8b35d6ffc6e4459cd71681955ca32c19ab99482495ad4/734",
    "shell_dialogs_unittests": "85fe698c2167629e676cec37774265631484016d92afb980a8733d91a0af55f5/488",
    "skia_unittests": "7be9bc18e40bdd3be867c46a96d95ff82bcc36d9f7998bd8d4208b5d01474dc8/488",
    "snapshot_unittests": "3545e5ab105fb10840c9b532534a39e7319a2c6a7f6b0f2e7e7a1b68f4bb91ee/488",
    "sql_unittests": "fbf66e0c3652bf28271ade9867550f7405fddef818e21cf402ffb688d14fe6ea/565",
    "storage_unittests": "dd4c9751ce3333e57b260f8e8a687c04f2607d4736391f7c80cf2600924ff3fb/646",
    "sync_integration_tests": "3704d45b96d9e2bedf3d268725f19a58b7c09725596c10fdadc306bfcb5269cc/647",
    "ui_base_unittests": "7eeb55f32ba5e8d3e9b544323f92e27e18755fe8a6905b973b1e8bc797aebed0/641",
    "ui_touch_selection_unittests": "172c2193a3eca5eb04d101b1da1da42c33930eadbc7c1ac77274a41b5191c489/488",
    "ui_unittests": "c5039a5bd2a3ae1c6524854c38f265c1ea99beaf4d62bd9b077ddbe4a30db31c/488",
    "unit_tests": "0ced188828696177872d0d0b60ca1262166f36605c408fa57a4ef871b47bd23d/1061",
    "url_unittests": "c57ff4ee618765136f4f7d967ec68837056044cf8ee68b916b9327fac7e7988c/488",
    "views_unittests": "d7480a452cdc2aef70995f2790de4d45d3f4a3b0bdd52475b017088be2610c68/488",
    "viz_unittests": "1626394b014af473aede20c535871a805f635eddc6ed9a7a5553e3f0fb689ab4/572",
    "vr_common_unittests": "601a5080c124e1fa03a4863f9f3e82074fb03b08324ef101c1da5d8ae6e46e34/566",
    "wm_unittests": "dd106f25ecc5a2cb73af1f08502811a9ac0637f49882923fef44eda67d0f30eb/488",
    "wtf_unittests": "945899557c73bc33212b6bbc27fa104259b1d64b92409056f1103c6743470caf/488",
    "xr_browser_tests": "70a45e04526f9acef5fa0cc873aeffd96dcc615f94a9ced09d90a63adacdc464/646",
    "zlib_unittests": "3907a1cc46e51ca6569d4eef0d0940f3f967789cd77713e60d9f5ad1877aff6c/489"
  },
  "swarming_command_lines_cwd": "out/3e1b-Linux_TSan_Buil",
  "swarming_command_lines_digest": "0ac3541af884a6da53be3c91b39ccb3662f24db53e932c9d1d7e21d5040c682d/94",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/85fdd6979b19a654ee91686d5ac88f7bed0e6b98
  • gitiles_ref:refs/branch-heads/7103
  • parent_buildername:Linux TSan Builder
  • parent_buildnumber:584
  • user_agent:recipe

Debug log (UTC)

[22:04:34.447] New invocation is queued and will start shortly
[22:04:34.447] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[22:04:35.501] Starting the invocation (attempt 1)
[22:04:35.532] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/85fdd6979b19a654ee91686d5ac88f7bed0e6b98"
[22:04:35.532] Popped gitiles commit info from properties and tags
[22:04:35.532] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[22:04:35.532] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[22:04:35.533] Buildbucket request:
{
  "requestId": "8881738138533693840",
  "builder": {
    "project": "chromium-m136",
    "bucket": "ci",
    "builder": "Linux TSan Tests"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/7103"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8881738138533693840",
            "job": "chromium-m136/Linux TSan Tests",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7103"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8716311830953752737"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8716311830953752737"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8716311830953752737"
                                              }
                                        ],
                                    "parent_build_id": "8716311830953752737",
                                    "parent_builder_group": "chromium.memory",
                                    "parent_buildername": "Linux TSan Builder",
                                    "parent_got_angle_revision": "ecc378cc61109732d174d6542c41fd523c331b13",
                                    "parent_got_dawn_revision": "1cffe7ec763900d104e4df62bc96d93f572157cb",
                                    "parent_got_revision": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98",
                                    "parent_got_revision_cp": "refs/branch-heads/7103@{#1722}",
                                    "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
                                    "parent_got_v8_revision": "b6178615ecae6d84b347cb7a1812cad9afca51f2",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#15}",
                                    "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
                                    "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
                                    "revision": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98",
                                    "swarm_hashes": {
                                          "absl_hardening_tests": "36d9e88be655cdeb6e08dc992155a96adc145d051f480c140073f9c17e21ccea/488",
                                          "accessibility_unittests": "e4e91ae83a5fe7eda5fe16c489f0c03affd38ae777e69b160514d0785c0ddb23/488",
                                          "angle_unittests": "8e68695a447fb8481cf2a4e1c5ae7ed60ab7a0cb611c62aa5d292ab2896916e9/489",
                                          "app_shell_unittests": "d66cdb80fab6ba9415d28d3ab91691a966809f83a0052623b7670a45404eed6e/649",
                                          "aura_unittests": "b3d37cfc0583d11df69827371145aec8363c4b95bf23837355a96c4616d446f7/488",
                                          "base_unittests": "1aca32194dc6197704f0e25ef21b4c14ababa184f92f39c21e5e46c07bc08d00/566",
                                          "blink_common_unittests": "4aceeeff73c708c7d211ae23c451120d8eb21ca95d2132654439c579e31bb221/566",
                                          "blink_fuzzer_unittests": "b6e11813ea55e9d3d5f153c682e8c811122ef10188170de15ec01fafd792f794/488",
                                          "blink_heap_unittests": "ddf6e147521363f8998a8e023309597d8ca13c695d021fde907268d074933e33/488",
                                          "blink_platform_unittests": "4965b88726925900b43970ea84ab202a9d09f2f70ded17a83f160c0f0839f54e/645",
                                          "blink_unittests": "a2c5cea4a8341ad9e4afb62f4c5c7b101c15b9597daaa293316e760be0e95ac2/729",
                                          "boringssl_crypto_tests": "4683f7679c42e32f9d75df888da52eef7f088b966861d5f7863d369e64b2db58/489",
                                          "boringssl_ssl_tests": "00941761c02c9c2dc7e96f4bb5925a749dbb7d66924d66998a79a7ea7a9f8c76/488",
                                          "capture_unittests": "9dba2c6bbea5aab77bb240b86c2358621ecc51b2aec0c575e8f732a71b3f24f4/567",
                                          "cast_unittests": "e55788aa8fe5ef062652b00308cf3a2e2d82d9986f42689ce0f445f49d7852de/488",
                                          "cc_unittests": "fc3bf0bb542a1bbd90d0035a16a4068cbe38910f5183de1eaa78ec77737b86af/648",
                                          "chrome_app_unittests": "eedbd24bd6080bc5a8bc4aa627657d596359e9ff6d097b8642f284686ecd08e6/566",
                                          "chromedriver_unittests": "a1a60b4aead1094f6b50380095ad4047fa0ba55ee48fc29fddf71a88c5071277/645",
                                          "components_browsertests": "9d6362b27b239006ffb0d8e3008b83f82e8a040de7d2750e08132d5ea0a9933d/732",
                                          "components_unittests": "54dee7e3bbdd9fe219830ceac36edf56e0d2d92d5012bb729e4ac18d854800e4/732",
                                          "compositor_unittests": "d60774c7e08988d63e9546927f38fe1e9938906fa7c342eec43c9adf62bfefff/564",
                                          "content_browsertests": "f92683039e6d9b19c2281408088cd8e1df15372c13c0c51234263456190d6e10/805",
                                          "content_unittests": "15df00cdf8c33ea53974fe117f874da079b3e4ed2300fc9aefa57f22b3feda04/725",
                                          "cronet_tests": "2f33d0e0d1a3e515dde139987f6733f3d2ab5485698ee5e8ca53e078349ef9c5/649",
                                          "cronet_unittests": "9682843cceb83cd83e164e5225d320a5b9a18492bd165611135540b1e6c69e70/565",
                                          "crypto_unittests": "43b3f1030ee54e35d1855406d2671327158cda5caa60eda3344954a744540f0b/488",
                                          "dbus_unittests": "f6c9597532bd98b42dce866e3a5f875e5ea747ace778c44daf9f1855608712a4/488",
                                          "device_unittests": "bb140ece649a976ca7015a18f5ac4f611d08d052e54a5843d0db65e0363199d2/565",
                                          "display_unittests": "15f1c4803ad0ec03e4bd724c3251dd59a46d014dc216c2127eef175662712091/488",
                                          "env_chromium_unittests": "27ebf5d2693277ed4b196be8b8fbcf23f6089a4a99b1e071b6e1bb225c604076/488",
                                          "events_unittests": "c76ccea81c8007e5add069c3f4e0dc65c1228cb6b3092e18307dbf9d6ebb1318/488",
                                          "extensions_browsertests": "552981fee294a32286e08c36eeac05f1bf89f3f3c6f4c58271d00e17818ee5a7/651",
                                          "extensions_unittests": "540bb7983e13f12b1061a7dba0c264b8f325d6791ed56d5bc08c919ec9e8988c/730",
                                          "filesystem_service_unittests": "aac43505adcf186852dccca109c4f8e63d18480412b7a3d1a5727f213a4bcaad/488",
                                          "fuzzing_unittests": "69c64d4850606623c0b5c7666cf1ea3866da025dac6acecacb1dc9d7e48abac8/488",
                                          "gcm_unit_tests": "678a4209d5ae18e46235348dc5cd9603448a41525341487291439da2e38311a7/565",
                                          "gfx_unittests": "a6fa9487005ce4d193620a93fdc3add7a5821da09bd5fc150a439d696e6f2b42/564",
                                          "gin_unittests": "aaabbd2ed1415f99db7d953f09ec178d08c497b9a18eeacaa0256700ac33c66a/565",
                                          "gl_tests": "35ffce76b0c48ffbc35e642c5cfe24a2ec8bff39a8c6bedf03edf157e10ce20a/488",
                                          "gl_unittests": "bf0754d1f6ea27b3c9796d3063e6bf223ee99c123b3d7c485b319ebca6dcc001/488",
                                          "google_apis_unittests": "80e9be1d35e39a6127b64b9ca72e2f68bc68991fb81f2bf9b3af498f3364f476/650",
                                          "gpu_unittests": "6cd75c4065c1cd9e7589079066a8240552d6cb9d7d6c68a744f7de8e0bde82bc/565",
                                          "gwp_asan_unittests": "181aedbcdad5fe4cada2853b056753ba07a375030a1a3f60e32489db9bbb1ee5/489",
                                          "headless_browsertests": "34ea2280b7c54727272ae59e5471320055a259b7468b5391da6245968c011df0/649",
                                          "headless_unittests": "acb8f90810e65b2a1f1c220dd01a0c792df4bee9c1d26347f200316a00691b63/488",
                                          "interactive_ui_tests": "4520c4a13170f2de31dd17330bcabc7df05caa813a26c8c2ef4235de69635763/973",
                                          "ipc_tests": "ae2b89f73843c615b767bb09fa5f6a7f6837c3ce4f2d6e19289266ee05a99dbf/488",
                                          "latency_unittests": "a48a766c02915f953d3b93fc1a11a022b328e892c9c7013b585184cf015c1696/488",
                                          "leveldb_unittests": "809b8ca1f0e16faa02fa0cc40316a37b234b13c5bfc9418cca8b9c33d5f51a20/488",
                                          "libjingle_xmpp_unittests": "63f2fb4017f6f4a78f1bca7858735cce63cf1117c6c2d1296375a6b04ecefe5d/488",
                                          "liburlpattern_unittests": "2bf5c9a1352a18263269570cab94d3d1128dd596f370b9235127e194b3e4dbb9/488",
                                          "media_unittests": "c82ce1cc04da9472b59f2e4af771dae67f5037e46c99c915406e47bc4c9a8879/568",
                                          "message_center_unittests": "933d16c7c77a37d6b181170b8469e03dffb4b9fb728880b4facc7af302212b92/488",
                                          "midi_unittests": "2bb347c7e0378f6ad154a3d8f4f893ddacadf739fcdc6f333bc76e823469d9d6/488",
                                          "mojo_unittests": "ed08a7c303629e4b8f121ffb2a6d4d3c718a60569011fcb4f2649036066d4396/566",
                                          "native_theme_unittests": "8191f7a2a604ecead9e4b8bcecadf707e19656a13b671aab623cc6e06d1eb796/488",
                                          "net_unittests": "c463f9818d37aa5d7352b95a5ce57c4689f864d13482a6492f58e6281de8d658/567",
                                          "openscreen_unittests": "4ddd5ed8dd031c7def0c5efaf5431218f6c2c20ecdf50e5a9ecd0ae189b985a7/489",
                                          "ozone_unittests": "cfebb380aaa2cef15eb166519c564a6ad9b599f39c776008c83bb1fdc4fdaead/488",
                                          "ozone_x11_unittests": "79babbcee57c8723c8b6d78f9675d8d48541d021f767e7cd99115f18a4f31785/488",
                                          "pdf_unittests": "e872ca863fad108694ae4649080f53b5fab204213e545ef2207c071e04742925/565",
                                          "perfetto_unittests": "c4fe82082c3d32526f932ec61d4de64f269d7b913b0a05143da231ff59c7fdda/488",
                                          "printing_unittests": "329955784942d9624a09770307bf28f2730f68b8796d61b6af245b6d4941d282/488",
                                          "pthreadpool_unittests": "62c9f1f9b901a9dfd9443bd59b228b485e40afe16ac9a5f0f9d82e92395658e8/488",
                                          "remoting_unittests": "3136d7112f2e2da44f25bf02472f24de85921a325432ad1a502d9bf211a4700a/565",
                                          "sandbox_linux_unittests": "9af1209f6fcba7f3f8d68de178ce43ed956e7417d665af3426933487a536ebb6/488",
                                          "services_unittests": "70f7b80984ff293ecec8b35d6ffc6e4459cd71681955ca32c19ab99482495ad4/734",
                                          "shell_dialogs_unittests": "85fe698c2167629e676cec37774265631484016d92afb980a8733d91a0af55f5/488",
                                          "skia_unittests": "7be9bc18e40bdd3be867c46a96d95ff82bcc36d9f7998bd8d4208b5d01474dc8/488",
                                          "snapshot_unittests": "3545e5ab105fb10840c9b532534a39e7319a2c6a7f6b0f2e7e7a1b68f4bb91ee/488",
                                          "sql_unittests": "fbf66e0c3652bf28271ade9867550f7405fddef818e21cf402ffb688d14fe6ea/565",
                                          "storage_unittests": "dd4c9751ce3333e57b260f8e8a687c04f2607d4736391f7c80cf2600924ff3fb/646",
                                          "sync_integration_tests": "3704d45b96d9e2bedf3d268725f19a58b7c09725596c10fdadc306bfcb5269cc/647",
                                          "ui_base_unittests": "7eeb55f32ba5e8d3e9b544323f92e27e18755fe8a6905b973b1e8bc797aebed0/641",
                                          "ui_touch_selection_unittests": "172c2193a3eca5eb04d101b1da1da42c33930eadbc7c1ac77274a41b5191c489/488",
                                          "ui_unittests": "c5039a5bd2a3ae1c6524854c38f265c1ea99beaf4d62bd9b077ddbe4a30db31c/488",
                                          "unit_tests": "0ced188828696177872d0d0b60ca1262166f36605c408fa57a4ef871b47bd23d/1061",
                                          "url_unittests": "c57ff4ee618765136f4f7d967ec68837056044cf8ee68b916b9327fac7e7988c/488",
                                          "views_unittests": "d7480a452cdc2aef70995f2790de4d45d3f4a3b0bdd52475b017088be2610c68/488",
                                          "viz_unittests": "1626394b014af473aede20c535871a805f635eddc6ed9a7a5553e3f0fb689ab4/572",
                                          "vr_common_unittests": "601a5080c124e1fa03a4863f9f3e82074fb03b08324ef101c1da5d8ae6e46e34/566",
                                          "wm_unittests": "dd106f25ecc5a2cb73af1f08502811a9ac0637f49882923fef44eda67d0f30eb/488",
                                          "wtf_unittests": "945899557c73bc33212b6bbc27fa104259b1d64b92409056f1103c6743470caf/488",
                                          "xr_browser_tests": "70a45e04526f9acef5fa0cc873aeffd96dcc615f94a9ced09d90a63adacdc464/646",
                                          "zlib_unittests": "3907a1cc46e51ca6569d4eef0d0940f3f967789cd77713e60d9f5ad1877aff6c/489"
                                        },
                                    "swarming_command_lines_cwd": "out/3e1b-Linux_TSan_Buil",
                                    "swarming_command_lines_digest": "0ac3541af884a6da53be3c91b39ccb3662f24db53e932c9d1d7e21d5040c682d/94",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7103",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98",
                              "tags": [
                                    "parent_buildername:Linux TSan Builder",
                                    "parent_buildnumber:584",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "27e662dc-2f47-452f-afd2-8de62cfade89",
                        "title": "Linux TSan Builder/584"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8716311830953752737"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8716311830953752737"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8716311830953752737"
                }
          ],
      "parent_build_id": "8716311830953752737",
      "parent_builder_group": "chromium.memory",
      "parent_buildername": "Linux TSan Builder",
      "parent_got_angle_revision": "ecc378cc61109732d174d6542c41fd523c331b13",
      "parent_got_dawn_revision": "1cffe7ec763900d104e4df62bc96d93f572157cb",
      "parent_got_revision": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98",
      "parent_got_revision_cp": "refs/branch-heads/7103@{#1722}",
      "parent_got_swiftshader_revision": "4982425ff1bdcb2ce52a360edde58a379119bfde",
      "parent_got_v8_revision": "b6178615ecae6d84b347cb7a1812cad9afca51f2",
      "parent_got_v8_revision_cp": "refs/branch-heads/13.6@{#15}",
      "parent_got_webrtc_revision": "2c8f5be6924d507ee74191b1aeadcec07f747f21",
      "parent_got_webrtc_revision_cp": "refs/heads/main@{#44272}",
      "swarm_hashes": {
            "absl_hardening_tests": "36d9e88be655cdeb6e08dc992155a96adc145d051f480c140073f9c17e21ccea/488",
            "accessibility_unittests": "e4e91ae83a5fe7eda5fe16c489f0c03affd38ae777e69b160514d0785c0ddb23/488",
            "angle_unittests": "8e68695a447fb8481cf2a4e1c5ae7ed60ab7a0cb611c62aa5d292ab2896916e9/489",
            "app_shell_unittests": "d66cdb80fab6ba9415d28d3ab91691a966809f83a0052623b7670a45404eed6e/649",
            "aura_unittests": "b3d37cfc0583d11df69827371145aec8363c4b95bf23837355a96c4616d446f7/488",
            "base_unittests": "1aca32194dc6197704f0e25ef21b4c14ababa184f92f39c21e5e46c07bc08d00/566",
            "blink_common_unittests": "4aceeeff73c708c7d211ae23c451120d8eb21ca95d2132654439c579e31bb221/566",
            "blink_fuzzer_unittests": "b6e11813ea55e9d3d5f153c682e8c811122ef10188170de15ec01fafd792f794/488",
            "blink_heap_unittests": "ddf6e147521363f8998a8e023309597d8ca13c695d021fde907268d074933e33/488",
            "blink_platform_unittests": "4965b88726925900b43970ea84ab202a9d09f2f70ded17a83f160c0f0839f54e/645",
            "blink_unittests": "a2c5cea4a8341ad9e4afb62f4c5c7b101c15b9597daaa293316e760be0e95ac2/729",
            "boringssl_crypto_tests": "4683f7679c42e32f9d75df888da52eef7f088b966861d5f7863d369e64b2db58/489",
            "boringssl_ssl_tests": "00941761c02c9c2dc7e96f4bb5925a749dbb7d66924d66998a79a7ea7a9f8c76/488",
            "capture_unittests": "9dba2c6bbea5aab77bb240b86c2358621ecc51b2aec0c575e8f732a71b3f24f4/567",
            "cast_unittests": "e55788aa8fe5ef062652b00308cf3a2e2d82d9986f42689ce0f445f49d7852de/488",
            "cc_unittests": "fc3bf0bb542a1bbd90d0035a16a4068cbe38910f5183de1eaa78ec77737b86af/648",
            "chrome_app_unittests": "eedbd24bd6080bc5a8bc4aa627657d596359e9ff6d097b8642f284686ecd08e6/566",
            "chromedriver_unittests": "a1a60b4aead1094f6b50380095ad4047fa0ba55ee48fc29fddf71a88c5071277/645",
            "components_browsertests": "9d6362b27b239006ffb0d8e3008b83f82e8a040de7d2750e08132d5ea0a9933d/732",
            "components_unittests": "54dee7e3bbdd9fe219830ceac36edf56e0d2d92d5012bb729e4ac18d854800e4/732",
            "compositor_unittests": "d60774c7e08988d63e9546927f38fe1e9938906fa7c342eec43c9adf62bfefff/564",
            "content_browsertests": "f92683039e6d9b19c2281408088cd8e1df15372c13c0c51234263456190d6e10/805",
            "content_unittests": "15df00cdf8c33ea53974fe117f874da079b3e4ed2300fc9aefa57f22b3feda04/725",
            "cronet_tests": "2f33d0e0d1a3e515dde139987f6733f3d2ab5485698ee5e8ca53e078349ef9c5/649",
            "cronet_unittests": "9682843cceb83cd83e164e5225d320a5b9a18492bd165611135540b1e6c69e70/565",
            "crypto_unittests": "43b3f1030ee54e35d1855406d2671327158cda5caa60eda3344954a744540f0b/488",
            "dbus_unittests": "f6c9597532bd98b42dce866e3a5f875e5ea747ace778c44daf9f1855608712a4/488",
            "device_unittests": "bb140ece649a976ca7015a18f5ac4f611d08d052e54a5843d0db65e0363199d2/565",
            "display_unittests": "15f1c4803ad0ec03e4bd724c3251dd59a46d014dc216c2127eef175662712091/488",
            "env_chromium_unittests": "27ebf5d2693277ed4b196be8b8fbcf23f6089a4a99b1e071b6e1bb225c604076/488",
            "events_unittests": "c76ccea81c8007e5add069c3f4e0dc65c1228cb6b3092e18307dbf9d6ebb1318/488",
            "extensions_browsertests": "552981fee294a32286e08c36eeac05f1bf89f3f3c6f4c58271d00e17818ee5a7/651",
            "extensions_unittests": "540bb7983e13f12b1061a7dba0c264b8f325d6791ed56d5bc08c919ec9e8988c/730",
            "filesystem_service_unittests": "aac43505adcf186852dccca109c4f8e63d18480412b7a3d1a5727f213a4bcaad/488",
            "fuzzing_unittests": "69c64d4850606623c0b5c7666cf1ea3866da025dac6acecacb1dc9d7e48abac8/488",
            "gcm_unit_tests": "678a4209d5ae18e46235348dc5cd9603448a41525341487291439da2e38311a7/565",
            "gfx_unittests": "a6fa9487005ce4d193620a93fdc3add7a5821da09bd5fc150a439d696e6f2b42/564",
            "gin_unittests": "aaabbd2ed1415f99db7d953f09ec178d08c497b9a18eeacaa0256700ac33c66a/565",
            "gl_tests": "35ffce76b0c48ffbc35e642c5cfe24a2ec8bff39a8c6bedf03edf157e10ce20a/488",
            "gl_unittests": "bf0754d1f6ea27b3c9796d3063e6bf223ee99c123b3d7c485b319ebca6dcc001/488",
            "google_apis_unittests": "80e9be1d35e39a6127b64b9ca72e2f68bc68991fb81f2bf9b3af498f3364f476/650",
            "gpu_unittests": "6cd75c4065c1cd9e7589079066a8240552d6cb9d7d6c68a744f7de8e0bde82bc/565",
            "gwp_asan_unittests": "181aedbcdad5fe4cada2853b056753ba07a375030a1a3f60e32489db9bbb1ee5/489",
            "headless_browsertests": "34ea2280b7c54727272ae59e5471320055a259b7468b5391da6245968c011df0/649",
            "headless_unittests": "acb8f90810e65b2a1f1c220dd01a0c792df4bee9c1d26347f200316a00691b63/488",
            "interactive_ui_tests": "4520c4a13170f2de31dd17330bcabc7df05caa813a26c8c2ef4235de69635763/973",
            "ipc_tests": "ae2b89f73843c615b767bb09fa5f6a7f6837c3ce4f2d6e19289266ee05a99dbf/488",
            "latency_unittests": "a48a766c02915f953d3b93fc1a11a022b328e892c9c7013b585184cf015c1696/488",
            "leveldb_unittests": "809b8ca1f0e16faa02fa0cc40316a37b234b13c5bfc9418cca8b9c33d5f51a20/488",
            "libjingle_xmpp_unittests": "63f2fb4017f6f4a78f1bca7858735cce63cf1117c6c2d1296375a6b04ecefe5d/488",
            "liburlpattern_unittests": "2bf5c9a1352a18263269570cab94d3d1128dd596f370b9235127e194b3e4dbb9/488",
            "media_unittests": "c82ce1cc04da9472b59f2e4af771dae67f5037e46c99c915406e47bc4c9a8879/568",
            "message_center_unittests": "933d16c7c77a37d6b181170b8469e03dffb4b9fb728880b4facc7af302212b92/488",
            "midi_unittests": "2bb347c7e0378f6ad154a3d8f4f893ddacadf739fcdc6f333bc76e823469d9d6/488",
            "mojo_unittests": "ed08a7c303629e4b8f121ffb2a6d4d3c718a60569011fcb4f2649036066d4396/566",
            "native_theme_unittests": "8191f7a2a604ecead9e4b8bcecadf707e19656a13b671aab623cc6e06d1eb796/488",
            "net_unittests": "c463f9818d37aa5d7352b95a5ce57c4689f864d13482a6492f58e6281de8d658/567",
            "openscreen_unittests": "4ddd5ed8dd031c7def0c5efaf5431218f6c2c20ecdf50e5a9ecd0ae189b985a7/489",
            "ozone_unittests": "cfebb380aaa2cef15eb166519c564a6ad9b599f39c776008c83bb1fdc4fdaead/488",
            "ozone_x11_unittests": "79babbcee57c8723c8b6d78f9675d8d48541d021f767e7cd99115f18a4f31785/488",
            "pdf_unittests": "e872ca863fad108694ae4649080f53b5fab204213e545ef2207c071e04742925/565",
            "perfetto_unittests": "c4fe82082c3d32526f932ec61d4de64f269d7b913b0a05143da231ff59c7fdda/488",
            "printing_unittests": "329955784942d9624a09770307bf28f2730f68b8796d61b6af245b6d4941d282/488",
            "pthreadpool_unittests": "62c9f1f9b901a9dfd9443bd59b228b485e40afe16ac9a5f0f9d82e92395658e8/488",
            "remoting_unittests": "3136d7112f2e2da44f25bf02472f24de85921a325432ad1a502d9bf211a4700a/565",
            "sandbox_linux_unittests": "9af1209f6fcba7f3f8d68de178ce43ed956e7417d665af3426933487a536ebb6/488",
            "services_unittests": "70f7b80984ff293ecec8b35d6ffc6e4459cd71681955ca32c19ab99482495ad4/734",
            "shell_dialogs_unittests": "85fe698c2167629e676cec37774265631484016d92afb980a8733d91a0af55f5/488",
            "skia_unittests": "7be9bc18e40bdd3be867c46a96d95ff82bcc36d9f7998bd8d4208b5d01474dc8/488",
            "snapshot_unittests": "3545e5ab105fb10840c9b532534a39e7319a2c6a7f6b0f2e7e7a1b68f4bb91ee/488",
            "sql_unittests": "fbf66e0c3652bf28271ade9867550f7405fddef818e21cf402ffb688d14fe6ea/565",
            "storage_unittests": "dd4c9751ce3333e57b260f8e8a687c04f2607d4736391f7c80cf2600924ff3fb/646",
            "sync_integration_tests": "3704d45b96d9e2bedf3d268725f19a58b7c09725596c10fdadc306bfcb5269cc/647",
            "ui_base_unittests": "7eeb55f32ba5e8d3e9b544323f92e27e18755fe8a6905b973b1e8bc797aebed0/641",
            "ui_touch_selection_unittests": "172c2193a3eca5eb04d101b1da1da42c33930eadbc7c1ac77274a41b5191c489/488",
            "ui_unittests": "c5039a5bd2a3ae1c6524854c38f265c1ea99beaf4d62bd9b077ddbe4a30db31c/488",
            "unit_tests": "0ced188828696177872d0d0b60ca1262166f36605c408fa57a4ef871b47bd23d/1061",
            "url_unittests": "c57ff4ee618765136f4f7d967ec68837056044cf8ee68b916b9327fac7e7988c/488",
            "views_unittests": "d7480a452cdc2aef70995f2790de4d45d3f4a3b0bdd52475b017088be2610c68/488",
            "viz_unittests": "1626394b014af473aede20c535871a805f635eddc6ed9a7a5553e3f0fb689ab4/572",
            "vr_common_unittests": "601a5080c124e1fa03a4863f9f3e82074fb03b08324ef101c1da5d8ae6e46e34/566",
            "wm_unittests": "dd106f25ecc5a2cb73af1f08502811a9ac0637f49882923fef44eda67d0f30eb/488",
            "wtf_unittests": "945899557c73bc33212b6bbc27fa104259b1d64b92409056f1103c6743470caf/488",
            "xr_browser_tests": "70a45e04526f9acef5fa0cc873aeffd96dcc615f94a9ced09d90a63adacdc464/646",
            "zlib_unittests": "3907a1cc46e51ca6569d4eef0d0940f3f967789cd77713e60d9f5ad1877aff6c/489"
          },
      "swarming_command_lines_cwd": "out/3e1b-Linux_TSan_Buil",
      "swarming_command_lines_digest": "0ac3541af884a6da53be3c91b39ccb3662f24db53e932c9d1d7e21d5040c682d/94",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98",
    "ref": "refs/branch-heads/7103"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8881738138533693840"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m136/Linux TSan Tests"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Linux TSan Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "584"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[22:04:36.047] Scheduled build:
{
  "id": "8716308076673624081",
  "builder": {
    "project": "chromium-m136",
    "bucket": "ci",
    "builder": "Linux TSan Tests"
  },
  "number": 583,
  "createdBy": "project:chromium-m136",
  "createTime": "2025-04-28T22:04:35.600409316Z",
  "updateTime": "2025-04-28T22:04:35.600409316Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98",
      "ref": "refs/branch-heads/7103"
    }
  }
}
[22:04:36.047] Task URL: https://cr-buildbucket.appspot.com/build/8716308076673624081
[22:04:36.047] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Linux TSan Tests:8881738138533693840:2:0) after 2m9s
[22:04:36.247] Received PubSub notification, asking Buildbucket for the build status
[22:04:36.268] Build status: SCHEDULED
[22:06:45.071] Handling timer "check-buildbucket-build-status" (chromium-m136/Linux TSan Tests:8881738138533693840:2:0)
[22:06:45.095] Build status: SCHEDULED
[22:06:45.095] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Linux TSan Tests:8881738138533693840:4:0) after 8m50s
[22:08:50.245] Received PubSub notification, asking Buildbucket for the build status
[22:08:50.310] Build status: STARTED
[22:15:35.121] Handling timer "check-buildbucket-build-status" (chromium-m136/Linux TSan Tests:8881738138533693840:4:0)
[22:15:35.174] Build status: STARTED
[22:15:35.174] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Linux TSan Tests:8881738138533693840:6:0) after 3m30s
[22:19:05.195] Handling timer "check-buildbucket-build-status" (chromium-m136/Linux TSan Tests:8881738138533693840:6:0)
[22:19:05.218] Build status: STARTED
[22:19:05.218] Scheduling timer "check-buildbucket-build-status" (chromium-m136/Linux TSan Tests:8881738138533693840:7:0) after 9m25s
[22:19:36.800] Received PubSub notification, asking Buildbucket for the build status
[22:19:36.832] Build:
{
  "id": "8716308076673624081",
  "builder": {
    "project": "chromium-m136",
    "bucket": "ci",
    "builder": "Linux TSan Tests"
  },
  "number": 583,
  "createdBy": "project:chromium-m136",
  "createTime": "2025-04-28T22:04:35.600409316Z",
  "startTime": "2025-04-28T22:08:48.998289086Z",
  "endTime": "2025-04-28T22:19:36.456574273Z",
  "updateTime": "2025-04-28T22:19:36.456574273Z",
  "status": "INFRA_FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "85fdd6979b19a654ee91686d5ac88f7bed0e6b98",
      "ref": "refs/branch-heads/7103"
    }
  }
}
[22:19:36.832] Invocation finished in 15m2.389459941s with status FAILED