Status: FAILED
Triggered by: chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 22 minutes
Actions:

Definition (rev 10f958521df17c41c0ad067545717a377f427240)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Linux TSan Tests"
>

Incoming triggers

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "38f656c70c4343b39a62473cada43558a084620f",
        "project": "chromium/src",
        "ref": "refs/branch-heads/7151"
      }
    ]
  },
  "branch": "refs/branch-heads/7151",
  "deps_revision_overrides": {
    "src": "38f656c70c4343b39a62473cada43558a084620f"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8709713667368472961"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8709713667368472961"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8709713667368472961"
    }
  ],
  "parent_build_id": "8709713667368472961",
  "parent_builder_group": "chromium.memory",
  "parent_buildername": "Linux TSan Builder",
  "parent_got_angle_revision": "df9c59dcacff7d186d00e3263a1aa68f8059137c",
  "parent_got_dawn_revision": "fbe707f88ccabca01031e47bf165bd9d499878dd",
  "parent_got_revision": "38f656c70c4343b39a62473cada43558a084620f",
  "parent_got_revision_cp": "refs/branch-heads/7151@{#3541}",
  "parent_got_swiftshader_revision": "7905fa19e456df5aa8e2233a7ec5832c9c6c287b",
  "parent_got_v8_revision": "cfefa0753b6454c7cb3dba261f6ff86bf5b19888",
  "parent_got_v8_revision_cp": "refs/branch-heads/13.7@{#37}",
  "parent_got_webrtc_revision": "cec4daea7ed5da94fc38d790bd12694c86865447",
  "parent_got_webrtc_revision_cp": "refs/branch-heads/7151@{#3}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "38f656c70c4343b39a62473cada43558a084620f",
  "swarm_hashes": {
    "absl_hardening_tests": "e6a60e18376e4d837232512006968edee4867d931abd3cf4dcd06b4e6452789a/488",
    "accessibility_unittests": "df15826d9fcf306e8f6ff1fa23e5bdc8ea66cb70fd719647154a9129778e6cb1/488",
    "angle_unittests": "1b715eb6a48d05b361dc3f995aa53ac0c20a22900319ea887accfc2658e21250/489",
    "app_shell_unittests": "fac27423f1529be77450d06176a1ec65191ee007a7add1e55a6065aa15c34cc1/649",
    "aura_unittests": "fbdf09171bef7ce5fc9e1e746fdc64162a6f1258334d9a205d16771ac2b6cdfc/488",
    "base_unittests": "fe21c21f3cfe664d2a77152655291433507aee4ce327e9603b48967a0fbd75ec/566",
    "blink_common_unittests": "7dee2c27408b3eba69fc3d5b9be67c96359779f073c319d4e8bbef052a10c597/566",
    "blink_fuzzer_unittests": "f1990862bf3c7c8f96f98f37e62692f36a2b1b3cbe258a7d6a900a91201b590e/488",
    "blink_heap_unittests": "f3ddb418e70e53d01d302c990c9c844c8aecb8f9b44b7e633b29cc143411728e/488",
    "blink_platform_unittests": "38944905ce046ccf99f4d649cf3db14acafb2102926c71e0c38eca0591925de8/645",
    "blink_unittests": "534d568882872d11121ff8c986aea74b15e75c639dbb9792da23e1473a0111db/729",
    "boringssl_crypto_tests": "a60e34f76e1627d75b83a5fd603a311e86f49b3b6ab9ba1dc4426253b9772d7e/489",
    "boringssl_ssl_tests": "4c9f43cd7edb973afc12c6a403f857f5410c8a5c3259f08823ee3011cd8be11a/488",
    "capture_unittests": "2b2b7794432d5c06ad26e4507024b6b9adf9bd0e4a7b917d0371b8be2d8154fe/567",
    "cast_unittests": "d793553d3e95569342b7ea11e4ed51c84438e3eb4a9d788e6deebdd7407089e9/488",
    "cc_unittests": "e53bbf2e018b0ff6439da55a9603a76e266a04f3bfd869214f3859e176ca006b/648",
    "chrome_app_unittests": "b1d283258737fb61fb0a97d4e98a4b8044abc90e158483cdd7da14e5a79522b9/566",
    "chromedriver_unittests": "dd4a8ab7dde0d571a58056f4bc86f4895f7289fa79c9d00fe2c62a5cbbdfd34a/645",
    "components_browsertests": "67933c1bbafbb1a0a0e6c177874879fad87d8a73d3c8f0b88c9b8f29356e11a4/732",
    "components_unittests": "90267b90ca39e911a18983b125832213096c8a1938c7a4925be495046b73b992/811",
    "compositor_unittests": "b0c4bcb50a1bedc8e280d55c5b8a7157e75f35c37db22ec626c0d6632aa524fa/564",
    "content_browsertests": "ba4a8b3da3db77ba1aad8fd15fde019dcd203e3d4939a10e6f80eb1cd83419d4/805",
    "content_unittests": "587c5923afc68438a3b0676cd6773fb86096f5dca08fb033a70c781b48614b9e/725",
    "cronet_tests": "0cb2e04959b7d1a2696d8c1857b993fcb76a802293b88fafe490eb9454ca94c6/649",
    "cronet_unittests": "3e1fea2494cea602d2ad13b8267dfe4fb86c1beef9d3966130b497d5d19bec2c/565",
    "crypto_unittests": "8438b6266adb7aa8ac2f5f5d6fc7382101471954f5d21b67985c4287f539cec7/488",
    "dbus_unittests": "4d2d92f01d8f1dc66b36bcfe9386ae4deea506aa16232973d77d635d83971fdf/488",
    "device_unittests": "d4fb70cbe4b98895cf520c439ed0d67af08ed14b25b3cba2e2bddb780eb435e7/565",
    "display_unittests": "2ab0989b69d8b82634bba2d812926102dbb91fe617460b29f5ac8bbfb2eb071c/488",
    "env_chromium_unittests": "d0b73deb9f1f73c884bf7f30b41987dfdf1b12b82c06c506ca3ec30a367c476b/488",
    "events_unittests": "44902c46304a7b8c7e7e562cfbdc50f8a52f718f992887bbe3f2236383fa40d2/488",
    "extensions_browsertests": "1d7421dfb57c36220055d28717d7d63b407ffdfe92505aa119c8f16827b8bf4c/651",
    "extensions_unittests": "71f4f15ee116d79e2f1de2fc3894b65383266431e3bbb4e1691ee66768d0e283/730",
    "filesystem_service_unittests": "6abd9d3eb8c62ca94418fc4478595830b5fd028a544328ae27c86aebee6700c5/488",
    "fuzzing_unittests": "315c39f31f666b140e371b4bdb7ff34ffc174f75e776fce2e1f762511790eb6b/488",
    "gcm_unit_tests": "70e52644152d4b5dd2192e362104225346a7eda80fe44bddc70b354d9e3d6f6b/565",
    "gfx_unittests": "cb1f2ac495a9bd1039d9408c40c51443c08074d56fcdfed5bc511c1a6cbdebad/564",
    "gin_unittests": "43d44e402c2c6b3fadd1a9f1415a89bdebfb02ff689fd0691373adea65de7102/565",
    "gl_tests": "e8f44be401ee73a107a78b2799e7c3be48913c025f732bb585b91416dbe4a950/488",
    "gl_unittests": "11d24d51480a515b514a36fa47faa1e7b6f9235871b179753a3bc0f88ebae1ff/488",
    "google_apis_unittests": "ce31abbf1e17df57561e975185855bd8a3a57b74bb56776c27f4e5e6771e694e/650",
    "gpu_unittests": "2eb9fe83e96028438b5c2217b1e40893c94da3446999deaedbf85cb0e920cc2d/565",
    "gwp_asan_unittests": "a66e68f50f5f2c196f517967d887aaf77673c010fcdce2a68fe1b2b041cea82b/489",
    "headless_browsertests": "5798667b63684a3fa480f88de9042adbb6828199088bd09f2e6f19a067b62fa9/649",
    "headless_unittests": "fb945a835e01f2128deeed4191069b01fbff26c74e2701e65a0939965ce84dc3/488",
    "interactive_ui_tests": "5d77083ca0aa66fcd1f61b67ec5c4c8174294e16d5012870bf9496a2a3b52ace/973",
    "ipc_tests": "6604c79ba86f5196e80808f9d2c1eb9499c566fc5f8b325d45ff4a8ed121a75d/488",
    "latency_unittests": "7aab69f92f90f464956a5d89647396fa90c11787ca7e10322259ab85973040ac/488",
    "leveldb_unittests": "8f009af2250904fdff4cc5c7b8a91921a85809c8df2c4461c2399a06e441dff4/488",
    "libjingle_xmpp_unittests": "0beaec07ef9a18a26973f0afdf9cf9aa200604b4c5fb5ee73aa801de4152c3b2/488",
    "liburlpattern_unittests": "ed4bc43adf3641b5b38bea4b79a864aa44269bf8a36cc00e7b361afed44a1b05/488",
    "media_unittests": "1b0141ff3bd5925d14860b658d27cab2e6077c269533be73e056a8047ebd2aa4/568",
    "message_center_unittests": "c169e85569c0d663922ab89d9dc2550a6550c71e2f492c17fda21574d7e129a5/488",
    "midi_unittests": "999157bb72e3d18d154d86c0dfa595c4409f9b7bd887db9508ea29fefe8e7974/488",
    "mojo_unittests": "12a89de20b8c442d664dfd2660e99a080042abc8fb87cfa50f056a2c0273838a/566",
    "native_theme_unittests": "6775d49d0999c1b866d1931e5d304cf6cda04e4ddcc88a2d45fcbd3adcd8c6d5/488",
    "net_unittests": "6a0fca6be632bc19433c5eac1c47a81ab6cbcafff479aa40f5b9b7fa5b15fdc3/567",
    "openscreen_unittests": "fa83d19be2c4069626a4844e2a5f02430a4666b174b66870795f6ed474d2d98b/489",
    "ozone_unittests": "468e1edc4b56b7866601eefed0406105f32ac34a3ce4513e9ca4065c68225515/488",
    "ozone_x11_unittests": "d334edd697efd651851c0f38570c6617dfd639b87f9f67bacf0a13d3edc46406/488",
    "pdf_unittests": "23b73d56eec3b8fd5ebed74670d716e060b0c8c47328914c49f4c91e0b3ced13/565",
    "perfetto_unittests": "5b329d82619f22e616e92249c2b5a23e6bb43864c5216404296b13a5737b6ae7/488",
    "printing_unittests": "bc3c015d096caabf70fe0162cffef8ebbd84dad2a073a6946a088e40ec7e8802/488",
    "pthreadpool_unittests": "1d0a7185bd0b5262bcd580a92d239ecf2f2556fa18a98625d78616d8ec725458/488",
    "remoting_unittests": "4e17c1636e84f7716f3b18af3edc2a92c57ce77ee8054a44b11bfe63face8125/565",
    "sandbox_linux_unittests": "ca81b69c3a071edba3981135bc075c8c1a77cd751cba7020663047c1ee2cc452/488",
    "services_unittests": "6b0b178d5ab99756cdaba3bc080db74fad73e7f7d0a865a3ec3fb5039cfc7dc4/734",
    "shell_dialogs_unittests": "b818fe1e1843f755442993fc65d1cc84aef4c76b1a4dcbc49d4da8b6b784837f/488",
    "skia_unittests": "9cffb204d8f6fd2aadb665a804d13e7dba851a1ae70519aaa029948ca35c4d17/488",
    "snapshot_unittests": "3a3671dde956f0827c796840e6e6456a90077baa8e544041fda031fd5da02f41/488",
    "sql_unittests": "e175ba5d958af117cab77995a3a4649aa4790b48892441e6de7ffa7b9decc470/565",
    "storage_unittests": "fa160533c6e7595e6b536a9b0775378eec11accd2a8cc5679a8d72d395a0ba95/646",
    "sync_integration_tests": "6050fd5a35f22c9eb191f2eb3f1cf755e3cb88abcf25b410873d5dae49736e14/647",
    "ui_base_unittests": "b95b06f81175907369cc1def43ea03ed488fe166e025d7df4311502b993998cf/641",
    "ui_touch_selection_unittests": "cd905114335907b982457a19f377f5a9b71878560f98860ba109cd522bdcdda2/488",
    "ui_unittests": "fcc64103033fee856f2f7ac86a7f3729af1e5ec4db7c5211377f15cb6ae68bd8/488",
    "unit_tests": "303468c2db84269cad25353698a1e88e2c33dfe6600b2ea802b83428d5ce9aec/1061",
    "url_unittests": "1c9d20d98846018099ef7c32c422f165d9a3c25947fe66adf873a1b802fbd6ec/488",
    "views_unittests": "12dadfcbca99ff5028754aaf35608f3c64c392fc0c1ef5c56d0ed4fb6faa3ccd/488",
    "viz_unittests": "643de0d362f7ae86098e31ee682a8e76477abe0ed5f2a9caefc9251387465cda/572",
    "vr_common_unittests": "3ce900b244ddf2d5f34d85ecbf714810b6b30e26285baf96ce3a56c52f80f101/566",
    "wm_unittests": "d85f708c0e69159a1bab60b73460bb742727127c9515c88de131dde4e4a7690d/488",
    "wtf_unittests": "7c992dd769ba380f599e9b4e2afe0918d7105207838e21e12bbb8e74395cfecd/488",
    "xr_browser_tests": "924b73b3a59cf8090ff2f884e94473da753cfcbe63dfa13b4b6ad8287eae0d23/646",
    "zlib_unittests": "c2a2770096ca66cbf30324ab5af18264fb8af65e90abf6b2426b79dcfdd19d3d/489"
  },
  "swarming_command_lines_cwd": "out/e073-Linux_TSan_Buil",
  "swarming_command_lines_digest": "0ac3541af884a6da53be3c91b39ccb3662f24db53e932c9d1d7e21d5040c682d/94",
  "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/38f656c70c4343b39a62473cada43558a084620f
  • gitiles_ref:refs/branch-heads/7151
  • parent_buildername:Linux TSan Builder
  • parent_buildnumber:1368
  • user_agent:recipe

Debug log (UTC)

[17:57:49.022] New invocation is queued and will start shortly
[17:57:49.022] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[17:57:50.284] Starting the invocation (attempt 1)
[17:57:50.320] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/38f656c70c4343b39a62473cada43558a084620f"
[17:57:50.320] Popped gitiles commit info from properties and tags
[17:57:50.320] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[17:57:50.320] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[17:57:50.321] Buildbucket request:
{
  "requestId": "8875140084655014448",
  "builder": {
    "project": "chromium-m137",
    "bucket": "ci",
    "builder": "Linux TSan Tests"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "38f656c70c4343b39a62473cada43558a084620f",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/7151"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8875140084655014448",
            "job": "chromium-m137/Linux TSan Tests",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "38f656c70c4343b39a62473cada43558a084620f",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7151"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "38f656c70c4343b39a62473cada43558a084620f"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8709713667368472961"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8709713667368472961"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8709713667368472961"
                                              }
                                        ],
                                    "parent_build_id": "8709713667368472961",
                                    "parent_builder_group": "chromium.memory",
                                    "parent_buildername": "Linux TSan Builder",
                                    "parent_got_angle_revision": "df9c59dcacff7d186d00e3263a1aa68f8059137c",
                                    "parent_got_dawn_revision": "fbe707f88ccabca01031e47bf165bd9d499878dd",
                                    "parent_got_revision": "38f656c70c4343b39a62473cada43558a084620f",
                                    "parent_got_revision_cp": "refs/branch-heads/7151@{#3541}",
                                    "parent_got_swiftshader_revision": "7905fa19e456df5aa8e2233a7ec5832c9c6c287b",
                                    "parent_got_v8_revision": "cfefa0753b6454c7cb3dba261f6ff86bf5b19888",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/13.7@{#37}",
                                    "parent_got_webrtc_revision": "cec4daea7ed5da94fc38d790bd12694c86865447",
                                    "parent_got_webrtc_revision_cp": "refs/branch-heads/7151@{#3}",
                                    "revision": "38f656c70c4343b39a62473cada43558a084620f",
                                    "swarm_hashes": {
                                          "absl_hardening_tests": "e6a60e18376e4d837232512006968edee4867d931abd3cf4dcd06b4e6452789a/488",
                                          "accessibility_unittests": "df15826d9fcf306e8f6ff1fa23e5bdc8ea66cb70fd719647154a9129778e6cb1/488",
                                          "angle_unittests": "1b715eb6a48d05b361dc3f995aa53ac0c20a22900319ea887accfc2658e21250/489",
                                          "app_shell_unittests": "fac27423f1529be77450d06176a1ec65191ee007a7add1e55a6065aa15c34cc1/649",
                                          "aura_unittests": "fbdf09171bef7ce5fc9e1e746fdc64162a6f1258334d9a205d16771ac2b6cdfc/488",
                                          "base_unittests": "fe21c21f3cfe664d2a77152655291433507aee4ce327e9603b48967a0fbd75ec/566",
                                          "blink_common_unittests": "7dee2c27408b3eba69fc3d5b9be67c96359779f073c319d4e8bbef052a10c597/566",
                                          "blink_fuzzer_unittests": "f1990862bf3c7c8f96f98f37e62692f36a2b1b3cbe258a7d6a900a91201b590e/488",
                                          "blink_heap_unittests": "f3ddb418e70e53d01d302c990c9c844c8aecb8f9b44b7e633b29cc143411728e/488",
                                          "blink_platform_unittests": "38944905ce046ccf99f4d649cf3db14acafb2102926c71e0c38eca0591925de8/645",
                                          "blink_unittests": "534d568882872d11121ff8c986aea74b15e75c639dbb9792da23e1473a0111db/729",
                                          "boringssl_crypto_tests": "a60e34f76e1627d75b83a5fd603a311e86f49b3b6ab9ba1dc4426253b9772d7e/489",
                                          "boringssl_ssl_tests": "4c9f43cd7edb973afc12c6a403f857f5410c8a5c3259f08823ee3011cd8be11a/488",
                                          "capture_unittests": "2b2b7794432d5c06ad26e4507024b6b9adf9bd0e4a7b917d0371b8be2d8154fe/567",
                                          "cast_unittests": "d793553d3e95569342b7ea11e4ed51c84438e3eb4a9d788e6deebdd7407089e9/488",
                                          "cc_unittests": "e53bbf2e018b0ff6439da55a9603a76e266a04f3bfd869214f3859e176ca006b/648",
                                          "chrome_app_unittests": "b1d283258737fb61fb0a97d4e98a4b8044abc90e158483cdd7da14e5a79522b9/566",
                                          "chromedriver_unittests": "dd4a8ab7dde0d571a58056f4bc86f4895f7289fa79c9d00fe2c62a5cbbdfd34a/645",
                                          "components_browsertests": "67933c1bbafbb1a0a0e6c177874879fad87d8a73d3c8f0b88c9b8f29356e11a4/732",
                                          "components_unittests": "90267b90ca39e911a18983b125832213096c8a1938c7a4925be495046b73b992/811",
                                          "compositor_unittests": "b0c4bcb50a1bedc8e280d55c5b8a7157e75f35c37db22ec626c0d6632aa524fa/564",
                                          "content_browsertests": "ba4a8b3da3db77ba1aad8fd15fde019dcd203e3d4939a10e6f80eb1cd83419d4/805",
                                          "content_unittests": "587c5923afc68438a3b0676cd6773fb86096f5dca08fb033a70c781b48614b9e/725",
                                          "cronet_tests": "0cb2e04959b7d1a2696d8c1857b993fcb76a802293b88fafe490eb9454ca94c6/649",
                                          "cronet_unittests": "3e1fea2494cea602d2ad13b8267dfe4fb86c1beef9d3966130b497d5d19bec2c/565",
                                          "crypto_unittests": "8438b6266adb7aa8ac2f5f5d6fc7382101471954f5d21b67985c4287f539cec7/488",
                                          "dbus_unittests": "4d2d92f01d8f1dc66b36bcfe9386ae4deea506aa16232973d77d635d83971fdf/488",
                                          "device_unittests": "d4fb70cbe4b98895cf520c439ed0d67af08ed14b25b3cba2e2bddb780eb435e7/565",
                                          "display_unittests": "2ab0989b69d8b82634bba2d812926102dbb91fe617460b29f5ac8bbfb2eb071c/488",
                                          "env_chromium_unittests": "d0b73deb9f1f73c884bf7f30b41987dfdf1b12b82c06c506ca3ec30a367c476b/488",
                                          "events_unittests": "44902c46304a7b8c7e7e562cfbdc50f8a52f718f992887bbe3f2236383fa40d2/488",
                                          "extensions_browsertests": "1d7421dfb57c36220055d28717d7d63b407ffdfe92505aa119c8f16827b8bf4c/651",
                                          "extensions_unittests": "71f4f15ee116d79e2f1de2fc3894b65383266431e3bbb4e1691ee66768d0e283/730",
                                          "filesystem_service_unittests": "6abd9d3eb8c62ca94418fc4478595830b5fd028a544328ae27c86aebee6700c5/488",
                                          "fuzzing_unittests": "315c39f31f666b140e371b4bdb7ff34ffc174f75e776fce2e1f762511790eb6b/488",
                                          "gcm_unit_tests": "70e52644152d4b5dd2192e362104225346a7eda80fe44bddc70b354d9e3d6f6b/565",
                                          "gfx_unittests": "cb1f2ac495a9bd1039d9408c40c51443c08074d56fcdfed5bc511c1a6cbdebad/564",
                                          "gin_unittests": "43d44e402c2c6b3fadd1a9f1415a89bdebfb02ff689fd0691373adea65de7102/565",
                                          "gl_tests": "e8f44be401ee73a107a78b2799e7c3be48913c025f732bb585b91416dbe4a950/488",
                                          "gl_unittests": "11d24d51480a515b514a36fa47faa1e7b6f9235871b179753a3bc0f88ebae1ff/488",
                                          "google_apis_unittests": "ce31abbf1e17df57561e975185855bd8a3a57b74bb56776c27f4e5e6771e694e/650",
                                          "gpu_unittests": "2eb9fe83e96028438b5c2217b1e40893c94da3446999deaedbf85cb0e920cc2d/565",
                                          "gwp_asan_unittests": "a66e68f50f5f2c196f517967d887aaf77673c010fcdce2a68fe1b2b041cea82b/489",
                                          "headless_browsertests": "5798667b63684a3fa480f88de9042adbb6828199088bd09f2e6f19a067b62fa9/649",
                                          "headless_unittests": "fb945a835e01f2128deeed4191069b01fbff26c74e2701e65a0939965ce84dc3/488",
                                          "interactive_ui_tests": "5d77083ca0aa66fcd1f61b67ec5c4c8174294e16d5012870bf9496a2a3b52ace/973",
                                          "ipc_tests": "6604c79ba86f5196e80808f9d2c1eb9499c566fc5f8b325d45ff4a8ed121a75d/488",
                                          "latency_unittests": "7aab69f92f90f464956a5d89647396fa90c11787ca7e10322259ab85973040ac/488",
                                          "leveldb_unittests": "8f009af2250904fdff4cc5c7b8a91921a85809c8df2c4461c2399a06e441dff4/488",
                                          "libjingle_xmpp_unittests": "0beaec07ef9a18a26973f0afdf9cf9aa200604b4c5fb5ee73aa801de4152c3b2/488",
                                          "liburlpattern_unittests": "ed4bc43adf3641b5b38bea4b79a864aa44269bf8a36cc00e7b361afed44a1b05/488",
                                          "media_unittests": "1b0141ff3bd5925d14860b658d27cab2e6077c269533be73e056a8047ebd2aa4/568",
                                          "message_center_unittests": "c169e85569c0d663922ab89d9dc2550a6550c71e2f492c17fda21574d7e129a5/488",
                                          "midi_unittests": "999157bb72e3d18d154d86c0dfa595c4409f9b7bd887db9508ea29fefe8e7974/488",
                                          "mojo_unittests": "12a89de20b8c442d664dfd2660e99a080042abc8fb87cfa50f056a2c0273838a/566",
                                          "native_theme_unittests": "6775d49d0999c1b866d1931e5d304cf6cda04e4ddcc88a2d45fcbd3adcd8c6d5/488",
                                          "net_unittests": "6a0fca6be632bc19433c5eac1c47a81ab6cbcafff479aa40f5b9b7fa5b15fdc3/567",
                                          "openscreen_unittests": "fa83d19be2c4069626a4844e2a5f02430a4666b174b66870795f6ed474d2d98b/489",
                                          "ozone_unittests": "468e1edc4b56b7866601eefed0406105f32ac34a3ce4513e9ca4065c68225515/488",
                                          "ozone_x11_unittests": "d334edd697efd651851c0f38570c6617dfd639b87f9f67bacf0a13d3edc46406/488",
                                          "pdf_unittests": "23b73d56eec3b8fd5ebed74670d716e060b0c8c47328914c49f4c91e0b3ced13/565",
                                          "perfetto_unittests": "5b329d82619f22e616e92249c2b5a23e6bb43864c5216404296b13a5737b6ae7/488",
                                          "printing_unittests": "bc3c015d096caabf70fe0162cffef8ebbd84dad2a073a6946a088e40ec7e8802/488",
                                          "pthreadpool_unittests": "1d0a7185bd0b5262bcd580a92d239ecf2f2556fa18a98625d78616d8ec725458/488",
                                          "remoting_unittests": "4e17c1636e84f7716f3b18af3edc2a92c57ce77ee8054a44b11bfe63face8125/565",
                                          "sandbox_linux_unittests": "ca81b69c3a071edba3981135bc075c8c1a77cd751cba7020663047c1ee2cc452/488",
                                          "services_unittests": "6b0b178d5ab99756cdaba3bc080db74fad73e7f7d0a865a3ec3fb5039cfc7dc4/734",
                                          "shell_dialogs_unittests": "b818fe1e1843f755442993fc65d1cc84aef4c76b1a4dcbc49d4da8b6b784837f/488",
                                          "skia_unittests": "9cffb204d8f6fd2aadb665a804d13e7dba851a1ae70519aaa029948ca35c4d17/488",
                                          "snapshot_unittests": "3a3671dde956f0827c796840e6e6456a90077baa8e544041fda031fd5da02f41/488",
                                          "sql_unittests": "e175ba5d958af117cab77995a3a4649aa4790b48892441e6de7ffa7b9decc470/565",
                                          "storage_unittests": "fa160533c6e7595e6b536a9b0775378eec11accd2a8cc5679a8d72d395a0ba95/646",
                                          "sync_integration_tests": "6050fd5a35f22c9eb191f2eb3f1cf755e3cb88abcf25b410873d5dae49736e14/647",
                                          "ui_base_unittests": "b95b06f81175907369cc1def43ea03ed488fe166e025d7df4311502b993998cf/641",
                                          "ui_touch_selection_unittests": "cd905114335907b982457a19f377f5a9b71878560f98860ba109cd522bdcdda2/488",
                                          "ui_unittests": "fcc64103033fee856f2f7ac86a7f3729af1e5ec4db7c5211377f15cb6ae68bd8/488",
                                          "unit_tests": "303468c2db84269cad25353698a1e88e2c33dfe6600b2ea802b83428d5ce9aec/1061",
                                          "url_unittests": "1c9d20d98846018099ef7c32c422f165d9a3c25947fe66adf873a1b802fbd6ec/488",
                                          "views_unittests": "12dadfcbca99ff5028754aaf35608f3c64c392fc0c1ef5c56d0ed4fb6faa3ccd/488",
                                          "viz_unittests": "643de0d362f7ae86098e31ee682a8e76477abe0ed5f2a9caefc9251387465cda/572",
                                          "vr_common_unittests": "3ce900b244ddf2d5f34d85ecbf714810b6b30e26285baf96ce3a56c52f80f101/566",
                                          "wm_unittests": "d85f708c0e69159a1bab60b73460bb742727127c9515c88de131dde4e4a7690d/488",
                                          "wtf_unittests": "7c992dd769ba380f599e9b4e2afe0918d7105207838e21e12bbb8e74395cfecd/488",
                                          "xr_browser_tests": "924b73b3a59cf8090ff2f884e94473da753cfcbe63dfa13b4b6ad8287eae0d23/646",
                                          "zlib_unittests": "c2a2770096ca66cbf30324ab5af18264fb8af65e90abf6b2426b79dcfdd19d3d/489"
                                        },
                                    "swarming_command_lines_cwd": "out/e073-Linux_TSan_Buil",
                                    "swarming_command_lines_digest": "0ac3541af884a6da53be3c91b39ccb3662f24db53e932c9d1d7e21d5040c682d/94",
                                    "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
                                  },
                              "ref": "refs/branch-heads/7151",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "38f656c70c4343b39a62473cada43558a084620f",
                              "tags": [
                                    "parent_buildername:Linux TSan Builder",
                                    "parent_buildnumber:1368",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "d2c84e0d-ab92-49ef-81eb-fbe9e29d5c26",
                        "title": "Linux TSan Builder/1368",
                        "url": "https://cr-buildbucket.appspot.com/build/8709713667368472961"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "38f656c70c4343b39a62473cada43558a084620f"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8709713667368472961"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8709713667368472961"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8709713667368472961"
                }
          ],
      "parent_build_id": "8709713667368472961",
      "parent_builder_group": "chromium.memory",
      "parent_buildername": "Linux TSan Builder",
      "parent_got_angle_revision": "df9c59dcacff7d186d00e3263a1aa68f8059137c",
      "parent_got_dawn_revision": "fbe707f88ccabca01031e47bf165bd9d499878dd",
      "parent_got_revision": "38f656c70c4343b39a62473cada43558a084620f",
      "parent_got_revision_cp": "refs/branch-heads/7151@{#3541}",
      "parent_got_swiftshader_revision": "7905fa19e456df5aa8e2233a7ec5832c9c6c287b",
      "parent_got_v8_revision": "cfefa0753b6454c7cb3dba261f6ff86bf5b19888",
      "parent_got_v8_revision_cp": "refs/branch-heads/13.7@{#37}",
      "parent_got_webrtc_revision": "cec4daea7ed5da94fc38d790bd12694c86865447",
      "parent_got_webrtc_revision_cp": "refs/branch-heads/7151@{#3}",
      "swarm_hashes": {
            "absl_hardening_tests": "e6a60e18376e4d837232512006968edee4867d931abd3cf4dcd06b4e6452789a/488",
            "accessibility_unittests": "df15826d9fcf306e8f6ff1fa23e5bdc8ea66cb70fd719647154a9129778e6cb1/488",
            "angle_unittests": "1b715eb6a48d05b361dc3f995aa53ac0c20a22900319ea887accfc2658e21250/489",
            "app_shell_unittests": "fac27423f1529be77450d06176a1ec65191ee007a7add1e55a6065aa15c34cc1/649",
            "aura_unittests": "fbdf09171bef7ce5fc9e1e746fdc64162a6f1258334d9a205d16771ac2b6cdfc/488",
            "base_unittests": "fe21c21f3cfe664d2a77152655291433507aee4ce327e9603b48967a0fbd75ec/566",
            "blink_common_unittests": "7dee2c27408b3eba69fc3d5b9be67c96359779f073c319d4e8bbef052a10c597/566",
            "blink_fuzzer_unittests": "f1990862bf3c7c8f96f98f37e62692f36a2b1b3cbe258a7d6a900a91201b590e/488",
            "blink_heap_unittests": "f3ddb418e70e53d01d302c990c9c844c8aecb8f9b44b7e633b29cc143411728e/488",
            "blink_platform_unittests": "38944905ce046ccf99f4d649cf3db14acafb2102926c71e0c38eca0591925de8/645",
            "blink_unittests": "534d568882872d11121ff8c986aea74b15e75c639dbb9792da23e1473a0111db/729",
            "boringssl_crypto_tests": "a60e34f76e1627d75b83a5fd603a311e86f49b3b6ab9ba1dc4426253b9772d7e/489",
            "boringssl_ssl_tests": "4c9f43cd7edb973afc12c6a403f857f5410c8a5c3259f08823ee3011cd8be11a/488",
            "capture_unittests": "2b2b7794432d5c06ad26e4507024b6b9adf9bd0e4a7b917d0371b8be2d8154fe/567",
            "cast_unittests": "d793553d3e95569342b7ea11e4ed51c84438e3eb4a9d788e6deebdd7407089e9/488",
            "cc_unittests": "e53bbf2e018b0ff6439da55a9603a76e266a04f3bfd869214f3859e176ca006b/648",
            "chrome_app_unittests": "b1d283258737fb61fb0a97d4e98a4b8044abc90e158483cdd7da14e5a79522b9/566",
            "chromedriver_unittests": "dd4a8ab7dde0d571a58056f4bc86f4895f7289fa79c9d00fe2c62a5cbbdfd34a/645",
            "components_browsertests": "67933c1bbafbb1a0a0e6c177874879fad87d8a73d3c8f0b88c9b8f29356e11a4/732",
            "components_unittests": "90267b90ca39e911a18983b125832213096c8a1938c7a4925be495046b73b992/811",
            "compositor_unittests": "b0c4bcb50a1bedc8e280d55c5b8a7157e75f35c37db22ec626c0d6632aa524fa/564",
            "content_browsertests": "ba4a8b3da3db77ba1aad8fd15fde019dcd203e3d4939a10e6f80eb1cd83419d4/805",
            "content_unittests": "587c5923afc68438a3b0676cd6773fb86096f5dca08fb033a70c781b48614b9e/725",
            "cronet_tests": "0cb2e04959b7d1a2696d8c1857b993fcb76a802293b88fafe490eb9454ca94c6/649",
            "cronet_unittests": "3e1fea2494cea602d2ad13b8267dfe4fb86c1beef9d3966130b497d5d19bec2c/565",
            "crypto_unittests": "8438b6266adb7aa8ac2f5f5d6fc7382101471954f5d21b67985c4287f539cec7/488",
            "dbus_unittests": "4d2d92f01d8f1dc66b36bcfe9386ae4deea506aa16232973d77d635d83971fdf/488",
            "device_unittests": "d4fb70cbe4b98895cf520c439ed0d67af08ed14b25b3cba2e2bddb780eb435e7/565",
            "display_unittests": "2ab0989b69d8b82634bba2d812926102dbb91fe617460b29f5ac8bbfb2eb071c/488",
            "env_chromium_unittests": "d0b73deb9f1f73c884bf7f30b41987dfdf1b12b82c06c506ca3ec30a367c476b/488",
            "events_unittests": "44902c46304a7b8c7e7e562cfbdc50f8a52f718f992887bbe3f2236383fa40d2/488",
            "extensions_browsertests": "1d7421dfb57c36220055d28717d7d63b407ffdfe92505aa119c8f16827b8bf4c/651",
            "extensions_unittests": "71f4f15ee116d79e2f1de2fc3894b65383266431e3bbb4e1691ee66768d0e283/730",
            "filesystem_service_unittests": "6abd9d3eb8c62ca94418fc4478595830b5fd028a544328ae27c86aebee6700c5/488",
            "fuzzing_unittests": "315c39f31f666b140e371b4bdb7ff34ffc174f75e776fce2e1f762511790eb6b/488",
            "gcm_unit_tests": "70e52644152d4b5dd2192e362104225346a7eda80fe44bddc70b354d9e3d6f6b/565",
            "gfx_unittests": "cb1f2ac495a9bd1039d9408c40c51443c08074d56fcdfed5bc511c1a6cbdebad/564",
            "gin_unittests": "43d44e402c2c6b3fadd1a9f1415a89bdebfb02ff689fd0691373adea65de7102/565",
            "gl_tests": "e8f44be401ee73a107a78b2799e7c3be48913c025f732bb585b91416dbe4a950/488",
            "gl_unittests": "11d24d51480a515b514a36fa47faa1e7b6f9235871b179753a3bc0f88ebae1ff/488",
            "google_apis_unittests": "ce31abbf1e17df57561e975185855bd8a3a57b74bb56776c27f4e5e6771e694e/650",
            "gpu_unittests": "2eb9fe83e96028438b5c2217b1e40893c94da3446999deaedbf85cb0e920cc2d/565",
            "gwp_asan_unittests": "a66e68f50f5f2c196f517967d887aaf77673c010fcdce2a68fe1b2b041cea82b/489",
            "headless_browsertests": "5798667b63684a3fa480f88de9042adbb6828199088bd09f2e6f19a067b62fa9/649",
            "headless_unittests": "fb945a835e01f2128deeed4191069b01fbff26c74e2701e65a0939965ce84dc3/488",
            "interactive_ui_tests": "5d77083ca0aa66fcd1f61b67ec5c4c8174294e16d5012870bf9496a2a3b52ace/973",
            "ipc_tests": "6604c79ba86f5196e80808f9d2c1eb9499c566fc5f8b325d45ff4a8ed121a75d/488",
            "latency_unittests": "7aab69f92f90f464956a5d89647396fa90c11787ca7e10322259ab85973040ac/488",
            "leveldb_unittests": "8f009af2250904fdff4cc5c7b8a91921a85809c8df2c4461c2399a06e441dff4/488",
            "libjingle_xmpp_unittests": "0beaec07ef9a18a26973f0afdf9cf9aa200604b4c5fb5ee73aa801de4152c3b2/488",
            "liburlpattern_unittests": "ed4bc43adf3641b5b38bea4b79a864aa44269bf8a36cc00e7b361afed44a1b05/488",
            "media_unittests": "1b0141ff3bd5925d14860b658d27cab2e6077c269533be73e056a8047ebd2aa4/568",
            "message_center_unittests": "c169e85569c0d663922ab89d9dc2550a6550c71e2f492c17fda21574d7e129a5/488",
            "midi_unittests": "999157bb72e3d18d154d86c0dfa595c4409f9b7bd887db9508ea29fefe8e7974/488",
            "mojo_unittests": "12a89de20b8c442d664dfd2660e99a080042abc8fb87cfa50f056a2c0273838a/566",
            "native_theme_unittests": "6775d49d0999c1b866d1931e5d304cf6cda04e4ddcc88a2d45fcbd3adcd8c6d5/488",
            "net_unittests": "6a0fca6be632bc19433c5eac1c47a81ab6cbcafff479aa40f5b9b7fa5b15fdc3/567",
            "openscreen_unittests": "fa83d19be2c4069626a4844e2a5f02430a4666b174b66870795f6ed474d2d98b/489",
            "ozone_unittests": "468e1edc4b56b7866601eefed0406105f32ac34a3ce4513e9ca4065c68225515/488",
            "ozone_x11_unittests": "d334edd697efd651851c0f38570c6617dfd639b87f9f67bacf0a13d3edc46406/488",
            "pdf_unittests": "23b73d56eec3b8fd5ebed74670d716e060b0c8c47328914c49f4c91e0b3ced13/565",
            "perfetto_unittests": "5b329d82619f22e616e92249c2b5a23e6bb43864c5216404296b13a5737b6ae7/488",
            "printing_unittests": "bc3c015d096caabf70fe0162cffef8ebbd84dad2a073a6946a088e40ec7e8802/488",
            "pthreadpool_unittests": "1d0a7185bd0b5262bcd580a92d239ecf2f2556fa18a98625d78616d8ec725458/488",
            "remoting_unittests": "4e17c1636e84f7716f3b18af3edc2a92c57ce77ee8054a44b11bfe63face8125/565",
            "sandbox_linux_unittests": "ca81b69c3a071edba3981135bc075c8c1a77cd751cba7020663047c1ee2cc452/488",
            "services_unittests": "6b0b178d5ab99756cdaba3bc080db74fad73e7f7d0a865a3ec3fb5039cfc7dc4/734",
            "shell_dialogs_unittests": "b818fe1e1843f755442993fc65d1cc84aef4c76b1a4dcbc49d4da8b6b784837f/488",
            "skia_unittests": "9cffb204d8f6fd2aadb665a804d13e7dba851a1ae70519aaa029948ca35c4d17/488",
            "snapshot_unittests": "3a3671dde956f0827c796840e6e6456a90077baa8e544041fda031fd5da02f41/488",
            "sql_unittests": "e175ba5d958af117cab77995a3a4649aa4790b48892441e6de7ffa7b9decc470/565",
            "storage_unittests": "fa160533c6e7595e6b536a9b0775378eec11accd2a8cc5679a8d72d395a0ba95/646",
            "sync_integration_tests": "6050fd5a35f22c9eb191f2eb3f1cf755e3cb88abcf25b410873d5dae49736e14/647",
            "ui_base_unittests": "b95b06f81175907369cc1def43ea03ed488fe166e025d7df4311502b993998cf/641",
            "ui_touch_selection_unittests": "cd905114335907b982457a19f377f5a9b71878560f98860ba109cd522bdcdda2/488",
            "ui_unittests": "fcc64103033fee856f2f7ac86a7f3729af1e5ec4db7c5211377f15cb6ae68bd8/488",
            "unit_tests": "303468c2db84269cad25353698a1e88e2c33dfe6600b2ea802b83428d5ce9aec/1061",
            "url_unittests": "1c9d20d98846018099ef7c32c422f165d9a3c25947fe66adf873a1b802fbd6ec/488",
            "views_unittests": "12dadfcbca99ff5028754aaf35608f3c64c392fc0c1ef5c56d0ed4fb6faa3ccd/488",
            "viz_unittests": "643de0d362f7ae86098e31ee682a8e76477abe0ed5f2a9caefc9251387465cda/572",
            "vr_common_unittests": "3ce900b244ddf2d5f34d85ecbf714810b6b30e26285baf96ce3a56c52f80f101/566",
            "wm_unittests": "d85f708c0e69159a1bab60b73460bb742727127c9515c88de131dde4e4a7690d/488",
            "wtf_unittests": "7c992dd769ba380f599e9b4e2afe0918d7105207838e21e12bbb8e74395cfecd/488",
            "xr_browser_tests": "924b73b3a59cf8090ff2f884e94473da753cfcbe63dfa13b4b6ad8287eae0d23/646",
            "zlib_unittests": "c2a2770096ca66cbf30324ab5af18264fb8af65e90abf6b2426b79dcfdd19d3d/489"
          },
      "swarming_command_lines_cwd": "out/e073-Linux_TSan_Buil",
      "swarming_command_lines_digest": "0ac3541af884a6da53be3c91b39ccb3662f24db53e932c9d1d7e21d5040c682d/94",
      "swarming_rts_command_lines_digest": "9484f33bed6c51d11cc0b1225b7532c3b2ccd522df35d72f0208c718535666e9/92"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "38f656c70c4343b39a62473cada43558a084620f",
    "ref": "refs/branch-heads/7151"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8875140084655014448"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m137/Linux TSan Tests"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Linux TSan Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "1368"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[17:57:50.929] Scheduled build:
{
  "id": "8709710022513927633",
  "builder": {
    "project": "chromium-m137",
    "bucket": "ci",
    "builder": "Linux TSan Tests"
  },
  "number": 1365,
  "createdBy": "project:chromium-m137",
  "createTime": "2025-07-10T17:57:50.390261596Z",
  "updateTime": "2025-07-10T17:57:50.390261596Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "38f656c70c4343b39a62473cada43558a084620f",
      "ref": "refs/branch-heads/7151"
    }
  }
}
[17:57:50.929] Task URL: https://cr-buildbucket.appspot.com/build/8709710022513927633
[17:57:50.929] Scheduling timer "check-buildbucket-build-status" (chromium-m137/Linux TSan Tests:8875140084655014448:2:0) after 8m10s
[17:57:52.068] Received PubSub notification, asking Buildbucket for the build status
[17:57:52.091] Build status: SCHEDULED
[17:57:58.375] Received PubSub notification, asking Buildbucket for the build status
[17:57:58.426] Build status: STARTED
[18:06:00.964] Handling timer "check-buildbucket-build-status" (chromium-m137/Linux TSan Tests:8875140084655014448:2:0)
[18:06:01.048] Build status: STARTED
[18:06:01.048] Scheduling timer "check-buildbucket-build-status" (chromium-m137/Linux TSan Tests:8875140084655014448:5:0) after 4m39s
[18:10:40.147] Handling timer "check-buildbucket-build-status" (chromium-m137/Linux TSan Tests:8875140084655014448:5:0)
[18:10:40.241] Build status: STARTED
[18:10:40.241] Scheduling timer "check-buildbucket-build-status" (chromium-m137/Linux TSan Tests:8875140084655014448:6:0) after 8m20s
[18:19:00.344] Handling timer "check-buildbucket-build-status" (chromium-m137/Linux TSan Tests:8875140084655014448:6:0)
[18:19:00.380] Build status: STARTED
[18:19:00.380] Scheduling timer "check-buildbucket-build-status" (chromium-m137/Linux TSan Tests:8875140084655014448:7:0) after 6m43s
[18:19:54.258] Received PubSub notification, asking Buildbucket for the build status
[18:19:54.291] Build:
{
  "id": "8709710022513927633",
  "builder": {
    "project": "chromium-m137",
    "bucket": "ci",
    "builder": "Linux TSan Tests"
  },
  "number": 1365,
  "createdBy": "project:chromium-m137",
  "createTime": "2025-07-10T17:57:50.390261596Z",
  "startTime": "2025-07-10T17:57:58.155390105Z",
  "endTime": "2025-07-10T18:19:53.609115413Z",
  "updateTime": "2025-07-10T18:19:53.609115413Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "38f656c70c4343b39a62473cada43558a084620f",
      "ref": "refs/branch-heads/7151"
    }
  }
}
[18:19:54.291] Invocation finished in 22m5.32688711s with status FAILED