Status: FAILED
Triggered by: chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 42 minutes
Actions:

Definition (rev cbb863c8fcfc493c6b34c33210ca32b6dd6fa019)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "Linux TSan Tests"
>

Incoming triggers

Incoming properties

{
  "$bootstrap/trigger": {
    "commits": [
      {
        "host": "chromium.googlesource.com",
        "id": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d",
        "project": "chromium/src",
        "ref": "refs/branch-heads/7871"
      }
    ]
  },
  "branch": "refs/branch-heads/7871",
  "deps_revision_overrides": {
    "src": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d"
  },
  "instruction_dependencies": [
    {
      "instruction_id": "bot_update_repro_instructions",
      "invocation_id": "build-8674739453017649857"
    },
    {
      "instruction_id": "generate_build_files_repro_instructions",
      "invocation_id": "build-8674739453017649857"
    },
    {
      "instruction_id": "compile_repro_instructions",
      "invocation_id": "build-8674739453017649857"
    }
  ],
  "parent_build_id": "8674739453017649857",
  "parent_builder_group": "chromium.memory",
  "parent_buildername": "Linux TSan Builder",
  "parent_got_angle_revision": "23e9441d4e3019c63c1b0bf9b00a413a6cbc2afa",
  "parent_got_dawn_revision": "d089fc91e7e4881362463faf8efe9ae435e34660",
  "parent_got_revision": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d",
  "parent_got_revision_cp": "refs/branch-heads/7871@{#4231}",
  "parent_got_swiftshader_revision": "fce27a96526f54c6d31fdccf57629788e3712220",
  "parent_got_v8_revision": "80533795232dad32e0ff68ff3a4fbb32acaa36ae",
  "parent_got_v8_revision_cp": "refs/branch-heads/15.0@{#48}",
  "parent_got_webrtc_revision": "1f975dfd761af6e5d76d28333191973b258d82a8",
  "parent_got_webrtc_revision_cp": "refs/branch-heads/7871@{#3}",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d",
  "swarm_hashes": {
    "absl_hardening_tests": "cd2974dd04928fd321ef60d47755f7ef600527b627c6461bff33fd7acf6961bb/488",
    "accessibility_unittests": "06eb92db7e3c750b98ede44655a73f746f300d40ff4b380327319e5fa892fb27/488",
    "angle_unittests": "2d5c58111c9d146bb1ff674b87bcc4f31f43a4a43dbab1d04c7fffc9a622a744/489",
    "aura_unittests": "c1c7ba0cd47eba80e52b82d212f5552c297b7604da113310fe82f77fdc562649/488",
    "base_unittests": "3a335303d05cb6fa9a3ddfbd218219d46d21a603447ba2f6b5e459e0805fba7e/566",
    "blink_common_unittests": "f7125eec4ce6e3a0cdbad181f6c995c66c0b02b4417c4177674ff4754fb9efb5/566",
    "blink_fuzzer_unittests": "fa37930624774f4cb79e388bda4ce850270974e53b8fa32325e45dfe5acd07c2/488",
    "blink_heap_unittests": "8aa112c814ef3521edd5f1d6ca3b0b963224870cd4be97f74641051cdd3a593c/488",
    "blink_platform_unittests": "55b0c709987bc18c962a5d43f2ba68a2af0defa86ed59fa8bac262207b48cb86/645",
    "blink_unittests": "c5442eee642b7e59afd3c6fb52eabbf7200c415003879e829bb6a9e113dc0c70/729",
    "boringssl_crypto_tests": "d478dd8b1a444902180d9c8d068f288a2aec81acbe6f966432f13626395ef04b/489",
    "boringssl_ssl_tests": "9ea4e7f165b413a9dbc04c037b65848f4c9ffbea8abaf9f3ab0b49e5f6bcad29/488",
    "capture_unittests": "9952dd7492ba4b551f5ab63977764fa2ac4f4bbd104e962e22cef7a9c7bdf0ae/567",
    "cast_unittests": "3a4d06a4d7c41ea2ebafe405fa019ebdf5e9a8794417f07b41c86983d46ae94c/488",
    "cc_unittests": "d6d7958a75233b3e1029315c8c30b0a6ec35c355b6cc99c51a2234ae7b2da2fb/648",
    "chrome_app_unittests": "98650c7301fbdcb1053f524ff68e965181b06c5aa615f1ca58a1b6f390a95264/566",
    "chromedriver_unittests": "46c8ff48b8cc429cd2c42be2109241bc71eabf0b8790f31de5ba350c7fd2cc5b/645",
    "components_browsertests": "620b99e9708eb7454255a4a792c77e8f970b62c823500e5842683333a78f68cb/732",
    "components_unittests": "3a7b719ec272f32c9892c0cbb7d38946ebdd23f1e7aa06632431642fc8d85be5/811",
    "compositor_unittests": "f482df7ec5f0b3d97ed29b66317e2a9ac553d40f3b89a798d0ad769583aa02ef/564",
    "content_browsertests": "a5fe19752cb11295bd4166ab6780e5dbfd770f7ced04900f7bea5da19ffc43f0/805",
    "content_unittests": "ca521f65542f388bd113213f291694122a6cc1efe92d7e4f7c14fe9593f39ee5/725",
    "crypto_unittests": "393eeb1a74b3682b3ef9eba80401be44b4ca990296e63dcec8d62cf9b930c3f4/568",
    "dbus_unittests": "f69feb0f2dbc6427eeceb0b4e32ee9ac8049e96cb5fb5a2c2948578c5bb8d031/488",
    "device_unittests": "09d6945d9b76126ab7ecb390a9aee09217192a6c99d83f557e3f619ba3972c6b/565",
    "display_unittests": "4cd83db1de0a5e7eacaa31bdc1e543958245d06c28907b6da98081e6846c40f2/488",
    "env_chromium_unittests": "055c6993d374fcb1a04d3db9264972275c9a5c62867cc87175f2e11f66688cd7/488",
    "events_unittests": "7ec07ee002e2ec41e8481b878224ab92a755673cadaa6c7a8ed0079de67001cb/488",
    "extensions_unittests": "2aef3387bcc65f98bdd9fa0b803b757309657f1a744dbaccee4344301a90eb9f/730",
    "filesystem_service_unittests": "331c1dfe1e28a8bbb8ca514853945e228a449acee8e0d929d4b7deb5c7462fbb/488",
    "fuzzing_unittests": "e7019b196a301f61a278cbbb68dce94ec1330815d4436361d63967b687159300/488",
    "gcm_unit_tests": "9f6142db2c16c15ba66f7b750ac7fb96abe3df0369db31e90d414dbf76d7cc58/565",
    "gfx_unittests": "9f209b9d75b5bef15e6035b41f49d8d46528a16038a71c9064551e3222f6adcb/564",
    "gin_unittests": "e6f9d11691e2e24b77950bf01ab21730733916f8a5eff5f10dfebbf19501d10e/565",
    "gl_tests": "06a9f6a317dac2fc3c1e818660016bd90e0e5d4f59c6771576d6f618407ba62f/488",
    "gl_unittests": "17c47ec85b560bcf926a50c5c79d9adc2900015dc9a9d62603d34cf78337b55b/488",
    "google_apis_unittests": "fd0adca83bc80080ff5ce6117865270d2ed2f56deb88f465e35f169a1ee61228/650",
    "gpu_unittests": "6d33898f65168f196a2156237189646324b3ee751b4336ed49f6c31fcd56ac75/565",
    "gtk_unittests": "a3746acdb6e767bd8b0966625090bc77037131f846bf0ee9dc3850f30646e918/488",
    "gwp_asan_unittests": "d5930c4d4e3869d10d25209705b7558730fc071491d30e1136c2e7d1c1392035/489",
    "headless_browsertests": "dac88429ef563d838c71e8f627020bce08530e02630fd59da300b23fc28393f7/733",
    "headless_unittests": "199f2e755a47a0535a2e89b8249d8e4e8ea923958ddce3b95530085c25600dd1/488",
    "interactive_ui_tests": "0cf8c18df6396cc0124d37ec3e48055ac30fdbc9f5042675266cbea4708f6604/1059",
    "ipc_tests": "bce8851be7bee08961898db20e306bd9a031b0cd976c44200bd7af585014b6fe/488",
    "latency_unittests": "066c795edef5b4dd84f48d48c951c0bc220e4846c0d6dac14deb3ecdc8caea4c/488",
    "leveldb_unittests": "5001d312ba5be99ecf35782e769d5fe97db3ab99acdd65428c934b414711bb53/488",
    "libjingle_xmpp_unittests": "23ac6783dc9961f9bf8a0e893a4fa22948ad847a3991a3dec661bdfa218801d8/488",
    "liburlpattern_unittests": "3c4f0be01fe12e516aa8b3fb0d9d4bc167f0193c212e4eb7d739c4f6db40f23c/488",
    "media_unittests": "523ffcbf75c07b92d3b384a5b9ae31fe9a9bfccd3a95d4a09cda3ecc5dc28afc/568",
    "message_center_unittests": "5f075b51cb59316fd348c6cd4df3368356f63968a375fdceddbb259af094d4f3/488",
    "midi_unittests": "feb90942935455c3d44f0e6a579ae4355d2b005cdbcfa84d953cc31f6f60be24/488",
    "mojo_unittests": "74842fb87940a1c90b7b7df8df0c6abd806edbb7e0109a20c63d247d87261728/566",
    "native_theme_unittests": "e289442ca4c9dd3e4c5da70f2983295fb3436dde21170ddb3fcf416958b687e2/488",
    "net_unittests": "0e958aa9e657fa8e7b47ec6daf237afb169262037dd274f2c94f5ad2ab41b686/567",
    "openscreen_unittests": "b250a27180a0168f37de2ba172e2623b09d99cadbf3e08552872463c27d7983f/489",
    "ozone_unittests": "63cfee8702ef4326412497ea8be921daaf029d77a05359da703af1b3edac05ba/488",
    "ozone_x11_unittests": "a482ab8d2eeaaa6be9afd3a40262a1ef0762d6bf814c05c4e65554cfad948c9d/488",
    "pdf_unittests": "330f6631b6ee1b138e04912f685dde476a6c7b1a9463f94f3eeb7957e09891e6/565",
    "perfetto_unittests": "ba502bfc79771fc37f529724bd252bd02834d95a4eaac2764b49d4f8334f0e0e/488",
    "printing_unittests": "2f95a6468b0136d753d18f58d070cd06600eb4d12a1703aaaf07baed2d5deed5/488",
    "pthreadpool_unittests": "c1dedd6e6fae4ad70ecd409b4d4e9b664e7fb82fedcfe48a79c288aa9674f91c/488",
    "puffin_unittests": "a70ecbacd9940e1e34f62b0b06934c71318d02f07d3a282dbf339d993873aaa9/572",
    "remoting_unittests": "199d5658e3afbaeb07c49fce241e1103a8b1e8a1919603f3d4846ff54425614a/565",
    "sandbox_linux_unittests": "778beae762479d29e7d327918b34600ea60c3e8102184f44ad7a064000fd924a/488",
    "services_unittests": "2a6c537c930c86eba4ae2d6dc207710d15445ab379a26f4a7b2c6937322b9fb9/813",
    "shell_dialogs_unittests": "75b92a392acfdc761100dba6fa7840ed6370a9d1f3188ffc4694737afdbb479b/488",
    "skia_unittests": "c2606f31bbf4f765ec345d1d475e8e4ced92a46ece5a62e44bb91718b613c677/488",
    "snapshot_unittests": "027ba1dfc1fc8312a951a601d6a109a94e1aa7595e212c9a48c8766b7deb60e4/488",
    "sql_unittests": "bef0ad740ff78ada4544b0994daf171a2134e4714b40fa967dd5d67c6b34ea35/565",
    "storage_unittests": "e182b53e65fd01562b2ec62836418bb6e7db43900569d6b373023e4eea136794/646",
    "sync_integration_tests": "8410bb8a8cb61c65566280e622aa25033694c9fa33e621cc09339580088366a5/647",
    "ui_base_unittests": "f1785e2dd4d3e936bd9fdee8f9ef413630b9644ca596285d502396bfd6b3426d/641",
    "ui_touch_selection_unittests": "06a4a456b1870a94848f1a5f391f7d82c6520d61aee896087e7bdb1573cb7175/488",
    "ui_unittests": "9d1410288600fc72ad248e9ca12323a5245d2d8c8510c5c599db4aac3015c63e/488",
    "unit_tests": "74698065c022149ad19b053af2c7c17ac65c21b24de78b3ee135fa4217bbf7c7/1061",
    "url_unittests": "d2bf9b33ef6fee49c06b753fc26bea3044b674508a55ab4e61d53b8cf7e20cea/488",
    "views_unittests": "0f44d378ca94c53d6ea3892a04dc8e1c10aa2ae580b76ca556cd94fc35e22f40/488",
    "viz_unittests": "1b942d4d06ee6d1dd557f85ec0ca4c731cf8c9839cb0da2e2258968e32df9e4f/572",
    "vr_common_unittests": "a0815d864225d14a749143627e06b65b4b7cff1e4495976c5193e94b320e73f6/566",
    "wm_unittests": "76baf0cff579e3bc8e8d9b49411ea1023827419e7b7e258db06da50aacb96f99/488",
    "wtf_unittests": "0331c8dbbba0088d87cb9a867c319c23084fa985b0d22ab7a7f4a70df19d6a27/488",
    "xr_browser_tests": "6deb7734037ff56dc832d4f0094adcf1815e8b0b1fcd18efbc92e883e60d84a6/646",
    "zlib_unittests": "02e2ca748a24056792b8da764732dabdaf190e08404baf06bbba2038abfdcfda/489"
  },
  "swarming_command_lines_cwd": "out/5450-Linux_TSan_Buil",
  "swarming_command_lines_digest": "c73c90069063bfddad5e88fa11e474b8b62f6b7f05003391547aeeffcf087149/94"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/c9e88cba0f781caeb9a4da5789170ee856bc4a7d
  • gitiles_ref:refs/branch-heads/7871
  • parent_buildername:Linux TSan Builder
  • parent_buildnumber:1160
  • user_agent:recipe

Debug log (UTC)

[19:18:24.549] New invocation is queued and will start shortly
[19:18:24.549] Triggered by user:chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[19:18:25.686] Starting the invocation (attempt 1)
[19:18:25.714] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/c9e88cba0f781caeb9a4da5789170ee856bc4a7d"
[19:18:25.714] Popped gitiles commit info from properties and tags
[19:18:25.715] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[19:18:25.715] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[19:18:25.716] Buildbucket request:
{
  "requestId": "8840164585151268160",
  "builder": {
    "project": "chromium-m150",
    "bucket": "ci",
    "builder": "Linux TSan Tests"
  },
  "properties": {
      "$bootstrap/trigger": {
            "commits": [
                  {
                        "host": "chromium.googlesource.com",
                        "id": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d",
                        "project": "chromium/src",
                        "ref": "refs/branch-heads/7871"
                      }
                ]
          },
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8840164585151268160",
            "job": "chromium-m150/Linux TSan Tests",
            "triggers": [
                  {
                        "gitiles": {
                              "properties": {
                                    "$bootstrap/trigger": {
                                          "commits": [
                                                {
                                                      "host": "chromium.googlesource.com",
                                                      "id": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d",
                                                      "project": "chromium/src",
                                                      "ref": "refs/branch-heads/7871"
                                                    }
                                              ]
                                        },
                                    "deps_revision_overrides": {
                                          "src": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d"
                                        },
                                    "instruction_dependencies": [
                                          {
                                                "instruction_id": "bot_update_repro_instructions",
                                                "invocation_id": "build-8674739453017649857"
                                              },
                                          {
                                                "instruction_id": "generate_build_files_repro_instructions",
                                                "invocation_id": "build-8674739453017649857"
                                              },
                                          {
                                                "instruction_id": "compile_repro_instructions",
                                                "invocation_id": "build-8674739453017649857"
                                              }
                                        ],
                                    "parent_build_id": "8674739453017649857",
                                    "parent_builder_group": "chromium.memory",
                                    "parent_buildername": "Linux TSan Builder",
                                    "parent_got_angle_revision": "23e9441d4e3019c63c1b0bf9b00a413a6cbc2afa",
                                    "parent_got_dawn_revision": "d089fc91e7e4881362463faf8efe9ae435e34660",
                                    "parent_got_revision": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d",
                                    "parent_got_revision_cp": "refs/branch-heads/7871@{#4231}",
                                    "parent_got_swiftshader_revision": "fce27a96526f54c6d31fdccf57629788e3712220",
                                    "parent_got_v8_revision": "80533795232dad32e0ff68ff3a4fbb32acaa36ae",
                                    "parent_got_v8_revision_cp": "refs/branch-heads/15.0@{#48}",
                                    "parent_got_webrtc_revision": "1f975dfd761af6e5d76d28333191973b258d82a8",
                                    "parent_got_webrtc_revision_cp": "refs/branch-heads/7871@{#3}",
                                    "revision": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d",
                                    "swarm_hashes": {
                                          "absl_hardening_tests": "cd2974dd04928fd321ef60d47755f7ef600527b627c6461bff33fd7acf6961bb/488",
                                          "accessibility_unittests": "06eb92db7e3c750b98ede44655a73f746f300d40ff4b380327319e5fa892fb27/488",
                                          "angle_unittests": "2d5c58111c9d146bb1ff674b87bcc4f31f43a4a43dbab1d04c7fffc9a622a744/489",
                                          "aura_unittests": "c1c7ba0cd47eba80e52b82d212f5552c297b7604da113310fe82f77fdc562649/488",
                                          "base_unittests": "3a335303d05cb6fa9a3ddfbd218219d46d21a603447ba2f6b5e459e0805fba7e/566",
                                          "blink_common_unittests": "f7125eec4ce6e3a0cdbad181f6c995c66c0b02b4417c4177674ff4754fb9efb5/566",
                                          "blink_fuzzer_unittests": "fa37930624774f4cb79e388bda4ce850270974e53b8fa32325e45dfe5acd07c2/488",
                                          "blink_heap_unittests": "8aa112c814ef3521edd5f1d6ca3b0b963224870cd4be97f74641051cdd3a593c/488",
                                          "blink_platform_unittests": "55b0c709987bc18c962a5d43f2ba68a2af0defa86ed59fa8bac262207b48cb86/645",
                                          "blink_unittests": "c5442eee642b7e59afd3c6fb52eabbf7200c415003879e829bb6a9e113dc0c70/729",
                                          "boringssl_crypto_tests": "d478dd8b1a444902180d9c8d068f288a2aec81acbe6f966432f13626395ef04b/489",
                                          "boringssl_ssl_tests": "9ea4e7f165b413a9dbc04c037b65848f4c9ffbea8abaf9f3ab0b49e5f6bcad29/488",
                                          "capture_unittests": "9952dd7492ba4b551f5ab63977764fa2ac4f4bbd104e962e22cef7a9c7bdf0ae/567",
                                          "cast_unittests": "3a4d06a4d7c41ea2ebafe405fa019ebdf5e9a8794417f07b41c86983d46ae94c/488",
                                          "cc_unittests": "d6d7958a75233b3e1029315c8c30b0a6ec35c355b6cc99c51a2234ae7b2da2fb/648",
                                          "chrome_app_unittests": "98650c7301fbdcb1053f524ff68e965181b06c5aa615f1ca58a1b6f390a95264/566",
                                          "chromedriver_unittests": "46c8ff48b8cc429cd2c42be2109241bc71eabf0b8790f31de5ba350c7fd2cc5b/645",
                                          "components_browsertests": "620b99e9708eb7454255a4a792c77e8f970b62c823500e5842683333a78f68cb/732",
                                          "components_unittests": "3a7b719ec272f32c9892c0cbb7d38946ebdd23f1e7aa06632431642fc8d85be5/811",
                                          "compositor_unittests": "f482df7ec5f0b3d97ed29b66317e2a9ac553d40f3b89a798d0ad769583aa02ef/564",
                                          "content_browsertests": "a5fe19752cb11295bd4166ab6780e5dbfd770f7ced04900f7bea5da19ffc43f0/805",
                                          "content_unittests": "ca521f65542f388bd113213f291694122a6cc1efe92d7e4f7c14fe9593f39ee5/725",
                                          "crypto_unittests": "393eeb1a74b3682b3ef9eba80401be44b4ca990296e63dcec8d62cf9b930c3f4/568",
                                          "dbus_unittests": "f69feb0f2dbc6427eeceb0b4e32ee9ac8049e96cb5fb5a2c2948578c5bb8d031/488",
                                          "device_unittests": "09d6945d9b76126ab7ecb390a9aee09217192a6c99d83f557e3f619ba3972c6b/565",
                                          "display_unittests": "4cd83db1de0a5e7eacaa31bdc1e543958245d06c28907b6da98081e6846c40f2/488",
                                          "env_chromium_unittests": "055c6993d374fcb1a04d3db9264972275c9a5c62867cc87175f2e11f66688cd7/488",
                                          "events_unittests": "7ec07ee002e2ec41e8481b878224ab92a755673cadaa6c7a8ed0079de67001cb/488",
                                          "extensions_unittests": "2aef3387bcc65f98bdd9fa0b803b757309657f1a744dbaccee4344301a90eb9f/730",
                                          "filesystem_service_unittests": "331c1dfe1e28a8bbb8ca514853945e228a449acee8e0d929d4b7deb5c7462fbb/488",
                                          "fuzzing_unittests": "e7019b196a301f61a278cbbb68dce94ec1330815d4436361d63967b687159300/488",
                                          "gcm_unit_tests": "9f6142db2c16c15ba66f7b750ac7fb96abe3df0369db31e90d414dbf76d7cc58/565",
                                          "gfx_unittests": "9f209b9d75b5bef15e6035b41f49d8d46528a16038a71c9064551e3222f6adcb/564",
                                          "gin_unittests": "e6f9d11691e2e24b77950bf01ab21730733916f8a5eff5f10dfebbf19501d10e/565",
                                          "gl_tests": "06a9f6a317dac2fc3c1e818660016bd90e0e5d4f59c6771576d6f618407ba62f/488",
                                          "gl_unittests": "17c47ec85b560bcf926a50c5c79d9adc2900015dc9a9d62603d34cf78337b55b/488",
                                          "google_apis_unittests": "fd0adca83bc80080ff5ce6117865270d2ed2f56deb88f465e35f169a1ee61228/650",
                                          "gpu_unittests": "6d33898f65168f196a2156237189646324b3ee751b4336ed49f6c31fcd56ac75/565",
                                          "gtk_unittests": "a3746acdb6e767bd8b0966625090bc77037131f846bf0ee9dc3850f30646e918/488",
                                          "gwp_asan_unittests": "d5930c4d4e3869d10d25209705b7558730fc071491d30e1136c2e7d1c1392035/489",
                                          "headless_browsertests": "dac88429ef563d838c71e8f627020bce08530e02630fd59da300b23fc28393f7/733",
                                          "headless_unittests": "199f2e755a47a0535a2e89b8249d8e4e8ea923958ddce3b95530085c25600dd1/488",
                                          "interactive_ui_tests": "0cf8c18df6396cc0124d37ec3e48055ac30fdbc9f5042675266cbea4708f6604/1059",
                                          "ipc_tests": "bce8851be7bee08961898db20e306bd9a031b0cd976c44200bd7af585014b6fe/488",
                                          "latency_unittests": "066c795edef5b4dd84f48d48c951c0bc220e4846c0d6dac14deb3ecdc8caea4c/488",
                                          "leveldb_unittests": "5001d312ba5be99ecf35782e769d5fe97db3ab99acdd65428c934b414711bb53/488",
                                          "libjingle_xmpp_unittests": "23ac6783dc9961f9bf8a0e893a4fa22948ad847a3991a3dec661bdfa218801d8/488",
                                          "liburlpattern_unittests": "3c4f0be01fe12e516aa8b3fb0d9d4bc167f0193c212e4eb7d739c4f6db40f23c/488",
                                          "media_unittests": "523ffcbf75c07b92d3b384a5b9ae31fe9a9bfccd3a95d4a09cda3ecc5dc28afc/568",
                                          "message_center_unittests": "5f075b51cb59316fd348c6cd4df3368356f63968a375fdceddbb259af094d4f3/488",
                                          "midi_unittests": "feb90942935455c3d44f0e6a579ae4355d2b005cdbcfa84d953cc31f6f60be24/488",
                                          "mojo_unittests": "74842fb87940a1c90b7b7df8df0c6abd806edbb7e0109a20c63d247d87261728/566",
                                          "native_theme_unittests": "e289442ca4c9dd3e4c5da70f2983295fb3436dde21170ddb3fcf416958b687e2/488",
                                          "net_unittests": "0e958aa9e657fa8e7b47ec6daf237afb169262037dd274f2c94f5ad2ab41b686/567",
                                          "openscreen_unittests": "b250a27180a0168f37de2ba172e2623b09d99cadbf3e08552872463c27d7983f/489",
                                          "ozone_unittests": "63cfee8702ef4326412497ea8be921daaf029d77a05359da703af1b3edac05ba/488",
                                          "ozone_x11_unittests": "a482ab8d2eeaaa6be9afd3a40262a1ef0762d6bf814c05c4e65554cfad948c9d/488",
                                          "pdf_unittests": "330f6631b6ee1b138e04912f685dde476a6c7b1a9463f94f3eeb7957e09891e6/565",
                                          "perfetto_unittests": "ba502bfc79771fc37f529724bd252bd02834d95a4eaac2764b49d4f8334f0e0e/488",
                                          "printing_unittests": "2f95a6468b0136d753d18f58d070cd06600eb4d12a1703aaaf07baed2d5deed5/488",
                                          "pthreadpool_unittests": "c1dedd6e6fae4ad70ecd409b4d4e9b664e7fb82fedcfe48a79c288aa9674f91c/488",
                                          "puffin_unittests": "a70ecbacd9940e1e34f62b0b06934c71318d02f07d3a282dbf339d993873aaa9/572",
                                          "remoting_unittests": "199d5658e3afbaeb07c49fce241e1103a8b1e8a1919603f3d4846ff54425614a/565",
                                          "sandbox_linux_unittests": "778beae762479d29e7d327918b34600ea60c3e8102184f44ad7a064000fd924a/488",
                                          "services_unittests": "2a6c537c930c86eba4ae2d6dc207710d15445ab379a26f4a7b2c6937322b9fb9/813",
                                          "shell_dialogs_unittests": "75b92a392acfdc761100dba6fa7840ed6370a9d1f3188ffc4694737afdbb479b/488",
                                          "skia_unittests": "c2606f31bbf4f765ec345d1d475e8e4ced92a46ece5a62e44bb91718b613c677/488",
                                          "snapshot_unittests": "027ba1dfc1fc8312a951a601d6a109a94e1aa7595e212c9a48c8766b7deb60e4/488",
                                          "sql_unittests": "bef0ad740ff78ada4544b0994daf171a2134e4714b40fa967dd5d67c6b34ea35/565",
                                          "storage_unittests": "e182b53e65fd01562b2ec62836418bb6e7db43900569d6b373023e4eea136794/646",
                                          "sync_integration_tests": "8410bb8a8cb61c65566280e622aa25033694c9fa33e621cc09339580088366a5/647",
                                          "ui_base_unittests": "f1785e2dd4d3e936bd9fdee8f9ef413630b9644ca596285d502396bfd6b3426d/641",
                                          "ui_touch_selection_unittests": "06a4a456b1870a94848f1a5f391f7d82c6520d61aee896087e7bdb1573cb7175/488",
                                          "ui_unittests": "9d1410288600fc72ad248e9ca12323a5245d2d8c8510c5c599db4aac3015c63e/488",
                                          "unit_tests": "74698065c022149ad19b053af2c7c17ac65c21b24de78b3ee135fa4217bbf7c7/1061",
                                          "url_unittests": "d2bf9b33ef6fee49c06b753fc26bea3044b674508a55ab4e61d53b8cf7e20cea/488",
                                          "views_unittests": "0f44d378ca94c53d6ea3892a04dc8e1c10aa2ae580b76ca556cd94fc35e22f40/488",
                                          "viz_unittests": "1b942d4d06ee6d1dd557f85ec0ca4c731cf8c9839cb0da2e2258968e32df9e4f/572",
                                          "vr_common_unittests": "a0815d864225d14a749143627e06b65b4b7cff1e4495976c5193e94b320e73f6/566",
                                          "wm_unittests": "76baf0cff579e3bc8e8d9b49411ea1023827419e7b7e258db06da50aacb96f99/488",
                                          "wtf_unittests": "0331c8dbbba0088d87cb9a867c319c23084fa985b0d22ab7a7f4a70df19d6a27/488",
                                          "xr_browser_tests": "6deb7734037ff56dc832d4f0094adcf1815e8b0b1fcd18efbc92e883e60d84a6/646",
                                          "zlib_unittests": "02e2ca748a24056792b8da764732dabdaf190e08404baf06bbba2038abfdcfda/489"
                                        },
                                    "swarming_command_lines_cwd": "out/5450-Linux_TSan_Buil",
                                    "swarming_command_lines_digest": "c73c90069063bfddad5e88fa11e474b8b62f6b7f05003391547aeeffcf087149/94"
                                  },
                              "ref": "refs/branch-heads/7871",
                              "repo": "https://chromium.googlesource.com/chromium/src",
                              "revision": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d",
                              "tags": [
                                    "parent_buildername:Linux TSan Builder",
                                    "parent_buildnumber:1160",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "6e1e57b3-20b5-4720-9217-5e345259ac77",
                        "title": "Linux TSan Builder/1160",
                        "url": "https://cr-buildbucket.appspot.com/build/8674739453017649857"
                      }
                ]
          },
      "deps_revision_overrides": {
            "src": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d"
          },
      "instruction_dependencies": [
            {
                  "instruction_id": "bot_update_repro_instructions",
                  "invocation_id": "build-8674739453017649857"
                },
            {
                  "instruction_id": "generate_build_files_repro_instructions",
                  "invocation_id": "build-8674739453017649857"
                },
            {
                  "instruction_id": "compile_repro_instructions",
                  "invocation_id": "build-8674739453017649857"
                }
          ],
      "parent_build_id": "8674739453017649857",
      "parent_builder_group": "chromium.memory",
      "parent_buildername": "Linux TSan Builder",
      "parent_got_angle_revision": "23e9441d4e3019c63c1b0bf9b00a413a6cbc2afa",
      "parent_got_dawn_revision": "d089fc91e7e4881362463faf8efe9ae435e34660",
      "parent_got_revision": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d",
      "parent_got_revision_cp": "refs/branch-heads/7871@{#4231}",
      "parent_got_swiftshader_revision": "fce27a96526f54c6d31fdccf57629788e3712220",
      "parent_got_v8_revision": "80533795232dad32e0ff68ff3a4fbb32acaa36ae",
      "parent_got_v8_revision_cp": "refs/branch-heads/15.0@{#48}",
      "parent_got_webrtc_revision": "1f975dfd761af6e5d76d28333191973b258d82a8",
      "parent_got_webrtc_revision_cp": "refs/branch-heads/7871@{#3}",
      "swarm_hashes": {
            "absl_hardening_tests": "cd2974dd04928fd321ef60d47755f7ef600527b627c6461bff33fd7acf6961bb/488",
            "accessibility_unittests": "06eb92db7e3c750b98ede44655a73f746f300d40ff4b380327319e5fa892fb27/488",
            "angle_unittests": "2d5c58111c9d146bb1ff674b87bcc4f31f43a4a43dbab1d04c7fffc9a622a744/489",
            "aura_unittests": "c1c7ba0cd47eba80e52b82d212f5552c297b7604da113310fe82f77fdc562649/488",
            "base_unittests": "3a335303d05cb6fa9a3ddfbd218219d46d21a603447ba2f6b5e459e0805fba7e/566",
            "blink_common_unittests": "f7125eec4ce6e3a0cdbad181f6c995c66c0b02b4417c4177674ff4754fb9efb5/566",
            "blink_fuzzer_unittests": "fa37930624774f4cb79e388bda4ce850270974e53b8fa32325e45dfe5acd07c2/488",
            "blink_heap_unittests": "8aa112c814ef3521edd5f1d6ca3b0b963224870cd4be97f74641051cdd3a593c/488",
            "blink_platform_unittests": "55b0c709987bc18c962a5d43f2ba68a2af0defa86ed59fa8bac262207b48cb86/645",
            "blink_unittests": "c5442eee642b7e59afd3c6fb52eabbf7200c415003879e829bb6a9e113dc0c70/729",
            "boringssl_crypto_tests": "d478dd8b1a444902180d9c8d068f288a2aec81acbe6f966432f13626395ef04b/489",
            "boringssl_ssl_tests": "9ea4e7f165b413a9dbc04c037b65848f4c9ffbea8abaf9f3ab0b49e5f6bcad29/488",
            "capture_unittests": "9952dd7492ba4b551f5ab63977764fa2ac4f4bbd104e962e22cef7a9c7bdf0ae/567",
            "cast_unittests": "3a4d06a4d7c41ea2ebafe405fa019ebdf5e9a8794417f07b41c86983d46ae94c/488",
            "cc_unittests": "d6d7958a75233b3e1029315c8c30b0a6ec35c355b6cc99c51a2234ae7b2da2fb/648",
            "chrome_app_unittests": "98650c7301fbdcb1053f524ff68e965181b06c5aa615f1ca58a1b6f390a95264/566",
            "chromedriver_unittests": "46c8ff48b8cc429cd2c42be2109241bc71eabf0b8790f31de5ba350c7fd2cc5b/645",
            "components_browsertests": "620b99e9708eb7454255a4a792c77e8f970b62c823500e5842683333a78f68cb/732",
            "components_unittests": "3a7b719ec272f32c9892c0cbb7d38946ebdd23f1e7aa06632431642fc8d85be5/811",
            "compositor_unittests": "f482df7ec5f0b3d97ed29b66317e2a9ac553d40f3b89a798d0ad769583aa02ef/564",
            "content_browsertests": "a5fe19752cb11295bd4166ab6780e5dbfd770f7ced04900f7bea5da19ffc43f0/805",
            "content_unittests": "ca521f65542f388bd113213f291694122a6cc1efe92d7e4f7c14fe9593f39ee5/725",
            "crypto_unittests": "393eeb1a74b3682b3ef9eba80401be44b4ca990296e63dcec8d62cf9b930c3f4/568",
            "dbus_unittests": "f69feb0f2dbc6427eeceb0b4e32ee9ac8049e96cb5fb5a2c2948578c5bb8d031/488",
            "device_unittests": "09d6945d9b76126ab7ecb390a9aee09217192a6c99d83f557e3f619ba3972c6b/565",
            "display_unittests": "4cd83db1de0a5e7eacaa31bdc1e543958245d06c28907b6da98081e6846c40f2/488",
            "env_chromium_unittests": "055c6993d374fcb1a04d3db9264972275c9a5c62867cc87175f2e11f66688cd7/488",
            "events_unittests": "7ec07ee002e2ec41e8481b878224ab92a755673cadaa6c7a8ed0079de67001cb/488",
            "extensions_unittests": "2aef3387bcc65f98bdd9fa0b803b757309657f1a744dbaccee4344301a90eb9f/730",
            "filesystem_service_unittests": "331c1dfe1e28a8bbb8ca514853945e228a449acee8e0d929d4b7deb5c7462fbb/488",
            "fuzzing_unittests": "e7019b196a301f61a278cbbb68dce94ec1330815d4436361d63967b687159300/488",
            "gcm_unit_tests": "9f6142db2c16c15ba66f7b750ac7fb96abe3df0369db31e90d414dbf76d7cc58/565",
            "gfx_unittests": "9f209b9d75b5bef15e6035b41f49d8d46528a16038a71c9064551e3222f6adcb/564",
            "gin_unittests": "e6f9d11691e2e24b77950bf01ab21730733916f8a5eff5f10dfebbf19501d10e/565",
            "gl_tests": "06a9f6a317dac2fc3c1e818660016bd90e0e5d4f59c6771576d6f618407ba62f/488",
            "gl_unittests": "17c47ec85b560bcf926a50c5c79d9adc2900015dc9a9d62603d34cf78337b55b/488",
            "google_apis_unittests": "fd0adca83bc80080ff5ce6117865270d2ed2f56deb88f465e35f169a1ee61228/650",
            "gpu_unittests": "6d33898f65168f196a2156237189646324b3ee751b4336ed49f6c31fcd56ac75/565",
            "gtk_unittests": "a3746acdb6e767bd8b0966625090bc77037131f846bf0ee9dc3850f30646e918/488",
            "gwp_asan_unittests": "d5930c4d4e3869d10d25209705b7558730fc071491d30e1136c2e7d1c1392035/489",
            "headless_browsertests": "dac88429ef563d838c71e8f627020bce08530e02630fd59da300b23fc28393f7/733",
            "headless_unittests": "199f2e755a47a0535a2e89b8249d8e4e8ea923958ddce3b95530085c25600dd1/488",
            "interactive_ui_tests": "0cf8c18df6396cc0124d37ec3e48055ac30fdbc9f5042675266cbea4708f6604/1059",
            "ipc_tests": "bce8851be7bee08961898db20e306bd9a031b0cd976c44200bd7af585014b6fe/488",
            "latency_unittests": "066c795edef5b4dd84f48d48c951c0bc220e4846c0d6dac14deb3ecdc8caea4c/488",
            "leveldb_unittests": "5001d312ba5be99ecf35782e769d5fe97db3ab99acdd65428c934b414711bb53/488",
            "libjingle_xmpp_unittests": "23ac6783dc9961f9bf8a0e893a4fa22948ad847a3991a3dec661bdfa218801d8/488",
            "liburlpattern_unittests": "3c4f0be01fe12e516aa8b3fb0d9d4bc167f0193c212e4eb7d739c4f6db40f23c/488",
            "media_unittests": "523ffcbf75c07b92d3b384a5b9ae31fe9a9bfccd3a95d4a09cda3ecc5dc28afc/568",
            "message_center_unittests": "5f075b51cb59316fd348c6cd4df3368356f63968a375fdceddbb259af094d4f3/488",
            "midi_unittests": "feb90942935455c3d44f0e6a579ae4355d2b005cdbcfa84d953cc31f6f60be24/488",
            "mojo_unittests": "74842fb87940a1c90b7b7df8df0c6abd806edbb7e0109a20c63d247d87261728/566",
            "native_theme_unittests": "e289442ca4c9dd3e4c5da70f2983295fb3436dde21170ddb3fcf416958b687e2/488",
            "net_unittests": "0e958aa9e657fa8e7b47ec6daf237afb169262037dd274f2c94f5ad2ab41b686/567",
            "openscreen_unittests": "b250a27180a0168f37de2ba172e2623b09d99cadbf3e08552872463c27d7983f/489",
            "ozone_unittests": "63cfee8702ef4326412497ea8be921daaf029d77a05359da703af1b3edac05ba/488",
            "ozone_x11_unittests": "a482ab8d2eeaaa6be9afd3a40262a1ef0762d6bf814c05c4e65554cfad948c9d/488",
            "pdf_unittests": "330f6631b6ee1b138e04912f685dde476a6c7b1a9463f94f3eeb7957e09891e6/565",
            "perfetto_unittests": "ba502bfc79771fc37f529724bd252bd02834d95a4eaac2764b49d4f8334f0e0e/488",
            "printing_unittests": "2f95a6468b0136d753d18f58d070cd06600eb4d12a1703aaaf07baed2d5deed5/488",
            "pthreadpool_unittests": "c1dedd6e6fae4ad70ecd409b4d4e9b664e7fb82fedcfe48a79c288aa9674f91c/488",
            "puffin_unittests": "a70ecbacd9940e1e34f62b0b06934c71318d02f07d3a282dbf339d993873aaa9/572",
            "remoting_unittests": "199d5658e3afbaeb07c49fce241e1103a8b1e8a1919603f3d4846ff54425614a/565",
            "sandbox_linux_unittests": "778beae762479d29e7d327918b34600ea60c3e8102184f44ad7a064000fd924a/488",
            "services_unittests": "2a6c537c930c86eba4ae2d6dc207710d15445ab379a26f4a7b2c6937322b9fb9/813",
            "shell_dialogs_unittests": "75b92a392acfdc761100dba6fa7840ed6370a9d1f3188ffc4694737afdbb479b/488",
            "skia_unittests": "c2606f31bbf4f765ec345d1d475e8e4ced92a46ece5a62e44bb91718b613c677/488",
            "snapshot_unittests": "027ba1dfc1fc8312a951a601d6a109a94e1aa7595e212c9a48c8766b7deb60e4/488",
            "sql_unittests": "bef0ad740ff78ada4544b0994daf171a2134e4714b40fa967dd5d67c6b34ea35/565",
            "storage_unittests": "e182b53e65fd01562b2ec62836418bb6e7db43900569d6b373023e4eea136794/646",
            "sync_integration_tests": "8410bb8a8cb61c65566280e622aa25033694c9fa33e621cc09339580088366a5/647",
            "ui_base_unittests": "f1785e2dd4d3e936bd9fdee8f9ef413630b9644ca596285d502396bfd6b3426d/641",
            "ui_touch_selection_unittests": "06a4a456b1870a94848f1a5f391f7d82c6520d61aee896087e7bdb1573cb7175/488",
            "ui_unittests": "9d1410288600fc72ad248e9ca12323a5245d2d8c8510c5c599db4aac3015c63e/488",
            "unit_tests": "74698065c022149ad19b053af2c7c17ac65c21b24de78b3ee135fa4217bbf7c7/1061",
            "url_unittests": "d2bf9b33ef6fee49c06b753fc26bea3044b674508a55ab4e61d53b8cf7e20cea/488",
            "views_unittests": "0f44d378ca94c53d6ea3892a04dc8e1c10aa2ae580b76ca556cd94fc35e22f40/488",
            "viz_unittests": "1b942d4d06ee6d1dd557f85ec0ca4c731cf8c9839cb0da2e2258968e32df9e4f/572",
            "vr_common_unittests": "a0815d864225d14a749143627e06b65b4b7cff1e4495976c5193e94b320e73f6/566",
            "wm_unittests": "76baf0cff579e3bc8e8d9b49411ea1023827419e7b7e258db06da50aacb96f99/488",
            "wtf_unittests": "0331c8dbbba0088d87cb9a867c319c23084fa985b0d22ab7a7f4a70df19d6a27/488",
            "xr_browser_tests": "6deb7734037ff56dc832d4f0094adcf1815e8b0b1fcd18efbc92e883e60d84a6/646",
            "zlib_unittests": "02e2ca748a24056792b8da764732dabdaf190e08404baf06bbba2038abfdcfda/489"
          },
      "swarming_command_lines_cwd": "out/5450-Linux_TSan_Buil",
      "swarming_command_lines_digest": "c73c90069063bfddad5e88fa11e474b8b62f6b7f05003391547aeeffcf087149/94"
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d",
    "ref": "refs/branch-heads/7871"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8840164585151268160"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium-m150/Linux TSan Tests"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "Linux TSan Builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "1160"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[19:18:26.276] Scheduled build:
{
  "id": "8674734523191799905",
  "builder": {
    "project": "chromium-m150",
    "bucket": "ci",
    "builder": "Linux TSan Tests"
  },
  "number": 1151,
  "createdBy": "project:chromium-m150",
  "createTime": "2026-07-31T19:18:25.797950653Z",
  "updateTime": "2026-07-31T19:18:25.797950653Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d",
      "ref": "refs/branch-heads/7871"
    }
  }
}
[19:18:26.276] Task URL: https://cr-buildbucket.appspot.com/build/8674734523191799905
[19:18:26.277] Scheduling timer "check-buildbucket-build-status" (chromium-m150/Linux TSan Tests:8840164585151268160:2:0) after 7m28s
[19:18:26.447] Received PubSub notification, asking Buildbucket for the build status
[19:18:26.466] Build status: SCHEDULED
[19:18:33.451] Received PubSub notification, asking Buildbucket for the build status
[19:18:33.471] Build status: STARTED
[19:25:54.394] Handling timer "check-buildbucket-build-status" (chromium-m150/Linux TSan Tests:8840164585151268160:2:0)
[19:25:54.440] Build status: STARTED
[19:25:54.440] Scheduling timer "check-buildbucket-build-status" (chromium-m150/Linux TSan Tests:8840164585151268160:5:0) after 5m27s
[19:31:21.514] Handling timer "check-buildbucket-build-status" (chromium-m150/Linux TSan Tests:8840164585151268160:5:0)
[19:31:21.535] Build status: STARTED
[19:31:21.535] Scheduling timer "check-buildbucket-build-status" (chromium-m150/Linux TSan Tests:8840164585151268160:6:0) after 4m49s
[19:36:10.553] Handling timer "check-buildbucket-build-status" (chromium-m150/Linux TSan Tests:8840164585151268160:6:0)
[19:36:10.571] Build status: STARTED
[19:36:10.571] Scheduling timer "check-buildbucket-build-status" (chromium-m150/Linux TSan Tests:8840164585151268160:7:0) after 6m8s
[19:42:18.661] Handling timer "check-buildbucket-build-status" (chromium-m150/Linux TSan Tests:8840164585151268160:7:0)
[19:42:18.720] Build status: STARTED
[19:42:18.720] Scheduling timer "check-buildbucket-build-status" (chromium-m150/Linux TSan Tests:8840164585151268160:8:0) after 5m8s
[19:47:26.722] Handling timer "check-buildbucket-build-status" (chromium-m150/Linux TSan Tests:8840164585151268160:8:0)
[19:47:26.743] Build status: STARTED
[19:47:26.744] Scheduling timer "check-buildbucket-build-status" (chromium-m150/Linux TSan Tests:8840164585151268160:9:0) after 5m25s
[19:52:51.748] Handling timer "check-buildbucket-build-status" (chromium-m150/Linux TSan Tests:8840164585151268160:9:0)
[19:52:51.766] Build status: STARTED
[19:52:51.766] Scheduling timer "check-buildbucket-build-status" (chromium-m150/Linux TSan Tests:8840164585151268160:10:0) after 9m34s
[20:01:22.944] Received PubSub notification, asking Buildbucket for the build status
[20:01:23.004] Build:
{
  "id": "8674734523191799905",
  "builder": {
    "project": "chromium-m150",
    "bucket": "ci",
    "builder": "Linux TSan Tests"
  },
  "number": 1151,
  "createdBy": "project:chromium-m150",
  "createTime": "2026-07-31T19:18:25.797950653Z",
  "startTime": "2026-07-31T19:18:33.236480136Z",
  "endTime": "2026-07-31T20:01:22.468949377Z",
  "updateTime": "2026-07-31T20:01:22.468949377Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "chromium/src",
      "id": "c9e88cba0f781caeb9a4da5789170ee856bc4a7d",
      "ref": "refs/branch-heads/7871"
    }
  }
}
[20:01:23.004] Invocation finished in 42m58.459623971s with status FAILED