Status: FAILED
Triggered by: v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com
Duration: 1 hour
Actions:

Definition (rev c922df7f3ebbf09d5a6f144d12699a27ed6f4f07)

buildbucket: <
  server: "cr-buildbucket.appspot.com"
  bucket: "ci"
  builder: "V8 NumFuzz - TSAN"
>

Incoming triggers

Incoming properties

{
  "newest_gitiles_revision": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
  "oldest_gitiles_revision": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
  "parent_build": "https://cr-buildbucket.appspot.com/build/8669861574406456961",
  "parent_buildername": "V8 Clusterfuzz Linux64 TSAN - release builder",
  "parent_gn_args": [
    "dcheck_always_on = false",
    "is_component_build = false",
    "is_debug = false",
    "is_tsan = true",
    "target_cpu = \"x64\"",
    "use_remoteexec = true",
    "v8_enable_google_benchmark = true",
    "v8_enable_test_features = true",
    "v8_enable_undefined_double = true"
  ],
  "parent_got_revision": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
  "parent_got_revision_cp": "refs/heads/main@{#110039}",
  "parent_test_spec": {
    "swarming_dimensions": {
      "os": "Ubuntu-22.04"
    },
    "swarming_task_attrs": {
      "expiration": 13800,
      "hard_timeout": 4800,
      "priority": 35
    },
    "tests": [
      [
        "d8testing_random_gc",
        2,
        null,
        "",
        [],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "allocation",
        [
          "--total-timeout-sec=4200",
          "--allocation-offset=1",
          "--extra-flags=--gc-interval=500"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "bytecode",
        [
          "--total-timeout-sec=4200",
          "--stress-bytecode-budget=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "marking",
        [
          "--total-timeout-sec=4200",
          "--stress-marking=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "delay",
        [
          "--total-timeout-sec=4200",
          "--stress-delay-tasks=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "interrupt",
        [
          "--total-timeout-sec=4200",
          "--stress-interrupt-budget=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "threads",
        [
          "--total-timeout-sec=4200",
          "--stress-thread-pool-size=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "stack",
        [
          "--total-timeout-sec=4200",
          "--stress-stack-size=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        5,
        null,
        "combined",
        [
          "--total-timeout-sec=4200",
          "--allocation-offset=1",
          "--scavenge-chaos=4",
          "--stress-bytecode-budget=1",
          "--stress-delay-tasks=4",
          "--stress-deopt=2",
          "--stress-compaction=2",
          "--stress-gc=4",
          "--stress-marking=4",
          "--stress-scavenge=4",
          "--stress-thread-pool-size=2",
          "--stress-stack-size=1",
          "--stress-interrupt-budget=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "scavenge",
        [
          "--total-timeout-sec=4200",
          "--stress-scavenge=1"
        ],
        {},
        {}
      ],
      [
        "numfuzz",
        1,
        null,
        "scavenge chaos",
        [
          "--total-timeout-sec=4200",
          "--scavenge-chaos=1"
        ],
        {},
        {}
      ]
    ]
  },
  "revision": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
  "swarm_hashes": {
    "d8_default": "aaac69e69bccac6e6cc47693c286454b400d0430f227786c2648bc6dfb70dee3/503",
    "run-num-fuzzer": "f07cb6b98fccb1e29471ac7fa0f4abab0b81825028e3e1351de735d676f2f896/503"
  }
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/9fb4444bfc464a5e54219acb04f0828b83dd15b5
  • gitiles_ref:refs/heads/main
  • parent_buildername:V8 Clusterfuzz Linux64 TSAN - release builder
  • parent_buildnumber:57514
  • user_agent:recipe

Debug log (UTC)

[14:28:59.195] New invocation is queued and will start shortly
[14:28:59.195] Triggered by user:v8-ci-builder@chops-service-accounts.iam.gserviceaccount.com
[14:29:00.256] Starting the invocation (attempt 1)
[14:29:00.286] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/9fb4444bfc464a5e54219acb04f0828b83dd15b5"
[14:29:00.286] Reconstructed gitiles commit from tags
[14:29:00.287] Preparing PubSub topic for "https://cr-buildbucket.appspot.com"
[14:29:00.287] PubSub topic is "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
[14:29:00.289] Buildbucket request:
{
  "requestId": "8835290557860489440",
  "builder": {
    "project": "v8",
    "bucket": "ci",
    "builder": "V8 NumFuzz - TSAN"
  },
  "properties": {
      "$recipe_engine/scheduler": {
            "hostname": "luci-scheduler.appspot.com",
            "invocation": "8835290557860489440",
            "job": "v8/V8 NumFuzz - TSAN",
            "triggers": [
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "147d71f2dac67ce8fe92756703a030f4e24ab24a",
                                    "oldest_gitiles_revision": "147d71f2dac67ce8fe92756703a030f4e24ab24a",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8669864250102259393",
                                    "parent_buildername": "V8 Clusterfuzz Linux64 TSAN - release builder",
                                    "parent_gn_args": [
                                          "dcheck_always_on = false",
                                          "is_component_build = false",
                                          "is_debug = false",
                                          "is_tsan = true",
                                          "target_cpu = \"x64\"",
                                          "use_remoteexec = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_enable_test_features = true",
                                          "v8_enable_undefined_double = true"
                                        ],
                                    "parent_got_revision": "147d71f2dac67ce8fe92756703a030f4e24ab24a",
                                    "parent_got_revision_cp": "refs/heads/main@{#110035}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-22.04"
                                              },
                                          "swarming_task_attrs": {
                                                "expiration": 13800,
                                                "hard_timeout": 4800,
                                                "priority": 35
                                              },
                                          "tests": [
                                                [
                                                      "d8testing_random_gc",
                                                      2,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "allocation",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--allocation-offset=1",
                                                            "--extra-flags=--gc-interval=500"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "bytecode",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-bytecode-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "marking",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-marking=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "delay",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-delay-tasks=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "interrupt",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "threads",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-thread-pool-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "stack",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-stack-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      5,
                                                      null,
                                                      "combined",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--allocation-offset=1",
                                                            "--scavenge-chaos=4",
                                                            "--stress-bytecode-budget=1",
                                                            "--stress-delay-tasks=4",
                                                            "--stress-deopt=2",
                                                            "--stress-compaction=2",
                                                            "--stress-gc=4",
                                                            "--stress-marking=4",
                                                            "--stress-scavenge=4",
                                                            "--stress-thread-pool-size=2",
                                                            "--stress-stack-size=1",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "scavenge",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-scavenge=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "scavenge chaos",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--scavenge-chaos=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "147d71f2dac67ce8fe92756703a030f4e24ab24a",
                                    "swarm_hashes": {
                                          "d8_default": "916731bbc66bded98e985a058d4b09104b875de5b693ede1ee556b9360a2d1aa/503",
                                          "run-num-fuzzer": "bd4661b154491cf682e8558a84e04686adefcd9fd1303a36c979c21d5262945c/503"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/147d71f2dac67ce8fe92756703a030f4e24ab24a",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Clusterfuzz Linux64 TSAN - release builder",
                                    "parent_buildnumber:57510",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "2ee24a84-8224-4954-9f02-6ed0050494b4",
                        "title": "V8 Clusterfuzz Linux64 TSAN - release builder/57510",
                        "url": "https://cr-buildbucket.appspot.com/build/8669864250102259393"
                      },
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "884695ac867e89279daa9a732ccff41c0acde934",
                                    "oldest_gitiles_revision": "884695ac867e89279daa9a732ccff41c0acde934",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8669863140809060881",
                                    "parent_buildername": "V8 Clusterfuzz Linux64 TSAN - release builder",
                                    "parent_gn_args": [
                                          "dcheck_always_on = false",
                                          "is_component_build = false",
                                          "is_debug = false",
                                          "is_tsan = true",
                                          "target_cpu = \"x64\"",
                                          "use_remoteexec = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_enable_test_features = true",
                                          "v8_enable_undefined_double = true"
                                        ],
                                    "parent_got_revision": "884695ac867e89279daa9a732ccff41c0acde934",
                                    "parent_got_revision_cp": "refs/heads/main@{#110036}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-22.04"
                                              },
                                          "swarming_task_attrs": {
                                                "expiration": 13800,
                                                "hard_timeout": 4800,
                                                "priority": 35
                                              },
                                          "tests": [
                                                [
                                                      "d8testing_random_gc",
                                                      2,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "allocation",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--allocation-offset=1",
                                                            "--extra-flags=--gc-interval=500"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "bytecode",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-bytecode-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "marking",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-marking=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "delay",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-delay-tasks=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "interrupt",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "threads",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-thread-pool-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "stack",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-stack-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      5,
                                                      null,
                                                      "combined",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--allocation-offset=1",
                                                            "--scavenge-chaos=4",
                                                            "--stress-bytecode-budget=1",
                                                            "--stress-delay-tasks=4",
                                                            "--stress-deopt=2",
                                                            "--stress-compaction=2",
                                                            "--stress-gc=4",
                                                            "--stress-marking=4",
                                                            "--stress-scavenge=4",
                                                            "--stress-thread-pool-size=2",
                                                            "--stress-stack-size=1",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "scavenge",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-scavenge=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "scavenge chaos",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--scavenge-chaos=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "884695ac867e89279daa9a732ccff41c0acde934",
                                    "swarm_hashes": {
                                          "d8_default": "916731bbc66bded98e985a058d4b09104b875de5b693ede1ee556b9360a2d1aa/503",
                                          "run-num-fuzzer": "bd4661b154491cf682e8558a84e04686adefcd9fd1303a36c979c21d5262945c/503"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/884695ac867e89279daa9a732ccff41c0acde934",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Clusterfuzz Linux64 TSAN - release builder",
                                    "parent_buildnumber:57511",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "e1888111-ac69-4e0c-bb60-b4f899f5e441",
                        "title": "V8 Clusterfuzz Linux64 TSAN - release builder/57511",
                        "url": "https://cr-buildbucket.appspot.com/build/8669863140809060881"
                      },
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "967a8d215579e31720d25d44c50758330b70bd7c",
                                    "oldest_gitiles_revision": "967a8d215579e31720d25d44c50758330b70bd7c",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8669862376772453617",
                                    "parent_buildername": "V8 Clusterfuzz Linux64 TSAN - release builder",
                                    "parent_gn_args": [
                                          "dcheck_always_on = false",
                                          "is_component_build = false",
                                          "is_debug = false",
                                          "is_tsan = true",
                                          "target_cpu = \"x64\"",
                                          "use_remoteexec = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_enable_test_features = true",
                                          "v8_enable_undefined_double = true"
                                        ],
                                    "parent_got_revision": "967a8d215579e31720d25d44c50758330b70bd7c",
                                    "parent_got_revision_cp": "refs/heads/main@{#110037}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-22.04"
                                              },
                                          "swarming_task_attrs": {
                                                "expiration": 13800,
                                                "hard_timeout": 4800,
                                                "priority": 35
                                              },
                                          "tests": [
                                                [
                                                      "d8testing_random_gc",
                                                      2,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "allocation",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--allocation-offset=1",
                                                            "--extra-flags=--gc-interval=500"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "bytecode",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-bytecode-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "marking",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-marking=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "delay",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-delay-tasks=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "interrupt",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "threads",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-thread-pool-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "stack",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-stack-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      5,
                                                      null,
                                                      "combined",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--allocation-offset=1",
                                                            "--scavenge-chaos=4",
                                                            "--stress-bytecode-budget=1",
                                                            "--stress-delay-tasks=4",
                                                            "--stress-deopt=2",
                                                            "--stress-compaction=2",
                                                            "--stress-gc=4",
                                                            "--stress-marking=4",
                                                            "--stress-scavenge=4",
                                                            "--stress-thread-pool-size=2",
                                                            "--stress-stack-size=1",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "scavenge",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-scavenge=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "scavenge chaos",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--scavenge-chaos=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "967a8d215579e31720d25d44c50758330b70bd7c",
                                    "swarm_hashes": {
                                          "d8_default": "41d24302a0178b3c6f9a2d6b453cfa23e09342eae89a7e1f5a22ee1f3e4739f4/503",
                                          "run-num-fuzzer": "92c9f6b2899489d941c550cbd94ecd1ca6b356fabae413bdabc7e7d0e052c7c1/503"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/967a8d215579e31720d25d44c50758330b70bd7c",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Clusterfuzz Linux64 TSAN - release builder",
                                    "parent_buildnumber:57512",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "cf7ef238-6441-4818-ba8f-8d79f2a2a86a",
                        "title": "V8 Clusterfuzz Linux64 TSAN - release builder/57512",
                        "url": "https://cr-buildbucket.appspot.com/build/8669862376772453617"
                      },
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "acd6087c56c9b224d6fe6ebbe57feca17f4f2ce1",
                                    "oldest_gitiles_revision": "acd6087c56c9b224d6fe6ebbe57feca17f4f2ce1",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8669862056629063457",
                                    "parent_buildername": "V8 Clusterfuzz Linux64 TSAN - release builder",
                                    "parent_gn_args": [
                                          "dcheck_always_on = false",
                                          "is_component_build = false",
                                          "is_debug = false",
                                          "is_tsan = true",
                                          "target_cpu = \"x64\"",
                                          "use_remoteexec = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_enable_test_features = true",
                                          "v8_enable_undefined_double = true"
                                        ],
                                    "parent_got_revision": "acd6087c56c9b224d6fe6ebbe57feca17f4f2ce1",
                                    "parent_got_revision_cp": "refs/heads/main@{#110038}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-22.04"
                                              },
                                          "swarming_task_attrs": {
                                                "expiration": 13800,
                                                "hard_timeout": 4800,
                                                "priority": 35
                                              },
                                          "tests": [
                                                [
                                                      "d8testing_random_gc",
                                                      2,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "allocation",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--allocation-offset=1",
                                                            "--extra-flags=--gc-interval=500"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "bytecode",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-bytecode-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "marking",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-marking=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "delay",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-delay-tasks=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "interrupt",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "threads",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-thread-pool-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "stack",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-stack-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      5,
                                                      null,
                                                      "combined",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--allocation-offset=1",
                                                            "--scavenge-chaos=4",
                                                            "--stress-bytecode-budget=1",
                                                            "--stress-delay-tasks=4",
                                                            "--stress-deopt=2",
                                                            "--stress-compaction=2",
                                                            "--stress-gc=4",
                                                            "--stress-marking=4",
                                                            "--stress-scavenge=4",
                                                            "--stress-thread-pool-size=2",
                                                            "--stress-stack-size=1",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "scavenge",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-scavenge=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "scavenge chaos",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--scavenge-chaos=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "acd6087c56c9b224d6fe6ebbe57feca17f4f2ce1",
                                    "swarm_hashes": {
                                          "d8_default": "784fc5d2965c8726bb3e3c632bfcd59a17e910782fa60192f70e34c9783d315f/503",
                                          "run-num-fuzzer": "4743be86a2d72a725eef61fac9a3cdb8da6adc178502f734e951d44480aa61b6/503"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/acd6087c56c9b224d6fe6ebbe57feca17f4f2ce1",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Clusterfuzz Linux64 TSAN - release builder",
                                    "parent_buildnumber:57513",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "6cdeebdc-d425-4022-9aca-d65e349a991d",
                        "title": "V8 Clusterfuzz Linux64 TSAN - release builder/57513",
                        "url": "https://cr-buildbucket.appspot.com/build/8669862056629063457"
                      },
                  {
                        "buildbucket": {
                              "properties": {
                                    "newest_gitiles_revision": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
                                    "oldest_gitiles_revision": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
                                    "parent_build": "https://cr-buildbucket.appspot.com/build/8669861574406456961",
                                    "parent_buildername": "V8 Clusterfuzz Linux64 TSAN - release builder",
                                    "parent_gn_args": [
                                          "dcheck_always_on = false",
                                          "is_component_build = false",
                                          "is_debug = false",
                                          "is_tsan = true",
                                          "target_cpu = \"x64\"",
                                          "use_remoteexec = true",
                                          "v8_enable_google_benchmark = true",
                                          "v8_enable_test_features = true",
                                          "v8_enable_undefined_double = true"
                                        ],
                                    "parent_got_revision": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
                                    "parent_got_revision_cp": "refs/heads/main@{#110039}",
                                    "parent_test_spec": {
                                          "swarming_dimensions": {
                                                "os": "Ubuntu-22.04"
                                              },
                                          "swarming_task_attrs": {
                                                "expiration": 13800,
                                                "hard_timeout": 4800,
                                                "priority": 35
                                              },
                                          "tests": [
                                                [
                                                      "d8testing_random_gc",
                                                      2,
                                                      null,
                                                      "",
                                                      [
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "allocation",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--allocation-offset=1",
                                                            "--extra-flags=--gc-interval=500"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "bytecode",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-bytecode-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "marking",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-marking=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "delay",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-delay-tasks=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "interrupt",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "threads",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-thread-pool-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "stack",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-stack-size=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      5,
                                                      null,
                                                      "combined",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--allocation-offset=1",
                                                            "--scavenge-chaos=4",
                                                            "--stress-bytecode-budget=1",
                                                            "--stress-delay-tasks=4",
                                                            "--stress-deopt=2",
                                                            "--stress-compaction=2",
                                                            "--stress-gc=4",
                                                            "--stress-marking=4",
                                                            "--stress-scavenge=4",
                                                            "--stress-thread-pool-size=2",
                                                            "--stress-stack-size=1",
                                                            "--stress-interrupt-budget=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "scavenge",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--stress-scavenge=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ],
                                                [
                                                      "numfuzz",
                                                      1,
                                                      null,
                                                      "scavenge chaos",
                                                      [
                                                            "--total-timeout-sec=4200",
                                                            "--scavenge-chaos=1"
                                                          ],
                                                      {
                                                          },
                                                      {
                                                          }
                                                    ]
                                              ]
                                        },
                                    "revision": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
                                    "swarm_hashes": {
                                          "d8_default": "aaac69e69bccac6e6cc47693c286454b400d0430f227786c2648bc6dfb70dee3/503",
                                          "run-num-fuzzer": "f07cb6b98fccb1e29471ac7fa0f4abab0b81825028e3e1351de735d676f2f896/503"
                                        }
                                  },
                              "tags": [
                                    "buildset:commit/gitiles/chromium.googlesource.com/v8/v8/+/9fb4444bfc464a5e54219acb04f0828b83dd15b5",
                                    "gitiles_ref:refs/heads/main",
                                    "parent_buildername:V8 Clusterfuzz Linux64 TSAN - release builder",
                                    "parent_buildnumber:57514",
                                    "user_agent:recipe"
                                  ]
                            },
                        "id": "d8758ef6-e3d3-4042-8132-efd53686ab76",
                        "title": "V8 Clusterfuzz Linux64 TSAN - release builder/57514",
                        "url": "https://cr-buildbucket.appspot.com/build/8669861574406456961"
                      }
                ]
          },
      "newest_gitiles_revision": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
      "oldest_gitiles_revision": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
      "parent_build": "https://cr-buildbucket.appspot.com/build/8669861574406456961",
      "parent_buildername": "V8 Clusterfuzz Linux64 TSAN - release builder",
      "parent_gn_args": [
            "dcheck_always_on = false",
            "is_component_build = false",
            "is_debug = false",
            "is_tsan = true",
            "target_cpu = \"x64\"",
            "use_remoteexec = true",
            "v8_enable_google_benchmark = true",
            "v8_enable_test_features = true",
            "v8_enable_undefined_double = true"
          ],
      "parent_got_revision": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
      "parent_got_revision_cp": "refs/heads/main@{#110039}",
      "parent_test_spec": {
            "swarming_dimensions": {
                  "os": "Ubuntu-22.04"
                },
            "swarming_task_attrs": {
                  "expiration": 13800,
                  "hard_timeout": 4800,
                  "priority": 35
                },
            "tests": [
                  [
                        "d8testing_random_gc",
                        2,
                        null,
                        "",
                        [
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "allocation",
                        [
                              "--total-timeout-sec=4200",
                              "--allocation-offset=1",
                              "--extra-flags=--gc-interval=500"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "bytecode",
                        [
                              "--total-timeout-sec=4200",
                              "--stress-bytecode-budget=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "marking",
                        [
                              "--total-timeout-sec=4200",
                              "--stress-marking=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "delay",
                        [
                              "--total-timeout-sec=4200",
                              "--stress-delay-tasks=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "interrupt",
                        [
                              "--total-timeout-sec=4200",
                              "--stress-interrupt-budget=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "threads",
                        [
                              "--total-timeout-sec=4200",
                              "--stress-thread-pool-size=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "stack",
                        [
                              "--total-timeout-sec=4200",
                              "--stress-stack-size=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        5,
                        null,
                        "combined",
                        [
                              "--total-timeout-sec=4200",
                              "--allocation-offset=1",
                              "--scavenge-chaos=4",
                              "--stress-bytecode-budget=1",
                              "--stress-delay-tasks=4",
                              "--stress-deopt=2",
                              "--stress-compaction=2",
                              "--stress-gc=4",
                              "--stress-marking=4",
                              "--stress-scavenge=4",
                              "--stress-thread-pool-size=2",
                              "--stress-stack-size=1",
                              "--stress-interrupt-budget=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "scavenge",
                        [
                              "--total-timeout-sec=4200",
                              "--stress-scavenge=1"
                            ],
                        {
                            },
                        {
                            }
                      ],
                  [
                        "numfuzz",
                        1,
                        null,
                        "scavenge chaos",
                        [
                              "--total-timeout-sec=4200",
                              "--scavenge-chaos=1"
                            ],
                        {
                            },
                        {
                            }
                      ]
                ]
          },
      "swarm_hashes": {
            "d8_default": "aaac69e69bccac6e6cc47693c286454b400d0430f227786c2648bc6dfb70dee3/503",
            "run-num-fuzzer": "f07cb6b98fccb1e29471ac7fa0f4abab0b81825028e3e1351de735d676f2f896/503"
          }
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "v8/v8",
    "id": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8835290557860489440"
    },
    {
      "key": "scheduler_job_id",
      "value": "v8/V8 NumFuzz - TSAN"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler"
    },
    {
      "key": "parent_buildername",
      "value": "V8 Clusterfuzz Linux64 TSAN - release builder"
    },
    {
      "key": "parent_buildnumber",
      "value": "57514"
    },
    {
      "key": "user_agent",
      "value": "recipe"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler/topics/scheduler.buildbucket.cr-buildbucket~appspot.gserviceaccount.com"
  }
}
[14:29:01.015] Scheduled build:
{
  "id": "8669860495945969265",
  "builder": {
    "project": "v8",
    "bucket": "ci",
    "builder": "V8 NumFuzz - TSAN"
  },
  "number": 34733,
  "createdBy": "project:v8",
  "createTime": "2026-09-23T14:29:00.399950637Z",
  "updateTime": "2026-09-23T14:29:00.399950637Z",
  "status": "SCHEDULED",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "v8/v8",
      "id": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
      "ref": "refs/heads/main"
    }
  }
}
[14:29:01.015] Task URL: https://cr-buildbucket.appspot.com/build/8669860495945969265
[14:29:01.015] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:2:0) after 4m7s
[14:29:01.937] Received PubSub notification, asking Buildbucket for the build status
[14:29:01.959] Build status: SCHEDULED
[14:29:09.602] Received PubSub notification, asking Buildbucket for the build status
[14:29:09.621] Build status: STARTED
[14:33:08.081] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:2:0)
[14:33:08.113] Build status: STARTED
[14:33:08.113] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:5:0) after 9m31s
[14:42:39.246] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:5:0)
[14:42:39.274] Build status: STARTED
[14:42:39.274] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:6:0) after 6m35s
[14:49:14.294] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:6:0)
[14:49:14.318] Build status: STARTED
[14:49:14.318] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:7:0) after 1m36s
[14:50:50.371] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:7:0)
[14:50:50.390] Build status: STARTED
[14:50:50.390] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:8:0) after 9m12s
[15:00:02.549] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:8:0)
[15:00:02.568] Build status: STARTED
[15:00:02.568] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:9:0) after 3m55s
[15:03:57.628] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:9:0)
[15:03:57.645] Build status: STARTED
[15:03:57.645] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:10:0) after 4m7s
[15:08:04.677] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:10:0)
[15:08:04.714] Build status: STARTED
[15:08:04.714] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:11:0) after 7m18s
[15:15:22.760] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:11:0)
[15:15:22.783] Build status: STARTED
[15:15:22.783] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:12:0) after 1m31s
[15:16:53.777] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:12:0)
[15:16:53.869] Build status: STARTED
[15:16:53.869] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:13:0) after 6m59s
[15:23:52.989] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:13:0)
[15:23:53.021] Build status: STARTED
[15:23:53.021] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:14:0) after 4m19s
[15:28:12.247] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:14:0)
[15:28:12.305] Build status: STARTED
[15:28:12.305] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:15:0) after 8m56s
[15:37:08.463] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:15:0)
[15:37:08.496] Build status: STARTED
[15:37:08.496] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:16:0) after 9m57s
[15:47:05.564] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:16:0)
[15:47:05.641] Build status: STARTED
[15:47:05.641] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:17:0) after 8m11s
[15:55:16.746] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:17:0)
[15:55:16.778] Build status: STARTED
[15:55:16.778] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:18:0) after 4m29s
[15:59:45.909] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:18:0)
[15:59:45.955] Build status: STARTED
[15:59:45.956] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:19:0) after 9m56s
[16:09:42.058] Handling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:19:0)
[16:09:42.076] Build status: STARTED
[16:09:42.076] Scheduling timer "check-buildbucket-build-status" (v8/V8 NumFuzz - TSAN:8835290557860489440:20:0) after 8m53s
[16:16:26.712] Received PubSub notification, asking Buildbucket for the build status
[16:16:26.783] Build:
{
  "id": "8669860495945969265",
  "builder": {
    "project": "v8",
    "bucket": "ci",
    "builder": "V8 NumFuzz - TSAN"
  },
  "number": 34733,
  "createdBy": "project:v8",
  "createTime": "2026-09-23T14:29:00.399950637Z",
  "startTime": "2026-09-23T14:29:09.411370656Z",
  "endTime": "2026-09-23T16:16:26.385771572Z",
  "updateTime": "2026-09-23T16:16:26.385771572Z",
  "status": "FAILURE",
  "input": {
    "gitilesCommit": {
      "host": "chromium.googlesource.com",
      "project": "v8/v8",
      "id": "9fb4444bfc464a5e54219acb04f0828b83dd15b5",
      "ref": "refs/heads/main"
    }
  }
}
[16:16:26.783] Invocation finished in 1h47m27.594402998s with status FAILED