Optimize virtual loss parameters

Perform match games to tune the following KataGo parameters:
- suppressVirtualLossExploreFactor
- suppressVirtualLossHindsight
- suppressVirtualLossLeakCatchUp
- numSearchThreads
- useNoisePruning

Relates to: lightvector/KataGo#864

Author: ChinChangYang

match.py

@@ -6,6 +6,7 @@
 from sgfmill import sgf
 from scipy.stats import bernoulli
 
+
 def spawn_process(command: "list[str]"):
     """Spawn a process with the command and wait until the process ends.
 
@@ -18,59 +19,91 @@ def spawn_process(command: "list[str]"):
     except Exception as e:
         print(f"Error occurred: {str(e)}")
 
+
 def get_command(x) -> str:
-    command = f'{x["exe"]} gtp -config {x["config"]} -model {x["model"]} -override-config maxVisits={x["maxVisits"]}'
+    command = f'{x["exe"]} gtp -config {x["config"]} -model {x["model"]}'
+    override_options = [
+        "maxVisits",
+        "suppressVirtualLossExploreFactor",
+        "suppressVirtualLossHindsight",
+        "suppressVirtualLossLeakCatchUp",
+        "numSearchThreads",
+        "useNoisePruning",
+    ]
+
+    for option in override_options:
+        if option in x:
+            command += f" -override-config {option}={x[option]}"
+
     return command
 
-def match(black_parameters, white_parameters, gogui_classpath, game_count, sgffile_prefix="match", verbose=True) -> int:
+
+def match(
+    black_parameters,
+    white_parameters,
+    gogui_classpath,
+    game_count,
+    sgffile_prefix="match",
+    verbose=True,
+) -> int:
     black_command = get_command(black_parameters)
     white_command = get_command(white_parameters)
     board_size = 19
     komi = 7.5
-    twogtp = ['java',
-              '-cp',
-              gogui_classpath,
-              'net.sf.gogui.tools.twogtp.Main',
-              '-black',
-              black_command,
-              '-white',
-              white_command,
-              '-size',
-              f'{board_size}',
-              '-komi',
-              f'{komi}',
-              '-auto',
-              '-sgffile',
-              f'{sgffile_prefix}-{game_count}']
+    twogtp = [
+        "java",
+        "-cp",
+        gogui_classpath,
+        "net.sf.gogui.tools.twogtp.Main",
+        "-black",
+        black_command,
+        "-white",
+        white_command,
+        "-size",
+        f"{board_size}",
+        "-komi",
+        f"{komi}",
+        "-auto",
+        "-sgffile",
+        f"{sgffile_prefix}-{game_count}",
+    ]
 
     spawn_process(twogtp)
-    sgffile = f'{sgffile_prefix}-{game_count}-0.sgf'
+    sgffile = f"{sgffile_prefix}-{game_count}-0.sgf"
     is_won = 0
 
     with open(sgffile, "rb") as f:
         sgf_string = f.read()
         game = sgf.Sgf_game.from_bytes(sgf_string)
         winner = game.get_winner()
 
-        if (winner == 'b'):
+        if winner == "b":
             # Black won
             is_won = -1
             if verbose:
-                print(f'Game {game_count}: Black won')
-        elif (winner == 'w'):
+                print(f"Game {game_count}: Black won")
+        elif winner == "w":
             # White won
             is_won = 1
             if verbose:
-                print(f'Game {game_count}: White won')
+                print(f"Game {game_count}: White won")
         else:
             # Draw
             is_won = 0
             if verbose:
-                print(f'Game {game_count}: Draw')
+                print(f"Game {game_count}: Draw")
 
     return is_won
 
-def match_games(black_parameters, white_parameters, game_count_start, games, sgffile_prefix="match", verbose=True) -> (int, int, int):
+
+def match_games(
+    black_parameters,
+    white_parameters,
+    game_count_start,
+    games,
+    sgffile_prefix="match",
+    verbose=True,
+) -> (int, int, int):
     results = []
     game_count_stop = game_count_start + games
 
@@ -81,7 +114,8 @@ def match_games(black_parameters, white_parameters, game_count_start, games, sgf
             "/Users/chinchangyang/Code/gogui/bin",
             game_count,
             sgffile_prefix=sgffile_prefix,
-            verbose=verbose)
+            verbose=verbose,
+        )
 
         results.append(result)
 
@@ -94,6 +128,7 @@ def match_games(black_parameters, white_parameters, game_count_start, games, sgf
 
     return (black_win, white_win, draw)
 
+
 def elo(M: float, N: float) -> float:
     """Calculate expected ELO
 
@@ -105,12 +140,13 @@ def elo(M: float, N: float) -> float:
         float: expected ELO
     """
     if N <= M:
-        return float('inf')
+        return float("inf")
     elif M <= 0:
-        return float('-inf')
+        return float("-inf")
     else:
         return -400 * math.log10(-1 + (N / M))
 
+
 def elo_range(M: int, N: int, a: float) -> float:
     """Calculate ELO standard deviation
 
@@ -122,9 +158,9 @@ def elo_range(M: int, N: int, a: float) -> float:
         float: ELO standard deviation
     """
     if N == M:
-        return (float('inf'), float('inf'))
+        return (float("inf"), float("inf"))
     elif M == 0:
-        return (float('-inf'), float('-inf'))
+        return (float("-inf"), float("-inf"))
     else:
         p = M / N  # mean
         var = bernoulli.var(p) / N  # variance of sample mean
@@ -134,70 +170,80 @@ def elo_range(M: int, N: int, a: float) -> float:
         elo_negative_delta = elo(M - delta, N)  # ELO with negative delta
         return (elo_negative_delta, elo_positive_delta)
 
+
 if __name__ == "__main__":
     t0 = time.time()
-    bot_a_name = 'b18c384nbt'
+    bot_a_name = "K1e10-Hoff-LCUoff-T8-NPon"
 
     bot_a_parameters = {
         "exe": "/Users/chinchangyang/Code/KataGo/cpp/build/katago",
         "config": "/Users/chinchangyang/.katago/default_gtp.cfg",
-        "model": "/Users/chinchangyang/Code/KataGo-Models/kata1-b18c384nbt-s7709731328-d3715293823.bin.gz",
-        "maxVisits": "2"
+        "model": "/Users/chinchangyang/.katago/default_model.bin.gz",
+        "suppressVirtualLossExploreFactor": "1e10",
+        "suppressVirtualLossHindsight": "false",
+        "suppressVirtualLossLeakCatchUp": "false",
+        "numSearchThreads": "8",
+        "useNoisePruning": "true",
     }
 
-    bot_b_name = 'b28c512nbt'
+    bot_b_name = "K8-Hoff-LCUoff-T4-NPoff"
 
     bot_b_parameters = {
-        "exe": "/Users/chinchangyang/Code/KataGo-CCY/cpp/build/katago",
-        "config": "/Users/chinchangyang/Code/KataGo-CCY/cpp/configs/misc/coreml_example.cfg",
-        "model": "/Users/chinchangyang/Code/KataGo-Models/b28c512nbt-s1436726784-d3907069532.bin.gz",
-        "maxVisits": "2"
+        "exe": "/Users/chinchangyang/Code/KataGo/cpp/build/katago",
+        "config": "/Users/chinchangyang/.katago/default_gtp.cfg",
+        "model": "/Users/chinchangyang/.katago/default_model.bin.gz",
+        "suppressVirtualLossExploreFactor": "8",  # {4, 64}
+        "suppressVirtualLossHindsight": "false",  # {false, true}
+        "suppressVirtualLossLeakCatchUp": "false",  # {false, true}
+        "numSearchThreads": "4",  # {8, 32}
+        "useNoisePruning": "false",  # {false, true}
     }
 
-    total_games = 1024
+    total_games = 100
     half_games = int(total_games / 2)
 
     black_win, white_win, draw = match_games(
-        bot_a_parameters,
-        bot_b_parameters,
-        0,
-        half_games)
+        bot_a_parameters, bot_b_parameters, 0, half_games
+    )
 
     bot_a_win = black_win
     bot_b_win = white_win
     both_draw = draw
 
-    print(f'{bot_a_name} Won: {bot_a_win}')
-    print(f'{bot_b_name} Won: {bot_b_win}')
-    print(f'Draw: {both_draw}')
+    print(f"{bot_a_name} Won: {bot_a_win}")
+    print(f"{bot_b_name} Won: {bot_b_win}")
+    print(f"Draw: {both_draw}")
 
     black_win, white_win, draw = match_games(
-        bot_b_parameters,
-        bot_a_parameters,
-        half_games,
-        half_games)
+        bot_b_parameters, bot_a_parameters, half_games, half_games
+    )
 
     bot_a_win = bot_a_win + white_win
     bot_b_win = bot_b_win + black_win
     both_draw = both_draw + draw
 
-    print(f'{bot_a_name} Won: {bot_a_win}')
-    print(f'{bot_b_name} Won: {bot_b_win}')
-    print(f'Draw: {both_draw}')
+    print(f"{bot_a_name} Won: {bot_a_win}")
+    print(f"{bot_b_name} Won: {bot_b_win}")
+    print(f"Draw: {both_draw}")
 
     bot_a_outcome = bot_a_win + (both_draw / 2)
     bot_b_outcome = bot_b_win + (both_draw / 2)
 
     print(
-        f'Expected ELO of {bot_a_name} parameters (from {total_games} games) = {elo(bot_a_outcome, total_games)}')
+        f"Expected ELO of {bot_a_name} parameters (from {total_games} games) = {elo(bot_a_outcome, total_games)}"
+    )
     print(
-        f'Expected ELO of {bot_b_name} parameters (from {total_games} games) = {elo(bot_b_outcome, total_games)}')
+        f"Expected ELO of {bot_b_name} parameters (from {total_games} games) = {elo(bot_b_outcome, total_games)}"
+    )
     print(
-        f'ELO range (+/- 1.0 standard deviation) of {bot_b_name} parameters = {elo_range(bot_b_outcome, total_games, 1.0)}')
+        f"ELO range (+/- 1.0 standard deviation) of {bot_b_name} parameters = {elo_range(bot_b_outcome, total_games, 1.0)}"
+    )
     print(
-        f'ELO range (+/- 2.0 standard deviation) of {bot_b_name} parameters = {elo_range(bot_b_outcome, total_games, 2.0)}')
+        f"ELO range (+/- 2.0 standard deviation) of {bot_b_name} parameters = {elo_range(bot_b_outcome, total_games, 2.0)}"
+    )
     print(
-        f'ELO range (+/- 3.0 standard deviation) of {bot_b_name} parameters = {elo_range(bot_b_outcome, total_games, 3.0)}')
+        f"ELO range (+/- 3.0 standard deviation) of {bot_b_name} parameters = {elo_range(bot_b_outcome, total_games, 3.0)}"
+    )
 
     elapsed = time.time() - t0
-    print(f'Elapsed: {str(datetime.timedelta(seconds=round(elapsed)))}')
+    print(f"Elapsed: {str(datetime.timedelta(seconds=round(elapsed)))}")