update

Author: hnyls2002

python/sglang/srt/model_executor/cuda_graph_runner.py

@@ -139,7 +139,7 @@ def get_batch_sizes_to_capture(model_runner: ModelRunner):
         gpu_mem = get_whatever_gpu_memory_capacity() / 1024
 
         if gpu_mem > 120:
-            capture_bs += list(range(160, 320, 8))
+            capture_bs += list(range(160, 256, 8))
 
     if max(capture_bs) > model_runner.req_to_token_pool.size:
         # In some case (e.g., with a small GPU or --max-running-requests), the #max-running-requests
@@ -196,6 +196,9 @@ def __init__(self, model_runner: ModelRunner):
 
         # Batch sizes to capture
         self.capture_bs, self.compile_bs = get_batch_sizes_to_capture(model_runner)
+
+        print(f"\x1b[32mx)bs={self.capture_bs}\x1b[0m")
+
         self.capture_forward_mode = ForwardMode.DECODE
         self.capture_hidden_mode = CaptureHiddenMode.NULL
         self.num_tokens_per_bs = 1