[Fix] Unload lora in HF_Runner if needed (sgl-project#5899)

Author: Qiaolin-Yu

python/sglang/test/runners.py

@@ -423,6 +423,10 @@ def forward_generation_raw(
                     )
                 del input_logits
 
+        if lora_paths is not None and lora_paths[i] is not None:
+            # Unload the LoRA adapter if it is used
+            model.unload()
+
         return ModelOutput(
             output_strs=output_strs,
             top_input_logprobs=top_input_logprobs,