Disable half2 for ExLlama when using HIP

modules/exllama.py

@@ -1,6 +1,8 @@
 import sys
 from pathlib import Path
 
+from torch import version as torch_version
+
 from modules import shared
 from modules.logging_colors import logger
 
@@ -51,6 +53,12 @@ def from_pretrained(self, path_to_model):
         if shared.args.gpu_split:
             config.set_auto_map(shared.args.gpu_split)
             config.gpu_peer_fix = True
+        if torch_version.hip:
+            config.rmsnorm_no_half2 = True
+            config.rope_no_half2 = True
+            config.matmul_no_half2 = True
+            config.silu_no_half2 = True
+
 
         model = ExLlama(config)
         tokenizer = ExLlamaTokenizer(str(tokenizer_model_path))

modules/exllama_hf.py

@@ -97,6 +97,11 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P
         if shared.args.gpu_split:
             config.set_auto_map(shared.args.gpu_split)
             config.gpu_peer_fix = True
+        if torch.version.hip:
+            config.rmsnorm_no_half2 = True
+            config.rope_no_half2 = True
+            config.matmul_no_half2 = True
+            config.silu_no_half2 = True
 
         # This slowes down a bit but align better with autogptq generation.
         # TODO: Should give user choice to tune the exllama config