local import patch_vllm()

ZX-ModelCloud · Dec 13, 2024 · 06708c7 · 06708c7
1 parent 95dedbe
commit 06708c7
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/gptqmodel/models/auto.py b/gptqmodel/models/auto.py
@@ -8,7 +8,6 @@
 from huggingface_hub import list_repo_files
 from transformers import AutoConfig
 
-from ..integration.integration_vllm import patch_vllm
 from ..quantization import QUANT_CONFIG_FILENAME
 from ..utils import BACKEND, EVAL
 from ..utils.logger import setup_logger
@@ -135,7 +134,9 @@ def load(
             **kwargs,
     ):
         if backend == BACKEND.VLLM:
+            from ..integration.integration_vllm import patch_vllm
             patch_vllm()
+
         is_quantized = False
         if hasattr(AutoConfig.from_pretrained(model_id_or_path, trust_remote_code=trust_remote_code), "quantization_config"):
             is_quantized = True