Merge branch 'HabanaAI:habana_main' into intern2-habana

HabanaAI · Nov 26, 2024 · 2ae1e39 · 2ae1e39
2 parents 19285e8 + 38c2d10
commit 2ae1e39
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 11 deletions.
diff --git a/requirements-hpu.txt b/requirements-hpu.txt
@@ -8,7 +8,5 @@ pandas
 tabulate
 setuptools>=61
 setuptools-scm>=8
-vllm-hpu-extension @ git+https://github.com/HabanaAI/vllm-hpu-extension.git@61334c5
+vllm-hpu-extension @ git+https://github.com/HabanaAI/vllm-hpu-extension.git@ac9740d
 neural-compressor @ git+https://github.com/intel/neural-compressor.git@b196432
-
-
diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py
@@ -117,13 +117,16 @@ class Fp8LinearMethod(LinearMethodBase):
 
     def __init__(self, quant_config: Fp8Config):
         self.quant_config = quant_config
+        self.cutlass_fp8_supported = False
         if current_platform.is_cuda_alike():
             self.cutlass_fp8_supported = cutlass_fp8_supported()
 
-        # For GPUs that lack FP8 hardware support, we can leverage the Marlin
-        # kernel for fast weight-only FP8 quantization
-        self.use_marlin = (not current_platform.has_device_capability(89)
-                           or envs.VLLM_TEST_FORCE_FP8_MARLIN)
+        self.use_marlin = False
+        if not current_platform.is_hpu():
+            # For GPUs that lack FP8 hardware support, we can leverage the
+            # Marlin kernel for fast weight-only FP8 quantization
+            self.use_marlin = (not current_platform.has_device_capability(89)
+                               or envs.VLLM_TEST_FORCE_FP8_MARLIN)
         # Disable marlin for rocm
         if current_platform.is_rocm():
             self.use_marlin = False

diff --git a/vllm/worker/hpu_model_runner.py b/vllm/worker/hpu_model_runner.py
@@ -1266,9 +1266,9 @@ def create_dummy_seq_group_metadata(self,
     def profile_run(self) -> None:
         num_layers = self.model_config.get_num_layers(self.parallel_config)
         kv_caches = [None] * num_layers
-        max_batch_size, max_seq_len = self.bucketing_ctx.get_max_prompt_shape()
-        max_seq_len = min(max_seq_len,
-                          self.max_num_batched_tokens // max_batch_size)
+        _, max_seq_len = self.bucketing_ctx.get_max_prompt_shape()
+        max_batch_size = min(self.max_num_seqs,
+                             self.max_num_batched_tokens // max_seq_len)
 
         self.warmup_scenario(max_batch_size, max_seq_len, True, kv_caches,
                              False, True)

diff --git a/vllm/worker/hpu_worker.py b/vllm/worker/hpu_worker.py
@@ -166,7 +166,9 @@ def determine_num_available_blocks(self) -> Tuple[int, int]:
         if is_fake_hpu():
             cache_block_size = self.get_cache_block_size_bytes()
             fake_hpu_cache_alloc = 4 * 2**30  # take 4 GiB flat on fake hpu
-            return fake_hpu_cache_alloc // cache_block_size, 0
+            num_fake_hpu_blocks = fake_hpu_cache_alloc // cache_block_size
+            self.model_runner.bucketing_ctx.num_hpu_blocks = num_fake_hpu_blocks
+            return num_fake_hpu_blocks, 0
         with HabanaMemoryProfiler() as m:
             self.model_runner.profile_run()
             torch.hpu.synchronize()
@@ -203,6 +205,8 @@ def determine_num_available_blocks(self) -> Tuple[int, int]:
         num_hpu_blocks = max(num_hpu_blocks, 0)
         num_cpu_blocks = max(num_cpu_blocks, 0)
 
+        self.model_runner.bucketing_ctx.num_hpu_blocks = num_hpu_blocks
+
         if self.model_runner.lora_manager:
             self.model_runner.remove_all_loras()