Remove dummy seq group data creation from loop (#301)

Remove dummy seq metadata from loop for Flat PA fix
HabanaAI · Sep 20, 2024 · cd7b1c1 · cd7b1c1
1 parent b62fba8
commit cd7b1c1
Showing 1 changed file with 5 additions and 3 deletions.
diff --git a/vllm/worker/habana_model_runner.py b/vllm/worker/habana_model_runner.py
@@ -1087,9 +1087,11 @@ def prepare_input_tensors(
         batch_size_padded = find_bucket(real_batch_size, bucket_cfg)
         batch_size_padding = batch_size_padded - real_batch_size
         seq_group_metadata_list = seq_group_metadata_list.copy()
-        seq_group_metadata_list.extend(
-            self.create_dummy_seq_group_metadata(0, 0, is_prompt)
-            for _ in range(batch_size_padding))
+        if batch_size_padding > 0:
+            dummy_seq_group_metadata = self.create_dummy_seq_group_metadata(
+                0, 0, is_prompt)
+            seq_group_metadata_list.extend(dummy_seq_group_metadata
+                                           for _ in range(batch_size_padding))
 
         prefill_reqs = []
         decode_reqs = []