From 987a213b0834b5f758eb4a5b4aa0915f6fbbf967 Mon Sep 17 00:00:00 2001 From: Nir David Date: Sun, 15 Dec 2024 17:08:00 +0200 Subject: [PATCH 1/6] Add inc fp8 qunatization documentation --- .../getting_started/gaudi-installation.rst | 2 +- docs/source/quantization/inc.rst | 71 +++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 docs/source/quantization/inc.rst diff --git a/docs/source/getting_started/gaudi-installation.rst b/docs/source/getting_started/gaudi-installation.rst index ee733afd27578..4457d3be7a12e 100644 --- a/docs/source/getting_started/gaudi-installation.rst +++ b/docs/source/getting_started/gaudi-installation.rst @@ -46,7 +46,7 @@ To verify that the Intel Gaudi software was correctly installed, run: $ hl-smi # verify that hl-smi is in your PATH and each Gaudi accelerator is visible $ apt list --installed | grep habana # verify that habanalabs-firmware-tools, habanalabs-graph, habanalabs-rdma-core, habanalabs-thunk and habanalabs-container-runtime are installed $ pip list | grep habana # verify that habana-torch-plugin, habana-torch-dataloader, habana-pyhlml and habana-media-loader are installed - $ pip list | grep neural # verify that neural_compressor is installed + $ pip list | grep neural # verify that neural_compressor_pt is installed Refer to `System Verification and Final Tests `__ for more details. diff --git a/docs/source/quantization/inc.rst b/docs/source/quantization/inc.rst new file mode 100644 index 0000000000000..ff8cbb48a6b6d --- /dev/null +++ b/docs/source/quantization/inc.rst @@ -0,0 +1,71 @@ +.. _INC: + +FP8 INC +================== + +vLLM supports FP8 (8-bit floating point) weight and activation quantization using INC (Intel Neural Compressor) on hardware acceleration of Intel Gaudi (HPU). +Currently, only Llama models quntization are supported. + +Please visit the Intel Gaudi documentation of `Run Inference Using FP8 `_. + +In order to run Inference it is required to have Measurements/Scales files: + +Retrieve Measurements +--------------------- + +To obtain measurement files: +* Use the "inc" quantization method (as parameter to the LLM object). +* Call shutdown_inc and shutdown methods of the model_executor in the end of the run. + +.. code-block:: python + + from vllm import LLM + llm = LLM("llama3.1/Meta-Llama-3.1-8B-Instruct", quantization="inc") + ... + # Call llm.generate on the required prompts and sampling params. + ... + llm.llm_engine.model_executor.shutdown_inc() + llm.llm_engine.model_executor.shutdown() + +.. note:: + + Make sure to supply the "QUANT_CONFIG" environment variable which points to the `Json config file `_ with MEASURE mode. + +Run Inference Using FP8 +----------------------- + +Intel Gaudi supports quantization of Linear Layers, KV-Cache and functions like Matmul and Softamx as shown in: +`Supported Modules `_. +`Supported Functions `_. + +In order to run Inference it requires to have Scales which located in scale files according to the `Json config file `_ dump_stats_path. +If none exist they can be generated during inference run using the measurement files (should be located in the same folder). + +To run inference (and obtain scale files): +* Use the "inc" quantization method (as parameter to the LLM object). +* Use the "fp8_inc" kv cache dtype (as parameter to the LLM object). +* Call shutdown method of the model_executor in the end of the run. + +.. code-block:: python + + from vllm import LLM + llm = LLM("llama3.1/Meta-Llama-3.1-8B-Instruct", quantization="inc", kv_cache_dtype="fp8_inc") + ... + # Call llm.generate on the required prompts and sampling params. + ... + llm.llm_engine.model_executor.shutdown() + +.. note:: + + Make sure to supply the "QUANT_CONFIG" environment variable which points to the `Json config file `_ with QUANTIZE mode. + +Specifying Device for the Model's Weights Uploading +--------------------------------------------------- + +It is possible to upload the (unquantized) weights on a different device before qunantizing them +and moving to the device on which the model will run. +Use the weights_load_device parameter for the LLM object to specify this device. +.. code-block:: python + from vllm import LLM + llm = LLM("llama3.1/Meta-Llama-3.1-8B-Instruct", quantization="inc", kv_cache_dtype="fp8_inc", weights_load_device="cpu") + From 15242afecb5f3482b30c1c4de0376475778948d1 Mon Sep 17 00:00:00 2001 From: Nir David Date: Mon, 16 Dec 2024 16:55:17 +0200 Subject: [PATCH 2/6] fix CR comments --- docs/source/quantization/inc.rst | 46 +++++++++++++++----------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/docs/source/quantization/inc.rst b/docs/source/quantization/inc.rst index ff8cbb48a6b6d..4996fe0b6ad47 100644 --- a/docs/source/quantization/inc.rst +++ b/docs/source/quantization/inc.rst @@ -1,21 +1,22 @@ .. _INC: FP8 INC -================== +======= vLLM supports FP8 (8-bit floating point) weight and activation quantization using INC (Intel Neural Compressor) on hardware acceleration of Intel Gaudi (HPU). -Currently, only Llama models quntization are supported. +Currently, quantization is supported only for Llama models. Please visit the Intel Gaudi documentation of `Run Inference Using FP8 `_. -In order to run Inference it is required to have Measurements/Scales files: +In order to run inference it is required to have measurements/scales files: -Retrieve Measurements ---------------------- +Obtain Measurements +------------------- To obtain measurement files: -* Use the "inc" quantization method (as parameter to the LLM object). -* Call shutdown_inc and shutdown methods of the model_executor in the end of the run. +* Set the "QUANT_CONFIG" environment variable which points to the `JSON config file `_ with MEASURE mode. +* Pass ``quantization=inc`` as parameter to the ``LLM`` object. +* Call ``shutdown_inc`` and ``shutdown`` methods of the ``model_executor`` at the end of the run. .. code-block:: python @@ -27,24 +28,23 @@ To obtain measurement files: llm.llm_engine.model_executor.shutdown_inc() llm.llm_engine.model_executor.shutdown() -.. note:: - - Make sure to supply the "QUANT_CONFIG" environment variable which points to the `Json config file `_ with MEASURE mode. - Run Inference Using FP8 ----------------------- -Intel Gaudi supports quantization of Linear Layers, KV-Cache and functions like Matmul and Softamx as shown in: +Intel Gaudi supports quantization of various modules and functions, including, but not limited to ``Linear``, ``KVCache``, ``Matmul`` and ``Softmax``. For more information, please refer to: `Supported Modules `_. `Supported Functions `_. -In order to run Inference it requires to have Scales which located in scale files according to the `Json config file `_ dump_stats_path. -If none exist they can be generated during inference run using the measurement files (should be located in the same folder). +In order to run inference it requires to have Scales which located in scale files according to the `JSON config file `_ ``dump_stats_path``. +If none exist, they can be generated during inference run using the measurement files (should be located in the same folder). To run inference (and obtain scale files): -* Use the "inc" quantization method (as parameter to the LLM object). -* Use the "fp8_inc" kv cache dtype (as parameter to the LLM object). -* Call shutdown method of the model_executor in the end of the run. +* Set the "QUANT_CONFIG" environment variable which points to the `JSON config file `_ with QUANTIZE mode. +* Pass ``quantization=inc`` as parameter to the ``LLM`` object. +* Pass ``fp8_inc`` as KV cache data type: + * Offline inference: pass ``kv_cache_dtype=fp8_inc`` as parameter to the ``LLM`` object. + * Online inference: pass ``--kv-cache-dtype=fp8_inc`` as command line parameter. +* Call shutdown method of the model_executor at the end of the run. .. code-block:: python @@ -55,17 +55,15 @@ To run inference (and obtain scale files): ... llm.llm_engine.model_executor.shutdown() -.. note:: - - Make sure to supply the "QUANT_CONFIG" environment variable which points to the `Json config file `_ with QUANTIZE mode. - Specifying Device for the Model's Weights Uploading --------------------------------------------------- -It is possible to upload the (unquantized) weights on a different device before qunantizing them -and moving to the device on which the model will run. -Use the weights_load_device parameter for the LLM object to specify this device. +It is possible to load the unquantized weights on a different device before quantizing them, +and moving to the device on which the model will run. This reduces the device memory footprint of model weights, as only quantized weights are stored in device memory. +To set the load device, use the ``weights_load_device`` parameter for the ``LLM`` object, or ``--weights-load-device`` command line parameter in online mode. + .. code-block:: python + from vllm import LLM llm = LLM("llama3.1/Meta-Llama-3.1-8B-Instruct", quantization="inc", kv_cache_dtype="fp8_inc", weights_load_device="cpu") From 6bb031c29c6c1a55bd4e7a2e089466484d1bb3ae Mon Sep 17 00:00:00 2001 From: Nir David Date: Thu, 19 Dec 2024 15:29:47 +0200 Subject: [PATCH 3/6] add more documentation changes --- docs/source/quantization/inc.rst | 65 +++++++++++++++----------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/docs/source/quantization/inc.rst b/docs/source/quantization/inc.rst index 4996fe0b6ad47..b1b68321ea656 100644 --- a/docs/source/quantization/inc.rst +++ b/docs/source/quantization/inc.rst @@ -3,47 +3,43 @@ FP8 INC ======= -vLLM supports FP8 (8-bit floating point) weight and activation quantization using INC (Intel Neural Compressor) on hardware acceleration of Intel Gaudi (HPU). +vLLM supports FP8 (8-bit floating point) weight and activation quantization using Intel® Neural Compressor (INC) on Intel® Gaudi® 2 and Intel® Gaudi® 3 AI accelerators. Currently, quantization is supported only for Llama models. -Please visit the Intel Gaudi documentation of `Run Inference Using FP8 `_. +Intel Gaudi supports quantization of various modules and functions, including, but not limited to ``Linear``, ``KVCache``, ``Matmul`` and ``Softmax``. For more information, please refer to: +`Supported Modules\Supported Functions\Custom Patched Modules `_. -In order to run inference it is required to have measurements/scales files: +.. note:: + Measurement files are required to run quantized models with vLLM on Gaudi accelerators. The FP8 model calibration procedure is described in the `vllm-hpu-extention `_ package. -Obtain Measurements -------------------- +.. note:: + ``QUANT_CONFIG`` is an environment variable that points to the measurement or quantization `JSON config file `_. + The measurement configuration file is used during the calibration procedure to collect measurements for a given model. The quantization configuration is used during inference. -To obtain measurement files: -* Set the "QUANT_CONFIG" environment variable which points to the `JSON config file `_ with MEASURE mode. -* Pass ``quantization=inc`` as parameter to the ``LLM`` object. -* Call ``shutdown_inc`` and ``shutdown`` methods of the ``model_executor`` at the end of the run. +Run Online Inference Using FP8 +------------------------------- -.. code-block:: python +Once you've completed the model calibration process and collected the measurements, you can run FP8 inference with vLLM using the following command: - from vllm import LLM - llm = LLM("llama3.1/Meta-Llama-3.1-8B-Instruct", quantization="inc") - ... - # Call llm.generate on the required prompts and sampling params. - ... - llm.llm_engine.model_executor.shutdown_inc() - llm.llm_engine.model_executor.shutdown() +.. code-block:: bash -Run Inference Using FP8 ------------------------ + export QUANT_CONFIG=/path/to/quant/config/inc/meta-llama-3.1-405b-instruct/maxabs_measure_g3.json + vllm serve meta-llama/Llama-3.1-405B-Instruct --quantization inc --kv-cache-dtype fp8_inc --weights-load-device cpu --tensor_paralel_size 8 -Intel Gaudi supports quantization of various modules and functions, including, but not limited to ``Linear``, ``KVCache``, ``Matmul`` and ``Softmax``. For more information, please refer to: -`Supported Modules `_. -`Supported Functions `_. - -In order to run inference it requires to have Scales which located in scale files according to the `JSON config file `_ ``dump_stats_path``. -If none exist, they can be generated during inference run using the measurement files (should be located in the same folder). - -To run inference (and obtain scale files): -* Set the "QUANT_CONFIG" environment variable which points to the `JSON config file `_ with QUANTIZE mode. -* Pass ``quantization=inc`` as parameter to the ``LLM`` object. -* Pass ``fp8_inc`` as KV cache data type: - * Offline inference: pass ``kv_cache_dtype=fp8_inc`` as parameter to the ``LLM`` object. - * Online inference: pass ``--kv-cache-dtype=fp8_inc`` as command line parameter. +.. tip:: + If you are just prototyping or testing your model with FP8, you can use the ``VLLM_SKIP_WARMUP=true`` environment variable to disable the warmup stage, which can take a long time. However, we do not recommend disabling this feature in production environments, as it causes a dramatic performance drop. + +.. tip:: + When using FP8 models, you may experience timeouts caused by the long compilation time of FP8 operations. To mitigate this problem, you can use these two environment variables: + ``VLLM_ENGINE_ITERATION_TIMEOUT_S`` - to adjust the vLLM server timeout. You can set the value in seconds, e.g., 600 equals 10 minutes. + ``VLLM_RPC_TIMEOUT`` - to adjust the RPC protocol timeout used by the OpenAI-compatible API. This value is in microseconds, e.g., 600000 equals 10 minutes. + +Run Offline Inference Using FP8 +------------------------------- + +To run offline inference (after completing the model calibration process): +* Set the "QUANT_CONFIG" environment variable to point to a JSON configuration file with QUANTIZE mode. +* Pass ``quantization=inc`` and ``kv_cache_dtype=fp8_inc`` as parameters to the ``LLM`` object. * Call shutdown method of the model_executor at the end of the run. .. code-block:: python @@ -58,12 +54,11 @@ To run inference (and obtain scale files): Specifying Device for the Model's Weights Uploading --------------------------------------------------- -It is possible to load the unquantized weights on a different device before quantizing them, -and moving to the device on which the model will run. This reduces the device memory footprint of model weights, as only quantized weights are stored in device memory. +It is possible to load the unquantized weights on a different device before quantizing them, then moving them to the device on which the model will run. +This reduces the device memory footprint of model weights, as only quantized weights are stored in device memory. To set the load device, use the ``weights_load_device`` parameter for the ``LLM`` object, or ``--weights-load-device`` command line parameter in online mode. .. code-block:: python from vllm import LLM llm = LLM("llama3.1/Meta-Llama-3.1-8B-Instruct", quantization="inc", kv_cache_dtype="fp8_inc", weights_load_device="cpu") - From 97a214e49af8ec2a472e59614f98c10e395f8f2e Mon Sep 17 00:00:00 2001 From: Nir David Date: Mon, 23 Dec 2024 11:50:39 +0200 Subject: [PATCH 4/6] some more CR fixes --- docs/source/quantization/inc.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/quantization/inc.rst b/docs/source/quantization/inc.rst index b1b68321ea656..4d9020f3186c1 100644 --- a/docs/source/quantization/inc.rst +++ b/docs/source/quantization/inc.rst @@ -27,10 +27,10 @@ Once you've completed the model calibration process and collected the measuremen vllm serve meta-llama/Llama-3.1-405B-Instruct --quantization inc --kv-cache-dtype fp8_inc --weights-load-device cpu --tensor_paralel_size 8 .. tip:: - If you are just prototyping or testing your model with FP8, you can use the ``VLLM_SKIP_WARMUP=true`` environment variable to disable the warmup stage, which can take a long time. However, we do not recommend disabling this feature in production environments, as it causes a dramatic performance drop. + If you are just prototyping or testing your model with FP8, you can use the ``VLLM_SKIP_WARMUP=true`` environment variable to disable the warmup stage, which can take a long time. However, we do not recommend disabling this feature in production environments as it causes a significant performance drop. .. tip:: - When using FP8 models, you may experience timeouts caused by the long compilation time of FP8 operations. To mitigate this problem, you can use these two environment variables: + When using FP8 models, you may experience timeouts caused by the long compilation time of FP8 operations. To mitigate this problem, you can use the below environment variables: ``VLLM_ENGINE_ITERATION_TIMEOUT_S`` - to adjust the vLLM server timeout. You can set the value in seconds, e.g., 600 equals 10 minutes. ``VLLM_RPC_TIMEOUT`` - to adjust the RPC protocol timeout used by the OpenAI-compatible API. This value is in microseconds, e.g., 600000 equals 10 minutes. @@ -56,7 +56,7 @@ Specifying Device for the Model's Weights Uploading It is possible to load the unquantized weights on a different device before quantizing them, then moving them to the device on which the model will run. This reduces the device memory footprint of model weights, as only quantized weights are stored in device memory. -To set the load device, use the ``weights_load_device`` parameter for the ``LLM`` object, or ``--weights-load-device`` command line parameter in online mode. +To set the device to upload weights, use the ``weights_load_device`` parameter for the ``LLM`` object, or ``--weights-load-device`` command line parameter when running online inference: .. code-block:: python From 9f1dd46ecf978854cb49fd2f71eab2091160ee3a Mon Sep 17 00:00:00 2001 From: Nir David Date: Tue, 7 Jan 2025 16:45:53 +0200 Subject: [PATCH 5/6] fix escaping backslash --- docs/source/quantization/inc.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/quantization/inc.rst b/docs/source/quantization/inc.rst index 4d9020f3186c1..76d5c662409df 100644 --- a/docs/source/quantization/inc.rst +++ b/docs/source/quantization/inc.rst @@ -7,7 +7,7 @@ vLLM supports FP8 (8-bit floating point) weight and activation quantization usin Currently, quantization is supported only for Llama models. Intel Gaudi supports quantization of various modules and functions, including, but not limited to ``Linear``, ``KVCache``, ``Matmul`` and ``Softmax``. For more information, please refer to: -`Supported Modules\Supported Functions\Custom Patched Modules `_. +`Supported Modules\\Supported Functions\\Custom Patched Modules `_. .. note:: Measurement files are required to run quantized models with vLLM on Gaudi accelerators. The FP8 model calibration procedure is described in the `vllm-hpu-extention `_ package. From bff9efa739ed73c07d029e9486556aa47cfe0135 Mon Sep 17 00:00:00 2001 From: Nir David Date: Thu, 9 Jan 2025 13:26:39 +0200 Subject: [PATCH 6/6] fix cr comments --- docs/source/index.rst | 1 + docs/source/quantization/inc.rst | 2 +- vllm/platforms/hpu.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index ebf1361976c5e..7c545a85ac2be 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -121,6 +121,7 @@ Documentation quantization/auto_awq quantization/bnb quantization/gguf + quantization/inc quantization/int8 quantization/fp8 quantization/fp8_e5m2_kvcache diff --git a/docs/source/quantization/inc.rst b/docs/source/quantization/inc.rst index 76d5c662409df..ad7e21af54c40 100644 --- a/docs/source/quantization/inc.rst +++ b/docs/source/quantization/inc.rst @@ -4,7 +4,7 @@ FP8 INC ======= vLLM supports FP8 (8-bit floating point) weight and activation quantization using Intel® Neural Compressor (INC) on Intel® Gaudi® 2 and Intel® Gaudi® 3 AI accelerators. -Currently, quantization is supported only for Llama models. +Currently, quantization is validated only in Llama models. Intel Gaudi supports quantization of various modules and functions, including, but not limited to ``Linear``, ``KVCache``, ``Matmul`` and ``Softmax``. For more information, please refer to: `Supported Modules\\Supported Functions\\Custom Patched Modules `_. diff --git a/vllm/platforms/hpu.py b/vllm/platforms/hpu.py index 314cd98212e9c..d75f146ecf4b8 100644 --- a/vllm/platforms/hpu.py +++ b/vllm/platforms/hpu.py @@ -13,6 +13,7 @@ class HpuPlatform(Platform): device_name: str = "hpu" device_type: str = "hpu" dispatch_key: str = "HPU" + supported_quantization: list[str] = ["inc"] @classmethod def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend: