QATzip initial release of version 0.2.3

Signed-off-by: Yu Yao <[email protected]>
intel · Dec 11, 2017 · 02f0e20 · 02f0e20
commit 02f0e20
Show file tree

Hide file tree

Showing 31 changed files with 10,222 additions and 0 deletions.
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,31 @@
+
+    BSD LICENSE
+
+    Copyright(c) 2016-2017 Intel Corporation.
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions
+    are met:
+
+      * Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+      * Redistributions in binary form must reproduce the above copyright
+        notice, this list of conditions and the following disclaimer in
+        the documentation and/or other materials provided with the
+        distribution.
+      * Neither the name of Intel Corporation nor the names of its
+        contributors may be used to endorse or promote products derived
+        from this software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/Makefile.init b/Makefile.init
@@ -0,0 +1,86 @@
+################################################################
+#   BSD LICENSE
+#
+#   Copyright(c) 2007-2017 Intel Corporation. All rights reserved.
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Intel Corporation nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################
+
+FUNCTEST_SRC_D = $(top_builddir)/test
+QZIP_UTIL_D = $(top_builddir)/utils
+QATZIP_LIB_D = $(top_builddir)/src
+QATZIP_LIB_STATIC = libqatzip.a
+QATZIP_LIB_SHARED = libqatzip.so
+
+QAT_INCLUDE  ?= -I$(ICP_ROOT)/quickassist/include    \
+                -I$(ICP_ROOT)/quickassist/include/dc \
+                -I$(ICP_ROOT)/quickassist/lookaside/access_layer/include
+USDM_INCLUDE ?= -I$(ICP_ROOT)/quickassist/utilities/libusdm_drv
+
+EXTRA_CFLAGS = $(QAT_INCLUDE) $(USDM_INCLUDE)   \
+               -I$(top_builddir)/include
+
+LIBADD = -lqat_s -lusdm_drv_s -lz -lpthread -lnuma
+
+default: $(QATZIP_LIB_STATIC) $(QATZIP_LIB_SHARED) qzip
+all: $(QATZIP_LIB_STATIC) $(QATZIP_LIB_SHARED) qzip test
+
+install: qzip $(QATZIP_LIB_SHARED)
+	$(INSTALL) -D -m 750 $(QATZIP_LIB_D)/$(QATZIP_LIB_STATIC) $(staticlib_dir)
+	$(INSTALL) -D -m 750 $(QATZIP_LIB_D)/$(QATZIP_LIB_SHARED) $(sharedlib_dir)
+	$(INSTALL) -D -m 750 $(top_builddir)/include/qatzip.h /usr/include/
+	$(LN_S) -f $(sharedlib_dir)/$(QATZIP_LIB_SHARED) /lib64/$(QATZIP_LIB_SHARED)
+	$(INSTALL) -D -m 777 $(QZIP_UTIL_D)/qzip $(bindir)
+
+uninstall:
+	$(RM) $(staticlib_dir)/$(QATZIP_LIB_STATIC)
+	$(RM) $(sharedlib_dir)/$(QATZIP_LIB_SHARED)
+	$(RM) /usr/include/qatzip.h
+	$(RM) /lib64/$(QATZIP_LIB_SHARED)
+	$(RM) $(bindir)/qzip
+
+$(QATZIP_LIB_STATIC):
+	$(MAKE) -C $(QATZIP_LIB_D) $(QATZIP_LIB_STATIC)
+
+$(QATZIP_LIB_SHARED):
+	$(MAKE) -C $(QATZIP_LIB_D) $(QATZIP_LIB_SHARED)
+
+qzip: $(QATZIP_LIB_STATIC)
+	$(MAKE) -C $(QZIP_UTIL_D) qzip
+
+test: $(QATZIP_LIB_STATIC)
+	$(MAKE) -C $(FUNCTEST_SRC_D) all
+
+clean:
+	$(MAKE) -C $(QATZIP_LIB_D) clean
+	$(MAKE) -C $(FUNCTEST_SRC_D) clean
+	$(MAKE) -C $(QZIP_UTIL_D) clean
+
+.PHONY: install uninstall $(QATZIP_LIB_STATIC) $(QATZIP_LIB_SHARED) qzip test clean
+export
diff --git a/README.md b/README.md
@@ -0,0 +1,215 @@
+# Intel&reg; QuickAssist Technology (QAT) QATzip Library
+
+## Table of Contents
+
+- [Introduction](#introduction)
+- [Licensing](#licensing)
+- [Features](#features)
+- [Hardware Requirements](#hardware-requirements)
+- [Software Requirements](#software-requirements)
+- [Additional Information](#additional-information)
+- [Limitations](#limitations)
+- [Installation Instructions](#installation-instructions)
+    - [Build Intel&reg; QuickAssist Technology Driver](#build-intel-quickassist-technology-driver)
+    - [Install QATzip](#install-qatzip)
+    - [Test QATzip](#test-qatzip)
+- [QATzip API manual](#qatzip-api-manual)
+- [Intended Audience](#intended-audience)
+- [Legal](#legal)
+
+## Introduction
+
+QATZip is a user space library which builds on top of the Intel&reg; QuickAssist
+Technology user space library, to provide extended accelerated compression and
+decompression services by offloading the actual compression and decompression
+request(s) to the Intel&reg; Chipset Series. QATZip produces data using the standard
+gzip\* format (RFC1952) with extended headers. The data can be decompressed with a
+compliant gzip\* implementation. QATzip is design to take full advantage of the
+performance provided by Intel&reg; QuickAssist Technology.
+
+## Licensing
+
+The Licensing of the files within this project is split as follows:
+
+Intel&reg; Quickassist Technology (QAT) QATzip - BSD License. Please see the `LICENSE`
+file contained in the top level folder. Further details can be found in the file headers
+of the relevant files.
+
+Example Intel&reg; Quickassist Technology Driver Configuration Files contained within the
+folder hierarchy `config_file` - Dual BSD/GPLv2 License.
+Please see the file headers of the configuration files, and the full GPLv2 license
+contained in the file `LICENSE.GPL` within the `config_file` folder.
+
+## Features
+
+* Acceleration of compression and decompression utilizing Intel&reg; QuickAssist Technology,
+including a utility to compress and decompress files.
+* Dynamic memory allocation for zero copy, by exposing qzMalloc() and qzFree() allowing
+working buffers to be pinned, contiguous buffers that can be used for DMA operations to
+and from the hardware.
+* Instance over-subscription, allowing a number of threads in the same process to
+seamlessly share a smaller number of hardware instances.
+* Memory allocation backed by huge page to provide access to pinned, contiguous memory.
+This is useful if there is contention for traditional kernel memory.
+* Configurable accelerator device sharing among processes.
+* Optional software failover for both compression and decompression services. QATzip may
+switch to software if there is insufficient system resources including acceleration
+instances or memory. This feature allows for a common software stack between server
+platforms that have acceleration devices and non-accelerated platforms.
+
+## Hardware Requirements
+
+This QATzip library supports compression and decompression offload to the following
+acceleration devices:
+
+* Intel&reg; C62X Series Chipset
+* [Intel&reg; Communications Chipset 8925 to 8955 Series][1]
+
+[1]:https://www.intel.com/content/www/us/en/ethernet-products/gigabit-server-adapters/quickassist-adapter-8950-brief.html
+
+## Software Requirements
+
+This release was validated on the following:
+
+* QATzip has been tested with the latest Intel&reg; QuickAssist Acceleration Driver.
+Please download the QAT driver from the link https://01.org/intel-quickassist-technology
+* QATzip has been tested by Intel&reg; on CentOS 7.2.1511 with kernel 3.10.0-327.el7.x86\_64
+* Zlib\* library of version 1.2.7 or higher
+* Suggest GCC\* of version 4.8.5 or higher
+
+## Additional Information
+
+The compression level in QATzip could be mapped to standard zlib\* as below:
+* QATzip level 1 - 4, similar to zlib\* level 1 - 4.
+* QATzip level 5 - 8, we map them to QATzip level 4.
+* QATzip level 9, we will use software zlib\* to compress as level 9.
+
+## Limitations
+
+* Currently QATzip only supports static Huffman tree encoding. Dynamic encoding is disabled.
+
+* The partitioned internal chunk size of 16 KB is disabled, this chunk is used for QAT hardware DMA.
+
+* For some certain standard zlib\* software in level 9 compressed input data, QATzip decompressor might
+generate incorrect output length.
+
+## Installation Instructions
+
+### Build Intel&reg; QuickAssist Technology Driver
+
+Please follow the instructions contained in:
+
+**For Intel&reg; C62X Series Chipset:**
+Intel&reg; QuickAssist Technology Software for Linux\* - Getting Started Guide - HW version 1.7 (336212)
+
+**For Intel&reg; Communications Chipset 89XX Series:**
+Intel&reg; Communications Chipset 89xx Series Software for Linux\* - Getting
+Started Guide (330750)
+
+These instructions can be found on the 01.org website in the following section:
+
+[Intel&reg; Quickassist Technology][7]
+
+[7]:https://01.org/packet-processing/intel%C2%AE-quickassist-technology-drivers-and-patches
+
+### Install QATzip
+
+**Set below environment variable**
+
+`ICP_ROOT`: the root directory of your QAT driver source tree
+
+`QATZIP_ROOT`: the root directory of your QATzip source tree
+
+**Enable huge page feature configure huge page**
+
+```bash
+    echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+    rmmod usdm_drv
+    insmod $ICP_ROOT/build/usdm_drv.ko max_huge_pages=1024 max_huge_pages_per_process=16
+```
+
+**Compile and Install QATzip**
+
+```bash
+    cd $QATZIP_ROOT
+    ./configure
+    make clean
+    make all install
+```
+
+For more configure options, please run "./configure -h" for help
+
+**Update the configuration files**
+
+copy the configure file(s) from directory of `$QATZIP_ROOT/config_file/$YOUR_PLATFORM/$CONFIG_TYPE/*.conf`
+to directory of `/etc`
+
+`YOUR_PLATFORM`: the QAT hardware platform, c6xx for Intel&reg; C62X Series
+Chipset, dh895xcc for Intel&reg; Communications Chipset 8925 to 8955 Series
+
+`CONFIG_TYPE`: tuned configure file(s) for different usage,
+`multiple_process_opt` for multiple process optimization,
+`multiple_thread_opt` for multiple thread optimization
+
+**Restart the QAT driver**
+
+```bash
+    service qat_service restart
+```
+
+With current configuration, each PCI-e device in C6XX platform could support
+up to at most 32 processes
+
+### Test QATzip
+
+Run the following command to check if the QATzip is setup correctly for
+compressing or decompressing files:
+
+```bash
+    qzip -k $your_input_file (add -h for help)
+
+    This compression and decompression util could support below options:
+    "  -A, --algorithm   set algorithm type, currently only support deflate",
+    "  -d, --decompress  decompress",
+    "  -h, --help        give this help",
+    "  -H, --huffmanhdr  set huffman header type",
+    "  -k, --keep        keep (don't delete) input files",
+    "  -V, --version     display version number",
+    "  -L, --level       set compression level",
+    "  -C, --chunksz     set chunk size",
+```
+
+## QATzip API Manual
+
+Please refer to file `QATzip-man.pdf` under the `docs` folder
+
+## Intended Audience
+
+The target audience is software developers, test and validation engineers,
+system integrators, end users and consumers for QATzip integrated Intel&reg;
+Quick Assist Technology
+
+## Legal
+
+Intel&reg; disclaims all express and implied warranties, including without
+limitation, the implied warranties of merchantability, fitness for a
+particular purpose, and non-infringement, as well as any warranty arising
+from course of performance, course of dealing, or usage in trade.
+
+This document contains information on products, services and/or processes in
+development.  All information provided here is subject to change without
+notice. Contact your Intel&reg; representative to obtain the latest forecast
+, schedule, specifications and roadmaps.
+
+The products and services described may contain defects or errors known as
+errata which may cause deviations from published specifications. Current
+characterized errata are available on request.
+
+Copies of documents which have an order number and are referenced in this
+document may be obtained by calling 1-800-548-4725 or by visiting
+www.intel.com/design/literature.htm.
+
+Intel, the Intel logo are trademarks of Intel Corporation in the U.S.
+and/or other countries.
+
+\*Other names and brands may be claimed as the property of others