Merge pull request #16 from unknownbrackets/dax

Allow compression to a DAX file

7zip/7zip.vcxproj

@@ -23,9 +23,14 @@
     <ClInclude Include="CPP\7zip\Archive\DeflateProps.h" />
     <ClInclude Include="CPP\7zip\Common\CWrappers.h" />
     <ClInclude Include="CPP\7zip\Common\FileStreams.h" />
+    <ClInclude Include="CPP\7zip\Common\InBuffer.h" />
     <ClInclude Include="CPP\7zip\Common\OutBuffer.h" />
     <ClInclude Include="CPP\7zip\Common\StreamUtils.h" />
+    <ClInclude Include="CPP\7zip\Compress\DeflateDecoder.h" />
     <ClInclude Include="CPP\7zip\Compress\DeflateEncoder.h" />
+    <ClInclude Include="CPP\7zip\Compress\LzOutWindow.h" />
+    <ClInclude Include="CPP\7zip\Compress\ZlibDecoder.h" />
+    <ClInclude Include="CPP\7zip\Compress\ZlibEncoder.h" />
     <ClInclude Include="CPP\Common\MyString.h" />
     <ClInclude Include="CPP\Common\StringConvert.h" />
     <ClInclude Include="CPP\Common\StringToInt.h" />
@@ -42,9 +47,15 @@
     <ClCompile Include="CPP\7zip\Archive\DeflateProps.cpp" />
     <ClCompile Include="CPP\7zip\Common\CWrappers.cpp" />
     <ClCompile Include="CPP\7zip\Common\FileStreams.cpp" />
+    <ClCompile Include="CPP\7zip\Common\InBuffer.cpp" />
     <ClCompile Include="CPP\7zip\Common\OutBuffer.cpp" />
     <ClCompile Include="CPP\7zip\Common\StreamUtils.cpp" />
+    <ClCompile Include="CPP\7zip\Compress\BitlDecoder.cpp" />
+    <ClCompile Include="CPP\7zip\Compress\DeflateDecoder.cpp" />
     <ClCompile Include="CPP\7zip\Compress\DeflateEncoder.cpp" />
+    <ClCompile Include="CPP\7zip\Compress\LzOutWindow.cpp" />
+    <ClCompile Include="CPP\7zip\Compress\ZlibDecoder.cpp" />
+    <ClCompile Include="CPP\7zip\Compress\ZlibEncoder.cpp" />
     <ClCompile Include="CPP\Common\MyString.cpp" />
     <ClCompile Include="CPP\Common\StringConvert.cpp" />
     <ClCompile Include="CPP\Common\StringToInt.cpp" />
@@ -56,6 +67,9 @@
     <ClCompile Include="C\Sort.c" />
     <ClCompile Include="deflate7z.cpp" />
   </ItemGroup>
+  <ItemGroup>
+    <None Include="Makefile" />
+  </ItemGroup>
   <PropertyGroup Label="Globals">
     <ProjectGuid>{6343B36E-030F-4A66-A588-758B2E2B5BEE}</ProjectGuid>
     <RootNamespace>7zip</RootNamespace>

7zip/7zip.vcxproj.filters

@@ -55,6 +55,21 @@
     <ClInclude Include="CPP\7zip\Common\StreamUtils.h">
       <Filter>7zip</Filter>
     </ClInclude>
+    <ClInclude Include="CPP\7zip\Compress\ZlibEncoder.h">
+      <Filter>7zip</Filter>
+    </ClInclude>
+    <ClInclude Include="CPP\7zip\Compress\ZlibDecoder.h">
+      <Filter>7zip</Filter>
+    </ClInclude>
+    <ClInclude Include="CPP\7zip\Compress\DeflateDecoder.h">
+      <Filter>7zip</Filter>
+    </ClInclude>
+    <ClInclude Include="CPP\7zip\Compress\LzOutWindow.h">
+      <Filter>7zip</Filter>
+    </ClInclude>
+    <ClInclude Include="CPP\7zip\Common\InBuffer.h">
+      <Filter>7zip</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="CPP\7zip\Archive\DeflateProps.cpp">
@@ -106,5 +121,26 @@
     <ClCompile Include="CPP\7zip\Common\StreamUtils.cpp">
       <Filter>7zip</Filter>
     </ClCompile>
+    <ClCompile Include="CPP\7zip\Compress\ZlibEncoder.cpp">
+      <Filter>7zip</Filter>
+    </ClCompile>
+    <ClCompile Include="CPP\7zip\Compress\ZlibDecoder.cpp">
+      <Filter>7zip</Filter>
+    </ClCompile>
+    <ClCompile Include="CPP\7zip\Compress\DeflateDecoder.cpp">
+      <Filter>7zip</Filter>
+    </ClCompile>
+    <ClCompile Include="CPP\7zip\Compress\LzOutWindow.cpp">
+      <Filter>7zip</Filter>
+    </ClCompile>
+    <ClCompile Include="CPP\7zip\Common\InBuffer.cpp">
+      <Filter>7zip</Filter>
+    </ClCompile>
+    <ClCompile Include="CPP\7zip\Compress\BitlDecoder.cpp">
+      <Filter>7zip</Filter>
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="Makefile" />
   </ItemGroup>
 </Project>

7zip/Makefile

@@ -1,5 +1,6 @@
 CC ?= gcc
 CXX ?= g++
+AR ?= ar
 
 CFLAGS += -W -Wall -Wextra -O2
 CXXFLAGS += -W -Wall -Wextra -std=c++11 -O2 -ICPP
@@ -8,9 +9,15 @@ CXXFLAGS += -W -Wall -Wextra -std=c++11 -O2 -ICPP
                CPP/7zip/Archive/DeflateProps.cpp \
                CPP/7zip/Common/CWrappers.cpp \
                CPP/7zip/Common/FileStreams.cpp \
+               CPP/7zip/Common/InBuffer.cpp \
                CPP/7zip/Common/OutBuffer.cpp \
                CPP/7zip/Common/StreamUtils.cpp \
+               CPP/7zip/Compress/BitlDecoder.cpp \
+               CPP/7zip/Compress/DeflateDecoder.cpp \
                CPP/7zip/Compress/DeflateEncoder.cpp \
+               CPP/7zip/Compress/LzOutWindow.cpp \
+               CPP/7zip/Compress/ZlibDecoder.cpp \
+               CPP/7zip/Compress/ZlibEncoder.cpp \
                CPP/Common/MyString.cpp \
                CPP/Common/StringConvert.cpp \
                CPP/Common/StringToInt.cpp \
@@ -32,7 +39,7 @@ CXXFLAGS += -W -Wall -Wextra -std=c++11 -O2 -ICPP
 	$(CC) -c $(CFLAGS) -o $@ $<
 
 7zip.a: $(7ZIP_CXX_OBJ) $(7ZIP_C_OBJ)
-	ar rcs $@ $^
+	$(AR) rcs $@ $^
 
 clean:
 	rm -f $(7ZIP_CXX_OBJ) $(7ZIP_C_OBJ) 7zip.a

7zip/deflate7z.cpp

@@ -14,6 +14,7 @@
 #include "CPP/7zip/IStream.h"
 #include "CPP/7zip/Archive/DeflateProps.h"
 #include "CPP/7zip/Compress/DeflateEncoder.h"
+#include "CPP/7zip/Compress/ZlibEncoder.h"
 
 #ifdef _WIN32
 #include <propvarutil.h>
@@ -118,7 +119,7 @@ HRESULT COutBlockStream::Write(const void *data, UInt32 size, UInt32 *processedS
 struct Context {
 	CInBlockStream *in;
 	COutBlockStream *out;
-	NCompress::NDeflate::NEncoder::CCOMCoder *coder;
+	ICompressCoder *coder;
 };
 
 void SetDefaults(Options *opts) {
@@ -127,6 +128,7 @@ void SetDefaults(Options *opts) {
 	opts->fastbytes = 0xFFFFFFFF;
 	opts->algo = 0xFFFFFFFF;
 	opts->matchcycles = 0xFFFFFFFF;
+	opts->useZlib = false;
 }
 
 static void SetupProperties(const Options *opts, NCompress::NDeflate::NEncoder::CCOMCoder *coder) {
@@ -149,8 +151,16 @@ void Alloc(Context **ctx, const Options *opts) {
 	c->in->AddRef();
 	c->out->AddRef();
 
-	c->coder = new NCompress::NDeflate::NEncoder::CCOMCoder();
-	SetupProperties(opts, c->coder);
+	if (opts->useZlib) {
+		NCompress::NZlib::CEncoder *coder = new NCompress::NZlib::CEncoder();
+		coder->Create();
+		SetupProperties(opts, coder->DeflateEncoderSpec);
+		c->coder = coder;
+	} else {
+		NCompress::NDeflate::NEncoder::CCOMCoder *coder = new NCompress::NDeflate::NEncoder::CCOMCoder();
+		SetupProperties(opts, coder);
+		c->coder = coder;
+	}
 }
 
 void Release(Context **ctx) {

7zip/deflate7z.h

@@ -10,6 +10,7 @@ namespace Deflate7z {
 		uint32_t fastbytes;
 		uint32_t algo;
 		uint32_t matchcycles;
+		bool useZlib;
 	};
 
 	struct Context;

README.md

@@ -48,6 +48,8 @@ Larger block sizes than the default will help compression, in the range of 2-3%.
 files may not be compatible with some software.  For example, [PPSSPP][] versions released
 after 2014-10-26 will support larger block sizes.
 
+Avoid DAX where CSOs using larger block sizes are supported, since DAX is less efficient.
+
 LZ4 support is mostly for experimentation.
 
 
@@ -79,7 +81,7 @@ Multiple files may be specified.  Inputs can be iso or cso files.
    --decompress    Write out to raw ISO, decompressing as needed
    --block=N       Specify a block size (default depends on iso size)
                    Many readers only support the 2048 size
-   --format=VER    Specify cso version (options: cso1, cso2, zso)
+   --format=VER    Specify cso version (options: cso1, cso2, zso, dax)
                    These are experimental, default is cso1
    --use-zlib      Enable trials with zlib for deflate compression
    --use-zopfli    Enable trials with Zopfli for deflate compression

cli/cli.cpp

@@ -27,7 +27,7 @@ void show_help(const char *arg0) {
 	fprintf(stderr, "   --decompress    Write out to raw ISO, decompressing as needed\n");
 	fprintf(stderr, "   --block=N       Specify a block size (default depends on iso size)\n");
 	fprintf(stderr, "                   Many readers only support the 2048 size\n");
-	fprintf(stderr, "   --format=VER    Specify cso version (options: cso1, cso2, zso)\n");
+	fprintf(stderr, "   --format=VER    Specify cso version (options: cso1, cso2, zso, dax)\n");
 	fprintf(stderr, "                   These are experimental, default is cso1\n");
 	// TODO: Bring this back once it's functional.
 	//fprintf(stderr, "   --smallest      Force compression of all sectors for smallest result\n");
@@ -173,6 +173,8 @@ int parse_args(Arguments &args, int argc, char *argv[]) {
 					args.flags_fmt = maxcso::TASKFLAG_FMT_CSO_2;
 				} else if (strcmp(val, "zso") == 0) {
 					args.flags_fmt = maxcso::TASKFLAG_FMT_ZSO;
+				} else if (strcmp(val, "dax") == 0) {
+					args.flags_fmt = maxcso::TASKFLAG_FMT_DAX;
 				} else {
 					show_help(argv[0]);
 					fprintf(stderr, "\nERROR: Unknown format %s, expecting cso1, cso2, or zso.\n", val);
@@ -231,11 +233,18 @@ int validate_args(const char *arg0, Arguments &args) {
 
 	if (args.crc) {
 		if (args.outputs.size()) {
-		show_help(arg0);
-		fprintf(stderr, "\nERROR: Output files not used with --crc.\n");
-		return 1;
+			show_help(arg0);
+			fprintf(stderr, "\nERROR: Output files not used with --crc.\n");
+			return 1;
 		}
 	} else {
+		std::string outputExt = ".cso";
+		if (args.flags_fmt & maxcso::TASKFLAG_FMT_DAX) {
+			outputExt = ".dax";
+		} else if (args.flags_fmt & maxcso::TASKFLAG_FMT_ZSO) {
+			outputExt = ".zso";
+		}
+
 		// Automatically write to .cso files if not specified.
 		for (size_t i = args.outputs.size(); i < args.inputs.size(); ++i) {
 			if (args.inputs[i].size() <= 4) {
@@ -244,10 +253,13 @@ int validate_args(const char *arg0, Arguments &args) {
 
 			std::string ext = args.inputs[i].substr(args.inputs[i].size() - 4);
 			std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
+			const bool inputRawExt = ext == ".iso";
+			const bool inputCompressedExt = ext == ".cso" || ext == ".zso" || ext == ".dax";
 
-			if (!args.decompress && ext == ".iso") {
-				args.outputs.push_back(args.inputs[i].substr(0, args.inputs[i].size() - 4) + ".cso");
-			} else if (args.decompress && (ext == ".cso" || ext == ".zso")) {
+			// Automatically switch extensions for convenience.
+			if (!args.decompress && (inputRawExt || inputCompressedExt) && ext != outputExt) {
+				args.outputs.push_back(args.inputs[i].substr(0, args.inputs[i].size() - 4) + outputExt);
+			} else if (args.decompress && inputCompressedExt) {
 				args.outputs.push_back(args.inputs[i].substr(0, args.inputs[i].size() - 4) + ".iso");
 			}
 		}
@@ -271,7 +283,7 @@ int validate_args(const char *arg0, Arguments &args) {
 	} else if (args.flags_fmt & maxcso::TASKFLAG_FMT_ZSO) {
 		args.flags_final = maxcso::TASKFLAG_NO_ZLIB | maxcso::TASKFLAG_NO_7ZIP | maxcso::TASKFLAG_NO_ZOPFLI | maxcso::TASKFLAG_NO_LZ4_HC_BRUTE;
 	} else {
-		// CSO v1, just disable lz4.
+		// CSO v1 or DAX, just disable lz4.
 		args.flags_final = maxcso::TASKFLAG_NO_ZOPFLI | maxcso::TASKFLAG_NO_LZ4;
 	}
 
@@ -297,6 +309,23 @@ int validate_args(const char *arg0, Arguments &args) {
 	}
 	args.flags_final |= args.flags_fmt;
 
+	if (args.flags_fmt & maxcso::TASKFLAG_FMT_DAX) {
+		// DAX has a fixed block size.
+		if (args.block_size != maxcso::DEFAULT_BLOCK_SIZE) {
+			show_help(arg0);
+			fprintf(stderr, "\nERROR: Block size must be default for DAX.\n");
+			return 1;
+		}
+
+		// Currently, compression will fail if no DEFLATE format is enabled for DAX.
+		uint32_t deflateFlags = maxcso::TASKFLAG_NO_ZLIB | maxcso::TASKFLAG_NO_ZLIB_DEFAULT | maxcso::TASKFLAG_NO_ZLIB_BRUTE | maxcso::TASKFLAG_NO_ZOPFLI | maxcso::TASKFLAG_NO_7ZIP;
+		if ((args.flags_final & deflateFlags) == deflateFlags) {
+			show_help(arg0);
+			fprintf(stderr, "\nERROR: DAX must use some kind of DEFLATE.\n");
+			return 1;
+		}
+	}
+
 	return 0;
 }
 

src/compress.cpp

@@ -3,6 +3,7 @@
 #include "compress.h"
 #include "uv_helper.h"
 #include "cso.h"
+#include "dax.h"
 #include "input.h"
 #include "output.h"
 #include "buffer_pool.h"
@@ -62,9 +63,13 @@ class CompressionTask {
 
 void CompressionTask::Enqueue() {
 	if (task_.block_size == DEFAULT_BLOCK_SIZE) {
-		// Start with a small block size.
-		// We'll re-evaluate later.
-		blockSize_ = SMALL_BLOCK_SIZE;
+		if (task_.flags & TASKFLAG_FMT_DAX) {
+			blockSize_ = DAX_FRAME_SIZE;
+		} else {
+			// Start with a small block size.
+			// We'll re-evaluate later.
+			blockSize_ = SMALL_BLOCK_SIZE;
+		}
 	} else {
 		if (task_.block_size > MAX_BLOCK_SIZE) {
 			Notify(TASK_INVALID_OPTION, "Block size too large");
@@ -148,6 +153,8 @@ void CompressionTask::BeginProcessing() {
 			fmt = CSO_FMT_CSO2;
 		} else if (task_.flags & TASKFLAG_FMT_ZSO) {
 			fmt = CSO_FMT_ZSO;
+		} else if (task_.flags & TASKFLAG_FMT_DAX) {
+			fmt = CSO_FMT_DAX;
 		}
 
 		// Now that we know the file size, check if we should resize the blockSize_.

src/compress.h

@@ -47,6 +47,7 @@ enum TaskFlags {
 	TASKFLAG_NO_ALL = TASKFLAG_NO_ZLIB | TASKFLAG_NO_ZOPFLI | TASKFLAG_NO_7ZIP | TASKFLAG_NO_LZ4,
 
 	TASKFLAG_DECOMPRESS = 0x400,
+	TASKFLAG_FMT_DAX = 0x800,
 };
 
 typedef std::function<void (const Task *, TaskStatus status, int64_t pos, int64_t total, int64_t written)> ProgressCallback;

src/cso.h

@@ -17,6 +17,7 @@ enum CSOFormat {
 	CSO_FMT_CSO1,
 	CSO_FMT_CSO2,
 	CSO_FMT_ZSO,
+	CSO_FMT_DAX,
 };
 
 #ifdef _MSC_VER

src/input.cpp

@@ -108,6 +108,7 @@ void Input::DetectFormat() {
 				finish_(false, "DAX uncompressed size not aligned to sector size");
 			} else {
 				size_ = header->uncompressed_size;
+				csoBlockSize_ = DAX_FRAME_SIZE;
 
 				const uint32_t frames = static_cast<uint32_t>((size_ + DAX_FRAME_SIZE - 1) >> DAX_FRAME_SHIFT);
 				daxIndex_ = new uint32_t[frames];
@@ -323,22 +324,39 @@ void Input::EnqueueDecompressSector(uint8_t *src, uint32_t len, uint32_t offset,
 			}
 		}
 	}, [this, actualBuf, offset](uv_work_t *req, int status) {
-		if (offset != 0) {
-			memmove(actualBuf, actualBuf + offset, SECTOR_SIZE);
-		}
-
 		if (!decompressError_.empty()) {
 			finish_(false, decompressError_.c_str());
 			pool.Release(actualBuf);
+			return;
 		} else if (status == -1) {
 			finish_(false, "Decompression work failed");
 			pool.Release(actualBuf);
-		} else {
-			callback_(pos_, actualBuf);
+			return;
+		}
 
+		// If the input has a larger block size than SECTOR_SIZE, queue each up here.
+		// Prevents double decompression of input blocks.
+		for (uint32_t suboffset = offset; suboffset < csoBlockSize_; suboffset += SECTOR_SIZE) {
+			const bool lastSubBlock = suboffset + SECTOR_SIZE >= csoBlockSize_;
+			uint8_t *const subBuf = lastSubBlock ? actualBuf : pool.Alloc();
+			if (lastSubBlock) {
+				memmove(actualBuf, actualBuf + suboffset, SECTOR_SIZE);
+			} else {
+				memmove(subBuf, actualBuf + suboffset, SECTOR_SIZE);
+			}
+
+			callback_(pos_, subBuf);
 			pos_ += SECTOR_SIZE;
-			ReadSector();
+
+			// We may have filled early.  In this case, we have to bail and re-decompress part of this block.
+			// Luckily, this isn't common.
+			if (paused_ && !lastSubBlock) {
+				pool.Release(actualBuf);
+				break;
+			}
 		}
+
+		ReadSector();
 	});
 }