From 0552eee65d8e7e986cab08470babbff71aa4fffa Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sat, 1 May 2021 13:37:47 -0700 Subject: [PATCH] Update 7-zip to 19.00. --- 7zip/Asm/x86/7zAsm.asm | 44 +- 7zip/Asm/x86/LzmaDecOpt.asm | 1258 +++++++++ 7zip/C/7zArcIn.c | 8 +- 7zip/C/7zDec.c | 12 +- 7zip/C/7zTypes.h | 5 +- 7zip/C/7zVersion.h | 8 +- 7zip/C/Alloc.c | 354 ++- 7zip/C/Alloc.h | 16 +- 7zip/C/Bcj2.c | 10 +- 7zip/C/Bcj2Enc.c | 9 +- 7zip/C/BwtSort.c | 4 +- 7zip/C/CpuArch.c | 28 +- 7zip/C/CpuArch.h | 11 +- 7zip/C/DllSecur.c | 25 +- 7zip/C/DllSecur.h | 3 +- 7zip/C/LzFind.c | 164 +- 7zip/C/LzFindMt.c | 117 +- 7zip/C/LzFindMt.h | 14 +- 7zip/C/Lzma2Dec.c | 210 +- 7zip/C/Lzma2Dec.h | 60 +- 7zip/C/Lzma2DecMt.c | 1082 +++++++ 7zip/C/Lzma2DecMt.h | 79 + 7zip/C/Lzma2Enc.c | 29 +- 7zip/C/Lzma86Enc.c | 8 +- 7zip/C/LzmaDec.c | 409 +-- 7zip/C/LzmaDec.h | 41 +- 7zip/C/LzmaEnc.c | 2508 ++++++++++------- 7zip/C/MtCoder.c | 95 +- 7zip/C/MtCoder.h | 32 +- 7zip/C/MtDec.c | 1138 ++++++++ 7zip/C/MtDec.h | 201 ++ 7zip/C/Ppmd7.c | 6 +- 7zip/C/Ppmd7.h | 6 +- 7zip/C/Ppmd7Dec.c | 4 +- 7zip/C/Ppmd8.c | 6 +- 7zip/C/Ppmd8.h | 6 +- 7zip/C/Ppmd8Dec.c | 4 +- 7zip/C/Util/7z/7zMain.c | 39 +- 7zip/C/Util/7zipInstall/7zipInstall.c | 76 +- 7zip/C/Util/7zipInstall/makefile | 3 +- 7zip/C/Util/7zipUninstall/7zipUninstall.c | 44 +- 7zip/C/Util/7zipUninstall/makefile | 1 + 7zip/C/Util/Lzma/LzmaUtil.c | 4 +- 7zip/C/Util/SfxSetup/SfxSetup.c | 16 +- 7zip/C/Util/SfxSetup/makefile | 1 + 7zip/C/Util/SfxSetup/makefile_con | 1 + 7zip/C/Xz.h | 196 +- 7zip/C/XzDec.c | 2113 ++++++++++++-- 7zip/C/XzEnc.c | 29 +- 7zip/C/XzIn.c | 8 +- 7zip/CPP/7zip/Aes.mak | 2 +- 7zip/CPP/7zip/Archive/7z/7zCompressionMode.h | 2 + 7zip/CPP/7zip/Archive/7z/7zDecode.cpp | 48 +- 7zip/CPP/7zip/Archive/7z/7zDecode.h | 4 +- 7zip/CPP/7zip/Archive/7z/7zEncode.cpp | 11 +- 7zip/CPP/7zip/Archive/7z/7zExtract.cpp | 10 +- 7zip/CPP/7zip/Archive/7z/7zHandler.cpp | 24 +- 7zip/CPP/7zip/Archive/7z/7zHandler.h | 39 +- 7zip/CPP/7zip/Archive/7z/7zHandlerOut.cpp | 31 +- 7zip/CPP/7zip/Archive/7z/7zIn.cpp | 21 +- 7zip/CPP/7zip/Archive/7z/7zIn.h | 10 + 7zip/CPP/7zip/Archive/7z/7zUpdate.cpp | 30 +- 7zip/CPP/7zip/Archive/ArHandler.cpp | 8 +- 7zip/CPP/7zip/Archive/Bz2Handler.cpp | 8 +- 7zip/CPP/7zip/Archive/Cab/CabHandler.cpp | 53 +- 7zip/CPP/7zip/Archive/Common/CoderMixer2.cpp | 1 + 7zip/CPP/7zip/Archive/Common/HandlerOut.cpp | 148 +- 7zip/CPP/7zip/Archive/Common/HandlerOut.h | 65 +- 7zip/CPP/7zip/Archive/DmgHandler.cpp | 84 +- 7zip/CPP/7zip/Archive/ElfHandler.cpp | 1 + 7zip/CPP/7zip/Archive/ExtHandler.cpp | 13 + 7zip/CPP/7zip/Archive/FatHandler.cpp | 11 +- 7zip/CPP/7zip/Archive/HfsHandler.cpp | 118 +- 7zip/CPP/7zip/Archive/IArchive.h | 10 + 7zip/CPP/7zip/Archive/LzmaHandler.cpp | 52 +- 7zip/CPP/7zip/Archive/Nsis/NsisIn.cpp | 13 +- 7zip/CPP/7zip/Archive/NtfsHandler.cpp | 23 +- 7zip/CPP/7zip/Archive/PeHandler.cpp | 38 +- 7zip/CPP/7zip/Archive/Rar/Rar5Handler.cpp | 205 +- 7zip/CPP/7zip/Archive/Rar/Rar5Handler.h | 2 +- 7zip/CPP/7zip/Archive/Rar/RarHandler.cpp | 30 +- 7zip/CPP/7zip/Archive/Rar/RarHandler.h | 2 +- 7zip/CPP/7zip/Archive/SquashfsHandler.cpp | 57 +- 7zip/CPP/7zip/Archive/SwfHandler.cpp | 2 +- 7zip/CPP/7zip/Archive/Tar/TarIn.cpp | 14 +- 7zip/CPP/7zip/Archive/UefiHandler.cpp | 14 +- 7zip/CPP/7zip/Archive/Wim/WimHandlerOut.cpp | 2 +- 7zip/CPP/7zip/Archive/Wim/WimIn.cpp | 18 +- 7zip/CPP/7zip/Archive/Wim/WimIn.h | 7 +- 7zip/CPP/7zip/Archive/XzHandler.cpp | 247 +- 7zip/CPP/7zip/Archive/Zip/ZipAddCommon.cpp | 2 +- 7zip/CPP/7zip/Archive/Zip/ZipHandler.cpp | 161 +- 7zip/CPP/7zip/Archive/Zip/ZipHandler.h | 7 + 7zip/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp | 3 +- 7zip/CPP/7zip/Archive/Zip/ZipIn.cpp | 50 +- 7zip/CPP/7zip/Archive/Zip/ZipIn.h | 8 + 7zip/CPP/7zip/Archive/Zip/ZipItem.h | 14 +- 7zip/CPP/7zip/Archive/Zip/ZipUpdate.cpp | 5 +- 7zip/CPP/7zip/Bundles/Alone/Alone.dsp | 81 +- 7zip/CPP/7zip/Bundles/Alone/makefile | 12 +- 7zip/CPP/7zip/Bundles/Alone/resource.rc | 4 + 7zip/CPP/7zip/Bundles/Alone7z/Alone.dsp | 81 +- 7zip/CPP/7zip/Bundles/Alone7z/makefile | 12 +- 7zip/CPP/7zip/Bundles/Alone7z/resource.rc | 4 + 7zip/CPP/7zip/Bundles/Fm/FM.dsp | 43 +- 7zip/CPP/7zip/Bundles/Fm/makefile | 11 - 7zip/CPP/7zip/Bundles/Format7z/makefile | 3 + .../CPP/7zip/Bundles/Format7zExtract/makefile | 4 + .../7zip/Bundles/Format7zExtractR/makefile | 3 + 7zip/CPP/7zip/Bundles/Format7zF/Arc.mak | 3 + 7zip/CPP/7zip/Bundles/Format7zF/Format7z.dsp | 40 + 7zip/CPP/7zip/Bundles/Format7zR/makefile | 3 + 7zip/CPP/7zip/Bundles/LzmaCon/makefile | 2 +- 7zip/CPP/7zip/Bundles/SFXCon/SFXCon.dsp | 34 + 7zip/CPP/7zip/Bundles/SFXCon/makefile | 7 +- 7zip/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp | 34 + 7zip/CPP/7zip/Bundles/SFXSetup/makefile | 7 + 7zip/CPP/7zip/Bundles/SFXWin/SFXWin.dsp | 26 + 7zip/CPP/7zip/Bundles/SFXWin/makefile | 6 + 7zip/CPP/7zip/Common/CreateCoder.cpp | 113 +- 7zip/CPP/7zip/Common/CreateCoder.h | 26 +- 7zip/CPP/7zip/Common/FilterCoder.cpp | 19 +- 7zip/CPP/7zip/Common/FilterCoder.h | 34 +- 7zip/CPP/7zip/Common/MethodProps.cpp | 13 +- 7zip/CPP/7zip/Compress/DeflateDecoder.cpp | 8 + 7zip/CPP/7zip/Compress/DeflateEncoder.cpp | 4 + 7zip/CPP/7zip/Compress/Lzma2Decoder.cpp | 371 ++- 7zip/CPP/7zip/Compress/Lzma2Decoder.h | 52 +- 7zip/CPP/7zip/Compress/Lzma2Encoder.cpp | 2 +- 7zip/CPP/7zip/Compress/LzmaDecoder.cpp | 11 +- 7zip/CPP/7zip/Compress/LzmaDecoder.h | 6 +- 7zip/CPP/7zip/Compress/LzmaEncoder.cpp | 6 +- 7zip/CPP/7zip/Compress/LzmsDecoder.cpp | 4 +- 7zip/CPP/7zip/Compress/Rar1Decoder.cpp | 408 +-- 7zip/CPP/7zip/Compress/Rar1Decoder.h | 50 +- 7zip/CPP/7zip/Compress/Rar2Decoder.cpp | 31 +- 7zip/CPP/7zip/Compress/Rar2Decoder.h | 11 +- 7zip/CPP/7zip/Compress/Rar3Decoder.cpp | 61 +- 7zip/CPP/7zip/Compress/Rar3Decoder.h | 6 +- 7zip/CPP/7zip/Compress/Rar5Decoder.cpp | 89 +- 7zip/CPP/7zip/Compress/Rar5Decoder.h | 14 +- 7zip/CPP/7zip/Compress/XpressDecoder.cpp | 19 +- 7zip/CPP/7zip/Compress/XzDecoder.cpp | 289 +- 7zip/CPP/7zip/Compress/XzDecoder.h | 97 +- 7zip/CPP/7zip/Crc.mak | 2 +- 7zip/CPP/7zip/Crc64.mak | 2 +- 7zip/CPP/7zip/Crypto/7zAes.cpp | 4 +- 7zip/CPP/7zip/Crypto/RandGen.cpp | 135 +- 7zip/CPP/7zip/Crypto/RandGen.h | 19 + 7zip/CPP/7zip/Crypto/Rar5Aes.cpp | 7 +- 7zip/CPP/7zip/Crypto/WzAes.cpp | 2 +- 7zip/CPP/7zip/Crypto/ZipCrypto.cpp | 2 +- 7zip/CPP/7zip/Crypto/ZipStrong.cpp | 9 +- 7zip/CPP/7zip/Crypto/ZipStrong.h | 5 +- 7zip/CPP/7zip/GuiCommon.rc | 3 + 7zip/CPP/7zip/Guid.txt | 2 + 7zip/CPP/7zip/ICoder.h | 10 +- 7zip/CPP/7zip/LzmaDec.mak | 5 + 7zip/CPP/7zip/UI/Agent/Agent.h | 4 +- .../CPP/7zip/UI/Common/ArchiveCommandLine.cpp | 216 +- 7zip/CPP/7zip/UI/Common/ArchiveCommandLine.h | 20 +- .../7zip/UI/Common/ArchiveExtractCallback.cpp | 26 +- 7zip/CPP/7zip/UI/Common/ArchiveName.cpp | 83 +- 7zip/CPP/7zip/UI/Common/ArchiveName.h | 5 +- .../7zip/UI/Common/ArchiveOpenCallback.cpp | 9 +- 7zip/CPP/7zip/UI/Common/Bench.cpp | 559 ++-- 7zip/CPP/7zip/UI/Common/Bench.h | 5 + 7zip/CPP/7zip/UI/Common/CompressCall.cpp | 4 + 7zip/CPP/7zip/UI/Common/CompressCall2.cpp | 5 +- 7zip/CPP/7zip/UI/Common/EnumDirItems.cpp | 169 ++ 7zip/CPP/7zip/UI/Common/EnumDirItems.h | 21 + 7zip/CPP/7zip/UI/Common/HashCalc.cpp | 2 +- 7zip/CPP/7zip/UI/Common/HashCalc.h | 9 +- 7zip/CPP/7zip/UI/Common/OpenArchive.cpp | 11 +- 7zip/CPP/7zip/UI/Common/PropIDUtils.cpp | 6 +- 7zip/CPP/7zip/UI/Common/Update.cpp | 99 +- 7zip/CPP/7zip/UI/Console/Console.dsp | 13 + 7zip/CPP/7zip/UI/Console/Console.mak | 9 +- 7zip/CPP/7zip/UI/Console/Console.manifest | 13 + .../UI/Console/ExtractCallbackConsole.cpp | 42 +- 7zip/CPP/7zip/UI/Console/HashCon.cpp | 4 +- 7zip/CPP/7zip/UI/Console/List.cpp | 131 +- 7zip/CPP/7zip/UI/Console/Main.cpp | 48 +- 7zip/CPP/7zip/UI/Console/MainAr.cpp | 10 +- 7zip/CPP/7zip/UI/Console/PercentPrinter.cpp | 8 +- .../7zip/UI/Console/UpdateCallbackConsole.cpp | 20 +- 7zip/CPP/7zip/UI/Console/makefile | 7 +- 7zip/CPP/7zip/UI/Console/resource.rc | 4 + 7zip/CPP/7zip/UI/Explorer/ContextMenu.cpp | 21 +- 7zip/CPP/7zip/UI/Explorer/Explorer.dsp | 17 + 7zip/CPP/7zip/UI/Explorer/makefile | 2 + 7zip/CPP/7zip/UI/Far/ExtractEngine.cpp | 2 + 7zip/CPP/7zip/UI/Far/Far.cpp | 19 +- .../CPP/7zip/UI/FileManager/7zFM.exe.manifest | 7 + .../7zip/UI/FileManager/AltStreamsFolder.cpp | 2 +- 7zip/CPP/7zip/UI/FileManager/EditDialog.cpp | 57 + 7zip/CPP/7zip/UI/FileManager/EditDialog.h | 25 + 7zip/CPP/7zip/UI/FileManager/EditDialog.rc | 15 + 7zip/CPP/7zip/UI/FileManager/EditDialogRes.h | 2 + .../7zip/UI/FileManager/ExtractCallback.cpp | 4 + 7zip/CPP/7zip/UI/FileManager/FM.cpp | 7 + 7zip/CPP/7zip/UI/FileManager/FM.dsp | 22 + 7zip/CPP/7zip/UI/FileManager/FM.mak | 17 + 7zip/CPP/7zip/UI/FileManager/FSFolder.cpp | 3 +- 7zip/CPP/7zip/UI/FileManager/FSFolderCopy.cpp | 2 - 7zip/CPP/7zip/UI/FileManager/LangUtils.cpp | 1 + .../7zip/UI/FileManager/ListViewDialog.cpp | 216 +- 7zip/CPP/7zip/UI/FileManager/ListViewDialog.h | 18 +- .../CPP/7zip/UI/FileManager/ListViewDialog.rc | 4 +- 7zip/CPP/7zip/UI/FileManager/Panel.cpp | 5 +- 7zip/CPP/7zip/UI/FileManager/PanelCopy.cpp | 76 +- 7zip/CPP/7zip/UI/FileManager/PanelCrc.cpp | 58 +- 7zip/CPP/7zip/UI/FileManager/PanelDrag.cpp | 124 +- .../7zip/UI/FileManager/PanelFolderChange.cpp | 1 + .../CPP/7zip/UI/FileManager/PanelItemOpen.cpp | 4 +- 7zip/CPP/7zip/UI/FileManager/PanelMenu.cpp | 72 +- .../7zip/UI/FileManager/PanelOperations.cpp | 14 +- .../7zip/UI/FileManager/PanelSplitFile.cpp | 8 +- .../7zip/UI/FileManager/ProgressDialog2.cpp | 30 +- .../CPP/7zip/UI/FileManager/ProgressDialog2.h | 70 +- 7zip/CPP/7zip/UI/FileManager/makefile | 12 +- 7zip/CPP/7zip/UI/FileManager/resource.rc | 1 + 7zip/CPP/7zip/UI/GUI/7zG.exe.manifest | 7 + 7zip/CPP/7zip/UI/GUI/BenchmarkDialog.cpp | 11 +- 7zip/CPP/7zip/UI/GUI/ExtractGUI.cpp | 81 +- 7zip/CPP/7zip/UI/GUI/GUI.cpp | 35 +- 7zip/CPP/7zip/UI/GUI/GUI.dsp | 41 + 7zip/CPP/7zip/UI/GUI/HashGUI.cpp | 202 +- 7zip/CPP/7zip/UI/GUI/HashGUI.h | 13 +- 7zip/CPP/7zip/UI/GUI/UpdateGUI.cpp | 6 +- 7zip/CPP/7zip/UI/GUI/makefile | 5 + 7zip/CPP/7zip/UI/GUI/resource.rc | 2 + 7zip/CPP/Build.mak | 64 +- 7zip/CPP/Common/ListFileUtils.cpp | 19 +- 7zip/CPP/Common/ListFileUtils.h | 6 +- 7zip/CPP/Common/MyBuffer2.h | 63 +- 7zip/CPP/Common/MyString.h | 5 +- 7zip/CPP/Common/MyWindows.h | 5 + 7zip/CPP/Common/StdOutStream.cpp | 59 + 7zip/CPP/Common/StdOutStream.h | 11 +- 7zip/CPP/Windows/Control/Dialog.cpp | 1 + 7zip/CPP/Windows/Control/Dialog.h | 3 + 7zip/CPP/Windows/MemoryLock.cpp | 40 + 7zip/CPP/Windows/MemoryLock.h | 2 + 7zip/CPP/Windows/Registry.cpp | 2 +- 7zip/CPP/Windows/Shell.cpp | 20 +- 7zip/DOC/7zFormat.txt | 6 +- 7zip/DOC/7zip.inf | 4 +- 7zip/DOC/7zip.nsi | 6 +- 7zip/DOC/7zip.wxs | 7 +- 7zip/DOC/License.txt | 2 +- 7zip/DOC/Methods.txt | 10 +- 7zip/DOC/lzma.txt | 8 +- 7zip/DOC/readme.txt | 58 +- 7zip/DOC/src-history.txt | 44 + 255 files changed, 13980 insertions(+), 4012 deletions(-) create mode 100644 7zip/Asm/x86/LzmaDecOpt.asm create mode 100644 7zip/C/Lzma2DecMt.c create mode 100644 7zip/C/Lzma2DecMt.h create mode 100644 7zip/C/MtDec.c create mode 100644 7zip/C/MtDec.h create mode 100644 7zip/CPP/7zip/LzmaDec.mak create mode 100644 7zip/CPP/7zip/UI/Console/Console.manifest create mode 100644 7zip/CPP/7zip/UI/FileManager/EditDialog.cpp create mode 100644 7zip/CPP/7zip/UI/FileManager/EditDialog.h create mode 100644 7zip/CPP/7zip/UI/FileManager/EditDialog.rc create mode 100644 7zip/CPP/7zip/UI/FileManager/EditDialogRes.h diff --git a/7zip/Asm/x86/7zAsm.asm b/7zip/Asm/x86/7zAsm.asm index 3707e90c5..067db822f 100644 --- a/7zip/Asm/x86/7zAsm.asm +++ b/7zip/Asm/x86/7zAsm.asm @@ -1,5 +1,5 @@ ; 7zAsm.asm -- ASM macros -; 2012-12-30 : Igor Pavlov : Public domain +; 2018-02-03 : Igor Pavlov : Public domain MY_ASM_START macro ifdef x64 @@ -52,6 +52,15 @@ endif x6 equ ESI x7 equ EDI + x0_W equ AX + x1_W equ CX + x2_W equ DX + x3_W equ BX + + x5_W equ BP + x6_W equ SI + x7_W equ DI + x0_L equ AL x1_L equ CL x2_L equ DL @@ -63,6 +72,10 @@ endif x3_H equ BH ifdef x64 + x5_L equ BPL + x6_L equ SIL + x7_L equ DIL + r0 equ RAX r1 equ RCX r2 equ RDX @@ -103,3 +116,32 @@ MY_POP_4_REGS macro pop r5 pop r3 endm + + +ifdef x64 + +; for WIN64-x64 ABI: + +REG_PARAM_0 equ r1 +REG_PARAM_1 equ r2 +REG_PARAM_2 equ r8 +REG_PARAM_3 equ r9 + +MY_PUSH_PRESERVED_REGS macro + MY_PUSH_4_REGS + push r12 + push r13 + push r14 + push r15 +endm + + +MY_POP_PRESERVED_REGS macro + pop r15 + pop r14 + pop r13 + pop r12 + MY_POP_4_REGS +endm + +endif diff --git a/7zip/Asm/x86/LzmaDecOpt.asm b/7zip/Asm/x86/LzmaDecOpt.asm new file mode 100644 index 000000000..8ebbc5f2a --- /dev/null +++ b/7zip/Asm/x86/LzmaDecOpt.asm @@ -0,0 +1,1258 @@ +; LzmaDecOpt.asm -- ASM version of LzmaDec_DecodeReal_3() function +; 2018-02-06: Igor Pavlov : Public domain +; +; 3 - is the code compatibility version of LzmaDec_DecodeReal_*() +; function for check at link time. +; That code is tightly coupled with LzmaDec_TryDummy() +; and with another functions in LzmaDec.c file. +; CLzmaDec structure, (probs) array layout, input and output of +; LzmaDec_DecodeReal_*() must be equal in both versions (C / ASM). + +ifndef x64 +; x64=1 +; .err +endif + +include 7zAsm.asm + +MY_ASM_START + +_TEXT$LZMADECOPT SEGMENT ALIGN(64) 'CODE' + +MY_ALIGN macro num:req + align num +endm + +MY_ALIGN_16 macro + MY_ALIGN 16 +endm + +MY_ALIGN_32 macro + MY_ALIGN 32 +endm + +MY_ALIGN_64 macro + MY_ALIGN 64 +endm + + +; _LZMA_SIZE_OPT equ 1 + +; _LZMA_PROB32 equ 1 + +ifdef _LZMA_PROB32 + PSHIFT equ 2 + PLOAD macro dest, mem + mov dest, dword ptr [mem] + endm + PSTORE macro src, mem + mov dword ptr [mem], src + endm +else + PSHIFT equ 1 + PLOAD macro dest, mem + movzx dest, word ptr [mem] + endm + PSTORE macro src, mem + mov word ptr [mem], @CatStr(src, _W) + endm +endif + +PMULT equ (1 SHL PSHIFT) +PMULT_HALF equ (1 SHL (PSHIFT - 1)) +PMULT_2 equ (1 SHL (PSHIFT + 1)) + + +; x0 range +; x1 pbPos / (prob) TREE +; x2 probBranch / prm (MATCHED) / pbPos / cnt +; x3 sym +;====== r4 === RSP +; x5 cod +; x6 t1 NORM_CALC / probs_state / dist +; x7 t0 NORM_CALC / prob2 IF_BIT_1 +; x8 state +; x9 match (MATCHED) / sym2 / dist2 / lpMask_reg +; x10 kBitModelTotal_reg +; r11 probs +; x12 offs (MATCHED) / dic / len_temp +; x13 processedPos +; x14 bit (MATCHED) / dicPos +; r15 buf + + +cod equ x5 +cod_L equ x5_L +range equ x0 +state equ x8 +state_R equ r8 +buf equ r15 +processedPos equ x13 +kBitModelTotal_reg equ x10 + +probBranch equ x2 +probBranch_R equ r2 +probBranch_W equ x2_W + +pbPos equ x1 +pbPos_R equ r1 + +cnt equ x2 +cnt_R equ r2 + +lpMask_reg equ x9 +dicPos equ r14 + +sym equ x3 +sym_R equ r3 +sym_L equ x3_L + +probs equ r11 +dic equ r12 + +t0 equ x7 +t0_W equ x7_W +t0_R equ r7 + +prob2 equ t0 +prob2_W equ t0_W + +t1 equ x6 +t1_R equ r6 + +probs_state equ t1 +probs_state_R equ t1_R + +prm equ r2 +match equ x9 +match_R equ r9 +offs equ x12 +offs_R equ r12 +bit equ x14 +bit_R equ r14 + +sym2 equ x9 +sym2_R equ r9 + +len_temp equ x12 + +dist equ sym +dist2 equ x9 + + + +kNumBitModelTotalBits equ 11 +kBitModelTotal equ (1 SHL kNumBitModelTotalBits) +kNumMoveBits equ 5 +kBitModelOffset equ ((1 SHL kNumMoveBits) - 1) +kTopValue equ (1 SHL 24) + +NORM_2 macro + ; movzx t0, BYTE PTR [buf] + shl cod, 8 + mov cod_L, BYTE PTR [buf] + shl range, 8 + ; or cod, t0 + inc buf +endm + + +NORM macro + cmp range, kTopValue + jae SHORT @F + NORM_2 +@@: +endm + + +; ---------- Branch MACROS ---------- + +UPDATE_0 macro probsArray:req, probOffset:req, probDisp:req + mov prob2, kBitModelTotal_reg + sub prob2, probBranch + shr prob2, kNumMoveBits + add probBranch, prob2 + PSTORE probBranch, probOffset * 1 + probsArray + probDisp * PMULT +endm + + +UPDATE_1 macro probsArray:req, probOffset:req, probDisp:req + sub prob2, range + sub cod, range + mov range, prob2 + mov prob2, probBranch + shr probBranch, kNumMoveBits + sub prob2, probBranch + PSTORE prob2, probOffset * 1 + probsArray + probDisp * PMULT +endm + + +CMP_COD macro probsArray:req, probOffset:req, probDisp:req + PLOAD probBranch, probOffset * 1 + probsArray + probDisp * PMULT + NORM + mov prob2, range + shr range, kNumBitModelTotalBits + imul range, probBranch + cmp cod, range +endm + + +IF_BIT_1_NOUP macro probsArray:req, probOffset:req, probDisp:req, toLabel:req + CMP_COD probsArray, probOffset, probDisp + jae toLabel +endm + + +IF_BIT_1 macro probsArray:req, probOffset:req, probDisp:req, toLabel:req + IF_BIT_1_NOUP probsArray, probOffset, probDisp, toLabel + UPDATE_0 probsArray, probOffset, probDisp +endm + + +IF_BIT_0_NOUP macro probsArray:req, probOffset:req, probDisp:req, toLabel:req + CMP_COD probsArray, probOffset, probDisp + jb toLabel +endm + + +; ---------- CMOV MACROS ---------- + +NORM_CALC macro prob:req + NORM + mov t0, range + shr range, kNumBitModelTotalBits + imul range, prob + sub t0, range + mov t1, cod + sub cod, range +endm + + +PUP macro prob:req, probPtr:req + sub t0, prob + ; only sar works for both 16/32 bit prob modes + sar t0, kNumMoveBits + add t0, prob + PSTORE t0, probPtr +endm + + +PUP_SUB macro prob:req, probPtr:req, symSub:req + sbb sym, symSub + PUP prob, probPtr +endm + + +PUP_COD macro prob:req, probPtr:req, symSub:req + mov t0, kBitModelOffset + cmovb cod, t1 + mov t1, sym + cmovb t0, kBitModelTotal_reg + PUP_SUB prob, probPtr, symSub +endm + + +BIT_0 macro prob:req, probNext:req + PLOAD prob, probs + 1 * PMULT + PLOAD probNext, probs + 1 * PMULT_2 + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, probs + 1 * PMULT_2 + PMULT + cmovae probNext, t0 + mov t0, kBitModelOffset + cmovb cod, t1 + cmovb t0, kBitModelTotal_reg + mov sym, 2 + PUP_SUB prob, probs + 1 * PMULT, 0 - 1 +endm + + +BIT_1 macro prob:req, probNext:req + PLOAD probNext, probs + sym_R * PMULT_2 + add sym, sym + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, probs + sym_R * PMULT + PMULT + cmovae probNext, t0 + PUP_COD prob, probs + t1_R * PMULT_HALF, 0 - 1 +endm + + +BIT_2 macro prob:req, symSub:req + add sym, sym + + NORM_CALC prob + + cmovae range, t0 + PUP_COD prob, probs + t1_R * PMULT_HALF, symSub +endm + + +; ---------- MATCHED LITERAL ---------- + +LITM_0 macro + mov offs, 256 * PMULT + shl match, (PSHIFT + 1) + mov bit, offs + and bit, match + PLOAD x1, probs + 256 * PMULT + bit_R * 1 + 1 * PMULT + lea prm, [probs + 256 * PMULT + bit_R * 1 + 1 * PMULT] + ; lea prm, [probs + 256 * PMULT + 1 * PMULT] + ; add prm, bit_R + xor offs, bit + add match, match + + NORM_CALC x1 + + cmovae offs, bit + mov bit, match + cmovae range, t0 + mov t0, kBitModelOffset + cmovb cod, t1 + cmovb t0, kBitModelTotal_reg + mov sym, 0 + PUP_SUB x1, prm, -2-1 +endm + + +LITM macro + and bit, offs + lea prm, [probs + offs_R * 1] + add prm, bit_R + PLOAD x1, prm + sym_R * PMULT + xor offs, bit + add sym, sym + add match, match + + NORM_CALC x1 + + cmovae offs, bit + mov bit, match + cmovae range, t0 + PUP_COD x1, prm + t1_R * PMULT_HALF, - 1 +endm + + +LITM_2 macro + and bit, offs + lea prm, [probs + offs_R * 1] + add prm, bit_R + PLOAD x1, prm + sym_R * PMULT + add sym, sym + + NORM_CALC x1 + + cmovae range, t0 + PUP_COD x1, prm + t1_R * PMULT_HALF, 256 - 1 +endm + + +; ---------- REVERSE BITS ---------- + +REV_0 macro prob:req, probNext:req + ; PLOAD prob, probs + 1 * PMULT + ; lea sym2_R, [probs + 2 * PMULT] + ; PLOAD probNext, probs + 2 * PMULT + PLOAD probNext, sym2_R + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, probs + 3 * PMULT + cmovae probNext, t0 + cmovb cod, t1 + mov t0, kBitModelOffset + cmovb t0, kBitModelTotal_reg + lea t1_R, [probs + 3 * PMULT] + cmovae sym2_R, t1_R + PUP prob, probs + 1 * PMULT +endm + + +REV_1 macro prob:req, probNext:req, step:req + add sym2_R, step * PMULT + PLOAD probNext, sym2_R + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, sym2_R + step * PMULT + cmovae probNext, t0 + cmovb cod, t1 + mov t0, kBitModelOffset + cmovb t0, kBitModelTotal_reg + lea t1_R, [sym2_R + step * PMULT] + cmovae sym2_R, t1_R + PUP prob, t1_R - step * PMULT_2 +endm + + +REV_2 macro prob:req, step:req + sub sym2_R, probs + shr sym2, PSHIFT + or sym, sym2 + + NORM_CALC prob + + cmovae range, t0 + lea t0, [sym - step] + cmovb sym, t0 + cmovb cod, t1 + mov t0, kBitModelOffset + cmovb t0, kBitModelTotal_reg + PUP prob, probs + sym2_R * PMULT +endm + + +REV_1_VAR macro prob:req + PLOAD prob, sym_R + mov probs, sym_R + add sym_R, sym2_R + + NORM_CALC prob + + cmovae range, t0 + lea t0_R, [sym_R + sym2_R] + cmovae sym_R, t0_R + mov t0, kBitModelOffset + cmovb cod, t1 + ; mov t1, kBitModelTotal + ; cmovb t0, t1 + cmovb t0, kBitModelTotal_reg + add sym2, sym2 + PUP prob, probs +endm + + + + +LIT_PROBS macro lpMaskParam:req + ; prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); + mov t0, processedPos + shl t0, 8 + add sym, t0 + and sym, lpMaskParam + add probs_state_R, pbPos_R + mov x1, LOC lc2 + lea sym, dword ptr[sym_R + 2 * sym_R] + add probs, Literal * PMULT + shl sym, x1_L + add probs, sym_R + UPDATE_0 probs_state_R, 0, IsMatch + inc processedPos +endm + + + +kNumPosBitsMax equ 4 +kNumPosStatesMax equ (1 SHL kNumPosBitsMax) + +kLenNumLowBits equ 3 +kLenNumLowSymbols equ (1 SHL kLenNumLowBits) +kLenNumHighBits equ 8 +kLenNumHighSymbols equ (1 SHL kLenNumHighBits) +kNumLenProbs equ (2 * kLenNumLowSymbols * kNumPosStatesMax + kLenNumHighSymbols) + +LenLow equ 0 +LenChoice equ LenLow +LenChoice2 equ (LenLow + kLenNumLowSymbols) +LenHigh equ (LenLow + 2 * kLenNumLowSymbols * kNumPosStatesMax) + +kNumStates equ 12 +kNumStates2 equ 16 +kNumLitStates equ 7 + +kStartPosModelIndex equ 4 +kEndPosModelIndex equ 14 +kNumFullDistances equ (1 SHL (kEndPosModelIndex SHR 1)) + +kNumPosSlotBits equ 6 +kNumLenToPosStates equ 4 + +kNumAlignBits equ 4 +kAlignTableSize equ (1 SHL kNumAlignBits) + +kMatchMinLen equ 2 +kMatchSpecLenStart equ (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) + +kStartOffset equ 1664 +SpecPos equ (-kStartOffset) +IsRep0Long equ (SpecPos + kNumFullDistances) +RepLenCoder equ (IsRep0Long + (kNumStates2 SHL kNumPosBitsMax)) +LenCoder equ (RepLenCoder + kNumLenProbs) +IsMatch equ (LenCoder + kNumLenProbs) +kAlign equ (IsMatch + (kNumStates2 SHL kNumPosBitsMax)) +IsRep equ (kAlign + kAlignTableSize) +IsRepG0 equ (IsRep + kNumStates) +IsRepG1 equ (IsRepG0 + kNumStates) +IsRepG2 equ (IsRepG1 + kNumStates) +PosSlot equ (IsRepG2 + kNumStates) +Literal equ (PosSlot + (kNumLenToPosStates SHL kNumPosSlotBits)) +NUM_BASE_PROBS equ (Literal + kStartOffset) + +if kAlign ne 0 + .err +endif + +if NUM_BASE_PROBS ne 1984 + .err +endif + + +PTR_FIELD equ dq ? + +CLzmaDec_Asm struct + lc db ? + lp db ? + pb db ? + _pad_ db ? + dicSize dd ? + + probs_Spec PTR_FIELD + probs_1664 PTR_FIELD + dic_Spec PTR_FIELD + dicBufSize PTR_FIELD + dicPos_Spec PTR_FIELD + buf_Spec PTR_FIELD + + range_Spec dd ? + code_Spec dd ? + processedPos_Spec dd ? + checkDicSize dd ? + rep0 dd ? + rep1 dd ? + rep2 dd ? + rep3 dd ? + state_Spec dd ? + remainLen dd ? +CLzmaDec_Asm ends + + +CLzmaDec_Asm_Loc struct + OLD_RSP PTR_FIELD + lzmaPtr PTR_FIELD + _pad0_ PTR_FIELD + _pad1_ PTR_FIELD + _pad2_ PTR_FIELD + dicBufSize PTR_FIELD + probs_Spec PTR_FIELD + dic_Spec PTR_FIELD + + limit PTR_FIELD + bufLimit PTR_FIELD + lc2 dd ? + lpMask dd ? + pbMask dd ? + checkDicSize dd ? + + _pad_ dd ? + remainLen dd ? + dicPos_Spec PTR_FIELD + rep0 dd ? + rep1 dd ? + rep2 dd ? + rep3 dd ? +CLzmaDec_Asm_Loc ends + + +GLOB_2 equ [sym_R].CLzmaDec_Asm. +GLOB equ [r1].CLzmaDec_Asm. +LOC_0 equ [r0].CLzmaDec_Asm_Loc. +LOC equ [RSP].CLzmaDec_Asm_Loc. + + +COPY_VAR macro name + mov t0, GLOB_2 name + mov LOC_0 name, t0 +endm + + +RESTORE_VAR macro name + mov t0, LOC name + mov GLOB name, t0 +endm + + + +IsMatchBranch_Pre macro reg + ; prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + mov pbPos, LOC pbMask + and pbPos, processedPos + shl pbPos, (kLenNumLowBits + 1 + PSHIFT) + lea probs_state_R, [probs + state_R] +endm + + +IsMatchBranch macro reg + IsMatchBranch_Pre + IF_BIT_1 probs_state_R, pbPos_R, IsMatch, IsMatch_label +endm + + +CheckLimits macro reg + cmp buf, LOC bufLimit + jae fin_OK + cmp dicPos, LOC limit + jae fin_OK +endm + + + +; RSP is (16x + 8) bytes aligned in WIN64-x64 +; LocalSize equ ((((SIZEOF CLzmaDec_Asm_Loc) + 7) / 16 * 16) + 8) + +PARAM_lzma equ REG_PARAM_0 +PARAM_limit equ REG_PARAM_1 +PARAM_bufLimit equ REG_PARAM_2 + +; MY_ALIGN_64 +MY_PROC LzmaDec_DecodeReal_3, 3 +MY_PUSH_PRESERVED_REGS + + lea r0, [RSP - (SIZEOF CLzmaDec_Asm_Loc)] + and r0, -128 + mov r5, RSP + mov RSP, r0 + mov LOC_0 Old_RSP, r5 + mov LOC_0 lzmaPtr, PARAM_lzma + + mov LOC_0 remainLen, 0 ; remainLen must be ZERO + + mov LOC_0 bufLimit, PARAM_bufLimit + mov sym_R, PARAM_lzma ; CLzmaDec_Asm_Loc pointer for GLOB_2 + mov dic, GLOB_2 dic_Spec + add PARAM_limit, dic + mov LOC_0 limit, PARAM_limit + + COPY_VAR(rep0) + COPY_VAR(rep1) + COPY_VAR(rep2) + COPY_VAR(rep3) + + mov dicPos, GLOB_2 dicPos_Spec + add dicPos, dic + mov LOC_0 dicPos_Spec, dicPos + mov LOC_0 dic_Spec, dic + + mov x1_L, GLOB_2 pb + mov t0, 1 + shl t0, x1_L + dec t0 + mov LOC_0 pbMask, t0 + + ; unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; + ; unsigned lc = p->prop.lc; + ; unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); + + mov x1_L, GLOB_2 lc + mov x2, 100h + mov t0, x2 + shr x2, x1_L + ; inc x1 + add x1_L, PSHIFT + mov LOC_0 lc2, x1 + mov x1_L, GLOB_2 lp + shl t0, x1_L + sub t0, x2 + mov LOC_0 lpMask, t0 + mov lpMask_reg, t0 + + ; mov probs, GLOB_2 probs_Spec + ; add probs, kStartOffset SHL PSHIFT + mov probs, GLOB_2 probs_1664 + mov LOC_0 probs_Spec, probs + + mov t0_R, GLOB_2 dicBufSize + mov LOC_0 dicBufSize, t0_R + + mov x1, GLOB_2 checkDicSize + mov LOC_0 checkDicSize, x1 + + mov processedPos, GLOB_2 processedPos_Spec + + mov state, GLOB_2 state_Spec + shl state, PSHIFT + + mov buf, GLOB_2 buf_Spec + mov range, GLOB_2 range_Spec + mov cod, GLOB_2 code_Spec + mov kBitModelTotal_reg, kBitModelTotal + xor sym, sym + + ; if (processedPos != 0 || checkDicSize != 0) + or x1, processedPos + jz @f + + add t0_R, dic + cmp dicPos, dic + cmovnz t0_R, dicPos + movzx sym, byte ptr[t0_R - 1] + +@@: + IsMatchBranch_Pre + cmp state, 4 * PMULT + jb lit_end + cmp state, kNumLitStates * PMULT + jb lit_matched_end + jmp lz_end + + + + +; ---------- LITERAL ---------- +MY_ALIGN_64 +lit_start: + xor state, state +lit_start_2: + LIT_PROBS lpMask_reg + + ifdef _LZMA_SIZE_OPT + + PLOAD x1, probs + 1 * PMULT + mov sym, 1 +MY_ALIGN_16 +lit_loop: + BIT_1 x1, x2 + mov x1, x2 + cmp sym, 127 + jbe lit_loop + + else + + BIT_0 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + + endif + + BIT_2 x2, 256 - 1 + + ; mov dic, LOC dic_Spec + mov probs, LOC probs_Spec + IsMatchBranch_Pre + mov byte ptr[dicPos], sym_L + inc dicPos + + CheckLimits +lit_end: + IF_BIT_0_NOUP probs_state_R, pbPos_R, IsMatch, lit_start + + ; jmp IsMatch_label + +; ---------- MATCHES ---------- +; MY_ALIGN_32 +IsMatch_label: + UPDATE_1 probs_state_R, pbPos_R, IsMatch + IF_BIT_1 probs_state_R, 0, IsRep, IsRep_label + + add probs, LenCoder * PMULT + add state, kNumStates * PMULT + +; ---------- LEN DECODE ---------- +len_decode: + mov len_temp, 8 - 1 - kMatchMinLen + IF_BIT_0_NOUP probs, 0, 0, len_mid_0 + UPDATE_1 probs, 0, 0 + add probs, (1 SHL (kLenNumLowBits + PSHIFT)) + mov len_temp, -1 - kMatchMinLen + IF_BIT_0_NOUP probs, 0, 0, len_mid_0 + UPDATE_1 probs, 0, 0 + add probs, LenHigh * PMULT - (1 SHL (kLenNumLowBits + PSHIFT)) + mov sym, 1 + PLOAD x1, probs + 1 * PMULT + +MY_ALIGN_32 +len8_loop: + BIT_1 x1, x2 + mov x1, x2 + cmp sym, 64 + jb len8_loop + + mov len_temp, (kLenNumHighSymbols - kLenNumLowSymbols * 2) - 1 - kMatchMinLen + jmp len_mid_2 + +MY_ALIGN_32 +len_mid_0: + UPDATE_0 probs, 0, 0 + add probs, pbPos_R + BIT_0 x2, x1 +len_mid_2: + BIT_1 x1, x2 + BIT_2 x2, len_temp + mov probs, LOC probs_Spec + cmp state, kNumStates * PMULT + jb copy_match + + +; ---------- DECODE DISTANCE ---------- + ; probs + PosSlot + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); + + mov t0, 3 + kMatchMinLen + cmp sym, 3 + kMatchMinLen + cmovb t0, sym + add probs, PosSlot * PMULT - (kMatchMinLen SHL (kNumPosSlotBits + PSHIFT)) + shl t0, (kNumPosSlotBits + PSHIFT) + add probs, t0_R + + ; sym = Len + ; mov LOC remainLen, sym + mov len_temp, sym + + ifdef _LZMA_SIZE_OPT + + PLOAD x1, probs + 1 * PMULT + mov sym, 1 +MY_ALIGN_16 +slot_loop: + BIT_1 x1, x2 + mov x1, x2 + cmp sym, 32 + jb slot_loop + + else + + BIT_0 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + + endif + + mov x1, sym + BIT_2 x2, 64-1 + + and sym, 3 + mov probs, LOC probs_Spec + cmp x1, 32 + kEndPosModelIndex / 2 + jb short_dist + + ; unsigned numDirectBits = (unsigned)(((distance >> 1) - 1)); + sub x1, (32 + 1 + kNumAlignBits) + ; distance = (2 | (distance & 1)); + or sym, 2 + PLOAD x2, probs + 1 * PMULT + shl sym, kNumAlignBits + 1 + lea sym2_R, [probs + 2 * PMULT] + + jmp direct_norm + ; lea t1, [sym_R + (1 SHL kNumAlignBits)] + ; cmp range, kTopValue + ; jb direct_norm + +; ---------- DIRECT DISTANCE ---------- +MY_ALIGN_32 +direct_loop: + shr range, 1 + mov t0, cod + sub cod, range + cmovs cod, t0 + cmovns sym, t1 + + comment ~ + sub cod, range + mov x2, cod + sar x2, 31 + lea sym, dword ptr [r2 + sym_R * 2 + 1] + and x2, range + add cod, x2 + ~ + dec x1 + je direct_end + + add sym, sym +direct_norm: + lea t1, [sym_R + (1 SHL kNumAlignBits)] + cmp range, kTopValue + jae near ptr direct_loop + ; we align for 32 here with "near ptr" command above + NORM_2 + jmp direct_loop + +MY_ALIGN_32 +direct_end: + ; prob = + kAlign; + ; distance <<= kNumAlignBits; + REV_0 x2, x1 + REV_1 x1, x2, 2 + REV_1 x2, x1, 4 + REV_2 x1, 8 + +decode_dist_end: + + ; if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) + + mov t0, LOC checkDicSize + test t0, t0 + cmove t0, processedPos + cmp sym, t0 + jae end_of_payload + + ; rep3 = rep2; + ; rep2 = rep1; + ; rep1 = rep0; + ; rep0 = distance + 1; + + inc sym + mov t0, LOC rep0 + mov t1, LOC rep1 + mov x1, LOC rep2 + mov LOC rep0, sym + ; mov sym, LOC remainLen + mov sym, len_temp + mov LOC rep1, t0 + mov LOC rep2, t1 + mov LOC rep3, x1 + + ; state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + cmp state, (kNumStates + kNumLitStates) * PMULT + mov state, kNumLitStates * PMULT + mov t0, (kNumLitStates + 3) * PMULT + cmovae state, t0 + + +; ---------- COPY MATCH ---------- +copy_match: + + ; len += kMatchMinLen; + ; add sym, kMatchMinLen + + ; if ((rem = limit - dicPos) == 0) + ; { + ; p->dicPos = dicPos; + ; return SZ_ERROR_DATA; + ; } + mov cnt_R, LOC limit + sub cnt_R, dicPos + jz fin_ERROR + + ; curLen = ((rem < len) ? (unsigned)rem : len); + cmp cnt_R, sym_R + ; cmovae cnt_R, sym_R ; 64-bit + cmovae cnt, sym ; 32-bit + + mov dic, LOC dic_Spec + mov x1, LOC rep0 + + mov t0_R, dicPos + add dicPos, cnt_R + ; processedPos += curLen; + add processedPos, cnt + ; len -= curLen; + sub sym, cnt + mov LOC remainLen, sym + + sub t0_R, dic + + ; pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); + sub t0_R, r1 + jae @f + + mov r1, LOC dicBufSize + add t0_R, r1 + sub r1, t0_R + cmp cnt_R, r1 + ja copy_match_cross +@@: + ; if (curLen <= dicBufSize - pos) + +; ---------- COPY MATCH FAST ---------- + ; Byte *dest = dic + dicPos; + ; mov r1, dic + ; ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; + ; sub t0_R, dicPos + ; dicPos += curLen; + + ; const Byte *lim = dest + curLen; + add t0_R, dic + movzx sym, byte ptr[t0_R] + add t0_R, cnt_R + neg cnt_R + ; lea r1, [dicPos - 1] +copy_common: + dec dicPos + ; cmp LOC rep0, 1 + ; je rep0Label + + ; t0_R - src_lim + ; r1 - dest_lim - 1 + ; cnt_R - (-cnt) + + IsMatchBranch_Pre + inc cnt_R + jz copy_end +MY_ALIGN_16 +@@: + mov byte ptr[cnt_R * 1 + dicPos], sym_L + movzx sym, byte ptr[cnt_R * 1 + t0_R] + inc cnt_R + jnz @b + +copy_end: +lz_end_match: + mov byte ptr[dicPos], sym_L + inc dicPos + + ; IsMatchBranch_Pre + CheckLimits +lz_end: + IF_BIT_1_NOUP probs_state_R, pbPos_R, IsMatch, IsMatch_label + + + +; ---------- LITERAL MATCHED ---------- + + LIT_PROBS LOC lpMask + + ; matchByte = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + mov x1, LOC rep0 + ; mov dic, LOC dic_Spec + mov LOC dicPos_Spec, dicPos + + ; state -= (state < 10) ? 3 : 6; + lea t0, [state_R - 6 * PMULT] + sub state, 3 * PMULT + cmp state, 7 * PMULT + cmovae state, t0 + + sub dicPos, dic + sub dicPos, r1 + jae @f + add dicPos, LOC dicBufSize +@@: + comment ~ + xor t0, t0 + sub dicPos, r1 + cmovb t0_R, LOC dicBufSize + ~ + + movzx match, byte ptr[dic + dicPos * 1] + + ifdef _LZMA_SIZE_OPT + + mov offs, 256 * PMULT + shl match, (PSHIFT + 1) + mov bit, match + mov sym, 1 +MY_ALIGN_16 +litm_loop: + LITM + cmp sym, 256 + jb litm_loop + sub sym, 256 + + else + + LITM_0 + LITM + LITM + LITM + LITM + LITM + LITM + LITM_2 + + endif + + mov probs, LOC probs_Spec + IsMatchBranch_Pre + ; mov dic, LOC dic_Spec + mov dicPos, LOC dicPos_Spec + mov byte ptr[dicPos], sym_L + inc dicPos + + CheckLimits +lit_matched_end: + IF_BIT_1_NOUP probs_state_R, pbPos_R, IsMatch, IsMatch_label + ; IsMatchBranch + mov lpMask_reg, LOC lpMask + sub state, 3 * PMULT + jmp lit_start_2 + + + +; ---------- REP 0 LITERAL ---------- +MY_ALIGN_32 +IsRep0Short_label: + UPDATE_0 probs_state_R, pbPos_R, IsRep0Long + + ; dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + mov dic, LOC dic_Spec + mov t0_R, dicPos + mov probBranch, LOC rep0 + sub t0_R, dic + + sub probs, RepLenCoder * PMULT + inc processedPos + ; state = state < kNumLitStates ? 9 : 11; + or state, 1 * PMULT + IsMatchBranch_Pre + + sub t0_R, probBranch_R + jae @f + add t0_R, LOC dicBufSize +@@: + movzx sym, byte ptr[dic + t0_R * 1] + jmp lz_end_match + + +MY_ALIGN_32 +IsRep_label: + UPDATE_1 probs_state_R, 0, IsRep + + ; The (checkDicSize == 0 && processedPos == 0) case was checked before in LzmaDec.c with kBadRepCode. + ; So we don't check it here. + + ; mov t0, processedPos + ; or t0, LOC checkDicSize + ; jz fin_ERROR_2 + + ; state = state < kNumLitStates ? 8 : 11; + cmp state, kNumLitStates * PMULT + mov state, 8 * PMULT + mov probBranch, 11 * PMULT + cmovae state, probBranch + + ; prob = probs + RepLenCoder; + add probs, RepLenCoder * PMULT + + IF_BIT_1 probs_state_R, 0, IsRepG0, IsRepG0_label + IF_BIT_0_NOUP probs_state_R, pbPos_R, IsRep0Long, IsRep0Short_label + UPDATE_1 probs_state_R, pbPos_R, IsRep0Long + jmp len_decode + +MY_ALIGN_32 +IsRepG0_label: + UPDATE_1 probs_state_R, 0, IsRepG0 + mov dist2, LOC rep0 + mov dist, LOC rep1 + mov LOC rep1, dist2 + + IF_BIT_1 probs_state_R, 0, IsRepG1, IsRepG1_label + mov LOC rep0, dist + jmp len_decode + +; MY_ALIGN_32 +IsRepG1_label: + UPDATE_1 probs_state_R, 0, IsRepG1 + mov dist2, LOC rep2 + mov LOC rep2, dist + + IF_BIT_1 probs_state_R, 0, IsRepG2, IsRepG2_label + mov LOC rep0, dist2 + jmp len_decode + +; MY_ALIGN_32 +IsRepG2_label: + UPDATE_1 probs_state_R, 0, IsRepG2 + mov dist, LOC rep3 + mov LOC rep3, dist2 + mov LOC rep0, dist + jmp len_decode + + + +; ---------- SPEC SHORT DISTANCE ---------- + +MY_ALIGN_32 +short_dist: + sub x1, 32 + 1 + jbe decode_dist_end + or sym, 2 + shl sym, x1_L + lea sym_R, [probs + sym_R * PMULT + SpecPos * PMULT + 1 * PMULT] + mov sym2, PMULT ; step +MY_ALIGN_32 +spec_loop: + REV_1_VAR x2 + dec x1 + jnz spec_loop + + mov probs, LOC probs_Spec + sub sym, sym2 + sub sym, SpecPos * PMULT + sub sym_R, probs + shr sym, PSHIFT + + jmp decode_dist_end + + +; ---------- COPY MATCH CROSS ---------- +copy_match_cross: + ; t0_R - src pos + ; r1 - len to dicBufSize + ; cnt_R - total copy len + + mov t1_R, t0_R ; srcPos + mov t0_R, dic + mov r1, LOC dicBufSize ; + neg cnt_R +@@: + movzx sym, byte ptr[t1_R * 1 + t0_R] + inc t1_R + mov byte ptr[cnt_R * 1 + dicPos], sym_L + inc cnt_R + cmp t1_R, r1 + jne @b + + movzx sym, byte ptr[t0_R] + sub t0_R, cnt_R + jmp copy_common + + + + +fin_ERROR: + mov LOC remainLen, len_temp +; fin_ERROR_2: + mov sym, 1 + jmp fin + +end_of_payload: + cmp sym, 0FFFFFFFFh ; -1 + jne fin_ERROR + + mov LOC remainLen, kMatchSpecLenStart + sub state, kNumStates * PMULT + +fin_OK: + xor sym, sym + +fin: + NORM + + mov r1, LOC lzmaPtr + + sub dicPos, LOC dic_Spec + mov GLOB dicPos_Spec, dicPos + mov GLOB buf_Spec, buf + mov GLOB range_Spec, range + mov GLOB code_Spec, cod + shr state, PSHIFT + mov GLOB state_Spec, state + mov GLOB processedPos_Spec, processedPos + + RESTORE_VAR(remainLen) + RESTORE_VAR(rep0) + RESTORE_VAR(rep1) + RESTORE_VAR(rep2) + RESTORE_VAR(rep3) + + mov x0, sym + + mov RSP, LOC Old_RSP + +MY_POP_PRESERVED_REGS +MY_ENDP + +_TEXT$LZMADECOPT ENDS + +end diff --git a/7zip/C/7zArcIn.c b/7zip/C/7zArcIn.c index e1b03d879..f74d0fad5 100644 --- a/7zip/C/7zArcIn.c +++ b/7zip/C/7zArcIn.c @@ -1,5 +1,5 @@ /* 7zArcIn.c -- 7z Input functions -2017-04-03 : Igor Pavlov : Public domain */ +2018-12-31 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -19,7 +19,7 @@ { MY_ALLOC(Byte, to, size, alloc); memcpy(to, from, size); } #define MY_ALLOC_ZE_AND_CPY(to, size, from, alloc) \ - { if ((size) == 0) p = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } } + { if ((size) == 0) to = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } } #define k7zMajorVersion 0 @@ -666,7 +666,7 @@ static SRes ReadUnpackInfo(CSzAr *p, MY_ALLOC(size_t, p->FoCodersOffsets, (size_t)numFolders + 1, alloc); MY_ALLOC(UInt32, p->FoStartPackStreamIndex, (size_t)numFolders + 1, alloc); MY_ALLOC(UInt32, p->FoToCoderUnpackSizes, (size_t)numFolders + 1, alloc); - MY_ALLOC(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc); + MY_ALLOC_ZE(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc); startBufPtr = sd.Data; @@ -1744,7 +1744,7 @@ size_t SzArEx_GetFullNameLen(const CSzArEx *p, size_t fileIndex) UInt16 *SzArEx_GetFullNameUtf16_Back(const CSzArEx *p, size_t fileIndex, UInt16 *dest) { - Bool needSlash; + BoolInt needSlash; if (!p->FileNameOffsets) { *(--dest) = 0; diff --git a/7zip/C/7zDec.c b/7zip/C/7zDec.c index 1ae87dfaf..7c4635211 100644 --- a/7zip/C/7zDec.c +++ b/7zip/C/7zDec.c @@ -1,5 +1,5 @@ /* 7zDec.c -- Decoding from 7z folder -2017-04-03 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -44,7 +44,7 @@ typedef struct const Byte *end; const Byte *begin; UInt64 processed; - Bool extra; + BoolInt extra; SRes res; const ILookInStream *inStream; } CByteInToLook; @@ -156,7 +156,7 @@ static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, I { SizeT inProcessed = (SizeT)lookahead, dicPos = state.dicPos; ELzmaStatus status; - res = LzmaDec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status); + res = LzmaDec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); lookahead -= inProcessed; inSize -= inProcessed; if (res != SZ_OK) @@ -218,7 +218,7 @@ static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, { SizeT inProcessed = (SizeT)lookahead, dicPos = state.decoder.dicPos; ELzmaStatus status; - res = Lzma2Dec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status); + res = Lzma2Dec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); lookahead -= inProcessed; inSize -= inProcessed; if (res != SZ_OK) @@ -269,7 +269,7 @@ static SRes SzDecodeCopy(UInt64 inSize, ILookInStream *inStream, Byte *outBuffer return SZ_OK; } -static Bool IS_MAIN_METHOD(UInt32 m) +static BoolInt IS_MAIN_METHOD(UInt32 m) { switch (m) { @@ -286,7 +286,7 @@ static Bool IS_MAIN_METHOD(UInt32 m) return False; } -static Bool IS_SUPPORTED_CODER(const CSzCoderInfo *c) +static BoolInt IS_SUPPORTED_CODER(const CSzCoderInfo *c) { return c->NumStreams == 1 diff --git a/7zip/C/7zTypes.h b/7zip/C/7zTypes.h index 29244b23e..65b3af63c 100644 --- a/7zip/C/7zTypes.h +++ b/7zip/C/7zTypes.h @@ -1,5 +1,5 @@ /* 7zTypes.h -- Basic types -2017-07-17 : Igor Pavlov : Public domain */ +2018-08-04 : Igor Pavlov : Public domain */ #ifndef __7Z_TYPES_H #define __7Z_TYPES_H @@ -103,7 +103,8 @@ typedef UInt32 SizeT; typedef size_t SizeT; #endif -typedef int Bool; +typedef int BoolInt; +/* typedef BoolInt Bool; */ #define True 1 #define False 0 diff --git a/7zip/C/7zVersion.h b/7zip/C/7zVersion.h index 69ef69853..c176823a4 100644 --- a/7zip/C/7zVersion.h +++ b/7zip/C/7zVersion.h @@ -1,7 +1,7 @@ -#define MY_VER_MAJOR 18 -#define MY_VER_MINOR 01 +#define MY_VER_MAJOR 19 +#define MY_VER_MINOR 00 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "18.01" +#define MY_VERSION_NUMBERS "19.00" #define MY_VERSION MY_VERSION_NUMBERS #ifdef MY_CPU_NAME @@ -10,7 +10,7 @@ #define MY_VERSION_CPU MY_VERSION #endif -#define MY_DATE "2018-01-28" +#define MY_DATE "2019-02-21" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov" diff --git a/7zip/C/Alloc.c b/7zip/C/Alloc.c index bda8b5bc9..bcede4b85 100644 --- a/7zip/C/Alloc.c +++ b/7zip/C/Alloc.c @@ -1,8 +1,10 @@ /* Alloc.c -- Memory allocation functions -2017-06-15 : Igor Pavlov : Public domain */ +2018-04-27 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include + #ifdef _WIN32 #include #endif @@ -14,12 +16,119 @@ /* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ #ifdef _SZ_ALLOC_DEBUG + #include int g_allocCount = 0; int g_allocCountMid = 0; int g_allocCountBig = 0; + + +#define CONVERT_INT_TO_STR(charType, tempSize) \ + unsigned char temp[tempSize]; unsigned i = 0; \ + while (val >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(val % 10)); val /= 10; } \ + *s++ = (charType)('0' + (unsigned)val); \ + while (i != 0) { i--; *s++ = temp[i]; } \ + *s = 0; + +static void ConvertUInt64ToString(UInt64 val, char *s) +{ + CONVERT_INT_TO_STR(char, 24); +} + +#define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) + +static void ConvertUInt64ToHex(UInt64 val, char *s) +{ + UInt64 v = val; + unsigned i; + for (i = 1;; i++) + { + v >>= 4; + if (v == 0) + break; + } + s[i] = 0; + do + { + unsigned t = (unsigned)(val & 0xF); + val >>= 4; + s[--i] = GET_HEX_CHAR(t); + } + while (i); +} + +#define DEBUG_OUT_STREAM stderr + +static void Print(const char *s) +{ + fputs(s, DEBUG_OUT_STREAM); +} + +static void PrintAligned(const char *s, size_t align) +{ + size_t len = strlen(s); + for(;;) + { + fputc(' ', DEBUG_OUT_STREAM); + if (len >= align) + break; + ++len; + } + Print(s); +} + +static void PrintLn() +{ + Print("\n"); +} + +static void PrintHex(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToHex(v, s); + PrintAligned(s, align); +} + +static void PrintDec(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToString(v, s); + PrintAligned(s, align); +} + +static void PrintAddr(void *p) +{ + PrintHex((UInt64)(size_t)(ptrdiff_t)p, 12); +} + + +#define PRINT_ALLOC(name, cnt, size, ptr) \ + Print(name " "); \ + PrintDec(cnt++, 10); \ + PrintHex(size, 10); \ + PrintAddr(ptr); \ + PrintLn(); + +#define PRINT_FREE(name, cnt, ptr) if (ptr) { \ + Print(name " "); \ + PrintDec(--cnt, 10); \ + PrintAddr(ptr); \ + PrintLn(); } + +#else + +#define PRINT_ALLOC(name, cnt, size, ptr) +#define PRINT_FREE(name, cnt, ptr) +#define Print(s) +#define PrintLn() +#define PrintHex(v, align) +#define PrintDec(v, align) +#define PrintAddr(p) + #endif + + void *MyAlloc(size_t size) { if (size == 0) @@ -27,7 +136,7 @@ void *MyAlloc(size_t size) #ifdef _SZ_ALLOC_DEBUG { void *p = malloc(size); - fprintf(stderr, "\nAlloc %10u bytes, count = %10d, addr = %8X", size, g_allocCount++, (unsigned)p); + PRINT_ALLOC("Alloc ", g_allocCount, size, p); return p; } #else @@ -37,10 +146,8 @@ void *MyAlloc(size_t size) void MyFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address) - fprintf(stderr, "\nFree; count = %10d, addr = %8X", --g_allocCount, (unsigned)address); - #endif + PRINT_FREE("Free ", g_allocCount, address); + free(address); } @@ -50,18 +157,16 @@ void *MidAlloc(size_t size) { if (size == 0) return NULL; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Mid %10d bytes; count = %10d", size, g_allocCountMid++); - #endif + + PRINT_ALLOC("Alloc-Mid", g_allocCountMid, size, NULL); + return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); } void MidFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address) - fprintf(stderr, "\nFree_Mid; count = %10d", --g_allocCountMid); - #endif + PRINT_FREE("Free-Mid", g_allocCountMid, address); + if (!address) return; VirtualFree(address, 0, MEM_RELEASE); @@ -96,9 +201,8 @@ void *BigAlloc(size_t size) { if (size == 0) return NULL; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Big %10u bytes; count = %10d", size, g_allocCountBig++); - #endif + + PRINT_ALLOC("Alloc-Big", g_allocCountBig, size, NULL); #ifdef _7ZIP_LARGE_PAGES { @@ -123,10 +227,7 @@ void *BigAlloc(size_t size) void BigFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address) - fprintf(stderr, "\nFree_Big; count = %10d", --g_allocCountBig); - #endif + PRINT_FREE("Free-Big", g_allocCountBig, address); if (!address) return; @@ -138,8 +239,217 @@ void BigFree(void *address) static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MyFree(address); } -ISzAlloc const g_Alloc = { SzAlloc, SzFree }; +const ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MidAlloc(size); } +static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MidFree(address); } +const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree }; static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); } static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); BigFree(address); } -ISzAlloc const g_BigAlloc = { SzBigAlloc, SzBigFree }; +const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; + + +/* + uintptr_t : C99 (optional) + : unsupported in VS6 +*/ + +#ifdef _WIN32 + typedef UINT_PTR UIntPtr; +#else + /* + typedef uintptr_t UIntPtr; + */ + typedef ptrdiff_t UIntPtr; +#endif + + +#define ADJUST_ALLOC_SIZE 0 +/* +#define ADJUST_ALLOC_SIZE (sizeof(void *) - 1) +*/ +/* + Use (ADJUST_ALLOC_SIZE = (sizeof(void *) - 1)), if + MyAlloc() can return address that is NOT multiple of sizeof(void *). +*/ + + +/* +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((char *)(p) - ((size_t)(UIntPtr)(p) & ((align) - 1)))) +*/ +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((((UIntPtr)(p)) & ~((UIntPtr)(align) - 1)))) + +#define MY_ALIGN_PTR_UP_PLUS(p, align) MY_ALIGN_PTR_DOWN(((char *)(p) + (align) + ADJUST_ALLOC_SIZE), align) + + +#if (_POSIX_C_SOURCE >= 200112L) && !defined(_WIN32) + #define USE_posix_memalign +#endif + +/* + This posix_memalign() is for test purposes only. + We also need special Free() function instead of free(), + if this posix_memalign() is used. +*/ + +/* +static int posix_memalign(void **ptr, size_t align, size_t size) +{ + size_t newSize = size + align; + void *p; + void *pAligned; + *ptr = NULL; + if (newSize < size) + return 12; // ENOMEM + p = MyAlloc(newSize); + if (!p) + return 12; // ENOMEM + pAligned = MY_ALIGN_PTR_UP_PLUS(p, align); + ((void **)pAligned)[-1] = p; + *ptr = pAligned; + return 0; +} +*/ + +/* + ALLOC_ALIGN_SIZE >= sizeof(void *) + ALLOC_ALIGN_SIZE >= cache_line_size +*/ + +#define ALLOC_ALIGN_SIZE ((size_t)1 << 7) + +static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) +{ + #ifndef USE_posix_memalign + + void *p; + void *pAligned; + size_t newSize; + UNUSED_VAR(pp); + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + + newSize = size + ALLOC_ALIGN_SIZE * 1 + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + p = MyAlloc(newSize); + + if (!p) + return NULL; + pAligned = MY_ALIGN_PTR_UP_PLUS(p, ALLOC_ALIGN_SIZE); + + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(p); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + ((void **)pAligned)[-1] = p; + + return pAligned; + + #else + + void *p; + UNUSED_VAR(pp); + if (posix_memalign(&p, ALLOC_ALIGN_SIZE, size)) + return NULL; + + Print(" posix_memalign="); PrintAddr(p); + PrintLn(); + + return p; + + #endif +} + + +static void SzAlignedFree(ISzAllocPtr pp, void *address) +{ + UNUSED_VAR(pp); + #ifndef USE_posix_memalign + if (address) + MyFree(((void **)address)[-1]); + #else + free(address); + #endif +} + + +const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree }; + + + +#define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *)) + +/* we align ptr to support cases where CAlignOffsetAlloc::offset is not multiply of sizeof(void *) */ +#define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1] +/* +#define REAL_BLOCK_PTR_VAR(p) ((void **)(p))[-1] +*/ + +static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) +{ + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + void *adr; + void *pAligned; + size_t newSize; + size_t extra; + size_t alignSize = (size_t)1 << p->numAlignBits; + + if (alignSize < sizeof(void *)) + alignSize = sizeof(void *); + + if (p->offset >= alignSize) + return NULL; + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + extra = p->offset & (sizeof(void *) - 1); + newSize = size + alignSize + extra + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + adr = ISzAlloc_Alloc(p->baseAlloc, newSize); + + if (!adr) + return NULL; + + pAligned = (char *)MY_ALIGN_PTR_DOWN((char *)adr + + alignSize - p->offset + extra + ADJUST_ALLOC_SIZE, alignSize) + p->offset; + + PrintLn(); + Print("- Aligned: "); + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(adr); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + REAL_BLOCK_PTR_VAR(pAligned) = adr; + + return pAligned; +} + + +static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address) +{ + if (address) + { + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + PrintLn(); + Print("- Aligned Free: "); + PrintLn(); + ISzAlloc_Free(p->baseAlloc, REAL_BLOCK_PTR_VAR(address)); + } +} + + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p) +{ + p->vt.Alloc = AlignOffsetAlloc_Alloc; + p->vt.Free = AlignOffsetAlloc_Free; +} diff --git a/7zip/C/Alloc.h b/7zip/C/Alloc.h index 3b88ef75b..648237646 100644 --- a/7zip/C/Alloc.h +++ b/7zip/C/Alloc.h @@ -1,5 +1,5 @@ /* Alloc.h -- Memory allocation functions -2017-04-03 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __COMMON_ALLOC_H #define __COMMON_ALLOC_H @@ -31,6 +31,20 @@ void BigFree(void *address); extern const ISzAlloc g_Alloc; extern const ISzAlloc g_BigAlloc; +extern const ISzAlloc g_MidAlloc; +extern const ISzAlloc g_AlignedAlloc; + + +typedef struct +{ + ISzAlloc vt; + ISzAllocPtr baseAlloc; + unsigned numAlignBits; /* ((1 << numAlignBits) >= sizeof(void *)) */ + size_t offset; /* (offset == (k * sizeof(void *)) && offset < (1 << numAlignBits) */ +} CAlignOffsetAlloc; + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p); + EXTERN_C_END diff --git a/7zip/C/Bcj2.c b/7zip/C/Bcj2.c index 0efff4e5d..9a0046a65 100644 --- a/7zip/C/Bcj2.c +++ b/7zip/C/Bcj2.c @@ -1,5 +1,5 @@ /* Bcj2.c -- BCJ2 Decoder (Converter for x86 code) -2017-04-03 : Igor Pavlov : Public domain */ +2018-04-28 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -232,10 +232,10 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) if (rem < 4) { - SizeT i; - SetUi32(p->temp, val); - for (i = 0; i < rem; i++) - dest[i] = p->temp[i]; + p->temp[0] = (Byte)val; if (rem > 0) dest[0] = (Byte)val; val >>= 8; + p->temp[1] = (Byte)val; if (rem > 1) dest[1] = (Byte)val; val >>= 8; + p->temp[2] = (Byte)val; if (rem > 2) dest[2] = (Byte)val; val >>= 8; + p->temp[3] = (Byte)val; p->dest = dest + rem; p->state = BCJ2_DEC_STATE_ORIG_0 + (unsigned)rem; break; diff --git a/7zip/C/Bcj2Enc.c b/7zip/C/Bcj2Enc.c index 98b775b70..bfbeb8e49 100644 --- a/7zip/C/Bcj2Enc.c +++ b/7zip/C/Bcj2Enc.c @@ -1,5 +1,5 @@ /* Bcj2Enc.c -- BCJ2 Encoder (Converter for x86 code) -2017-04-03 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -12,7 +12,6 @@ #define PRF(x) #endif -#include #include #include "Bcj2.h" @@ -53,7 +52,7 @@ void Bcj2Enc_Init(CBcj2Enc *p) p->probs[i] = kBitModelTotal >> 1; } -static Bool MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) +static BoolInt MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) { if ((UInt32)p->low < (UInt32)0xFF000000 || (UInt32)(p->low >> 32) != 0) { @@ -166,7 +165,7 @@ static void Bcj2Enc_Encode_2(CBcj2Enc *p) { Byte context = (Byte)(num == 0 ? p->prevByte : src[-1]); - Bool needConvert; + BoolInt needConvert; p->bufs[BCJ2_STREAM_MAIN] = dest + 1; p->ip += (UInt32)num + 1; @@ -254,7 +253,7 @@ void Bcj2Enc_Encode(CBcj2Enc *p) { const Byte *src = p->src; const Byte *srcLim = p->srcLim; - unsigned finishMode = p->finishMode; + EBcj2Enc_FinishMode finishMode = p->finishMode; p->src = p->temp; p->srcLim = p->temp + p->tempPos; diff --git a/7zip/C/BwtSort.c b/7zip/C/BwtSort.c index aa3192018..cc2f4b29c 100644 --- a/7zip/C/BwtSort.c +++ b/7zip/C/BwtSort.c @@ -1,5 +1,5 @@ /* BwtSort.c -- BWT block sorting -2017-04-03 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -448,7 +448,7 @@ UInt32 BlockSort(UInt32 *Indices, const Byte *data, UInt32 blockSize) groupSize = ((Indices[i] & ~0xC0000000) >> kNumBitsMax); { - Bool finishedGroup = ((Indices[i] & 0x80000000) == 0); + BoolInt finishedGroup = ((Indices[i] & 0x80000000) == 0); if ((Indices[i] & 0x40000000) != 0) { groupSize += ((Indices[(size_t)i + 1] >> kNumBitsMax) << kNumExtra0Bits); diff --git a/7zip/C/CpuArch.c b/7zip/C/CpuArch.c index 554ffa4fc..02e482e08 100644 --- a/7zip/C/CpuArch.c +++ b/7zip/C/CpuArch.c @@ -1,5 +1,5 @@ /* CpuArch.c -- CPU specific code -2016-02-25: Igor Pavlov : Public domain */ +2018-02-18: Igor Pavlov : Public domain */ #include "Precomp.h" @@ -115,7 +115,7 @@ void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d) #endif } -Bool x86cpuid_CheckAndRead(Cx86cpuid *p) +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p) { CHECK_CPUID_IS_SUPPORTED MyCPUID(0, &p->maxFunc, &p->vendor[0], &p->vendor[2], &p->vendor[1]); @@ -144,7 +144,7 @@ int x86cpuid_GetFirm(const Cx86cpuid *p) return -1; } -Bool CPU_Is_InOrder() +BoolInt CPU_Is_InOrder() { Cx86cpuid p; int firm; @@ -175,7 +175,7 @@ Bool CPU_Is_InOrder() #if !defined(MY_CPU_AMD64) && defined(_WIN32) #include -static Bool CPU_Sys_Is_SSE_Supported() +static BoolInt CPU_Sys_Is_SSE_Supported() { OSVERSIONINFO vi; vi.dwOSVersionInfoSize = sizeof(vi); @@ -188,7 +188,7 @@ static Bool CPU_Sys_Is_SSE_Supported() #define CHECK_SYS_SSE_SUPPORT #endif -Bool CPU_Is_Aes_Supported() +BoolInt CPU_Is_Aes_Supported() { Cx86cpuid p; CHECK_SYS_SSE_SUPPORT @@ -197,4 +197,22 @@ Bool CPU_Is_Aes_Supported() return (p.c >> 25) & 1; } +BoolInt CPU_IsSupported_PageGB() +{ + Cx86cpuid cpuid; + if (!x86cpuid_CheckAndRead(&cpuid)) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000000, &d[0], &d[1], &d[2], &d[3]); + if (d[0] < 0x80000001) + return False; + } + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000001, &d[0], &d[1], &d[2], &d[3]); + return (d[3] >> 26) & 1; + } +} + #endif diff --git a/7zip/C/CpuArch.h b/7zip/C/CpuArch.h index 51dd607b0..bd4293880 100644 --- a/7zip/C/CpuArch.h +++ b/7zip/C/CpuArch.h @@ -1,5 +1,5 @@ /* CpuArch.h -- CPU specific code -2017-06-30 : Igor Pavlov : Public domain */ +2018-02-18 : Igor Pavlov : Public domain */ #ifndef __CPU_ARCH_H #define __CPU_ARCH_H @@ -174,7 +174,7 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #ifndef MY_CPU_NAME #ifdef MY_CPU_LE #define MY_CPU_NAME "LE" - #elif MY_CPU_BE + #elif defined(MY_CPU_BE) #define MY_CPU_NAME "BE" #else /* @@ -318,15 +318,16 @@ enum void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d); -Bool x86cpuid_CheckAndRead(Cx86cpuid *p); +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p); int x86cpuid_GetFirm(const Cx86cpuid *p); #define x86cpuid_GetFamily(ver) (((ver >> 16) & 0xFF0) | ((ver >> 8) & 0xF)) #define x86cpuid_GetModel(ver) (((ver >> 12) & 0xF0) | ((ver >> 4) & 0xF)) #define x86cpuid_GetStepping(ver) (ver & 0xF) -Bool CPU_Is_InOrder(); -Bool CPU_Is_Aes_Supported(); +BoolInt CPU_Is_InOrder(); +BoolInt CPU_Is_Aes_Supported(); +BoolInt CPU_IsSupported_PageGB(); #endif diff --git a/7zip/C/DllSecur.c b/7zip/C/DllSecur.c index 90352ce10..5ea108ab8 100644 --- a/7zip/C/DllSecur.c +++ b/7zip/C/DllSecur.c @@ -1,5 +1,5 @@ /* DllSecur.c -- DLL loading security -2016-10-04 : Igor Pavlov : Public domain */ +2018-02-21 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -28,10 +28,31 @@ static const char * const g_Dlls = "CRYPTBASE\0" "OLEACC\0" "CLBCATQ\0" + "VERSION\0" ; #endif +void My_SetDefaultDllDirectories() +{ + #ifndef UNDER_CE + + OSVERSIONINFO vi; + vi.dwOSVersionInfoSize = sizeof(vi); + GetVersionEx(&vi); + if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) + { + Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + if (setDllDirs) + if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) + return; + } + + #endif +} + + void LoadSecurityDlls() { #ifndef UNDER_CE @@ -70,7 +91,7 @@ void LoadSecurityDlls() for (;;) { char c = *dll++; - buf[pos + k] = c; + buf[pos + k] = (Byte)c; k++; if (c == 0) break; diff --git a/7zip/C/DllSecur.h b/7zip/C/DllSecur.h index e01f2247d..e2a049ad2 100644 --- a/7zip/C/DllSecur.h +++ b/7zip/C/DllSecur.h @@ -1,5 +1,5 @@ /* DllSecur.h -- DLL loading for security -2016-06-08 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __DLL_SECUR_H #define __DLL_SECUR_H @@ -10,6 +10,7 @@ EXTERN_C_BEGIN #ifdef _WIN32 +void My_SetDefaultDllDirectories(); void LoadSecurityDlls(); #endif diff --git a/7zip/C/LzFind.c b/7zip/C/LzFind.c index 60fa124bd..3caf17bf4 100644 --- a/7zip/C/LzFind.c +++ b/7zip/C/LzFind.c @@ -1,5 +1,5 @@ /* LzFind.c -- Match finder for LZ algorithms -2017-06-10 : Igor Pavlov : Public domain */ +2018-07-08 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -138,7 +138,7 @@ static void MatchFinder_SetDefaultSettings(CMatchFinder *p) void MatchFinder_Construct(CMatchFinder *p) { - UInt32 i; + unsigned i; p->bufferBase = NULL; p->directInput = 0; p->hash = NULL; @@ -147,7 +147,7 @@ void MatchFinder_Construct(CMatchFinder *p) for (i = 0; i < 256; i++) { - UInt32 r = i; + UInt32 r = (UInt32)i; unsigned j; for (j = 0; j < 8; j++) r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); @@ -369,6 +369,8 @@ static void MatchFinder_Normalize(CMatchFinder *p) MatchFinder_ReduceOffsets(p, subValue); } + +MY_NO_INLINE static void MatchFinder_CheckLimits(CMatchFinder *p) { if (p->pos == kMaxValForNormalize) @@ -380,10 +382,16 @@ static void MatchFinder_CheckLimits(CMatchFinder *p) MatchFinder_SetLimits(p); } -static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, + +/* + (lenLimit > maxLen) +*/ +MY_FORCE_INLINE +static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, UInt32 maxLen) + UInt32 *distances, unsigned maxLen) { + /* son[_cyclicBufferPos] = curMatch; for (;;) { @@ -401,7 +409,8 @@ static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, break; if (maxLen < len) { - *distances++ = maxLen = len; + maxLen = len; + *distances++ = len; *distances++ = delta - 1; if (len == lenLimit) return distances; @@ -409,15 +418,58 @@ static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, } } } + */ + + const Byte *lim = cur + lenLimit; + son[_cyclicBufferPos] = curMatch; + do + { + UInt32 delta = pos - curMatch; + if (delta >= _cyclicBufferSize) + break; + { + ptrdiff_t diff; + curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; + diff = (ptrdiff_t)0 - delta; + if (cur[maxLen] == cur[maxLen + diff]) + { + const Byte *c = cur; + while (*c == c[diff]) + { + if (++c == lim) + { + distances[0] = (UInt32)(lim - cur); + distances[1] = delta - 1; + return distances + 2; + } + } + { + unsigned len = (unsigned)(c - cur); + if (maxLen < len) + { + maxLen = len; + distances[0] = (UInt32)len; + distances[1] = delta - 1; + distances += 2; + } + } + } + } + } + while (--cutValue); + + return distances; } + +MY_FORCE_INLINE UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, UInt32 *distances, UInt32 maxLen) { - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; for (;;) { UInt32 delta = pos - curMatch; @@ -427,9 +479,10 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt return distances; } { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); + UInt32 pair0 = pair[0]; if (pb[len] == cur[len]) { if (++len != lenLimit && pb[len] == cur[len]) @@ -438,11 +491,12 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt break; if (maxLen < len) { - *distances++ = maxLen = len; + maxLen = (UInt32)len; + *distances++ = (UInt32)len; *distances++ = delta - 1; if (len == lenLimit) { - *ptr1 = pair[0]; + *ptr1 = pair0; *ptr0 = pair[1]; return distances; } @@ -469,9 +523,9 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue) { - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; for (;;) { UInt32 delta = pos - curMatch; @@ -481,9 +535,9 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const return; } { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); if (pb[len] == cur[len]) { while (++len != lenLimit) @@ -521,13 +575,13 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const p->buffer++; \ if (++p->pos == p->posLimit) MatchFinder_CheckLimits(p); -#define MOVE_POS_RET MOVE_POS return offset; +#define MOVE_POS_RET MOVE_POS return (UInt32)offset; static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } #define GET_MATCHES_HEADER2(minLen, ret_op) \ - UInt32 lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \ - lenLimit = p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ + unsigned lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \ + lenLimit = (unsigned)p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ cur = p->buffer; #define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return 0) @@ -536,22 +590,22 @@ static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } #define MF_PARAMS(p) p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue #define GET_MATCHES_FOOTER(offset, maxLen) \ - offset = (UInt32)(GetMatchesSpec1(lenLimit, curMatch, MF_PARAMS(p), \ - distances + offset, maxLen) - distances); MOVE_POS_RET; + offset = (unsigned)(GetMatchesSpec1((UInt32)lenLimit, curMatch, MF_PARAMS(p), \ + distances + offset, (UInt32)maxLen) - distances); MOVE_POS_RET; #define SKIP_FOOTER \ - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; #define UPDATE_maxLen { \ ptrdiff_t diff = (ptrdiff_t)0 - d2; \ const Byte *c = cur + maxLen; \ const Byte *lim = cur + lenLimit; \ for (; c != lim; c++) if (*(c + diff) != *c) break; \ - maxLen = (UInt32)(c - cur); } + maxLen = (unsigned)(c - cur); } static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; + unsigned offset; GET_MATCHES_HEADER(2) HASH2_CALC; curMatch = p->hash[hv]; @@ -562,7 +616,7 @@ static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; + unsigned offset; GET_MATCHES_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv]; @@ -573,7 +627,8 @@ UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, d2, maxLen, offset, pos; + UInt32 h2, d2, pos; + unsigned maxLen, offset; UInt32 *hash; GET_MATCHES_HEADER(3) @@ -595,12 +650,12 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) { UPDATE_maxLen - distances[0] = maxLen; + distances[0] = (UInt32)maxLen; distances[1] = d2 - 1; offset = 2; if (maxLen == lenLimit) { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS_RET; } } @@ -610,7 +665,8 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, d2, d3, maxLen, offset, pos; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen, offset; UInt32 *hash; GET_MATCHES_HEADER(4) @@ -619,12 +675,12 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; + d2 = pos - hash [h2]; d3 = pos - (hash + kFix3HashSize)[h3]; curMatch = (hash + kFix4HashSize)[hv]; - hash[ h2] = pos; + hash [h2] = pos; (hash + kFix3HashSize)[h3] = pos; (hash + kFix4HashSize)[hv] = pos; @@ -633,7 +689,8 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) { - distances[0] = maxLen = 2; + maxLen = 2; + distances[0] = 2; distances[1] = d2 - 1; offset = 2; } @@ -649,10 +706,10 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (offset != 0) { UPDATE_maxLen - distances[(size_t)offset - 2] = maxLen; + distances[(size_t)offset - 2] = (UInt32)maxLen; if (maxLen == lenLimit) { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS_RET; } } @@ -675,13 +732,13 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; + d2 = pos - hash [h2]; d3 = pos - (hash + kFix3HashSize)[h3]; d4 = pos - (hash + kFix4HashSize)[h4]; curMatch = (hash + kFix5HashSize)[hv]; - hash[ h2] = pos; + hash [h2] = pos; (hash + kFix3HashSize)[h3] = pos; (hash + kFix4HashSize)[h4] = pos; (hash + kFix5HashSize)[hv] = pos; @@ -742,7 +799,8 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, d2, d3, maxLen, offset, pos; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen, offset; UInt32 *hash; GET_MATCHES_HEADER(4) @@ -751,12 +809,11 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; + d2 = pos - hash [h2]; d3 = pos - (hash + kFix3HashSize)[h3]; - curMatch = (hash + kFix4HashSize)[hv]; - hash[ h2] = pos; + hash [h2] = pos; (hash + kFix3HashSize)[h3] = pos; (hash + kFix4HashSize)[hv] = pos; @@ -765,7 +822,8 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) { - distances[0] = maxLen = 2; + maxLen = 2; + distances[0] = 2; distances[1] = d2 - 1; offset = 2; } @@ -781,7 +839,7 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (offset != 0) { UPDATE_maxLen - distances[(size_t)offset - 2] = maxLen; + distances[(size_t)offset - 2] = (UInt32)maxLen; if (maxLen == lenLimit) { p->son[p->cyclicBufferPos] = curMatch; @@ -792,7 +850,7 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (maxLen < 3) maxLen = 3; - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), + offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), distances + offset, maxLen) - (distances)); MOVE_POS_RET } @@ -809,13 +867,13 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; + d2 = pos - hash [h2]; d3 = pos - (hash + kFix3HashSize)[h3]; d4 = pos - (hash + kFix4HashSize)[h4]; curMatch = (hash + kFix5HashSize)[hv]; - hash[ h2] = pos; + hash [h2] = pos; (hash + kFix3HashSize)[h3] = pos; (hash + kFix4HashSize)[h4] = pos; (hash + kFix5HashSize)[hv] = pos; @@ -878,12 +936,12 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; + unsigned offset; GET_MATCHES_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), + offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), distances, 2) - (distances)); MOVE_POS_RET } @@ -941,7 +999,7 @@ static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) HASH4_CALC; hash = p->hash; curMatch = (hash + kFix4HashSize)[hv]; - hash[ h2] = + hash [h2] = (hash + kFix3HashSize)[h3] = (hash + kFix4HashSize)[hv] = p->pos; SKIP_FOOTER @@ -960,7 +1018,7 @@ static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) HASH5_CALC; hash = p->hash; curMatch = (hash + kFix5HashSize)[hv]; - hash[ h2] = + hash [h2] = (hash + kFix3HashSize)[h3] = (hash + kFix4HashSize)[h4] = (hash + kFix5HashSize)[hv] = p->pos; @@ -980,7 +1038,7 @@ static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) HASH4_CALC; hash = p->hash; curMatch = (hash + kFix4HashSize)[hv]; - hash[ h2] = + hash [h2] = (hash + kFix3HashSize)[h3] = (hash + kFix4HashSize)[hv] = p->pos; p->son[p->cyclicBufferPos] = curMatch; @@ -1000,7 +1058,7 @@ static void Hc5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) HASH5_CALC; hash = p->hash; curMatch = hash + kFix5HashSize)[hv]; - hash[ h2] = + hash [h2] = (hash + kFix3HashSize)[h3] = (hash + kFix4HashSize)[h4] = (hash + kFix5HashSize)[hv] = p->pos; diff --git a/7zip/C/LzFindMt.c b/7zip/C/LzFindMt.c index 65c9ffd73..bb0f42c30 100644 --- a/7zip/C/LzFindMt.c +++ b/7zip/C/LzFindMt.c @@ -1,5 +1,5 @@ /* LzFindMt.c -- multithreaded Match finder for LZ algorithms -2017-06-10 : Igor Pavlov : Public domain */ +2018-12-29 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -232,38 +232,57 @@ static void MatchFinderMt_GetNextBlock_Hash(CMatchFinderMt *p) #define kEmptyHashValue 0 -/* #define MFMT_GM_INLINE */ +#define MFMT_GM_INLINE #ifdef MFMT_GM_INLINE -#define NO_INLINE MY_FAST_CALL +/* + we use size_t for _cyclicBufferPos instead of UInt32 + to eliminate "movsx" BUG in old MSVC x64 compiler. +*/ -static Int32 NO_INLINE GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, - UInt32 *_distances, UInt32 _maxLen, const UInt32 *hash, Int32 limit, UInt32 size, UInt32 *posRes) +MY_NO_INLINE +static UInt32 *GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, + UInt32 *distances, UInt32 _maxLen, const UInt32 *hash, const UInt32 *limit, UInt32 size, UInt32 *posRes) { do { - UInt32 *distances = _distances + 1; - UInt32 curMatch = pos - *hash++; + UInt32 *_distances = ++distances; + UInt32 delta = *hash++; - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; UInt32 cutValue = _cutValue; - UInt32 maxLen = _maxLen; - for (;;) + unsigned maxLen = (unsigned)_maxLen; + + /* + if (size > 1) { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) + UInt32 delta = *hash; + if (delta < _cyclicBufferSize) { - *ptr0 = *ptr1 = kEmptyHashValue; - break; + UInt32 cyc1 = _cyclicBufferPos + 1; + CLzRef *pair = son + ((size_t)(cyc1 - delta + ((delta > cyc1) ? _cyclicBufferSize : 0)) << 1); + Byte b = *(cur + 1 - delta); + _distances[0] = pair[0]; + _distances[1] = b; } + } + */ + if (cutValue == 0 || delta >= _cyclicBufferSize) + { + *ptr0 = *ptr1 = kEmptyHashValue; + } + else + for(;;) + { { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((_cyclicBufferPos < delta) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); + UInt32 pair0 = *pair; if (pb[len] == cur[len]) { if (++len != lenLimit && pb[len] == cur[len]) @@ -272,54 +291,66 @@ static Int32 NO_INLINE GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte * break; if (maxLen < len) { - *distances++ = maxLen = len; + maxLen = len; + *distances++ = (UInt32)len; *distances++ = delta - 1; if (len == lenLimit) { - *ptr1 = pair[0]; - *ptr0 = pair[1]; + UInt32 pair1 = pair[1]; + *ptr1 = pair0; + *ptr0 = pair1; break; } } } - if (pb[len] < cur[len]) - { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; + UInt32 curMatch = pos - delta; + // delta = pos - *pair; + // delta = pos - pair[((UInt32)pb[len] - (UInt32)cur[len]) >> 31]; + if (pb[len] < cur[len]) + { + delta = pos - pair[1]; + *ptr1 = curMatch; + ptr1 = pair + 1; + len1 = len; + } + else + { + delta = pos - *pair; + *ptr0 = curMatch; + ptr0 = pair; + len0 = len; + } } } + if (--cutValue == 0 || delta >= _cyclicBufferSize) + { + *ptr0 = *ptr1 = kEmptyHashValue; + break; + } } pos++; _cyclicBufferPos++; cur++; { UInt32 num = (UInt32)(distances - _distances); - *_distances = num - 1; - _distances += num; - limit -= num; + _distances[-1] = num; } } - while (limit > 0 && --size != 0); + while (distances < limit && --size != 0); *posRes = pos; - return limit; + return distances; } #endif + + static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) { UInt32 numProcessed = 0; UInt32 curPos = 2; - UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); + UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); // * 2 distances[1] = p->hashNumAvail; @@ -369,8 +400,10 @@ static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) #else { UInt32 posRes; - curPos = limit - GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, - distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, (Int32)(limit - curPos), size, &posRes); + curPos = (UInt32)(GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, + distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, + distances + limit, + size, &posRes) - distances); p->hashBufPos += posRes - pos; cyclicBufferPos += posRes - pos; p->buffer += posRes - pos; diff --git a/7zip/C/LzFindMt.h b/7zip/C/LzFindMt.h index a6645f53c..ef431e3f5 100644 --- a/7zip/C/LzFindMt.h +++ b/7zip/C/LzFindMt.h @@ -1,5 +1,5 @@ /* LzFindMt.h -- multithreaded Match finder for LZ algorithms -2017-04-03 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_MT_H #define __LZ_FIND_MT_H @@ -19,10 +19,10 @@ EXTERN_C_BEGIN typedef struct _CMtSync { - Bool wasCreated; - Bool needStart; - Bool exit; - Bool stopWriting; + BoolInt wasCreated; + BoolInt needStart; + BoolInt exit; + BoolInt stopWriting; CThread thread; CAutoResetEvent canStart; @@ -30,8 +30,8 @@ typedef struct _CMtSync CAutoResetEvent wasStopped; CSemaphore freeSemaphore; CSemaphore filledSemaphore; - Bool csWasInitialized; - Bool csWasEntered; + BoolInt csWasInitialized; + BoolInt csWasEntered; CCriticalSection cs; UInt32 numProcessedBlocks; } CMtSync; diff --git a/7zip/C/Lzma2Dec.c b/7zip/C/Lzma2Dec.c index 63853c6df..4e138a4ae 100644 --- a/7zip/C/Lzma2Dec.c +++ b/7zip/C/Lzma2Dec.c @@ -1,5 +1,5 @@ /* Lzma2Dec.c -- LZMA2 Decoder -2017-04-03 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ /* #define SHOW_DEBUG_INFO */ @@ -14,28 +14,22 @@ #include "Lzma2Dec.h" /* -00000000 - EOS -00000001 U U - Uncompressed Reset Dic -00000010 U U - Uncompressed No Reset -100uuuuu U U P P - LZMA no reset -101uuuuu U U P P - LZMA reset state -110uuuuu U U P P S - LZMA reset state + new prop -111uuuuu U U P P S - LZMA reset state + new prop + reset dic +00000000 - End of data +00000001 U U - Uncompressed, reset dic, need reset state and set new prop +00000010 U U - Uncompressed, no reset +100uuuuu U U P P - LZMA, no reset +101uuuuu U U P P - LZMA, reset state +110uuuuu U U P P S - LZMA, reset state + set new prop +111uuuuu U U P P S - LZMA, reset state + set new prop, reset dic u, U - Unpack Size P - Pack Size S - Props */ -#define LZMA2_CONTROL_LZMA (1 << 7) -#define LZMA2_CONTROL_COPY_NO_RESET 2 #define LZMA2_CONTROL_COPY_RESET_DIC 1 -#define LZMA2_CONTROL_EOF 0 -#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & LZMA2_CONTROL_LZMA) == 0) - -#define LZMA2_GET_LZMA_MODE(p) (((p)->control >> 5) & 3) -#define LZMA2_IS_THERE_PROP(mode) ((mode) >= 2) +#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & (1 << 7)) == 0) #define LZMA2_LCLP_MAX 4 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) @@ -91,9 +85,11 @@ SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) void Lzma2Dec_Init(CLzma2Dec *p) { p->state = LZMA2_STATE_CONTROL; - p->needInitDic = True; - p->needInitState = True; - p->needInitProp = True; + p->needInitLevel = 0xE0; + p->isExtraMode = False; + p->unpackSize = 0; + + // p->decoder.dicPos = 0; // we can use it instead of full init LzmaDec_Init(&p->decoder); } @@ -102,19 +98,26 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) switch (p->state) { case LZMA2_STATE_CONTROL: + p->isExtraMode = False; p->control = b; - PRF(printf("\n %4X ", (unsigned)p->decoder.dicPos)); - PRF(printf(" %2X", (unsigned)b)); + PRF(printf("\n %8X", (unsigned)p->decoder.dicPos)); + PRF(printf(" %02X", (unsigned)b)); if (b == 0) return LZMA2_STATE_FINISHED; if (LZMA2_IS_UNCOMPRESSED_STATE(p)) { - if (b > 2) + if (b == LZMA2_CONTROL_COPY_RESET_DIC) + p->needInitLevel = 0xC0; + else if (b > 2 || p->needInitLevel == 0xE0) return LZMA2_STATE_ERROR; - p->unpackSize = 0; } else + { + if (b < p->needInitLevel) + return LZMA2_STATE_ERROR; + p->needInitLevel = 0; p->unpackSize = (UInt32)(b & 0x1F) << 16; + } return LZMA2_STATE_UNPACK0; case LZMA2_STATE_UNPACK0: @@ -124,8 +127,8 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) case LZMA2_STATE_UNPACK1: p->unpackSize |= (UInt32)b; p->unpackSize++; - PRF(printf(" %8u", (unsigned)p->unpackSize)); - return (LZMA2_IS_UNCOMPRESSED_STATE(p)) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; + PRF(printf(" %7u", (unsigned)p->unpackSize)); + return LZMA2_IS_UNCOMPRESSED_STATE(p) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; case LZMA2_STATE_PACK0: p->packSize = (UInt32)b << 8; @@ -134,9 +137,9 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) case LZMA2_STATE_PACK1: p->packSize |= (UInt32)b; p->packSize++; - PRF(printf(" %8u", (unsigned)p->packSize)); - return LZMA2_IS_THERE_PROP(LZMA2_GET_LZMA_MODE(p)) ? LZMA2_STATE_PROP: - (p->needInitProp ? LZMA2_STATE_ERROR : LZMA2_STATE_DATA); + // if (p->packSize < 5) return LZMA2_STATE_ERROR; + PRF(printf(" %5u", (unsigned)p->packSize)); + return (p->control & 0x40) ? LZMA2_STATE_PROP : LZMA2_STATE_DATA; case LZMA2_STATE_PROP: { @@ -145,13 +148,12 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) return LZMA2_STATE_ERROR; lc = b % 9; b /= 9; - p->decoder.prop.pb = b / 5; + p->decoder.prop.pb = (Byte)(b / 5); lp = b % 5; if (lc + lp > LZMA2_LCLP_MAX) return LZMA2_STATE_ERROR; - p->decoder.prop.lc = lc; - p->decoder.prop.lp = lp; - p->needInitProp = False; + p->decoder.prop.lc = (Byte)lc; + p->decoder.prop.lp = (Byte)lp; return LZMA2_STATE_DATA; } } @@ -167,7 +169,7 @@ static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, const Byte *src, SizeT s p->processedPos += (UInt32)size; } -void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState); +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState); SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, @@ -230,12 +232,7 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, if (p->state == LZMA2_STATE_DATA) { - Bool initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); - if (initDic) - p->needInitProp = p->needInitState = True; - else if (p->needInitDic) - break; - p->needInitDic = False; + BoolInt initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); LzmaDec_InitDicAndState(&p->decoder, initDic, False); } @@ -257,23 +254,17 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, if (p->state == LZMA2_STATE_DATA) { - unsigned mode = LZMA2_GET_LZMA_MODE(p); - Bool initDic = (mode == 3); - Bool initState = (mode != 0); - if ((!initDic && p->needInitDic) || (!initState && p->needInitState)) - break; - + BoolInt initDic = (p->control >= 0xE0); + BoolInt initState = (p->control >= 0xA0); LzmaDec_InitDicAndState(&p->decoder, initDic, initState); - p->needInitDic = False; - p->needInitState = False; p->state = LZMA2_STATE_DATA_CONT; } if (inCur > p->packSize) inCur = (SizeT)p->packSize; - - res = LzmaDec_DecodeToDic(&p->decoder, dicPos + outCur, src, &inCur, curFinishMode, status); + res = LzmaDec_DecodeToDic(&p->decoder, dicPos + outCur, src, &inCur, curFinishMode, status); + src += inCur; *srcLen += inCur; p->packSize -= (UInt32)inCur; @@ -310,6 +301,129 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, } + + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, + const Byte *src, SizeT *srcLen, + int checkFinishBlock) +{ + SizeT inSize = *srcLen; + *srcLen = 0; + + while (p->state != LZMA2_STATE_ERROR) + { + if (p->state == LZMA2_STATE_FINISHED) + return (ELzma2ParseStatus)LZMA_STATUS_FINISHED_WITH_MARK; + + if (outSize == 0 && !checkFinishBlock) + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + + if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT) + { + if (*srcLen == inSize) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + (*srcLen)++; + + p->state = Lzma2Dec_UpdateState(p, *src++); + + if (p->state == LZMA2_STATE_UNPACK0) + { + // if (p->decoder.dicPos != 0) + if (p->control == LZMA2_CONTROL_COPY_RESET_DIC || p->control >= 0xE0) + return LZMA2_PARSE_STATUS_NEW_BLOCK; + // if (outSize == 0) return LZMA_STATUS_NOT_FINISHED; + } + + // The following code can be commented. + // It's not big problem, if we read additional input bytes. + // It will be stopped later in LZMA2_STATE_DATA / LZMA2_STATE_DATA_CONT state. + + if (outSize == 0 && p->state != LZMA2_STATE_FINISHED) + { + // checkFinishBlock is true. So we expect that block must be finished, + // We can return LZMA_STATUS_NOT_SPECIFIED or LZMA_STATUS_NOT_FINISHED here + // break; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + } + + if (p->state == LZMA2_STATE_DATA) + return LZMA2_PARSE_STATUS_NEW_CHUNK; + + continue; + } + + if (outSize == 0) + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + + { + SizeT inCur = inSize - *srcLen; + + if (LZMA2_IS_UNCOMPRESSED_STATE(p)) + { + if (inCur == 0) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + if (inCur > p->unpackSize) + inCur = p->unpackSize; + if (inCur > outSize) + inCur = outSize; + p->decoder.dicPos += inCur; + src += inCur; + *srcLen += inCur; + outSize -= inCur; + p->unpackSize -= (UInt32)inCur; + p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT; + } + else + { + p->isExtraMode = True; + + if (inCur == 0) + { + if (p->packSize != 0) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + } + else if (p->state == LZMA2_STATE_DATA) + { + p->state = LZMA2_STATE_DATA_CONT; + if (*src != 0) + { + // first byte of lzma chunk must be Zero + *srcLen += 1; + p->packSize--; + break; + } + } + + if (inCur > p->packSize) + inCur = (SizeT)p->packSize; + + src += inCur; + *srcLen += inCur; + p->packSize -= (UInt32)inCur; + + if (p->packSize == 0) + { + SizeT rem = outSize; + if (rem > p->unpackSize) + rem = p->unpackSize; + p->decoder.dicPos += rem; + p->unpackSize -= (UInt32)rem; + outSize -= rem; + if (p->unpackSize == 0) + p->state = LZMA2_STATE_CONTROL; + } + } + } + } + + p->state = LZMA2_STATE_ERROR; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; +} + + + + SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT outSize = *destLen, inSize = *srcLen; diff --git a/7zip/C/Lzma2Dec.h b/7zip/C/Lzma2Dec.h index 917af990d..b8ddeac89 100644 --- a/7zip/C/Lzma2Dec.h +++ b/7zip/C/Lzma2Dec.h @@ -1,5 +1,5 @@ /* Lzma2Dec.h -- LZMA2 Decoder -2017-04-03 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __LZMA2_DEC_H #define __LZMA2_DEC_H @@ -12,25 +12,24 @@ EXTERN_C_BEGIN typedef struct { - CLzmaDec decoder; - UInt32 packSize; - UInt32 unpackSize; unsigned state; Byte control; - Bool needInitDic; - Bool needInitState; - Bool needInitProp; + Byte needInitLevel; + Byte isExtraMode; + Byte _pad_; + UInt32 packSize; + UInt32 unpackSize; + CLzmaDec decoder; } CLzma2Dec; #define Lzma2Dec_Construct(p) LzmaDec_Construct(&(p)->decoder) -#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc); -#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc); +#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc) +#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc) SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); void Lzma2Dec_Init(CLzma2Dec *p); - /* finishMode: It has meaning only if the decoding reaches output limit (*destLen or dicLimit). @@ -53,6 +52,47 @@ SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); +/* ---------- LZMA2 block and chunk parsing ---------- */ + +/* +Lzma2Dec_Parse() parses compressed data stream up to next independent block or next chunk data. +It can return LZMA_STATUS_* code or LZMA2_PARSE_STATUS_* code: + - LZMA2_PARSE_STATUS_NEW_BLOCK - there is new block, and 1 additional byte (control byte of next block header) was read from input. + - LZMA2_PARSE_STATUS_NEW_CHUNK - there is new chunk, and only lzma2 header of new chunk was read. + CLzma2Dec::unpackSize contains unpack size of that chunk +*/ + +typedef enum +{ +/* + LZMA_STATUS_NOT_SPECIFIED // data error + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED // + LZMA_STATUS_NEEDS_MORE_INPUT + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK // unused +*/ + LZMA2_PARSE_STATUS_NEW_BLOCK = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + 1, + LZMA2_PARSE_STATUS_NEW_CHUNK +} ELzma2ParseStatus; + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, // output size + const Byte *src, SizeT *srcLen, + int checkFinishBlock // set (checkFinishBlock = 1), if it must read full input data, if decoder.dicPos reaches blockMax position. + ); + +/* +LZMA2 parser doesn't decode LZMA chunks, so we must read + full input LZMA chunk to decode some part of LZMA chunk. + +Lzma2Dec_GetUnpackExtra() returns the value that shows + max possible number of output bytes that can be output by decoder + at current input positon. +*/ + +#define Lzma2Dec_GetUnpackExtra(p) ((p)->isExtraMode ? (p)->unpackSize : 0); + + /* ---------- One Call Interface ---------- */ /* diff --git a/7zip/C/Lzma2DecMt.c b/7zip/C/Lzma2DecMt.c new file mode 100644 index 000000000..988643d95 --- /dev/null +++ b/7zip/C/Lzma2DecMt.c @@ -0,0 +1,1082 @@ +/* Lzma2DecMt.c -- LZMA2 Decoder Multi-thread +2019-02-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) +#define PRF_STR_INT_2(s, d1, d2) PRF(printf("\n" s " %d %d\n", (unsigned)d1, (unsigned)d2)) + +// #define _7ZIP_ST + +#include "Alloc.h" + +#include "Lzma2Dec.h" +#include "Lzma2DecMt.h" + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +#define LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT (1 << 28) + +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p) +{ + p->inBufSize_ST = 1 << 20; + p->outStep_ST = 1 << 20; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->outBlockMax = LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT; + p->inBlockMax = p->outBlockMax + p->outBlockMax / 16; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CLzma2DecMtThread ---------- */ + +typedef struct +{ + CLzma2Dec dec; + Byte dec_created; + Byte needInit; + + Byte *outBuf; + size_t outBufSize; + + EMtDecParseState state; + ELzma2ParseStatus parseStatus; + + size_t inPreSize; + size_t outPreSize; + + size_t inCodeSize; + size_t outCodeSize; + SRes codeRes; + + CAlignOffsetAlloc alloc; + + Byte mtPad[1 << 7]; +} CLzma2DecMtThread; + +#endif + + +/* ---------- CLzma2DecMt ---------- */ + +typedef struct +{ + // ISzAllocPtr alloc; + ISzAllocPtr allocMid; + + CAlignOffsetAlloc alignOffsetAlloc; + CLzma2DecMtProps props; + Byte prop; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + + BoolInt finishMode; + BoolInt outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + BoolInt readWasFinished; + SRes readRes; + + Byte *inBuf; + size_t inBufSize; + Byte dec_created; + CLzma2Dec dec; + + size_t inPos; + size_t inLim; + + #ifndef _7ZIP_ST + UInt64 outProcessed_Parse; + BoolInt mtc_WasConstructed; + CMtDec mtc; + CLzma2DecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CLzma2DecMt; + + + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CLzma2DecMt *p = (CLzma2DecMt *)ISzAlloc_Alloc(alloc, sizeof(CLzma2DecMt)); + if (!p) + return NULL; + + // p->alloc = alloc; + p->allocMid = allocMid; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + p->alignOffsetAlloc.baseAlloc = alloc; + + p->inBuf = NULL; + p->inBufSize = 0; + p->dec_created = False; + + // Lzma2DecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + t->dec_created = False; + t->outBuf = NULL; + t->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_FreeOutBufs(CLzma2DecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->outBuf) + { + ISzAlloc_Free(p->allocMid, t->outBuf); + t->outBuf = NULL; + t->outBufSize = 0; + } + } +} + +#endif + + +static void Lzma2DecMt_FreeSt(CLzma2DecMt *p) +{ + if (p->dec_created) + { + Lzma2Dec_Free(&p->dec, &p->alignOffsetAlloc.vt); + p->dec_created = False; + } + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void Lzma2DecMt_Destroy(CLzma2DecMtHandle pp) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + Lzma2DecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + Lzma2Dec_FreeProbs(&t->dec, &t->alloc.vt); // p->alloc !!! + t->dec_created = False; + } + } + } + Lzma2DecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CLzma2DecMt *me = (CLzma2DecMt *)obj; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + PRF_STR_INT_2("Parse", coderIndex, cc->srcSize); + + cc->state = MTDEC_PARSE_CONTINUE; + + if (cc->startCall) + { + if (!t->dec_created) + { + Lzma2Dec_Construct(&t->dec); + t->dec_created = True; + AlignOffsetAlloc_CreateVTable(&t->alloc); + { + /* (1 << 12) is expected size of one way in data cache. + We optimize alignment for cache line size of 128 bytes and smaller */ + const unsigned kNumAlignBits = 12; + const unsigned kNumCacheLineBits = 7; /* <= kNumAlignBits */ + t->alloc.numAlignBits = kNumAlignBits; + t->alloc.offset = ((UInt32)coderIndex * ((1 << 11) + (1 << 8) + (1 << 6))) & ((1 << kNumAlignBits) - (1 << kNumCacheLineBits)); + t->alloc.baseAlloc = me->alignOffsetAlloc.baseAlloc; + } + } + Lzma2Dec_Init(&t->dec); + + t->inPreSize = 0; + t->outPreSize = 0; + // t->blockWasFinished = False; + // t->finishedWithMark = False; + t->parseStatus = (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; + t->state = MTDEC_PARSE_CONTINUE; + + t->inCodeSize = 0; + t->outCodeSize = 0; + t->codeRes = SZ_OK; + + // (cc->srcSize == 0) is allowed + } + + { + ELzma2ParseStatus status; + BoolInt overflow; + UInt32 unpackRem = 0; + + int checkFinishBlock = True; + size_t limit = me->props.outBlockMax; + if (me->outSize_Defined) + { + UInt64 rem = me->outSize - me->outProcessed_Parse; + if (limit >= rem) + { + limit = (size_t)rem; + if (!me->finishMode) + checkFinishBlock = False; + } + } + + // checkFinishBlock = False, if we want to decode partial data + // that must be finished at position <= outBlockMax. + + { + const SizeT srcOrig = cc->srcSize; + SizeT srcSize_Point = 0; + SizeT dicPos_Point = 0; + + cc->srcSize = 0; + overflow = False; + + for (;;) + { + SizeT srcCur = srcOrig - cc->srcSize; + + status = Lzma2Dec_Parse(&t->dec, + limit - t->dec.decoder.dicPos, + cc->src + cc->srcSize, &srcCur, + checkFinishBlock); + + cc->srcSize += srcCur; + + if (status == LZMA2_PARSE_STATUS_NEW_CHUNK) + { + if (t->dec.unpackSize > me->props.outBlockMax - t->dec.decoder.dicPos) + { + overflow = True; + break; + } + continue; + } + + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + if (t->dec.decoder.dicPos == 0) + continue; + // we decode small blocks in one thread + if (t->dec.decoder.dicPos >= (1 << 14)) + break; + dicPos_Point = t->dec.decoder.dicPos; + srcSize_Point = cc->srcSize; + continue; + } + + if ((int)status == LZMA_STATUS_NOT_FINISHED && checkFinishBlock + // && limit == t->dec.decoder.dicPos + // && limit == me->props.outBlockMax + ) + { + overflow = True; + break; + } + + unpackRem = Lzma2Dec_GetUnpackExtra(&t->dec); + break; + } + + if (dicPos_Point != 0 + && (int)status != LZMA2_PARSE_STATUS_NEW_BLOCK + && (int)status != LZMA_STATUS_FINISHED_WITH_MARK + && (int)status != LZMA_STATUS_NOT_SPECIFIED) + { + // we revert to latest newBlock state + status = LZMA2_PARSE_STATUS_NEW_BLOCK; + unpackRem = 0; + t->dec.decoder.dicPos = dicPos_Point; + cc->srcSize = srcSize_Point; + overflow = False; + } + } + + t->inPreSize += cc->srcSize; + t->parseStatus = status; + + if (overflow) + cc->state = MTDEC_PARSE_OVERFLOW; + else + { + size_t dicPos = t->dec.decoder.dicPos; + + if ((int)status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + cc->state = MTDEC_PARSE_NEW; + cc->srcSize--; // we don't need control byte of next block + t->inPreSize--; + } + else + { + cc->state = MTDEC_PARSE_END; + if ((int)status != LZMA_STATUS_FINISHED_WITH_MARK) + { + // (status == LZMA_STATUS_NOT_SPECIFIED) + // (status == LZMA_STATUS_NOT_FINISHED) + if (unpackRem != 0) + { + /* we also reserve space for max possible number of output bytes of current LZMA chunk */ + SizeT rem = limit - dicPos; + if (rem > unpackRem) + rem = unpackRem; + dicPos += rem; + } + } + } + + me->outProcessed_Parse += dicPos; + } + + cc->outPos = dicPos; + t->outPreSize = (size_t)dicPos; + } + + t->state = cc->state; + return; + } +} + + +static SRes Lzma2DecMt_MtCallback_PreCode(void *pp, unsigned coderIndex) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + Byte *dest = t->outBuf; + + if (t->inPreSize == 0) + { + t->codeRes = SZ_ERROR_DATA; + return t->codeRes; + } + + if (!dest || t->outBufSize < t->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + t->outBuf = NULL; + t->outBufSize = 0; + } + + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, t->outPreSize + // + (1 << 28) + ); + // Sleep(200); + if (!dest) + return SZ_ERROR_MEM; + t->outBuf = dest; + t->outBufSize = t->outPreSize; + } + + t->dec.decoder.dic = dest; + t->dec.decoder.dicBufSize = t->outPreSize; + + t->needInit = True; + + return Lzma2Dec_AllocateProbs(&t->dec, me->prop, &t->alloc.vt); // alloc.vt +} + + +static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + UNUSED_VAR(srcFinished) + + PRF_STR_INT_2("Code", coderIndex, srcSize); + + *inCodePos = t->inCodeSize; + *outCodePos = 0; + *stop = True; + + if (t->needInit) + { + Lzma2Dec_Init(&t->dec); + t->needInit = False; + } + + { + ELzmaStatus status; + size_t srcProcessed = srcSize; + BoolInt blockWasFinished = + ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK); + + SRes res = Lzma2Dec_DecodeToDic(&t->dec, + t->outPreSize, + src, &srcProcessed, + blockWasFinished ? LZMA_FINISH_END : LZMA_FINISH_ANY, + &status); + + t->codeRes = res; + + t->inCodeSize += srcProcessed; + *inCodePos = t->inCodeSize; + t->outCodeSize = t->dec.decoder.dicPos; + *outCodePos = t->dec.decoder.dicPos; + + if (res != SZ_OK) + return res; + + if (srcProcessed == srcSize) + *stop = False; + + if (blockWasFinished) + { + if (srcSize != srcProcessed) + return SZ_ERROR_FAIL; + + if (t->inPreSize == t->inCodeSize) + { + if (t->outPreSize != t->outCodeSize) + return SZ_ERROR_FAIL; + *stop = True; + } + } + else + { + if (t->outPreSize == t->outCodeSize) + *stop = True; + } + + return SZ_OK; + } +} + + +#define LZMA2DECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, + BoolInt *needContinue, BoolInt *canRecode) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + const CLzma2DecMtThread *t = &me->coders[coderIndex]; + size_t size = t->outCodeSize; + const Byte *data = t->outBuf; + BoolInt needContinue2 = True; + + PRF_STR_INT_2("Write", coderIndex, srcSize); + + *needContinue = False; + *canRecode = True; + UNUSED_VAR(src) + UNUSED_VAR(srcSize) + + if ( + // t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + t->state == MTDEC_PARSE_OVERFLOW + || t->state == MTDEC_PARSE_END) + needContinue2 = False; + + + if (!needWriteToStream) + return SZ_OK; + + me->mtc.inProcessed += t->inCodeSize; + + if (t->codeRes == SZ_OK) + if ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK) + if (t->outPreSize != t->outCodeSize + || t->inPreSize != t->inCodeSize) + return SZ_ERROR_FAIL; + + *canRecode = False; + + if (me->outStream) + { + for (;;) + { + size_t cur = size; + size_t written; + if (cur > LZMA2DECMT_STREAM_WRITE_STEP) + cur = LZMA2DECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + me->outProcessed += written; + // me->mtc.writtenTotal += written; + if (written != cur) + return SZ_ERROR_WRITE; + data += cur; + size -= cur; + if (size == 0) + { + *needContinue = needContinue2; + return SZ_OK; + } + RINOK(MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0)); + } + } + + return SZ_ERROR_FAIL; + /* + if (size > me->outBufSize) + return SZ_ERROR_OUTPUT_EOF; + memcpy(me->outBuf, data, size); + me->outBufSize -= size; + me->outBuf += size; + *needContinue = needContinue2; + return SZ_OK; + */ +} + +#endif + + +static SRes Lzma2Dec_Prepare_ST(CLzma2DecMt *p) +{ + if (!p->dec_created) + { + Lzma2Dec_Construct(&p->dec); + p->dec_created = True; + } + + RINOK(Lzma2Dec_Allocate(&p->dec, p->prop, &p->alignOffsetAlloc.vt)); + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + Lzma2Dec_Init(&p->dec); + + return SZ_OK; +} + + +static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p + #ifndef _7ZIP_ST + , BoolInt tMode + #endif + ) +{ + SizeT wrPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CLzma2Dec *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + Lzma2DecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + RINOK(Lzma2Dec_Prepare_ST(p)); + + dec = &p->dec; + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + wrPos = dec->decoder.dicPos; + + for (;;) + { + SizeT dicPos; + SizeT size; + ELzmaFinishMode finishMode; + SizeT inProcessed; + ELzmaStatus status; + SRes res; + + SizeT outProcessed; + BoolInt outFinished; + BoolInt needStop; + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + // p->readProcessed += inLim; + // inLim -= 5; p->readWasFinished = True; // for test + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + dicPos = dec->decoder.dicPos; + { + SizeT next = dec->decoder.dicBufSize; + if (next - wrPos > p->props.outStep_ST) + next = wrPos + p->props.outStep_ST; + size = next - dicPos; + } + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (SizeT)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + inProcessed = inLim - inPos; + + res = Lzma2Dec_DecodeToDic(dec, dicPos + size, inData + inPos, &inProcessed, finishMode, &status); + + inPos += inProcessed; + p->inProcessed += inProcessed; + outProcessed = dec->decoder.dicPos - dicPos; + p->outProcessed += outProcessed; + + outFinished = (p->outSize_Defined && p->outSize <= p->outProcessed); + + needStop = (res != SZ_OK + || (inProcessed == 0 && outProcessed == 0) + || status == LZMA_STATUS_FINISHED_WITH_MARK + || (!p->finishMode && outFinished)); + + if (needStop || outProcessed >= size) + { + SRes res2; + { + size_t writeSize = dec->decoder.dicPos - wrPos; + size_t written = ISeqOutStream_Write(p->outStream, dec->decoder.dic + wrPos, writeSize); + res2 = (written == writeSize) ? SZ_OK : SZ_ERROR_WRITE; + } + + if (dec->decoder.dicPos == dec->decoder.dicBufSize) + dec->decoder.dicPos = 0; + wrPos = dec->decoder.dicPos; + + RINOK(res2); + + if (needStop) + { + if (res != SZ_OK) + return res; + + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + { + if (p->finishMode) + { + if (p->outSize_Defined && p->outSize != p->outProcessed) + return SZ_ERROR_DATA; + } + return SZ_OK; + } + + if (!p->finishMode && outFinished) + return SZ_OK; + + if (status == LZMA_STATUS_NEEDS_MORE_INPUT) + return SZ_ERROR_INPUT_EOF; + + return SZ_ERROR_DATA; + } + } + + if (p->progress) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + RINOK(ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed)); + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + } +} + + + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, const UInt64 *outDataSize, int finishMode, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + UInt64 *inProcessed, + // UInt64 *outProcessed, + int *isMT, + ICompressProgress *progress) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + #ifndef _7ZIP_ST + BoolInt tMode; + #endif + + *inProcessed = 0; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->readWasFinished = False; + + *isMT = False; + + + #ifndef _7ZIP_ST + + tMode = False; + + // p->mtc.parseRes = SZ_OK; + + // p->mtc.numFilledThreads = 0; + // p->mtc.crossStart = 0; + // p->mtc.crossEnd = 0; + // p->mtc.allocError_for_Read_BlockIndex = 0; + // p->mtc.isAllocError = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback vt; + + Lzma2DecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + + // p->outBuf = NULL; + // p->outBufSize = 0; + /* + if (!outStream) + { + // p->outBuf = outBuf; + // p->outBufSize = *outBufSize; + // *outBufSize = 0; + return SZ_ERROR_PARAM; + } + */ + + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->alignOffsetAlloc.baseAlloc; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.inBufSize = p->props.inBufSize_MT; + + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = Lzma2DecMt_MtCallback_Parse; + vt.PreCode = Lzma2DecMt_MtCallback_PreCode; + vt.Code = Lzma2DecMt_MtCallback_Code; + vt.Write = Lzma2DecMt_MtCallback_Write; + + { + BoolInt needContinue = False; + + SRes res = MtDec_Code(&p->mtc); + + /* + if (!outStream) + *outBufSize = p->outBuf - outBuf; + */ + + *inProcessed = p->mtc.inProcessed; + + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + res = p->mtc.mtProgress.res; + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + if (res == SZ_OK) + return p->mtc.readRes; + return res; + } + + tMode = True; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->inProcessed = p->mtc.inProcessed; + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = Lzma2Dec_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + ); + + *inProcessed = p->inProcessed; + + // res = SZ_OK; // for test + if (res == SZ_OK && p->readRes != SZ_OK) + res = p->readRes; + + /* + #ifndef _7ZIP_ST + if (res == SZ_OK && tMode && p->mtc.parseRes != SZ_OK) + res = p->mtc.parseRes; + #endif + */ + + return res; + } +} + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->inPos = 0; + p->inLim = 0; + + return Lzma2Dec_Prepare_ST(p); +} + + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + ELzmaFinishMode finishMode; + SRes readRes; + size_t size = *outSize; + + *outSize = 0; + *inStreamProcessed = 0; + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (size_t)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + readRes = SZ_OK; + + for (;;) + { + SizeT inCur; + SizeT outCur; + ELzmaStatus status; + SRes res; + + if (p->inPos == p->inLim && readRes == SZ_OK) + { + p->inPos = 0; + p->inLim = p->props.inBufSize_ST; + readRes = ISeqInStream_Read(p->inStream, p->inBuf, &p->inLim); + } + + inCur = p->inLim - p->inPos; + outCur = size; + + res = Lzma2Dec_DecodeToBuf(&p->dec, data, &outCur, + p->inBuf + p->inPos, &inCur, finishMode, &status); + + p->inPos += inCur; + p->inProcessed += inCur; + *inStreamProcessed += inCur; + p->outProcessed += outCur; + *outSize += outCur; + size -= outCur; + data += outCur; + + if (res != 0) + return res; + + /* + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + return readRes; + + if (size == 0 && status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (p->finishMode && p->outSize_Defined && p->outProcessed >= p->outSize) + return SZ_ERROR_DATA; + return readRes; + } + */ + + if (inCur == 0 && outCur == 0) + return readRes; + } +} diff --git a/7zip/C/Lzma2DecMt.h b/7zip/C/Lzma2DecMt.h new file mode 100644 index 000000000..7791c310b --- /dev/null +++ b/7zip/C/Lzma2DecMt.h @@ -0,0 +1,79 @@ +/* Lzma2DecMt.h -- LZMA2 Decoder Multi-thread +2018-02-17 : Igor Pavlov : Public domain */ + +#ifndef __LZMA2_DEC_MT_H +#define __LZMA2_DEC_MT_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t outBlockMax; + size_t inBlockMax; + #endif +} CLzma2DecMtProps; + +/* init to single-thread mode */ +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p); + + +/* ---------- CLzma2DecMtHandle Interface ---------- */ + +/* Lzma2DecMt_ * functions can return the following exit codes: +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_PARAM - Incorrect paramater in props + SZ_ERROR_WRITE - ISeqOutStream write callback error + // SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output + SZ_ERROR_PROGRESS - some break from progress callback + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) +*/ + +typedef void * CLzma2DecMtHandle; + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void Lzma2DecMt_Destroy(CLzma2DecMtHandle p); + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - if lzma2 stream must be finished + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + + // out variables: + UInt64 *inProcessed, + int *isMT, /* out: (*isMT == 0), if single thread decoding was used */ + + // UInt64 *outProcessed, + ICompressProgress *progress); + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream); + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed); + + +EXTERN_C_END + +#endif diff --git a/7zip/C/Lzma2Enc.c b/7zip/C/Lzma2Enc.c index e970944cd..5c1ad4931 100644 --- a/7zip/C/Lzma2Enc.c +++ b/7zip/C/Lzma2Enc.c @@ -1,5 +1,5 @@ /* Lzma2Enc.c -- LZMA2 Encoder -2017-08-28 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -115,7 +115,7 @@ SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 k ISzAllocPtr alloc, ISzAllocPtr allocBig); SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); void LzmaEnc_Finish(CLzmaEncHandle pp); @@ -133,7 +133,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, size_t packSize = packSizeLimit; UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX; unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0); - Bool useCopyBlock; + BoolInt useCopyBlock; SRes res; *packSizeRes = 0; @@ -369,9 +369,11 @@ typedef struct ISeqOutStream *outStream; Byte *outBuf; - size_t outBufSize; + size_t outBuf_Rem; /* remainder in outBuf */ + + size_t outBufSize; /* size of allocated outBufs[i] */ size_t outBufsDataSizes[MTCODER__BLOCKS_MAX]; - Bool mtCoder_WasConstructed; + BoolInt mtCoder_WasConstructed; CMtCoder mtCoder; Byte *outBufs[MTCODER__BLOCKS_MAX]; @@ -666,7 +668,7 @@ static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned out if (!dest) { - dest = ISzAlloc_Alloc(me->alloc, me->outBufSize); + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); if (!dest) return SZ_ERROR_MEM; me->outBufs[outBufIndex] = dest; @@ -674,7 +676,8 @@ static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned out MtProgressThunk_CreateVTable(&progressThunk); progressThunk.mtProgress = &me->mtCoder.mtProgress; - progressThunk.index = coderIndex; + progressThunk.inSize = 0; + progressThunk.outSize = 0; res = Lzma2Enc_EncodeMt1(me, &me->coders[coderIndex], @@ -698,10 +701,10 @@ static SRes Lzma2Enc_MtCallback_Write(void *pp, unsigned outBufIndex) if (me->outStream) return ISeqOutStream_Write(me->outStream, data, size) == size ? SZ_OK : SZ_ERROR_WRITE; - if (size > me->outBufSize) + if (size > me->outBuf_Rem) return SZ_ERROR_OUTPUT_EOF; memcpy(me->outBuf, data, size); - me->outBufSize -= size; + me->outBuf_Rem -= size; me->outBuf += size; return SZ_OK; } @@ -720,10 +723,10 @@ SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, CLzma2Enc *p = (CLzma2Enc *)pp; if (inStream && inData) - return E_INVALIDARG; + return SZ_ERROR_PARAM; if (outStream && outBuf) - return E_INVALIDARG; + return SZ_ERROR_PARAM; { unsigned i; @@ -748,11 +751,11 @@ SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, p->outStream = outStream; p->outBuf = NULL; - p->outBufSize = 0; + p->outBuf_Rem = 0; if (!outStream) { p->outBuf = outBuf; - p->outBufSize = *outBufSize; + p->outBuf_Rem = *outBufSize; *outBufSize = 0; } diff --git a/7zip/C/Lzma86Enc.c b/7zip/C/Lzma86Enc.c index a15b58d6c..2617bab8e 100644 --- a/7zip/C/Lzma86Enc.c +++ b/7zip/C/Lzma86Enc.c @@ -1,5 +1,5 @@ /* Lzma86Enc.c -- LZMA + x86 (BCJ) Filter Encoder -2016-05-16 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -18,7 +18,7 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, { size_t outSize2 = *destLen; Byte *filteredStream; - Bool useFilter; + BoolInt useFilter; int mainResult = SZ_ERROR_OUTPUT_EOF; CLzmaEncProps props; LzmaEncProps_Init(&props); @@ -56,7 +56,7 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, { size_t minSize = 0; - Bool bestIsFiltered = False; + BoolInt bestIsFiltered = False; /* passes for SZ_FILTER_AUTO: 0 - BCJ + LZMA @@ -71,7 +71,7 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, size_t outSizeProcessed = outSize2 - LZMA86_HEADER_SIZE; size_t outPropsSize = 5; SRes curRes; - Bool curModeIsFiltered = (numPasses > 1 && i == numPasses - 1); + BoolInt curModeIsFiltered = (numPasses > 1 && i == numPasses - 1); if (curModeIsFiltered && !bestIsFiltered) break; if (useFilter && i == 0) diff --git a/7zip/C/LzmaDec.c b/7zip/C/LzmaDec.c index e96fa975b..ba3e1dd50 100644 --- a/7zip/C/LzmaDec.c +++ b/7zip/C/LzmaDec.c @@ -1,12 +1,13 @@ /* LzmaDec.c -- LZMA Decoder -2017-04-03 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" -#include "LzmaDec.h" - #include +/* #include "CpuArch.h" */ +#include "LzmaDec.h" + #define kNumTopBits 24 #define kTopValue ((UInt32)1 << kNumTopBits) @@ -18,15 +19,22 @@ #define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } -#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); #define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); #define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ { UPDATE_0(p); i = (i + i); A0; } else \ { UPDATE_1(p); i = (i + i) + 1; A1; } -#define GET_BIT(p, i) GET_BIT2(p, i, ; , ;) -#define TREE_GET_BIT(probs, i) { GET_BIT((probs + i), i); } +#define TREE_GET_BIT(probs, i) { GET_BIT2(probs + i, i, ;, ;); } + +#define REV_BIT(p, i, A0, A1) IF_BIT_0(p + i) \ + { UPDATE_0(p + i); A0; } else \ + { UPDATE_1(p + i); A1; } +#define REV_BIT_VAR( p, i, m) REV_BIT(p, i, i += m; m += m, m += m; i += m; ) +#define REV_BIT_CONST(p, i, m) REV_BIT(p, i, i += m; , i += m * 2; ) +#define REV_BIT_LAST( p, i, m) REV_BIT(p, i, i -= m , ; ) + #define TREE_DECODE(probs, limit, i) \ { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; } @@ -46,16 +54,19 @@ i -= 0x40; } #endif -#define NORMAL_LITER_DEC GET_BIT(prob + symbol, symbol) +#define NORMAL_LITER_DEC TREE_GET_BIT(prob, symbol) #define MATCHED_LITER_DEC \ - matchByte <<= 1; \ - bit = (matchByte & offs); \ - probLit = prob + offs + bit + symbol; \ - GET_BIT2(probLit, symbol, offs &= ~bit, offs &= bit) + matchByte += matchByte; \ + bit = offs; \ + offs &= matchByte; \ + probLit = prob + (offs + bit + symbol); \ + GET_BIT2(probLit, symbol, offs ^= bit; , ;) + + #define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } -#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0_CHECK range = bound; #define UPDATE_1_CHECK range -= bound; code -= bound; #define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \ @@ -66,25 +77,28 @@ { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } +#define REV_BIT_CHECK(p, i, m) IF_BIT_0_CHECK(p + i) \ + { UPDATE_0_CHECK; i += m; m += m; } else \ + { UPDATE_1_CHECK; m += m; i += m; } + + #define kNumPosBitsMax 4 #define kNumPosStatesMax (1 << kNumPosBitsMax) #define kLenNumLowBits 3 #define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) #define kLenNumHighBits 8 #define kLenNumHighSymbols (1 << kLenNumHighBits) -#define LenChoice 0 -#define LenChoice2 (LenChoice + 1) -#define LenLow (LenChoice2 + 1) -#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) -#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) +#define LenLow 0 +#define LenHigh (LenLow + 2 * (kNumPosStatesMax << kLenNumLowBits)) #define kNumLenProbs (LenHigh + kLenNumHighSymbols) +#define LenChoice LenLow +#define LenChoice2 (LenLow + (1 << kLenNumLowBits)) #define kNumStates 12 +#define kNumStates2 16 #define kNumLitStates 7 #define kStartPosModelIndex 4 @@ -98,54 +112,117 @@ #define kAlignTableSize (1 << kNumAlignBits) #define kMatchMinLen 2 -#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) +#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) + +/* External ASM code needs same CLzmaProb array layout. So don't change it. */ -#define IsMatch 0 -#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) +/* (probs_1664) is faster and better for code size at some platforms */ +/* +#ifdef MY_CPU_X86_OR_AMD64 +*/ +#define kStartOffset 1664 +#define GET_PROBS p->probs_1664 +/* +#define GET_PROBS p->probs + kStartOffset +#else +#define kStartOffset 0 +#define GET_PROBS p->probs +#endif +*/ + +#define SpecPos (-kStartOffset) +#define IsRep0Long (SpecPos + kNumFullDistances) +#define RepLenCoder (IsRep0Long + (kNumStates2 << kNumPosBitsMax)) +#define LenCoder (RepLenCoder + kNumLenProbs) +#define IsMatch (LenCoder + kNumLenProbs) +#define Align (IsMatch + (kNumStates2 << kNumPosBitsMax)) +#define IsRep (Align + kAlignTableSize) #define IsRepG0 (IsRep + kNumStates) #define IsRepG1 (IsRepG0 + kNumStates) #define IsRepG2 (IsRepG1 + kNumStates) -#define IsRep0Long (IsRepG2 + kNumStates) -#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) -#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) -#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) -#define LenCoder (Align + kAlignTableSize) -#define RepLenCoder (LenCoder + kNumLenProbs) -#define Literal (RepLenCoder + kNumLenProbs) - -#define LZMA_BASE_SIZE 1846 -#define LZMA_LIT_SIZE 0x300 +#define PosSlot (IsRepG2 + kNumStates) +#define Literal (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define NUM_BASE_PROBS (Literal + kStartOffset) -#if Literal != LZMA_BASE_SIZE -StopCompilingDueBUG +#if Align != 0 && kStartOffset != 0 + #error Stop_Compiling_Bad_LZMA_kAlign #endif -#define LzmaProps_GetNumProbs(p) (Literal + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) +#if NUM_BASE_PROBS != 1984 + #error Stop_Compiling_Bad_LZMA_PROBS +#endif + + +#define LZMA_LIT_SIZE 0x300 + +#define LzmaProps_GetNumProbs(p) (NUM_BASE_PROBS + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) + + +#define CALC_POS_STATE(processedPos, pbMask) (((processedPos) & (pbMask)) << 4) +#define COMBINED_PS_STATE (posState + state) +#define GET_LEN_STATE (posState) #define LZMA_DIC_MIN (1 << 12) -/* First LZMA-symbol is always decoded. -And it decodes new LZMA-symbols while (buf < bufLimit), but "buf" is without last normalization +/* +p->remainLen : shows status of LZMA decoder: + < kMatchSpecLenStart : normal remain + = kMatchSpecLenStart : finished + = kMatchSpecLenStart + 1 : need init range coder + = kMatchSpecLenStart + 2 : need init range coder and state +*/ + +/* ---------- LZMA_DECODE_REAL ---------- */ +/* +LzmaDec_DecodeReal_3() can be implemented in external ASM file. +3 - is the code compatibility version of that function for check at link time. +*/ + +#define LZMA_DECODE_REAL LzmaDec_DecodeReal_3 + +/* +LZMA_DECODE_REAL() +In: + RangeCoder is normalized + if (p->dicPos == limit) + { + LzmaDec_TryDummy() was called before to exclude LITERAL and MATCH-REP cases. + So first symbol can be only MATCH-NON-REP. And if that MATCH-NON-REP symbol + is not END_OF_PAYALOAD_MARKER, then function returns error code. + } + +Processing: + first LZMA symbol will be decoded in any case + All checks for limits are at the end of main loop, + It will decode new LZMA-symbols while (p->buf < bufLimit && dicPos < limit), + RangeCoder is still without last normalization when (p->buf < bufLimit) is being checked. + Out: + RangeCoder is normalized Result: SZ_OK - OK SZ_ERROR_DATA - Error p->remainLen: < kMatchSpecLenStart : normal remain = kMatchSpecLenStart : finished - = kMatchSpecLenStart + 1 : Flush marker (unused now) - = kMatchSpecLenStart + 2 : State Init Marker (unused now) */ -static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte *bufLimit) -{ - CLzmaProb *probs = p->probs; - unsigned state = p->state; +#ifdef _LZMA_DEC_OPT + +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit); + +#else + +static +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +{ + CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3]; unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; - unsigned lpMask = ((unsigned)1 << (p->prop.lp)) - 1; unsigned lc = p->prop.lc; + unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); Byte *dic = p->dic; SizeT dicBufSize = p->dicBufSize; @@ -164,17 +241,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte CLzmaProb *prob; UInt32 bound; unsigned ttt; - unsigned posState = processedPos & pbMask; + unsigned posState = CALC_POS_STATE(processedPos, pbMask); - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0(prob) { unsigned symbol; UPDATE_0(prob); prob = probs + Literal; if (processedPos != 0 || checkDicSize != 0) - prob += ((UInt32)LZMA_LIT_SIZE * (((processedPos & lpMask) << lc) + - (dic[(dicPos == 0 ? dicBufSize : dicPos) - 1] >> (8 - lc)))); + prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); processedPos++; if (state < kNumLitStates) @@ -240,13 +316,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte else { UPDATE_1(prob); + /* + // that case was checked before with kBadRepCode if (checkDicSize == 0 && processedPos == 0) return SZ_ERROR_DATA; + */ prob = probs + IsRepG0 + state; IF_BIT_0(prob) { UPDATE_0(prob); - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0(prob) { UPDATE_0(prob); @@ -299,7 +378,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; offset = 0; lim = (1 << kLenNumLowBits); } @@ -310,15 +389,15 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; - lim = (1 << kLenNumMidBits); + lim = (1 << kLenNumLowBits); } else { UPDATE_1(probLen); probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; + offset = kLenNumLowSymbols * 2; lim = (1 << kLenNumHighBits); } } @@ -331,7 +410,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; len = 1; TREE_GET_BIT(probLen, len); TREE_GET_BIT(probLen, len); @@ -345,7 +424,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); len = 1; TREE_GET_BIT(probLen, len); TREE_GET_BIT(probLen, len); @@ -356,7 +435,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte UPDATE_1(probLen); probLen = prob + LenHigh; TREE_DECODE(probLen, (1 << kLenNumHighBits), len); - len += kLenNumLowSymbols + kLenNumMidSymbols; + len += kLenNumLowSymbols * 2; } } } @@ -376,16 +455,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte if (posSlot < kEndPosModelIndex) { distance <<= numDirectBits; - prob = probs + SpecPos + distance - posSlot - 1; + prob = probs + SpecPos; { - UInt32 mask = 1; - unsigned i = 1; + UInt32 m = 1; + distance++; do { - GET_BIT2(prob + i, i, ; , distance |= mask); - mask <<= 1; + REV_BIT_VAR(prob, distance, m); } - while (--numDirectBits != 0); + while (--numDirectBits); + distance -= m; } } else @@ -412,19 +491,20 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte } */ } - while (--numDirectBits != 0); + while (--numDirectBits); prob = probs + Align; distance <<= kNumAlignBits; { unsigned i = 1; - GET_BIT2(prob + i, i, ; , distance |= 1); - GET_BIT2(prob + i, i, ; , distance |= 2); - GET_BIT2(prob + i, i, ; , distance |= 4); - GET_BIT2(prob + i, i, ; , distance |= 8); + REV_BIT_CONST(prob, i, 1); + REV_BIT_CONST(prob, i, 2); + REV_BIT_CONST(prob, i, 4); + REV_BIT_LAST (prob, i, 8); + distance |= i; } if (distance == (UInt32)0xFFFFFFFF) { - len += kMatchSpecLenStart; + len = kMatchSpecLenStart; state -= kNumStates; break; } @@ -435,20 +515,12 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte rep2 = rep1; rep1 = rep0; rep0 = distance + 1; - if (checkDicSize == 0) - { - if (distance >= processedPos) - { - p->dicPos = dicPos; - return SZ_ERROR_DATA; - } - } - else if (distance >= checkDicSize) + state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) { p->dicPos = dicPos; return SZ_ERROR_DATA; } - state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; } len += kMatchMinLen; @@ -467,7 +539,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte curLen = ((rem < len) ? (unsigned)rem : len); pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); - processedPos += curLen; + processedPos += (UInt32)curLen; len -= curLen; if (curLen <= dicBufSize - pos) @@ -475,7 +547,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte Byte *dest = dic + dicPos; ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; const Byte *lim = dest + curLen; - dicPos += curLen; + dicPos += (SizeT)curLen; do *(dest) = (Byte)*(dest + src); while (++dest != lim); @@ -500,17 +572,18 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte p->buf = buf; p->range = range; p->code = code; - p->remainLen = len; + p->remainLen = (UInt32)len; p->dicPos = dicPos; p->processedPos = processedPos; p->reps[0] = rep0; p->reps[1] = rep1; p->reps[2] = rep2; p->reps[3] = rep3; - p->state = state; + p->state = (UInt32)state; return SZ_OK; } +#endif static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) { @@ -519,7 +592,7 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) Byte *dic = p->dic; SizeT dicPos = p->dicPos; SizeT dicBufSize = p->dicBufSize; - unsigned len = p->remainLen; + unsigned len = (unsigned)p->remainLen; SizeT rep0 = p->reps[0]; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ SizeT rem = limit - dicPos; if (rem < len) @@ -528,8 +601,8 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len) p->checkDicSize = p->prop.dicSize; - p->processedPos += len; - p->remainLen -= len; + p->processedPos += (UInt32)len; + p->remainLen -= (UInt32)len; while (len != 0) { len--; @@ -540,6 +613,14 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) } } + +#define kRange0 0xFFFFFFFF +#define kBound0 ((kRange0 >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1)) +#define kBadRepCode (kBound0 + (((kRange0 - kBound0) >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1))) +#if kBadRepCode != (0xC0000000 - 0x400) + #error Stop_Compiling_Bad_LZMA_Check +#endif + static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) { do @@ -550,9 +631,13 @@ static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte UInt32 rem = p->prop.dicSize - p->processedPos; if (limit - p->dicPos > rem) limit2 = p->dicPos + rem; + + if (p->processedPos == 0) + if (p->code >= kBadRepCode) + return SZ_ERROR_DATA; } - - RINOK(LzmaDec_DecodeReal(p, limit2, bufLimit)); + + RINOK(LZMA_DECODE_REAL(p, limit2, bufLimit)); if (p->checkDicSize == 0 && p->processedPos >= p->prop.dicSize) p->checkDicSize = p->prop.dicSize; @@ -561,9 +646,6 @@ static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte } while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart); - if (p->remainLen > kMatchSpecLenStart) - p->remainLen = kMatchSpecLenStart; - return 0; } @@ -580,17 +662,17 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS UInt32 range = p->range; UInt32 code = p->code; const Byte *bufLimit = buf + inSize; - const CLzmaProb *probs = p->probs; - unsigned state = p->state; + const CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; ELzmaDummy res; { const CLzmaProb *prob; UInt32 bound; unsigned ttt; - unsigned posState = (p->processedPos) & ((1 << p->prop.pb) - 1); + unsigned posState = CALC_POS_STATE(p->processedPos, (1 << p->prop.pb) - 1); - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK @@ -618,10 +700,11 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS { unsigned bit; const CLzmaProb *probLit; - matchByte <<= 1; - bit = (matchByte & offs); - probLit = prob + offs + bit + symbol; - GET_BIT2_CHECK(probLit, symbol, offs &= ~bit, offs &= bit) + matchByte += matchByte; + bit = offs; + offs &= matchByte; + probLit = prob + (offs + bit + symbol); + GET_BIT2_CHECK(probLit, symbol, offs ^= bit; , ; ) } while (symbol < 0x100); } @@ -648,7 +731,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK; - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK; @@ -691,7 +774,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(probLen) { UPDATE_0_CHECK; - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; offset = 0; limit = 1 << kLenNumLowBits; } @@ -702,15 +785,15 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(probLen) { UPDATE_0_CHECK; - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; - limit = 1 << kLenNumMidBits; + limit = 1 << kLenNumLowBits; } else { UPDATE_1_CHECK; probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; + offset = kLenNumLowSymbols * 2; limit = 1 << kLenNumHighBits; } } @@ -722,7 +805,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS { unsigned posSlot; prob = probs + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << + ((len < kNumLenToPosStates - 1 ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); if (posSlot >= kStartPosModelIndex) @@ -733,7 +816,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS if (posSlot < kEndPosModelIndex) { - prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits) - posSlot - 1; + prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits); } else { @@ -745,17 +828,18 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS code -= range & (((code - range) >> 31) - 1); /* if (code >= range) code -= range; */ } - while (--numDirectBits != 0); + while (--numDirectBits); prob = probs + Align; numDirectBits = kNumAlignBits; } { unsigned i = 1; + unsigned m = 1; do { - GET_BIT_CHECK(prob + i, i); + REV_BIT_CHECK(prob, i, m); } - while (--numDirectBits != 0); + while (--numDirectBits); } } } @@ -766,20 +850,19 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS } -void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState) +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState) { - p->needFlush = 1; - p->remainLen = 0; + p->remainLen = kMatchSpecLenStart + 1; p->tempBufSize = 0; if (initDic) { p->processedPos = 0; p->checkDicSize = 0; - p->needInitState = 1; + p->remainLen = kMatchSpecLenStart + 2; } if (initState) - p->needInitState = 1; + p->remainLen = kMatchSpecLenStart + 2; } void LzmaDec_Init(CLzmaDec *p) @@ -788,53 +871,54 @@ void LzmaDec_Init(CLzmaDec *p) LzmaDec_InitDicAndState(p, True, True); } -static void LzmaDec_InitStateReal(CLzmaDec *p) -{ - SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); - SizeT i; - CLzmaProb *probs = p->probs; - for (i = 0; i < numProbs; i++) - probs[i] = kBitModelTotal >> 1; - p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; - p->state = 0; - p->needInitState = 0; -} SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT inSize = *srcLen; (*srcLen) = 0; - LzmaDec_WriteRem(p, dicLimit); *status = LZMA_STATUS_NOT_SPECIFIED; - while (p->remainLen != kMatchSpecLenStart) + if (p->remainLen > kMatchSpecLenStart) { - int checkEndMarkNow; + for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) + p->tempBuf[p->tempBufSize++] = *src++; + if (p->tempBufSize != 0 && p->tempBuf[0] != 0) + return SZ_ERROR_DATA; + if (p->tempBufSize < RC_INIT_SIZE) + { + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + p->code = + ((UInt32)p->tempBuf[1] << 24) + | ((UInt32)p->tempBuf[2] << 16) + | ((UInt32)p->tempBuf[3] << 8) + | ((UInt32)p->tempBuf[4]); + p->range = 0xFFFFFFFF; + p->tempBufSize = 0; + + if (p->remainLen > kMatchSpecLenStart + 1) + { + SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); + SizeT i; + CLzmaProb *probs = p->probs; + for (i = 0; i < numProbs; i++) + probs[i] = kBitModelTotal >> 1; + p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; + p->state = 0; + } - if (p->needFlush) - { - for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) - p->tempBuf[p->tempBufSize++] = *src++; - if (p->tempBufSize < RC_INIT_SIZE) - { - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (p->tempBuf[0] != 0) - return SZ_ERROR_DATA; - p->code = - ((UInt32)p->tempBuf[1] << 24) - | ((UInt32)p->tempBuf[2] << 16) - | ((UInt32)p->tempBuf[3] << 8) - | ((UInt32)p->tempBuf[4]); - p->range = 0xFFFFFFFF; - p->needFlush = 0; - p->tempBufSize = 0; - } + p->remainLen = 0; + } + + LzmaDec_WriteRem(p, dicLimit); + + while (p->remainLen != kMatchSpecLenStart) + { + int checkEndMarkNow = 0; - checkEndMarkNow = 0; if (p->dicPos >= dicLimit) { if (p->remainLen == 0 && p->code == 0) @@ -855,9 +939,6 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr checkEndMarkNow = 1; } - if (p->needInitState) - LzmaDec_InitStateReal(p); - if (p->tempBufSize == 0) { SizeT processed; @@ -898,10 +979,10 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr p->tempBufSize = rem; if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) { - int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, rem); + int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, (SizeT)rem); if (dummyRes == DUMMY_ERROR) { - (*srcLen) += lookAhead; + (*srcLen) += (SizeT)lookAhead; *status = LZMA_STATUS_NEEDS_MORE_INPUT; return SZ_OK; } @@ -924,17 +1005,20 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr return SZ_ERROR_FAIL; /* some internal error */ lookAhead -= rem; } - (*srcLen) += lookAhead; + (*srcLen) += (SizeT)lookAhead; src += lookAhead; - inSize -= lookAhead; + inSize -= (SizeT)lookAhead; p->tempBufSize = 0; } } - if (p->code == 0) - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return (p->code == 0) ? SZ_OK : SZ_ERROR_DATA; + + if (p->code != 0) + return SZ_ERROR_DATA; + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return SZ_OK; } + SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT outSize = *destLen; @@ -1011,10 +1095,10 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size) if (d >= (9 * 5 * 5)) return SZ_ERROR_UNSUPPORTED; - p->lc = d % 9; + p->lc = (Byte)(d % 9); d /= 9; - p->pb = d / 5; - p->lp = d % 5; + p->pb = (Byte)(d / 5); + p->lp = (Byte)(d % 5); return SZ_OK; } @@ -1026,9 +1110,10 @@ static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAl { LzmaDec_FreeProbs(p, alloc); p->probs = (CLzmaProb *)ISzAlloc_Alloc(alloc, numProbs * sizeof(CLzmaProb)); - p->numProbs = numProbs; if (!p->probs) return SZ_ERROR_MEM; + p->probs_1664 = p->probs + 1664; + p->numProbs = numProbs; } return SZ_OK; } diff --git a/7zip/C/LzmaDec.h b/7zip/C/LzmaDec.h index d6af92203..1f0927ab1 100644 --- a/7zip/C/LzmaDec.h +++ b/7zip/C/LzmaDec.h @@ -1,5 +1,5 @@ /* LzmaDec.h -- LZMA Decoder -2017-04-03 : Igor Pavlov : Public domain */ +2018-04-21 : Igor Pavlov : Public domain */ #ifndef __LZMA_DEC_H #define __LZMA_DEC_H @@ -12,11 +12,13 @@ EXTERN_C_BEGIN /* _LZMA_PROB32 can increase the speed on some CPUs, but memory usage for CLzmaDec::probs will be doubled in that case */ +typedef #ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 + UInt32 #else -#define CLzmaProb UInt16 + UInt16 #endif + CLzmaProb; /* ---------- LZMA Properties ---------- */ @@ -25,7 +27,10 @@ EXTERN_C_BEGIN typedef struct _CLzmaProps { - unsigned lc, lp, pb; + Byte lc; + Byte lp; + Byte pb; + Byte _pad_; UInt32 dicSize; } CLzmaProps; @@ -47,32 +52,34 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size); typedef struct { + /* Don't change this structure. ASM code can use it. */ CLzmaProps prop; CLzmaProb *probs; + CLzmaProb *probs_1664; Byte *dic; - const Byte *buf; - UInt32 range, code; - SizeT dicPos; SizeT dicBufSize; + SizeT dicPos; + const Byte *buf; + UInt32 range; + UInt32 code; UInt32 processedPos; UInt32 checkDicSize; - unsigned state; UInt32 reps[4]; - unsigned remainLen; - int needFlush; - int needInitState; + UInt32 state; + UInt32 remainLen; + UInt32 numProbs; unsigned tempBufSize; Byte tempBuf[LZMA_REQUIRED_INPUT_MAX]; } CLzmaDec; -#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; } +#define LzmaDec_Construct(p) { (p)->dic = NULL; (p)->probs = NULL; } void LzmaDec_Init(CLzmaDec *p); /* There are two types of LZMA streams: - 0) Stream with end mark. That end mark adds about 6 bytes to compressed size. - 1) Stream without end mark. You must know exact uncompressed size to decompress such stream. */ + - Stream with end mark. That end mark adds about 6 bytes to compressed size. + - Stream without end mark. You must know exact uncompressed size to decompress such stream. */ typedef enum { @@ -132,8 +139,8 @@ LzmaDec_Allocate* can return: SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc); -SRes LzmaDec_Allocate(CLzmaDec *state, const Byte *prop, unsigned propsSize, ISzAllocPtr alloc); -void LzmaDec_Free(CLzmaDec *state, ISzAllocPtr alloc); +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); +void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc); /* ---------- Dictionary Interface ---------- */ @@ -142,7 +149,7 @@ void LzmaDec_Free(CLzmaDec *state, ISzAllocPtr alloc); You must work with CLzmaDec variables directly in this interface. STEPS: - LzmaDec_Constr() + LzmaDec_Construct() LzmaDec_Allocate() for (each new stream) { diff --git a/7zip/C/LzmaEnc.c b/7zip/C/LzmaEnc.c index 9b7e69104..46a0db000 100644 --- a/7zip/C/LzmaEnc.c +++ b/7zip/C/LzmaEnc.c @@ -1,5 +1,5 @@ /* LzmaEnc.c -- LZMA Encoder -2017-06-22 : Igor Pavlov : Public domain */ +2019-01-10: Igor Pavlov : Public domain */ #include "Precomp.h" @@ -26,15 +26,6 @@ static unsigned g_STAT_OFFSET = 0; #define kLzmaMaxHistorySize ((UInt32)3 << 29) /* #define kLzmaMaxHistorySize ((UInt32)7 << 29) */ -#define kBlockSizeMax ((1 << LZMA_NUM_BLOCK_SIZE_BITS) - 1) - -#define kBlockSize (9 << 10) -#define kUnpackBlockSize (1 << 18) -#define kMatchArraySize (1 << 21) -#define kMatchRecordMaxSize ((LZMA_MATCH_LEN_MAX * 2 + 3) * LZMA_MATCH_LEN_MAX) - -#define kNumMaxDirectBits (31) - #define kNumTopBits 24 #define kTopValue ((UInt32)1 << kNumTopBits) @@ -47,6 +38,8 @@ static unsigned g_STAT_OFFSET = 0; #define kNumBitPriceShiftBits 4 #define kBitPrice (1 << kNumBitPriceShiftBits) +#define REP_LEN_COUNT 64 + void LzmaEncProps_Init(CLzmaEncProps *p) { p->level = 5; @@ -111,9 +104,9 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2) #define BSR2_RET(pos, res) { unsigned long zz; _BitScanReverse(&zz, (pos)); res = (zz + zz) + ((pos >> (zz - 1)) & 1); } -static UInt32 GetPosSlot1(UInt32 pos) +static unsigned GetPosSlot1(UInt32 pos) { - UInt32 res; + unsigned res; BSR2_RET(pos, res); return res; } @@ -146,18 +139,18 @@ static void LzmaEnc_FastPosInit(Byte *g_FastPos) /* we can use ((limit - pos) >> 31) only if (pos < ((UInt32)1 << 31)) */ /* -#define BSR2_RET(pos, res) { UInt32 zz = 6 + ((kNumLogBits - 1) & \ +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ (0 - (((((UInt32)1 << (kNumLogBits + 6)) - 1) - pos) >> 31))); \ res = p->g_FastPos[pos >> zz] + (zz * 2); } */ /* -#define BSR2_RET(pos, res) { UInt32 zz = 6 + ((kNumLogBits - 1) & \ +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ (0 - (((((UInt32)1 << (kNumLogBits)) - 1) - (pos >> 6)) >> 31))); \ res = p->g_FastPos[pos >> zz] + (zz * 2); } */ -#define BSR2_RET(pos, res) { UInt32 zz = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ +#define BSR2_RET(pos, res) { unsigned zz = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ res = p->g_FastPos[pos >> zz] + (zz * 2); } /* @@ -168,32 +161,35 @@ static void LzmaEnc_FastPosInit(Byte *g_FastPos) #define GetPosSlot1(pos) p->g_FastPos[pos] #define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } -#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos]; else BSR2_RET(pos, res); } +#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos & (kNumFullDistances - 1)]; else BSR2_RET(pos, res); } #endif #define LZMA_NUM_REPS 4 -typedef unsigned CState; +typedef UInt16 CState; +typedef UInt16 CExtra; typedef struct { UInt32 price; - CState state; - int prev1IsChar; - int prev2; - - UInt32 posPrev2; - UInt32 backPrev2; - - UInt32 posPrev; - UInt32 backPrev; - UInt32 backs[LZMA_NUM_REPS]; + CExtra extra; + // 0 : normal + // 1 : LIT : MATCH + // > 1 : MATCH (extra-1) : LIT : REP0 (len) + UInt32 len; + UInt32 dist; + UInt32 reps[LZMA_NUM_REPS]; } COptimal; -#define kNumOpts (1 << 12) + +// 18.06 +#define kNumOpts (1 << 11) +#define kPackReserve (kNumOpts * 8) +// #define kNumOpts (1 << 12) +// #define kPackReserve (1 + kNumOpts * 2) #define kNumLenToPosStates 4 #define kNumPosSlotBits 6 @@ -201,22 +197,21 @@ typedef struct #define kDicLogSizeMax 32 #define kDistTableSizeMax (kDicLogSizeMax * 2) - #define kNumAlignBits 4 #define kAlignTableSize (1 << kNumAlignBits) #define kAlignMask (kAlignTableSize - 1) #define kStartPosModelIndex 4 #define kEndPosModelIndex 14 -#define kNumPosModels (kEndPosModelIndex - kStartPosModelIndex) - #define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) +typedef #ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 + UInt32 #else -#define CLzmaProb UInt16 + UInt16 #endif + CLzmaProb; #define LZMA_PB_MAX 4 #define LZMA_LC_MAX 8 @@ -224,15 +219,11 @@ typedef struct #define LZMA_NUM_PB_STATES_MAX (1 << LZMA_PB_MAX) - #define kLenNumLowBits 3 #define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) #define kLenNumHighBits 8 #define kLenNumHighSymbols (1 << kLenNumHighBits) - -#define kLenNumSymbolsTotal (kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) +#define kLenNumSymbolsTotal (kLenNumLowSymbols * 2 + kLenNumHighSymbols) #define LZMA_MATCH_LEN_MIN 2 #define LZMA_MATCH_LEN_MAX (LZMA_MATCH_LEN_MIN + kLenNumSymbolsTotal - 1) @@ -242,27 +233,31 @@ typedef struct typedef struct { - CLzmaProb choice; - CLzmaProb choice2; - CLzmaProb low[LZMA_NUM_PB_STATES_MAX << kLenNumLowBits]; - CLzmaProb mid[LZMA_NUM_PB_STATES_MAX << kLenNumMidBits]; + CLzmaProb low[LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)]; CLzmaProb high[kLenNumHighSymbols]; } CLenEnc; typedef struct { - CLenEnc p; - UInt32 tableSize; + unsigned tableSize; UInt32 prices[LZMA_NUM_PB_STATES_MAX][kLenNumSymbolsTotal]; - UInt32 counters[LZMA_NUM_PB_STATES_MAX]; + // UInt32 prices1[LZMA_NUM_PB_STATES_MAX][kLenNumLowSymbols * 2]; + // UInt32 prices2[kLenNumSymbolsTotal]; } CLenPriceEnc; +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices[posState][(size_t)(len) - LZMA_MATCH_LEN_MIN]) + +/* +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices2[(size_t)(len) - 2] + ((p)->prices1[posState][((len) - 2) & (kLenNumLowSymbols * 2 - 1)] & (((len) - 2 - kLenNumLowSymbols * 2) >> 9))) +*/ typedef struct { UInt32 range; - Byte cache; + unsigned cache; UInt64 low; UInt64 cacheSize; Byte *buf; @@ -278,69 +273,77 @@ typedef struct { CLzmaProb *litProbs; - UInt32 state; + unsigned state; UInt32 reps[LZMA_NUM_REPS]; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; CLzmaProb isRep[kNumStates]; CLzmaProb isRepG0[kNumStates]; CLzmaProb isRepG1[kNumStates]; CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb posEncoders[kNumFullDistances]; - CLenPriceEnc lenEnc; - CLenPriceEnc repLenEnc; + CLenEnc lenProbs; + CLenEnc repLenProbs; + } CSaveState; +typedef UInt32 CProbPrice; + + typedef struct { void *matchFinderObj; IMatchFinder matchFinder; - UInt32 optimumEndIndex; - UInt32 optimumCurrentIndex; + unsigned optCur; + unsigned optEnd; - UInt32 longestMatchLength; - UInt32 numPairs; + unsigned longestMatchLen; + unsigned numPairs; UInt32 numAvail; - UInt32 numFastBytes; - UInt32 additionalOffset; + unsigned state; + unsigned numFastBytes; + unsigned additionalOffset; UInt32 reps[LZMA_NUM_REPS]; - UInt32 state; + unsigned lpMask, pbMask; + CLzmaProb *litProbs; + CRangeEnc rc; + + UInt32 backRes; unsigned lc, lp, pb; - unsigned lpMask, pbMask; unsigned lclp; - CLzmaProb *litProbs; - - Bool fastMode; - Bool writeEndMark; - Bool finished; - Bool multiThread; - Bool needInit; + BoolInt fastMode; + BoolInt writeEndMark; + BoolInt finished; + BoolInt multiThread; + BoolInt needInit; + // BoolInt _maxMode; UInt64 nowPos64; - UInt32 matchPriceCount; - UInt32 alignPriceCount; + unsigned matchPriceCount; + // unsigned alignPriceCount; + int repLenEncCounter; - UInt32 distTableSize; + unsigned distTableSize; UInt32 dictSize; SRes result; - CRangeEnc rc; - #ifndef _7ZIP_ST - Bool mtMode; + BoolInt mtMode; + // begin of CMatchFinderMt is used in LZ thread CMatchFinderMt matchFinderMt; + // end of CMatchFinderMt is used in BT and HASH threads #endif CMatchFinder matchFinderBase; @@ -349,33 +352,37 @@ typedef struct Byte pad[128]; #endif - COptimal opt[kNumOpts]; - - #ifndef LZMA_LOG_BSR - Byte g_FastPos[1 << kNumLogBits]; - #endif + // LZ thread + CProbPrice ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; - UInt32 ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2 + 1]; + UInt32 alignPrices[kAlignTableSize]; UInt32 posSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; UInt32 distancesPrices[kNumLenToPosStates][kNumFullDistances]; - UInt32 alignPrices[kAlignTableSize]; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; CLzmaProb isRep[kNumStates]; CLzmaProb isRepG0[kNumStates]; CLzmaProb isRepG1[kNumStates]; CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; - CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb posEncoders[kNumFullDistances]; + CLenEnc lenProbs; + CLenEnc repLenProbs; + + #ifndef LZMA_LOG_BSR + Byte g_FastPos[1 << kNumLogBits]; + #endif + CLenPriceEnc lenEnc; CLenPriceEnc repLenEnc; + COptimal opt[kNumOpts]; + CSaveState saveState; #ifndef _7ZIP_ST @@ -384,58 +391,62 @@ typedef struct } CLzmaEnc; + +#define COPY_ARR(dest, src, arr) memcpy(dest->arr, src->arr, sizeof(src->arr)); + void LzmaEnc_SaveState(CLzmaEncHandle pp) { CLzmaEnc *p = (CLzmaEnc *)pp; CSaveState *dest = &p->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; + dest->state = p->state; + + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; + + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << p->lclp) * sizeof(CLzmaProb)); } + void LzmaEnc_RestoreState(CLzmaEncHandle pp) { CLzmaEnc *dest = (CLzmaEnc *)pp; const CSaveState *p = &dest->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; + dest->state = p->state; - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; + + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); + memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << dest->lclp) * sizeof(CLzmaProb)); } + + SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) { CLzmaEnc *p = (CLzmaEnc *)pp; @@ -462,9 +473,10 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) p->lp = props.lp; p->pb = props.pb; p->fastMode = (props.algo == 0); + // p->_maxMode = True; p->matchFinderBase.btMode = (Byte)(props.btMode ? 1 : 0); { - UInt32 numHashBytes = 4; + unsigned numHashBytes = 4; if (props.btMode) { if (props.numHashBytes < 2) @@ -501,14 +513,19 @@ void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) } +#define kState_Start 0 +#define kState_LitAfterMatch 4 +#define kState_LitAfterRep 5 +#define kState_MatchAfterLit 7 +#define kState_RepAfterLit 8 -static const int kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; -static const int kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; -static const int kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; -static const int kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; - -#define IsCharState(s) ((s) < 7) +static const Byte kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; +static const Byte kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; +static const Byte kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; +static const Byte kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; +#define IsLitState(s) ((s) < 7) +#define GetLenToPosState2(len) (((len) < kNumLenToPosStates - 1) ? (len) : kNumLenToPosStates - 1) #define GetLenToPosState(len) (((len) < kNumLenToPosStates + 1) ? (len) - 2 : kNumLenToPosStates - 1) #define kInfinityPrice (1 << 30) @@ -519,9 +536,11 @@ static void RangeEnc_Construct(CRangeEnc *p) p->bufBase = NULL; } -#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed_sizet(p) ((size_t)(p)->processed + ((p)->buf - (p)->bufBase) + (size_t)(p)->cacheSize) #define RC_BUF_SIZE (1 << 16) + static int RangeEnc_Alloc(CRangeEnc *p, ISzAllocPtr alloc) { if (!p->bufBase) @@ -543,10 +562,10 @@ static void RangeEnc_Free(CRangeEnc *p, ISzAllocPtr alloc) static void RangeEnc_Init(CRangeEnc *p) { /* Stream.Init(); */ - p->low = 0; p->range = 0xFFFFFFFF; - p->cacheSize = 1; p->cache = 0; + p->low = 0; + p->cacheSize = 0; p->buf = p->bufBase; @@ -554,7 +573,7 @@ static void RangeEnc_Init(CRangeEnc *p) p->res = SZ_OK; } -static void RangeEnc_FlushStream(CRangeEnc *p) +MY_NO_INLINE static void RangeEnc_FlushStream(CRangeEnc *p) { size_t num; if (p->res != SZ_OK) @@ -566,25 +585,36 @@ static void RangeEnc_FlushStream(CRangeEnc *p) p->buf = p->bufBase; } -static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) +MY_NO_INLINE static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) { - if ((UInt32)p->low < (UInt32)0xFF000000 || (unsigned)(p->low >> 32) != 0) + UInt32 low = (UInt32)p->low; + unsigned high = (unsigned)(p->low >> 32); + p->low = (UInt32)(low << 8); + if (low < (UInt32)0xFF000000 || high != 0) { - Byte temp = p->cache; - do { Byte *buf = p->buf; - *buf++ = (Byte)(temp + (Byte)(p->low >> 32)); + *buf++ = (Byte)(p->cache + high); + p->cache = (unsigned)(low >> 24); p->buf = buf; if (buf == p->bufLim) RangeEnc_FlushStream(p); - temp = 0xFF; + if (p->cacheSize == 0) + return; + } + high += 0xFF; + for (;;) + { + Byte *buf = p->buf; + *buf++ = (Byte)(high); + p->buf = buf; + if (buf == p->bufLim) + RangeEnc_FlushStream(p); + if (--p->cacheSize == 0) + return; } - while (--p->cacheSize != 0); - p->cache = (Byte)((UInt32)p->low >> 24); } p->cacheSize++; - p->low = (UInt32)p->low << 8; } static void RangeEnc_FlushData(CRangeEnc *p) @@ -594,78 +624,121 @@ static void RangeEnc_FlushData(CRangeEnc *p) RangeEnc_ShiftLow(p); } -static void RangeEnc_EncodeDirectBits(CRangeEnc *p, UInt32 value, unsigned numBits) -{ - do - { - p->range >>= 1; - p->low += p->range & (0 - ((value >> --numBits) & 1)); - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); - } - } - while (numBits != 0); -} +#define RC_NORM(p) if (range < kTopValue) { range <<= 8; RangeEnc_ShiftLow(p); } -static void RangeEnc_EncodeBit(CRangeEnc *p, CLzmaProb *prob, UInt32 symbol) -{ - UInt32 ttt = *prob; - UInt32 newBound = (p->range >> kNumBitModelTotalBits) * ttt; - if (symbol == 0) - { - p->range = newBound; - ttt += (kBitModelTotal - ttt) >> kNumMoveBits; - } - else - { - p->low += newBound; - p->range -= newBound; - ttt -= ttt >> kNumMoveBits; +#define RC_BIT_PRE(p, prob) \ + ttt = *(prob); \ + newBound = (range >> kNumBitModelTotalBits) * ttt; + +// #define _LZMA_ENC_USE_BRANCH + +#ifdef _LZMA_ENC_USE_BRANCH + +#define RC_BIT(p, prob, bit) { \ + RC_BIT_PRE(p, prob) \ + if (bit == 0) { range = newBound; ttt += (kBitModelTotal - ttt) >> kNumMoveBits; } \ + else { (p)->low += newBound; range -= newBound; ttt -= ttt >> kNumMoveBits; } \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ } - *prob = (CLzmaProb)ttt; - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); + +#else + +#define RC_BIT(p, prob, bit) { \ + UInt32 mask; \ + RC_BIT_PRE(p, prob) \ + mask = 0 - (UInt32)bit; \ + range &= mask; \ + mask &= newBound; \ + range -= mask; \ + (p)->low += mask; \ + mask = (UInt32)bit - 1; \ + range += newBound & mask; \ + mask &= (kBitModelTotal - ((1 << kNumMoveBits) - 1)); \ + mask += ((1 << kNumMoveBits) - 1); \ + ttt += (Int32)(mask - ttt) >> kNumMoveBits; \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ } + +#endif + + + + +#define RC_BIT_0_BASE(p, prob) \ + range = newBound; *(prob) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); + +#define RC_BIT_1_BASE(p, prob) \ + range -= newBound; (p)->low += newBound; *(prob) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); \ + +#define RC_BIT_0(p, prob) \ + RC_BIT_0_BASE(p, prob) \ + RC_NORM(p) + +#define RC_BIT_1(p, prob) \ + RC_BIT_1_BASE(p, prob) \ + RC_NORM(p) + +static void RangeEnc_EncodeBit_0(CRangeEnc *p, CLzmaProb *prob) +{ + UInt32 range, ttt, newBound; + range = p->range; + RC_BIT_PRE(p, prob) + RC_BIT_0(p, prob) + p->range = range; } -static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol) +static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 sym) { - symbol |= 0x100; + UInt32 range = p->range; + sym |= 0x100; do { - RangeEnc_EncodeBit(p, probs + (symbol >> 8), (symbol >> 7) & 1); - symbol <<= 1; + UInt32 ttt, newBound; + // RangeEnc_EncodeBit(p, probs + (sym >> 8), (sym >> 7) & 1); + CLzmaProb *prob = probs + (sym >> 8); + UInt32 bit = (sym >> 7) & 1; + sym <<= 1; + RC_BIT(p, prob, bit); } - while (symbol < 0x10000); + while (sym < 0x10000); + p->range = range; } -static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol, UInt32 matchByte) +static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 sym, UInt32 matchByte) { + UInt32 range = p->range; UInt32 offs = 0x100; - symbol |= 0x100; + sym |= 0x100; do { + UInt32 ttt, newBound; + CLzmaProb *prob; + UInt32 bit; matchByte <<= 1; - RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (symbol >> 8)), (symbol >> 7) & 1); - symbol <<= 1; - offs &= ~(matchByte ^ symbol); + // RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (sym >> 8)), (sym >> 7) & 1); + prob = probs + (offs + (matchByte & offs) + (sym >> 8)); + bit = (sym >> 7) & 1; + sym <<= 1; + offs &= ~(matchByte ^ sym); + RC_BIT(p, prob, bit); } - while (symbol < 0x10000); + while (sym < 0x10000); + p->range = range; } -static void LzmaEnc_InitPriceTables(UInt32 *ProbPrices) + + +static void LzmaEnc_InitPriceTables(CProbPrice *ProbPrices) { UInt32 i; - for (i = (1 << kNumMoveReducingBits) / 2; i < kBitModelTotal; i += (1 << kNumMoveReducingBits)) + for (i = 0; i < (kBitModelTotal >> kNumMoveReducingBits); i++) { - const int kCyclesBits = kNumBitPriceShiftBits; - UInt32 w = i; - UInt32 bitCount = 0; - int j; + const unsigned kCyclesBits = kNumBitPriceShiftBits; + UInt32 w = (i << kNumMoveReducingBits) + (1 << (kNumMoveReducingBits - 1)); + unsigned bitCount = 0; + unsigned j; for (j = 0; j < kCyclesBits; j++) { w = w * w; @@ -676,567 +749,655 @@ static void LzmaEnc_InitPriceTables(UInt32 *ProbPrices) bitCount++; } } - ProbPrices[i >> kNumMoveReducingBits] = ((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + ProbPrices[i] = (CProbPrice)((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + // printf("\n%3d: %5d", i, ProbPrices[i]); } } -#define GET_PRICE(prob, symbol) \ - p->ProbPrices[((prob) ^ (((-(int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; +#define GET_PRICE(prob, bit) \ + p->ProbPrices[((prob) ^ (unsigned)(((-(int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; -#define GET_PRICEa(prob, symbol) \ - ProbPrices[((prob) ^ ((-((int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; +#define GET_PRICEa(prob, bit) \ + ProbPrices[((prob) ^ (unsigned)((-((int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; #define GET_PRICE_0(prob) p->ProbPrices[(prob) >> kNumMoveReducingBits] #define GET_PRICE_1(prob) p->ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] -#define GET_PRICE_0a(prob) ProbPrices[(prob) >> kNumMoveReducingBits] -#define GET_PRICE_1a(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] +#define GET_PRICEa_0(prob) ProbPrices[(prob) >> kNumMoveReducingBits] +#define GET_PRICEa_1(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] -static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 symbol, const UInt32 *ProbPrices) + +static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 sym, const CProbPrice *ProbPrices) { UInt32 price = 0; - symbol |= 0x100; + sym |= 0x100; do { - price += GET_PRICEa(probs[symbol >> 8], (symbol >> 7) & 1); - symbol <<= 1; + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); } - while (symbol < 0x10000); + while (sym >= 2); return price; } -static UInt32 LitEnc_GetPriceMatched(const CLzmaProb *probs, UInt32 symbol, UInt32 matchByte, const UInt32 *ProbPrices) + +static UInt32 LitEnc_Matched_GetPrice(const CLzmaProb *probs, UInt32 sym, UInt32 matchByte, const CProbPrice *ProbPrices) { UInt32 price = 0; UInt32 offs = 0x100; - symbol |= 0x100; + sym |= 0x100; do { matchByte <<= 1; - price += GET_PRICEa(probs[offs + (matchByte & offs) + (symbol >> 8)], (symbol >> 7) & 1); - symbol <<= 1; - offs &= ~(matchByte ^ symbol); + price += GET_PRICEa(probs[offs + (matchByte & offs) + (sym >> 8)], (sym >> 7) & 1); + sym <<= 1; + offs &= ~(matchByte ^ sym); } - while (symbol < 0x10000); + while (sym < 0x10000); return price; } -static void RcTree_Encode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) -{ - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0;) - { - UInt32 bit; - i--; - bit = (symbol >> i) & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); - m = (m << 1) | bit; - } -} - -static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) +static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, unsigned numBits, unsigned sym) { - UInt32 m = 1; - int i; - for (i = 0; i < numBitLevels; i++) + UInt32 range = rc->range; + unsigned m = 1; + do { - UInt32 bit = symbol & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); + UInt32 ttt, newBound; + unsigned bit = sym & 1; + // RangeEnc_EncodeBit(rc, probs + m, bit); + sym >>= 1; + RC_BIT(rc, probs + m, bit); m = (m << 1) | bit; - symbol >>= 1; - } -} - -static UInt32 RcTree_GetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, const UInt32 *ProbPrices) -{ - UInt32 price = 0; - symbol |= (1 << numBitLevels); - while (symbol != 1) - { - price += GET_PRICEa(probs[symbol >> 1], symbol & 1); - symbol >>= 1; } - return price; + while (--numBits); + rc->range = range; } -static UInt32 RcTree_ReverseGetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, const UInt32 *ProbPrices) -{ - UInt32 price = 0; - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0; i--) - { - UInt32 bit = symbol & 1; - symbol >>= 1; - price += GET_PRICEa(probs[m], bit); - m = (m << 1) | bit; - } - return price; -} static void LenEnc_Init(CLenEnc *p) { unsigned i; - p->choice = p->choice2 = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumLowBits); i++) + for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)); i++) p->low[i] = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumMidBits); i++) - p->mid[i] = kProbInitValue; for (i = 0; i < kLenNumHighSymbols; i++) p->high[i] = kProbInitValue; } -static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState) +static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, unsigned sym, unsigned posState) { - if (symbol < kLenNumLowSymbols) - { - RangeEnc_EncodeBit(rc, &p->choice, 0); - RcTree_Encode(rc, p->low + (posState << kLenNumLowBits), kLenNumLowBits, symbol); - } - else + UInt32 range, ttt, newBound; + CLzmaProb *probs = p->low; + range = rc->range; + RC_BIT_PRE(rc, probs); + if (sym >= kLenNumLowSymbols) { - RangeEnc_EncodeBit(rc, &p->choice, 1); - if (symbol < kLenNumLowSymbols + kLenNumMidSymbols) - { - RangeEnc_EncodeBit(rc, &p->choice2, 0); - RcTree_Encode(rc, p->mid + (posState << kLenNumMidBits), kLenNumMidBits, symbol - kLenNumLowSymbols); - } - else + RC_BIT_1(rc, probs); + probs += kLenNumLowSymbols; + RC_BIT_PRE(rc, probs); + if (sym >= kLenNumLowSymbols * 2) { - RangeEnc_EncodeBit(rc, &p->choice2, 1); - RcTree_Encode(rc, p->high, kLenNumHighBits, symbol - kLenNumLowSymbols - kLenNumMidSymbols); + RC_BIT_1(rc, probs); + rc->range = range; + // RcTree_Encode(rc, p->high, kLenNumHighBits, sym - kLenNumLowSymbols * 2); + LitEnc_Encode(rc, p->high, sym - kLenNumLowSymbols * 2); + return; } + sym -= kLenNumLowSymbols; } -} -static void LenEnc_SetPrices(CLenEnc *p, UInt32 posState, UInt32 numSymbols, UInt32 *prices, const UInt32 *ProbPrices) -{ - UInt32 a0 = GET_PRICE_0a(p->choice); - UInt32 a1 = GET_PRICE_1a(p->choice); - UInt32 b0 = a1 + GET_PRICE_0a(p->choice2); - UInt32 b1 = a1 + GET_PRICE_1a(p->choice2); - UInt32 i = 0; - for (i = 0; i < kLenNumLowSymbols; i++) + // RcTree_Encode(rc, probs + (posState << kLenNumLowBits), kLenNumLowBits, sym); { - if (i >= numSymbols) - return; - prices[i] = a0 + RcTree_GetPrice(p->low + (posState << kLenNumLowBits), kLenNumLowBits, i, ProbPrices); - } - for (; i < kLenNumLowSymbols + kLenNumMidSymbols; i++) - { - if (i >= numSymbols) - return; - prices[i] = b0 + RcTree_GetPrice(p->mid + (posState << kLenNumMidBits), kLenNumMidBits, i - kLenNumLowSymbols, ProbPrices); + unsigned m; + unsigned bit; + RC_BIT_0(rc, probs); + probs += (posState << (1 + kLenNumLowBits)); + bit = (sym >> 2) ; RC_BIT(rc, probs + 1, bit); m = (1 << 1) + bit; + bit = (sym >> 1) & 1; RC_BIT(rc, probs + m, bit); m = (m << 1) + bit; + bit = sym & 1; RC_BIT(rc, probs + m, bit); + rc->range = range; } - for (; i < numSymbols; i++) - prices[i] = b1 + RcTree_GetPrice(p->high, kLenNumHighBits, i - kLenNumLowSymbols - kLenNumMidSymbols, ProbPrices); } -static void MY_FAST_CALL LenPriceEnc_UpdateTable(CLenPriceEnc *p, UInt32 posState, const UInt32 *ProbPrices) +static void SetPrices_3(const CLzmaProb *probs, UInt32 startPrice, UInt32 *prices, const CProbPrice *ProbPrices) { - LenEnc_SetPrices(&p->p, posState, p->tableSize, p->prices[posState], ProbPrices); - p->counters[posState] = p->tableSize; + unsigned i; + for (i = 0; i < 8; i += 2) + { + UInt32 price = startPrice; + UInt32 prob; + price += GET_PRICEa(probs[1 ], (i >> 2)); + price += GET_PRICEa(probs[2 + (i >> 2)], (i >> 1) & 1); + prob = probs[4 + (i >> 1)]; + prices[i ] = price + GET_PRICEa_0(prob); + prices[i + 1] = price + GET_PRICEa_1(prob); + } } -static void LenPriceEnc_UpdateTables(CLenPriceEnc *p, UInt32 numPosStates, const UInt32 *ProbPrices) -{ - UInt32 posState; - for (posState = 0; posState < numPosStates; posState++) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); -} -static void LenEnc_Encode2(CLenPriceEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState, Bool updatePrice, const UInt32 *ProbPrices) +MY_NO_INLINE static void MY_FAST_CALL LenPriceEnc_UpdateTables( + CLenPriceEnc *p, + unsigned numPosStates, + const CLenEnc *enc, + const CProbPrice *ProbPrices) { - LenEnc_Encode(&p->p, rc, symbol, posState); - if (updatePrice) - if (--p->counters[posState] == 0) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); -} + UInt32 b; + + { + unsigned prob = enc->low[0]; + UInt32 a, c; + unsigned posState; + b = GET_PRICEa_1(prob); + a = GET_PRICEa_0(prob); + c = b + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); + for (posState = 0; posState < numPosStates; posState++) + { + UInt32 *prices = p->prices[posState]; + const CLzmaProb *probs = enc->low + (posState << (1 + kLenNumLowBits)); + SetPrices_3(probs, a, prices, ProbPrices); + SetPrices_3(probs + kLenNumLowSymbols, c, prices + kLenNumLowSymbols, ProbPrices); + } + } + + /* + { + unsigned i; + UInt32 b; + a = GET_PRICEa_0(enc->low[0]); + for (i = 0; i < kLenNumLowSymbols; i++) + p->prices2[i] = a; + a = GET_PRICEa_1(enc->low[0]); + b = a + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); + for (i = kLenNumLowSymbols; i < kLenNumLowSymbols * 2; i++) + p->prices2[i] = b; + a += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + } + */ + + // p->counter = numSymbols; + // p->counter = 64; + { + unsigned i = p->tableSize; + + if (i > kLenNumLowSymbols * 2) + { + const CLzmaProb *probs = enc->high; + UInt32 *prices = p->prices[0] + kLenNumLowSymbols * 2; + i -= kLenNumLowSymbols * 2 - 1; + i >>= 1; + b += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + do + { + /* + p->prices2[i] = a + + // RcTree_GetPrice(enc->high, kLenNumHighBits, i - kLenNumLowSymbols * 2, ProbPrices); + LitEnc_GetPrice(probs, i - kLenNumLowSymbols * 2, ProbPrices); + */ + // UInt32 price = a + RcTree_GetPrice(probs, kLenNumHighBits - 1, sym, ProbPrices); + unsigned sym = --i + (1 << (kLenNumHighBits - 1)); + UInt32 price = b; + do + { + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); + } + while (sym >= 2); + { + unsigned prob = probs[(size_t)i + (1 << (kLenNumHighBits - 1))]; + prices[(size_t)i * 2 ] = price + GET_PRICEa_0(prob); + prices[(size_t)i * 2 + 1] = price + GET_PRICEa_1(prob); + } + } + while (i); + { + unsigned posState; + size_t num = (p->tableSize - kLenNumLowSymbols * 2) * sizeof(p->prices[0][0]); + for (posState = 1; posState < numPosStates; posState++) + memcpy(p->prices[posState] + kLenNumLowSymbols * 2, p->prices[0] + kLenNumLowSymbols * 2, num); + } + } + } +} -static void MovePos(CLzmaEnc *p, UInt32 num) -{ +/* #ifdef SHOW_STAT g_STAT_OFFSET += num; printf("\n MovePos %u", num); #endif +*/ - if (num != 0) - { - p->additionalOffset += num; - p->matchFinder.Skip(p->matchFinderObj, num); - } -} +#define MOVE_POS(p, num) { \ + p->additionalOffset += (num); \ + p->matchFinder.Skip(p->matchFinderObj, (UInt32)(num)); } + -static UInt32 ReadMatchDistances(CLzmaEnc *p, UInt32 *numDistancePairsRes) +static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) { - UInt32 lenRes = 0, numPairs; + unsigned numPairs; + + p->additionalOffset++; p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); + *numPairsRes = numPairs; #ifdef SHOW_STAT printf("\n i = %u numPairs = %u ", g_STAT_OFFSET, numPairs / 2); g_STAT_OFFSET++; { - UInt32 i; + unsigned i; for (i = 0; i < numPairs; i += 2) printf("%2u %6u | ", p->matches[i], p->matches[i + 1]); } #endif - if (numPairs > 0) + if (numPairs == 0) + return 0; { - lenRes = p->matches[(size_t)numPairs - 2]; - if (lenRes == p->numFastBytes) + unsigned len = p->matches[(size_t)numPairs - 2]; + if (len != p->numFastBytes) + return len; { UInt32 numAvail = p->numAvail; if (numAvail > LZMA_MATCH_LEN_MAX) numAvail = LZMA_MATCH_LEN_MAX; { - const Byte *pbyCur = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - const Byte *pby = pbyCur + lenRes; + const Byte *p1 = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + const Byte *p2 = p1 + len; ptrdiff_t dif = (ptrdiff_t)-1 - p->matches[(size_t)numPairs - 1]; - const Byte *pbyLim = pbyCur + numAvail; - for (; pby != pbyLim && *pby == pby[dif]; pby++); - lenRes = (UInt32)(pby - pbyCur); + const Byte *lim = p1 + numAvail; + for (; p2 != lim && *p2 == p2[dif]; p2++) + {} + return (unsigned)(p2 - p1); } } } - p->additionalOffset++; - *numDistancePairsRes = numPairs; - return lenRes; } +#define MARK_LIT ((UInt32)(Int32)-1) -#define MakeAsChar(p) (p)->backPrev = (UInt32)(-1); (p)->prev1IsChar = False; -#define MakeAsShortRep(p) (p)->backPrev = 0; (p)->prev1IsChar = False; -#define IsShortRep(p) ((p)->backPrev == 0) +#define MakeAs_Lit(p) { (p)->dist = MARK_LIT; (p)->extra = 0; } +#define MakeAs_ShortRep(p) { (p)->dist = 0; (p)->extra = 0; } +#define IsShortRep(p) ((p)->dist == 0) -static UInt32 GetRepLen1Price(CLzmaEnc *p, UInt32 state, UInt32 posState) -{ - return - GET_PRICE_0(p->isRepG0[state]) + - GET_PRICE_0(p->isRep0Long[state][posState]); -} -static UInt32 GetPureRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 state, UInt32 posState) +#define GetPrice_ShortRep(p, state, posState) \ + ( GET_PRICE_0(p->isRepG0[state]) + GET_PRICE_0(p->isRep0Long[state][posState])) + +#define GetPrice_Rep_0(p, state, posState) ( \ + GET_PRICE_1(p->isMatch[state][posState]) \ + + GET_PRICE_1(p->isRep0Long[state][posState])) \ + + GET_PRICE_1(p->isRep[state]) \ + + GET_PRICE_0(p->isRepG0[state]) + +MY_FORCE_INLINE +static UInt32 GetPrice_PureRep(const CLzmaEnc *p, unsigned repIndex, size_t state, size_t posState) { UInt32 price; + UInt32 prob = p->isRepG0[state]; if (repIndex == 0) { - price = GET_PRICE_0(p->isRepG0[state]); + price = GET_PRICE_0(prob); price += GET_PRICE_1(p->isRep0Long[state][posState]); } else { - price = GET_PRICE_1(p->isRepG0[state]); + price = GET_PRICE_1(prob); + prob = p->isRepG1[state]; if (repIndex == 1) - price += GET_PRICE_0(p->isRepG1[state]); + price += GET_PRICE_0(prob); else { - price += GET_PRICE_1(p->isRepG1[state]); + price += GET_PRICE_1(prob); price += GET_PRICE(p->isRepG2[state], repIndex - 2); } } return price; } -static UInt32 GetRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 len, UInt32 state, UInt32 posState) -{ - return p->repLenEnc.prices[posState][(size_t)len - LZMA_MATCH_LEN_MIN] + - GetPureRepPrice(p, repIndex, state, posState); -} -static UInt32 Backward(CLzmaEnc *p, UInt32 *backRes, UInt32 cur) +static unsigned Backward(CLzmaEnc *p, unsigned cur) { - UInt32 posMem = p->opt[cur].posPrev; - UInt32 backMem = p->opt[cur].backPrev; - p->optimumEndIndex = cur; - do + unsigned wr = cur + 1; + p->optEnd = wr; + + for (;;) { - if (p->opt[cur].prev1IsChar) + UInt32 dist = p->opt[cur].dist; + unsigned len = (unsigned)p->opt[cur].len; + unsigned extra = (unsigned)p->opt[cur].extra; + cur -= len; + + if (extra) { - MakeAsChar(&p->opt[posMem]) - p->opt[posMem].posPrev = posMem - 1; - if (p->opt[cur].prev2) + wr--; + p->opt[wr].len = (UInt32)len; + cur -= extra; + len = extra; + if (extra == 1) + { + p->opt[wr].dist = dist; + dist = MARK_LIT; + } + else { - p->opt[(size_t)posMem - 1].prev1IsChar = False; - p->opt[(size_t)posMem - 1].posPrev = p->opt[cur].posPrev2; - p->opt[(size_t)posMem - 1].backPrev = p->opt[cur].backPrev2; + p->opt[wr].dist = 0; + len--; + wr--; + p->opt[wr].dist = MARK_LIT; + p->opt[wr].len = 1; } } + + if (cur == 0) { - UInt32 posPrev = posMem; - UInt32 backCur = backMem; - - backMem = p->opt[posPrev].backPrev; - posMem = p->opt[posPrev].posPrev; - - p->opt[posPrev].backPrev = backCur; - p->opt[posPrev].posPrev = cur; - cur = posPrev; + p->backRes = dist; + p->optCur = wr; + return len; } + + wr--; + p->opt[wr].dist = dist; + p->opt[wr].len = (UInt32)len; } - while (cur != 0); - *backRes = p->opt[0].backPrev; - p->optimumCurrentIndex = p->opt[0].posPrev; - return p->optimumCurrentIndex; } -#define LIT_PROBS(pos, prevByte) (p->litProbs + ((((pos) & p->lpMask) << p->lc) + ((prevByte) >> (8 - p->lc))) * (UInt32)0x300) - -static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) -{ - UInt32 lenEnd, cur; - UInt32 reps[LZMA_NUM_REPS], repLens[LZMA_NUM_REPS]; - UInt32 *matches; - { - UInt32 numAvail, mainLen, numPairs, repMaxIndex, i, posState, len; - UInt32 matchPrice, repMatchPrice, normalMatchPrice; - const Byte *data; - Byte curByte, matchByte; +#define LIT_PROBS(pos, prevByte) \ + (p->litProbs + (UInt32)3 * (((((pos) << 8) + (prevByte)) & p->lpMask) << p->lc)) - if (p->optimumEndIndex != p->optimumCurrentIndex) - { - const COptimal *opt = &p->opt[p->optimumCurrentIndex]; - UInt32 lenRes = opt->posPrev - p->optimumCurrentIndex; - *backRes = opt->backPrev; - p->optimumCurrentIndex = opt->posPrev; - return lenRes; - } - p->optimumCurrentIndex = p->optimumEndIndex = 0; - - if (p->additionalOffset == 0) - mainLen = ReadMatchDistances(p, &numPairs); - else - { - mainLen = p->longestMatchLength; - numPairs = p->numPairs; - } - numAvail = p->numAvail; - if (numAvail < 2) - { - *backRes = (UInt32)(-1); - return 1; - } - if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; +static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) +{ + unsigned last, cur; + UInt32 reps[LZMA_NUM_REPS]; + unsigned repLens[LZMA_NUM_REPS]; + UInt32 *matches; - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - repMaxIndex = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 lenTest; - const Byte *data2; - reps[i] = p->reps[i]; - data2 = data - reps[i] - 1; - if (data[0] != data2[0] || data[1] != data2[1]) + UInt32 numAvail; + unsigned numPairs, mainLen, repMaxIndex, i, posState; + UInt32 matchPrice, repMatchPrice; + const Byte *data; + Byte curByte, matchByte; + + p->optCur = p->optEnd = 0; + + if (p->additionalOffset == 0) + mainLen = ReadMatchDistances(p, &numPairs); + else { - repLens[i] = 0; - continue; + mainLen = p->longestMatchLen; + numPairs = p->numPairs; } - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - repLens[i] = lenTest; - if (lenTest > repLens[repMaxIndex]) - repMaxIndex = i; - } - if (repLens[repMaxIndex] >= p->numFastBytes) - { - UInt32 lenRes; - *backRes = repMaxIndex; - lenRes = repLens[repMaxIndex]; - MovePos(p, lenRes - 1); - return lenRes; - } - - matches = p->matches; - if (mainLen >= p->numFastBytes) - { - *backRes = matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); - return mainLen; - } - curByte = *data; - matchByte = *(data - (reps[0] + 1)); - - if (mainLen < 2 && curByte != matchByte && repLens[repMaxIndex] < 2) - { - *backRes = (UInt32)-1; - return 1; - } - - p->opt[0].state = (CState)p->state; - - posState = (position & p->pbMask); - - { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + - (!IsCharState(p->state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); - } - - MakeAsChar(&p->opt[1]); - - matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); - repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); - - if (matchByte == curByte) - { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, p->state, posState); - if (shortRepPrice < p->opt[1].price) + + numAvail = p->numAvail; + if (numAvail < 2) { - p->opt[1].price = shortRepPrice; - MakeAsShortRep(&p->opt[1]); + p->backRes = MARK_LIT; + return 1; } - } - lenEnd = ((mainLen >= repLens[repMaxIndex]) ? mainLen : repLens[repMaxIndex]); - - if (lenEnd < 2) - { - *backRes = p->opt[1].backPrev; - return 1; - } - - p->opt[1].posPrev = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) - p->opt[0].backs[i] = reps[i]; - - len = lenEnd; - do - p->opt[len--].price = kInfinityPrice; - while (len >= 2); - - for (i = 0; i < LZMA_NUM_REPS; i++) - { - UInt32 repLen = repLens[i]; - UInt32 price; - if (repLen < 2) - continue; - price = repMatchPrice + GetPureRepPrice(p, i, p->state, posState); - do + if (numAvail > LZMA_MATCH_LEN_MAX) + numAvail = LZMA_MATCH_LEN_MAX; + + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + repMaxIndex = 0; + + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][(size_t)repLen - 2]; - COptimal *opt = &p->opt[repLen]; - if (curAndLenPrice < opt->price) + unsigned len; + const Byte *data2; + reps[i] = p->reps[i]; + data2 = data - reps[i]; + if (data[0] != data2[0] || data[1] != data2[1]) { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = i; - opt->prev1IsChar = False; + repLens[i] = 0; + continue; } + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + repLens[i] = len; + if (len > repLens[repMaxIndex]) + repMaxIndex = i; } - while (--repLen >= 2); - } - - normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); + + if (repLens[repMaxIndex] >= p->numFastBytes) + { + unsigned len; + p->backRes = (UInt32)repMaxIndex; + len = repLens[repMaxIndex]; + MOVE_POS(p, len - 1) + return len; + } + + matches = p->matches; + + if (mainLen >= p->numFastBytes) + { + p->backRes = matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) + return mainLen; + } + + curByte = *data; + matchByte = *(data - reps[0]); - len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2); - if (len <= mainLen) - { - UInt32 offs = 0; - while (len > matches[offs]) - offs += 2; - for (; ; len++) + last = repLens[repMaxIndex]; + if (last <= mainLen) + last = mainLen; + + if (last < 2 && curByte != matchByte) + { + p->backRes = MARK_LIT; + return 1; + } + + p->opt[0].state = (CState)p->state; + + posState = (position & p->pbMask); + { - COptimal *opt; - UInt32 distance = matches[(size_t)offs + 1]; + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + + (!IsLitState(p->state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + } - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][(size_t)len - LZMA_MATCH_LEN_MIN]; - UInt32 lenToPosState = GetLenToPosState(len); - if (distance < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][distance]; - else + MakeAs_Lit(&p->opt[1]); + + matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); + repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); + + // 18.06 + if (matchByte == curByte && repLens[0] == 0) + { + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, p->state, posState); + if (shortRepPrice < p->opt[1].price) { - UInt32 slot; - GetPosSlot2(distance, slot); - curAndLenPrice += p->alignPrices[distance & kAlignMask] + p->posSlotPrices[lenToPosState][slot]; + p->opt[1].price = shortRepPrice; + MakeAs_ShortRep(&p->opt[1]); } - opt = &p->opt[len]; - if (curAndLenPrice < opt->price) + if (last < 2) { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = distance + LZMA_NUM_REPS; - opt->prev1IsChar = False; + p->backRes = p->opt[1].dist; + return 1; } - if (len == matches[offs]) + } + + p->opt[1].len = 1; + + p->opt[0].reps[0] = reps[0]; + p->opt[0].reps[1] = reps[1]; + p->opt[0].reps[2] = reps[2]; + p->opt[0].reps[3] = reps[3]; + + // ---------- REP ---------- + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + unsigned repLen = repLens[i]; + UInt32 price; + if (repLen < 2) + continue; + price = repMatchPrice + GetPrice_PureRep(p, i, p->state, posState); + do { - offs += 2; - if (offs == numPairs) - break; + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, repLen); + COptimal *opt = &p->opt[repLen]; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)repLen; + opt->dist = (UInt32)i; + opt->extra = 0; + } } + while (--repLen >= 2); } - } + + + // ---------- MATCH ---------- + { + unsigned len = repLens[0] + 1; + if (len <= mainLen) + { + unsigned offs = 0; + UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); + + if (len < 2) + len = 2; + else + while (len > matches[offs]) + offs += 2; + + for (; ; len++) + { + COptimal *opt; + UInt32 dist = matches[(size_t)offs + 1]; + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); + unsigned lenToPosState = GetLenToPosState(len); + + if (dist < kNumFullDistances) + price += p->distancesPrices[lenToPosState][dist & (kNumFullDistances - 1)]; + else + { + unsigned slot; + GetPosSlot2(dist, slot); + price += p->alignPrices[dist & kAlignMask]; + price += p->posSlotPrices[lenToPosState][slot]; + } + + opt = &p->opt[len]; + + if (price < opt->price) + { + opt->price = price; + opt->len = (UInt32)len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } + + if (len == matches[offs]) + { + offs += 2; + if (offs == numPairs) + break; + } + } + } + } + - cur = 0; + cur = 0; #ifdef SHOW_STAT2 /* if (position >= 0) */ { unsigned i; printf("\n pos = %4X", position); - for (i = cur; i <= lenEnd; i++) + for (i = cur; i <= last; i++) printf("\nprice[%4X] = %u", position - cur + i, p->opt[i].price); } #endif - } + + + // ---------- Optimal Parsing ---------- + for (;;) { - UInt32 numAvail; - UInt32 numAvailFull, newLen, numPairs, posPrev, state, posState, startLen; - UInt32 curPrice, curAnd1Price, matchPrice, repMatchPrice; - Bool nextIsChar; + unsigned numAvail; + UInt32 numAvailFull; + unsigned newLen, numPairs, prev, state, posState, startLen; + UInt32 litPrice, matchPrice, repMatchPrice; + BoolInt nextIsLit; Byte curByte, matchByte; const Byte *data; - COptimal *curOpt; - COptimal *nextOpt; + COptimal *curOpt, *nextOpt; - cur++; - if (cur == lenEnd) - return Backward(p, backRes, cur); + if (++cur == last) + break; + + // 18.06 + if (cur >= kNumOpts - 64) + { + unsigned j, best; + UInt32 price = p->opt[cur].price; + best = cur; + for (j = cur + 1; j <= last; j++) + { + UInt32 price2 = p->opt[j].price; + if (price >= price2) + { + price = price2; + best = j; + } + } + { + unsigned delta = best - cur; + if (delta != 0) + { + MOVE_POS(p, delta); + } + } + cur = best; + break; + } newLen = ReadMatchDistances(p, &numPairs); + if (newLen >= p->numFastBytes) { p->numPairs = numPairs; - p->longestMatchLength = newLen; - return Backward(p, backRes, cur); + p->longestMatchLen = newLen; + break; } - position++; + curOpt = &p->opt[cur]; - posPrev = curOpt->posPrev; - if (curOpt->prev1IsChar) - { - posPrev--; - if (curOpt->prev2) - { - state = p->opt[curOpt->posPrev2].state; - if (curOpt->backPrev2 < LZMA_NUM_REPS) - state = kRepNextStates[state]; - else - state = kMatchNextStates[state]; - } - else - state = p->opt[posPrev].state; - state = kLiteralNextStates[state]; - } - else - state = p->opt[posPrev].state; - if (posPrev == cur - 1) + + position++; + + // we need that check here, if skip_items in p->opt are possible + /* + if (curOpt->price >= kInfinityPrice) + continue; + */ + + prev = cur - curOpt->len; + + if (curOpt->len == 1) { + state = (unsigned)p->opt[prev].state; if (IsShortRep(curOpt)) state = kShortRepNextStates[state]; else @@ -1244,357 +1405,499 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) } else { - UInt32 pos; const COptimal *prevOpt; - if (curOpt->prev1IsChar && curOpt->prev2) + UInt32 b0; + UInt32 dist = curOpt->dist; + + if (curOpt->extra) { - posPrev = curOpt->posPrev2; - pos = curOpt->backPrev2; - state = kRepNextStates[state]; + prev -= (unsigned)curOpt->extra; + state = kState_RepAfterLit; + if (curOpt->extra == 1) + state = (dist < LZMA_NUM_REPS ? kState_RepAfterLit : kState_MatchAfterLit); } else { - pos = curOpt->backPrev; - if (pos < LZMA_NUM_REPS) + state = (unsigned)p->opt[prev].state; + if (dist < LZMA_NUM_REPS) state = kRepNextStates[state]; else state = kMatchNextStates[state]; } - prevOpt = &p->opt[posPrev]; - if (pos < LZMA_NUM_REPS) + + prevOpt = &p->opt[prev]; + b0 = prevOpt->reps[0]; + + if (dist < LZMA_NUM_REPS) { - UInt32 i; - reps[0] = prevOpt->backs[pos]; - for (i = 1; i <= pos; i++) - reps[i] = prevOpt->backs[(size_t)i - 1]; - for (; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[i]; + if (dist == 0) + { + reps[0] = b0; + reps[1] = prevOpt->reps[1]; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; + } + else + { + reps[1] = b0; + b0 = prevOpt->reps[1]; + if (dist == 1) + { + reps[0] = b0; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; + } + else + { + reps[2] = b0; + reps[0] = prevOpt->reps[dist]; + reps[3] = prevOpt->reps[dist ^ 1]; + } + } } else { - UInt32 i; - reps[0] = (pos - LZMA_NUM_REPS); - for (i = 1; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[(size_t)i - 1]; + reps[0] = (dist - LZMA_NUM_REPS + 1); + reps[1] = b0; + reps[2] = prevOpt->reps[1]; + reps[3] = prevOpt->reps[2]; } } + curOpt->state = (CState)state; + curOpt->reps[0] = reps[0]; + curOpt->reps[1] = reps[1]; + curOpt->reps[2] = reps[2]; + curOpt->reps[3] = reps[3]; - curOpt->backs[0] = reps[0]; - curOpt->backs[1] = reps[1]; - curOpt->backs[2] = reps[2]; - curOpt->backs[3] = reps[3]; - - curPrice = curOpt->price; - nextIsChar = False; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; curByte = *data; - matchByte = *(data - (reps[0] + 1)); + matchByte = *(data - reps[0]); posState = (position & p->pbMask); - curAnd1Price = curPrice + GET_PRICE_0(p->isMatch[state][posState]); + /* + The order of Price checks: + < LIT + <= SHORT_REP + < LIT : REP_0 + < REP [ : LIT : REP_0 ] + < MATCH [ : LIT : REP_0 ] + */ + { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - curAnd1Price += - (!IsCharState(state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + UInt32 curPrice = curOpt->price; + unsigned prob = p->isMatch[state][posState]; + matchPrice = curPrice + GET_PRICE_1(prob); + litPrice = curPrice + GET_PRICE_0(prob); } nextOpt = &p->opt[(size_t)cur + 1]; - - if (curAnd1Price < nextOpt->price) + nextIsLit = False; + + // here we can allow skip_items in p->opt, if we don't check (nextOpt->price < kInfinityPrice) + // 18.new.06 + if ((nextOpt->price < kInfinityPrice + // && !IsLitState(state) + && matchByte == curByte) + || litPrice > nextOpt->price + ) + litPrice = 0; + else { - nextOpt->price = curAnd1Price; - nextOpt->posPrev = cur; - MakeAsChar(nextOpt); - nextIsChar = True; + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + litPrice += (!IsLitState(state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + + if (litPrice < nextOpt->price) + { + nextOpt->price = litPrice; + nextOpt->len = 1; + MakeAs_Lit(nextOpt); + nextIsLit = True; + } } - matchPrice = curPrice + GET_PRICE_1(p->isMatch[state][posState]); repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); - if (matchByte == curByte && !(nextOpt->posPrev < cur && nextOpt->backPrev == 0)) + numAvailFull = p->numAvail; { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, state, posState); - if (shortRepPrice <= nextOpt->price) + unsigned temp = kNumOpts - 1 - cur; + if (numAvailFull > temp) + numAvailFull = (UInt32)temp; + } + + // 18.06 + // ---------- SHORT_REP ---------- + if (IsLitState(state)) // 18.new + if (matchByte == curByte) + if (repMatchPrice < nextOpt->price) // 18.new + // if (numAvailFull < 2 || data[1] != *(data - reps[0] + 1)) + if ( + // nextOpt->price >= kInfinityPrice || + nextOpt->len < 2 // we can check nextOpt->len, if skip items are not allowed in p->opt + || (nextOpt->dist != 0 + // && nextOpt->extra <= 1 // 17.old + ) + ) + { + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, state, posState); + // if (shortRepPrice <= nextOpt->price) // 17.old + if (shortRepPrice < nextOpt->price) // 18.new { nextOpt->price = shortRepPrice; - nextOpt->posPrev = cur; - MakeAsShortRep(nextOpt); - nextIsChar = True; + nextOpt->len = 1; + MakeAs_ShortRep(nextOpt); + nextIsLit = False; } } - numAvailFull = p->numAvail; - { - UInt32 temp = kNumOpts - 1 - cur; - if (temp < numAvailFull) - numAvailFull = temp; - } - + if (numAvailFull < 2) continue; numAvail = (numAvailFull <= p->numFastBytes ? numAvailFull : p->numFastBytes); - if (!nextIsChar && matchByte != curByte) /* speed optimization */ + // numAvail <= p->numFastBytes + + // ---------- LIT : REP_0 ---------- + + if (!nextIsLit + && litPrice != 0 // 18.new + && matchByte != curByte + && numAvailFull > 2) { - /* try Literal + rep0 */ - UInt32 temp; - UInt32 lenTest2; - const Byte *data2 = data - reps[0] - 1; - UInt32 limit = p->numFastBytes + 1; - if (limit > numAvailFull) - limit = numAvailFull; - - for (temp = 1; temp < limit && data[temp] == data2[temp]; temp++); - lenTest2 = temp - 1; - if (lenTest2 >= 2) + const Byte *data2 = data - reps[0]; + if (data[1] == data2[1] && data[2] == data2[2]) { - UInt32 state2 = kLiteralNextStates[state]; - UInt32 posStateNext = (position + 1) & p->pbMask; - UInt32 nextRepMatchPrice = curAnd1Price + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - /* for (; lenTest2 >= 2; lenTest2--) */ + unsigned len; + unsigned limit = p->numFastBytes + 1; + if (limit > numAvailFull) + limit = numAvailFull; + for (len = 3; len < limit && data[len] == data2[len]; len++) + {} + { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) + unsigned state2 = kLiteralNextStates[state]; + unsigned posState2 = (position + 1) & p->pbMask; + UInt32 price = litPrice + GetPrice_Rep_0(p, state2, posState2); { - opt->price = curAndLenPrice; - opt->posPrev = cur + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = False; + unsigned offset = cur + len; + + if (last < offset) + last = offset; + + // do + { + UInt32 price2; + COptimal *opt; + len--; + // price2 = price + GetPrice_Len_Rep_0(p, len, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len); + + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len; + opt->dist = 0; + opt->extra = 1; + } + } + // while (len >= 3); } } } } startLen = 2; /* speed optimization */ + { - UInt32 repIndex; - for (repIndex = 0; repIndex < LZMA_NUM_REPS; repIndex++) - { - UInt32 lenTest; - UInt32 lenTestTemp; - UInt32 price; - const Byte *data2 = data - reps[repIndex] - 1; - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - while (lenEnd < cur + lenTest) - p->opt[++lenEnd].price = kInfinityPrice; - lenTestTemp = lenTest; - price = repMatchPrice + GetPureRepPrice(p, repIndex, state, posState); - do + // ---------- REP ---------- + unsigned repIndex = 0; // 17.old + // unsigned repIndex = IsLitState(state) ? 0 : 1; // 18.notused + for (; repIndex < LZMA_NUM_REPS; repIndex++) { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][(size_t)lenTest - 2]; - COptimal *opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) + unsigned len; + UInt32 price; + const Byte *data2 = data - reps[repIndex]; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; + + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + + // if (len < startLen) continue; // 18.new: speed optimization + { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = repIndex; - opt->prev1IsChar = False; + unsigned offset = cur + len; + if (last < offset) + last = offset; + } + { + unsigned len2 = len; + price = repMatchPrice + GetPrice_PureRep(p, repIndex, state, posState); + do + { + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, len2); + COptimal *opt = &p->opt[cur + len2]; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len2; + opt->dist = (UInt32)repIndex; + opt->extra = 0; + } + } + while (--len2 >= 2); } - } - while (--lenTest >= 2); - lenTest = lenTestTemp; - - if (repIndex == 0) - startLen = lenTest + 1; - /* if (_maxMode) */ + if (repIndex == 0) startLen = len + 1; // 17.old + // startLen = len + 1; // 18.new + + /* if (_maxMode) */ { - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; + // ---------- REP : LIT : REP_0 ---------- + // numFastBytes + 1 + numFastBytes + + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; if (limit > numAvailFull) limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) + + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) { - UInt32 nextRepMatchPrice; - UInt32 state2 = kRepNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = - price + p->repLenEnc.prices[posState][(size_t)lenTest - 2] + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[(size_t)lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (position + lenTest + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); + unsigned state2 = kRepNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + price += GET_PRICE_LEN(&p->repLenEnc, posState, len) + + GET_PRICE_0(p->isMatch[state2][posState2]) + + LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); - /* for (; lenTest2 >= 2; lenTest2--) */ + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterRep; + posState2 = (posState2 + 1) & p->pbMask; + + + price += GetPrice_Rep_0(p, state2, posState2); + + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} + + len2 -= len; + // if (len2 >= 3) + { { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + lenTest + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) + unsigned offset = cur + len + len2; + + if (last < offset) + last = offset; + // do { - opt->price = curAndLenPrice; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = repIndex; + UInt32 price2; + COptimal *opt; + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); + + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len2; + opt->extra = (CExtra)(len + 1); + opt->dist = (UInt32)repIndex; + } } + // while (len2 >= 3); } } + } } + } } - } - /* for (UInt32 lenTest = 2; lenTest <= newLen; lenTest++) */ + + + // ---------- MATCH ---------- + /* for (unsigned len = 2; len <= newLen; len++) */ if (newLen > numAvail) { newLen = numAvail; for (numPairs = 0; newLen > matches[numPairs]; numPairs += 2); - matches[numPairs] = newLen; + matches[numPairs] = (UInt32)newLen; numPairs += 2; } + + // startLen = 2; /* speed optimization */ + if (newLen >= startLen) { UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[state]); - UInt32 offs, curBack, posSlot; - UInt32 lenTest; - while (lenEnd < cur + newLen) - p->opt[++lenEnd].price = kInfinityPrice; + UInt32 dist; + unsigned offs, posSlot, len; + + { + unsigned offset = cur + newLen; + if (last < offset) + last = offset; + } offs = 0; while (startLen > matches[offs]) offs += 2; - curBack = matches[(size_t)offs + 1]; - GetPosSlot2(curBack, posSlot); - for (lenTest = /*2*/ startLen; ; lenTest++) + dist = matches[(size_t)offs + 1]; + + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); + + for (len = /*2*/ startLen; ; len++) { - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][(size_t)lenTest - LZMA_MATCH_LEN_MIN]; + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); { - UInt32 lenToPosState = GetLenToPosState(lenTest); - COptimal *opt; - if (curBack < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][curBack]; - else - curAndLenPrice += p->posSlotPrices[lenToPosState][posSlot] + p->alignPrices[curBack & kAlignMask]; - - opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = curBack + LZMA_NUM_REPS; - opt->prev1IsChar = False; - } + COptimal *opt; + unsigned lenNorm = len - 2; + lenNorm = GetLenToPosState2(lenNorm); + if (dist < kNumFullDistances) + price += p->distancesPrices[lenNorm][dist & (kNumFullDistances - 1)]; + else + price += p->posSlotPrices[lenNorm][posSlot] + p->alignPrices[dist & kAlignMask]; + + opt = &p->opt[cur + len]; + if (price < opt->price) + { + opt->price = price; + opt->len = (UInt32)len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } } - if (/*_maxMode && */lenTest == matches[offs]) + if (len == matches[offs]) { - /* Try Match + Literal + Rep0 */ - const Byte *data2 = data - curBack - 1; - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; + // if (p->_maxMode) { + // MATCH : LIT : REP_0 + + const Byte *data2 = data - dist - 1; + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; if (limit > numAvailFull) limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) + + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) { - UInt32 nextRepMatchPrice; - UInt32 state2 = kMatchNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = curAndLenPrice + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[(size_t)lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (posStateNext + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - - /* for (; lenTest2 >= 2; lenTest2--) */ + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} + + len2 -= len; + + // if (len2 >= 3) + { + unsigned state2 = kMatchNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + unsigned offset; + price += GET_PRICE_0(p->isMatch[state2][posState2]); + price += LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); + + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterMatch; + + posState2 = (posState2 + 1) & p->pbMask; + price += GetPrice_Rep_0(p, state2, posState2); + + offset = cur + len + len2; + + if (last < offset) + last = offset; + // do { - UInt32 offset = cur + lenTest + 1 + lenTest2; - UInt32 curAndLenPrice2; + UInt32 price2; COptimal *opt; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice2 = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); opt = &p->opt[offset]; - if (curAndLenPrice2 < opt->price) + // offset--; + if (price2 < opt->price) { - opt->price = curAndLenPrice2; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = curBack + LZMA_NUM_REPS; + opt->price = price2; + opt->len = (UInt32)len2; + opt->extra = (CExtra)(len + 1); + opt->dist = dist + LZMA_NUM_REPS; } } + // while (len2 >= 3); } + + } + offs += 2; if (offs == numPairs) break; - curBack = matches[(size_t)offs + 1]; - if (curBack >= kNumFullDistances) - GetPosSlot2(curBack, posSlot); + dist = matches[(size_t)offs + 1]; + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); } } } } + + do + p->opt[last].price = kInfinityPrice; + while (--last); + + return Backward(p, cur); } + + #define ChangePair(smallDist, bigDist) (((bigDist) >> 7) > (smallDist)) -static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) + + +static unsigned GetOptimumFast(CLzmaEnc *p) { - UInt32 numAvail, mainLen, mainDist, numPairs, repIndex, repLen, i; + UInt32 numAvail, mainDist; + unsigned mainLen, numPairs, repIndex, repLen, i; const Byte *data; - const UInt32 *matches; if (p->additionalOffset == 0) mainLen = ReadMatchDistances(p, &numPairs); else { - mainLen = p->longestMatchLength; + mainLen = p->longestMatchLen; numPairs = p->numPairs; } numAvail = p->numAvail; - *backRes = (UInt32)-1; + p->backRes = MARK_LIT; if (numAvail < 2) return 1; + // if (mainLen < 2 && p->state == 0) return 1; // 18.06.notused if (numAvail > LZMA_MATCH_LEN_MAX) numAvail = LZMA_MATCH_LEN_MAX; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - repLen = repIndex = 0; + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 len; - const Byte *data2 = data - p->reps[i] - 1; + unsigned len; + const Byte *data2 = data - p->reps[i]; if (data[0] != data2[0] || data[1] != data2[1]) continue; - for (len = 2; len < numAvail && data[len] == data2[len]; len++); + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} if (len >= p->numFastBytes) { - *backRes = i; - MovePos(p, len - 1); + p->backRes = (UInt32)i; + MOVE_POS(p, len - 1) return len; } if (len > repLen) @@ -1604,84 +1907,152 @@ static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) } } - matches = p->matches; if (mainLen >= p->numFastBytes) { - *backRes = matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); + p->backRes = p->matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) return mainLen; } mainDist = 0; /* for GCC */ + if (mainLen >= 2) { - mainDist = matches[(size_t)numPairs - 1]; - while (numPairs > 2 && mainLen == matches[(size_t)numPairs - 4] + 1) + mainDist = p->matches[(size_t)numPairs - 1]; + while (numPairs > 2) { - if (!ChangePair(matches[(size_t)numPairs - 3], mainDist)) + UInt32 dist2; + if (mainLen != p->matches[(size_t)numPairs - 4] + 1) + break; + dist2 = p->matches[(size_t)numPairs - 3]; + if (!ChangePair(dist2, mainDist)) break; numPairs -= 2; - mainLen = matches[(size_t)numPairs - 2]; - mainDist = matches[(size_t)numPairs - 1]; + mainLen--; + mainDist = dist2; } if (mainLen == 2 && mainDist >= 0x80) mainLen = 1; } - if (repLen >= 2 && ( - (repLen + 1 >= mainLen) || - (repLen + 2 >= mainLen && mainDist >= (1 << 9)) || - (repLen + 3 >= mainLen && mainDist >= (1 << 15)))) + if (repLen >= 2) + if ( repLen + 1 >= mainLen + || (repLen + 2 >= mainLen && mainDist >= (1 << 9)) + || (repLen + 3 >= mainLen && mainDist >= (1 << 15))) { - *backRes = repIndex; - MovePos(p, repLen - 1); + p->backRes = (UInt32)repIndex; + MOVE_POS(p, repLen - 1) return repLen; } if (mainLen < 2 || numAvail <= 2) return 1; - p->longestMatchLength = ReadMatchDistances(p, &p->numPairs); - if (p->longestMatchLength >= 2) { - UInt32 newDistance = matches[(size_t)p->numPairs - 1]; - if ((p->longestMatchLength >= mainLen && newDistance < mainDist) || - (p->longestMatchLength == mainLen + 1 && !ChangePair(mainDist, newDistance)) || - (p->longestMatchLength > mainLen + 1) || - (p->longestMatchLength + 1 >= mainLen && mainLen >= 3 && ChangePair(newDistance, mainDist))) - return 1; + unsigned len1 = ReadMatchDistances(p, &p->numPairs); + p->longestMatchLen = len1; + + if (len1 >= 2) + { + UInt32 newDist = p->matches[(size_t)p->numPairs - 1]; + if ( (len1 >= mainLen && newDist < mainDist) + || (len1 == mainLen + 1 && !ChangePair(mainDist, newDist)) + || (len1 > mainLen + 1) + || (len1 + 1 >= mainLen && mainLen >= 3 && ChangePair(newDist, mainDist))) + return 1; + } } data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 len, limit; - const Byte *data2 = data - p->reps[i] - 1; + unsigned len, limit; + const Byte *data2 = data - p->reps[i]; if (data[0] != data2[0] || data[1] != data2[1]) continue; limit = mainLen - 1; - for (len = 2; len < limit && data[len] == data2[len]; len++); - if (len >= limit) - return 1; + for (len = 2;; len++) + { + if (len >= limit) + return 1; + if (data[len] != data2[len]) + break; + } + } + + p->backRes = mainDist + LZMA_NUM_REPS; + if (mainLen != 2) + { + MOVE_POS(p, mainLen - 2) } - *backRes = mainDist + LZMA_NUM_REPS; - MovePos(p, mainLen - 2); return mainLen; } -static void WriteEndMarker(CLzmaEnc *p, UInt32 posState) + + + +static void WriteEndMarker(CLzmaEnc *p, unsigned posState) { - UInt32 len; - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); + UInt32 range; + range = p->rc.range; + { + UInt32 ttt, newBound; + CLzmaProb *prob = &p->isMatch[p->state][posState]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_1(&p->rc, prob) + prob = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_0(&p->rc, prob) + } p->state = kMatchNextStates[p->state]; - len = LZMA_MATCH_LEN_MIN; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, (1 << kNumPosSlotBits) - 1); - RangeEnc_EncodeDirectBits(&p->rc, (((UInt32)1 << 30) - 1) >> kNumAlignBits, 30 - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); + + p->rc.range = range; + LenEnc_Encode(&p->lenProbs, &p->rc, 0, posState); + range = p->rc.range; + + { + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[0], (1 << kNumPosSlotBits) - 1); + CLzmaProb *probs = p->posSlotEncoder[0]; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < (1 << kNumPosSlotBits)); + } + { + // RangeEnc_EncodeDirectBits(&p->rc, ((UInt32)1 << (30 - kNumAlignBits)) - 1, 30 - kNumAlignBits); UInt32 range = p->range; + unsigned numBits = 30 - kNumAlignBits; + do + { + range >>= 1; + p->rc.low += range; + RC_NORM(&p->rc) + } + while (--numBits); + } + + { + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); + CLzmaProb *probs = p->posAlignEncoder; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < kAlignTableSize); + } + p->rc.range = range; } + static SRes CheckErrors(CLzmaEnc *p) { if (p->result != SZ_OK) @@ -1695,7 +2066,8 @@ static SRes CheckErrors(CLzmaEnc *p) return p->result; } -static SRes Flush(CLzmaEnc *p, UInt32 nowPos) + +MY_NO_INLINE static SRes Flush(CLzmaEnc *p, UInt32 nowPos) { /* ReleaseMFStream(); */ p->finished = True; @@ -1706,47 +2078,126 @@ static SRes Flush(CLzmaEnc *p, UInt32 nowPos) return CheckErrors(p); } -static void FillAlignPrices(CLzmaEnc *p) + +MY_NO_INLINE static void FillAlignPrices(CLzmaEnc *p) { - UInt32 i; - for (i = 0; i < kAlignTableSize; i++) - p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); - p->alignPriceCount = 0; + unsigned i; + const CProbPrice *ProbPrices = p->ProbPrices; + const CLzmaProb *probs = p->posAlignEncoder; + // p->alignPriceCount = 0; + for (i = 0; i < kAlignTableSize / 2; i++) + { + UInt32 price = 0; + unsigned sym = i; + unsigned m = 1; + unsigned bit; + UInt32 prob; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + prob = probs[m]; + p->alignPrices[i ] = price + GET_PRICEa_0(prob); + p->alignPrices[i + 8] = price + GET_PRICEa_1(prob); + // p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); + } } -static void FillDistancesPrices(CLzmaEnc *p) + +MY_NO_INLINE static void FillDistancesPrices(CLzmaEnc *p) { + // int y; for (y = 0; y < 100; y++) { + UInt32 tempPrices[kNumFullDistances]; - UInt32 i, lenToPosState; - for (i = kStartPosModelIndex; i < kNumFullDistances; i++) - { - UInt32 posSlot = GetPosSlot1(i); - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base - posSlot - 1, footerBits, i - base, p->ProbPrices); + unsigned i, lps; + + const CProbPrice *ProbPrices = p->ProbPrices; + p->matchPriceCount = 0; + + for (i = kStartPosModelIndex / 2; i < kNumFullDistances / 2; i++) + { + unsigned posSlot = GetPosSlot1(i); + unsigned footerBits = (posSlot >> 1) - 1; + unsigned base = ((2 | (posSlot & 1)) << footerBits); + const CLzmaProb *probs = p->posEncoders + (size_t)base * 2; + // tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base, footerBits, i - base, p->ProbPrices); + UInt32 price = 0; + unsigned m = 1; + unsigned sym = i; + unsigned offset = (unsigned)1 << footerBits; + base += i; + + if (footerBits) + do + { + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[m], bit); + m = (m << 1) + bit; + } + while (--footerBits); + + { + unsigned prob = probs[m]; + tempPrices[base ] = price + GET_PRICEa_0(prob); + tempPrices[base + offset] = price + GET_PRICEa_1(prob); + } } - for (lenToPosState = 0; lenToPosState < kNumLenToPosStates; lenToPosState++) + for (lps = 0; lps < kNumLenToPosStates; lps++) { - UInt32 posSlot; - const CLzmaProb *encoder = p->posSlotEncoder[lenToPosState]; - UInt32 *posSlotPrices = p->posSlotPrices[lenToPosState]; - for (posSlot = 0; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] = RcTree_GetPrice(encoder, kNumPosSlotBits, posSlot, p->ProbPrices); - for (posSlot = kEndPosModelIndex; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] += ((((posSlot >> 1) - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + unsigned slot; + unsigned distTableSize2 = (p->distTableSize + 1) >> 1; + UInt32 *posSlotPrices = p->posSlotPrices[lps]; + const CLzmaProb *probs = p->posSlotEncoder[lps]; + + for (slot = 0; slot < distTableSize2; slot++) + { + // posSlotPrices[slot] = RcTree_GetPrice(encoder, kNumPosSlotBits, slot, p->ProbPrices); + UInt32 price; + unsigned bit; + unsigned sym = slot + (1 << (kNumPosSlotBits - 1)); + unsigned prob; + bit = sym & 1; sym >>= 1; price = GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + prob = probs[(size_t)slot + (1 << (kNumPosSlotBits - 1))]; + posSlotPrices[(size_t)slot * 2 ] = price + GET_PRICEa_0(prob); + posSlotPrices[(size_t)slot * 2 + 1] = price + GET_PRICEa_1(prob); + } + + { + UInt32 delta = ((UInt32)((kEndPosModelIndex / 2 - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + for (slot = kEndPosModelIndex / 2; slot < distTableSize2; slot++) + { + posSlotPrices[(size_t)slot * 2 ] += delta; + posSlotPrices[(size_t)slot * 2 + 1] += delta; + delta += ((UInt32)1 << kNumBitPriceShiftBits); + } + } { - UInt32 *distancesPrices = p->distancesPrices[lenToPosState]; - for (i = 0; i < kStartPosModelIndex; i++) - distancesPrices[i] = posSlotPrices[i]; - for (; i < kNumFullDistances; i++) - distancesPrices[i] = posSlotPrices[GetPosSlot1(i)] + tempPrices[i]; + UInt32 *dp = p->distancesPrices[lps]; + + dp[0] = posSlotPrices[0]; + dp[1] = posSlotPrices[1]; + dp[2] = posSlotPrices[2]; + dp[3] = posSlotPrices[3]; + + for (i = 4; i < kNumFullDistances; i += 2) + { + UInt32 slotPrice = posSlotPrices[GetPosSlot1(i)]; + dp[i ] = slotPrice + tempPrices[i]; + dp[i + 1] = slotPrice + tempPrices[i + 1]; + } } } - p->matchPriceCount = 0; + // } } + + void LzmaEnc_Construct(CLzmaEnc *p) { RangeEnc_Construct(&p->rc); @@ -1770,6 +2221,7 @@ void LzmaEnc_Construct(CLzmaEnc *p) LzmaEnc_InitPriceTables(p->ProbPrices); p->litProbs = NULL; p->saveState.litProbs = NULL; + } CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc) @@ -1806,7 +2258,8 @@ void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig) ISzAlloc_Free(alloc, p); } -static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize, UInt32 maxUnpackSize) + +static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpackSize) { UInt32 nowPos32, startPos32; if (p->needInit) @@ -1824,13 +2277,13 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize if (p->nowPos64 == 0) { - UInt32 numPairs; + unsigned numPairs; Byte curByte; if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) return Flush(p, nowPos32); ReadMatchDistances(p, &numPairs); - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][0], 0); - p->state = kLiteralNextStates[p->state]; + RangeEnc_EncodeBit_0(&p->rc, &p->isMatch[kState_Start][0]); + // p->state = kLiteralNextStates[p->state]; curByte = *(p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset); LitEnc_Encode(&p->rc, p->litProbs, curByte); p->additionalOffset--; @@ -1838,123 +2291,253 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize } if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) != 0) + for (;;) { - UInt32 pos, len, posState; - + UInt32 dist; + unsigned len, posState; + UInt32 range, ttt, newBound; + CLzmaProb *probs; + if (p->fastMode) - len = GetOptimumFast(p, &pos); + len = GetOptimumFast(p); else - len = GetOptimum(p, nowPos32, &pos); + { + unsigned oci = p->optCur; + if (p->optEnd == oci) + len = GetOptimum(p, nowPos32); + else + { + const COptimal *opt = &p->opt[oci]; + len = opt->len; + p->backRes = opt->dist; + p->optCur = oci + 1; + } + } + + posState = (unsigned)nowPos32 & p->pbMask; + range = p->rc.range; + probs = &p->isMatch[p->state][posState]; + + RC_BIT_PRE(&p->rc, probs) + + dist = p->backRes; #ifdef SHOW_STAT2 - printf("\n pos = %4X, len = %u pos = %u", nowPos32, len, pos); + printf("\n pos = %6X, len = %3u pos = %6u", nowPos32, len, dist); #endif - posState = nowPos32 & p->pbMask; - if (len == 1 && pos == (UInt32)-1) + if (dist == MARK_LIT) { Byte curByte; - CLzmaProb *probs; const Byte *data; + unsigned state; - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 0); + RC_BIT_0(&p->rc, probs); + p->rc.range = range; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; - curByte = *data; probs = LIT_PROBS(nowPos32, *(data - 1)); - if (IsCharState(p->state)) + curByte = *data; + state = p->state; + p->state = kLiteralNextStates[state]; + if (IsLitState(state)) LitEnc_Encode(&p->rc, probs, curByte); else - LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0] - 1)); - p->state = kLiteralNextStates[p->state]; + LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0])); } else { - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - if (pos < LZMA_NUM_REPS) + RC_BIT_1(&p->rc, probs); + probs = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, probs) + + if (dist < LZMA_NUM_REPS) { - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 1); - if (pos == 0) + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG0[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 0) { - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 0); - RangeEnc_EncodeBit(&p->rc, &p->isRep0Long[p->state][posState], ((len == 1) ? 0 : 1)); + RC_BIT_0(&p->rc, probs); + probs = &p->isRep0Long[p->state][posState]; + RC_BIT_PRE(&p->rc, probs) + if (len != 1) + { + RC_BIT_1_BASE(&p->rc, probs); + } + else + { + RC_BIT_0_BASE(&p->rc, probs); + p->state = kShortRepNextStates[p->state]; + } } else { - UInt32 distance = p->reps[pos]; - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 1); - if (pos == 1) - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 0); + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG1[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 1) + { + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[1]; + } else { - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRepG2[p->state], pos - 2); - if (pos == 3) + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG2[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 2) + { + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[2]; + } + else + { + RC_BIT_1_BASE(&p->rc, probs); + dist = p->reps[3]; p->reps[3] = p->reps[2]; + } p->reps[2] = p->reps[1]; } p->reps[1] = p->reps[0]; - p->reps[0] = distance; + p->reps[0] = dist; } - if (len == 1) - p->state = kShortRepNextStates[p->state]; - else + + RC_NORM(&p->rc) + + p->rc.range = range; + + if (len != 1) { - LenEnc_Encode2(&p->repLenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); + LenEnc_Encode(&p->repLenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + --p->repLenEncCounter; p->state = kRepNextStates[p->state]; } } else { - UInt32 posSlot; - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); + unsigned posSlot; + RC_BIT_0(&p->rc, probs); + p->rc.range = range; p->state = kMatchNextStates[p->state]; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - pos -= LZMA_NUM_REPS; - GetPosSlot(pos, posSlot); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, posSlot); + + LenEnc_Encode(&p->lenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + // --p->lenEnc.counter; + + dist -= LZMA_NUM_REPS; + p->reps[3] = p->reps[2]; + p->reps[2] = p->reps[1]; + p->reps[1] = p->reps[0]; + p->reps[0] = dist + 1; - if (posSlot >= kStartPosModelIndex) + p->matchPriceCount++; + GetPosSlot(dist, posSlot); + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], posSlot); { - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - UInt32 posReduced = pos - base; + UInt32 sym = (UInt32)posSlot + (1 << kNumPosSlotBits); + range = p->rc.range; + probs = p->posSlotEncoder[GetLenToPosState(len)]; + do + { + CLzmaProb *prob = probs + (sym >> kNumPosSlotBits); + UInt32 bit = (sym >> (kNumPosSlotBits - 1)) & 1; + sym <<= 1; + RC_BIT(&p->rc, prob, bit); + } + while (sym < (1 << kNumPosSlotBits * 2)); + p->rc.range = range; + } + + if (dist >= kStartPosModelIndex) + { + unsigned footerBits = ((posSlot >> 1) - 1); - if (posSlot < kEndPosModelIndex) - RcTree_ReverseEncode(&p->rc, p->posEncoders + base - posSlot - 1, footerBits, posReduced); + if (dist < kNumFullDistances) + { + unsigned base = ((2 | (posSlot & 1)) << footerBits); + RcTree_ReverseEncode(&p->rc, p->posEncoders + base, footerBits, (unsigned)(dist /* - base */)); + } else { - RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); - p->alignPriceCount++; + UInt32 pos2 = (dist | 0xF) << (32 - footerBits); + range = p->rc.range; + // RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); + /* + do + { + range >>= 1; + p->rc.low += range & (0 - ((dist >> --footerBits) & 1)); + RC_NORM(&p->rc) + } + while (footerBits > kNumAlignBits); + */ + do + { + range >>= 1; + p->rc.low += range & (0 - (pos2 >> 31)); + pos2 += pos2; + RC_NORM(&p->rc) + } + while (pos2 != 0xF0000000); + + + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); + + { + unsigned m = 1; + unsigned bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); + p->rc.range = range; + // p->alignPriceCount++; + } } } - p->reps[3] = p->reps[2]; - p->reps[2] = p->reps[1]; - p->reps[1] = p->reps[0]; - p->reps[0] = pos; - p->matchPriceCount++; } } + + nowPos32 += (UInt32)len; p->additionalOffset -= len; - nowPos32 += len; + if (p->additionalOffset == 0) { UInt32 processed; + if (!p->fastMode) { - if (p->matchPriceCount >= (1 << 7)) + /* + if (p->alignPriceCount >= 16) // kAlignTableSize + FillAlignPrices(p); + if (p->matchPriceCount >= 128) FillDistancesPrices(p); - if (p->alignPriceCount >= kAlignTableSize) + if (p->lenEnc.counter <= 0) + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + */ + if (p->matchPriceCount >= 64) + { FillAlignPrices(p); + // { int y; for (y = 0; y < 100; y++) { + FillDistancesPrices(p); + // }} + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + } + if (p->repLenEncCounter <= 0) + { + p->repLenEncCounter = REP_LEN_COUNT; + LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); + } } + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) break; processed = nowPos32 - startPos32; - if (useLimits) + + if (maxPackSize) { - if (processed + kNumOpts + 300 >= maxUnpackSize || - RangeEnc_GetProcessed(&p->rc) + kNumOpts * 2 >= maxPackSize) + if (processed + kNumOpts + 300 >= maxUnpackSize + || RangeEnc_GetProcessed_sizet(&p->rc) + kPackReserve >= maxPackSize) break; } else if (processed >= (1 << 17)) @@ -1964,10 +2547,13 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize } } } + p->nowPos64 += nowPos32 - startPos32; return Flush(p, nowPos32); } + + #define kBigHashDicLimit ((UInt32)1 << 24) static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) @@ -2004,7 +2590,10 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, #ifndef _7ZIP_ST if (p->mtMode) { - RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)); + RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, + LZMA_MATCH_LEN_MAX + + 1 /* 18.04 */ + , allocBig)); p->matchFinderObj = &p->matchFinderMt; p->matchFinderBase.bigHash = (Byte)( (p->dictSize > kBigHashDicLimit && p->matchFinderBase.hashMask >= 0xFFFFFF) ? 1 : 0); @@ -2024,17 +2613,21 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, void LzmaEnc_Init(CLzmaEnc *p) { - UInt32 i; + unsigned i; p->state = 0; - for (i = 0 ; i < LZMA_NUM_REPS; i++) - p->reps[i] = 0; + p->reps[0] = + p->reps[1] = + p->reps[2] = + p->reps[3] = 1; RangeEnc_Init(&p->rc); + for (i = 0; i < (1 << kNumAlignBits); i++) + p->posAlignEncoder[i] = kProbInitValue; for (i = 0; i < kNumStates; i++) { - UInt32 j; + unsigned j; for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++) { p->isMatch[i][j] = kProbInitValue; @@ -2046,41 +2639,47 @@ void LzmaEnc_Init(CLzmaEnc *p) p->isRepG2[i] = kProbInitValue; } - { - UInt32 num = (UInt32)0x300 << (p->lp + p->lc); - CLzmaProb *probs = p->litProbs; - for (i = 0; i < num; i++) - probs[i] = kProbInitValue; - } - { for (i = 0; i < kNumLenToPosStates; i++) { CLzmaProb *probs = p->posSlotEncoder[i]; - UInt32 j; + unsigned j; for (j = 0; j < (1 << kNumPosSlotBits); j++) probs[j] = kProbInitValue; } } { - for (i = 0; i < kNumFullDistances - kEndPosModelIndex; i++) + for (i = 0; i < kNumFullDistances; i++) p->posEncoders[i] = kProbInitValue; } - LenEnc_Init(&p->lenEnc.p); - LenEnc_Init(&p->repLenEnc.p); + { + UInt32 num = (UInt32)0x300 << (p->lp + p->lc); + UInt32 k; + CLzmaProb *probs = p->litProbs; + for (k = 0; k < num; k++) + probs[k] = kProbInitValue; + } - for (i = 0; i < (1 << kNumAlignBits); i++) - p->posAlignEncoder[i] = kProbInitValue; - p->optimumEndIndex = 0; - p->optimumCurrentIndex = 0; + LenEnc_Init(&p->lenProbs); + LenEnc_Init(&p->repLenProbs); + + p->optEnd = 0; + p->optCur = 0; + + { + for (i = 0; i < kNumOpts; i++) + p->opt[i].price = kInfinityPrice; + } + p->additionalOffset = 0; p->pbMask = (1 << p->pb) - 1; - p->lpMask = (1 << p->lp) - 1; + p->lpMask = ((UInt32)0x100 << p->lp) - ((unsigned)0x100 >> p->lc); } + void LzmaEnc_InitPrices(CLzmaEnc *p) { if (!p->fastMode) @@ -2092,14 +2691,17 @@ void LzmaEnc_InitPrices(CLzmaEnc *p) p->lenEnc.tableSize = p->repLenEnc.tableSize = p->numFastBytes + 1 - LZMA_MATCH_LEN_MIN; - LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, p->ProbPrices); - LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, p->ProbPrices); + + p->repLenEncCounter = REP_LEN_COUNT; + + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); } static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - UInt32 i; - for (i = 0; i < (UInt32)kDicLogSizeMaxCompress; i++) + unsigned i; + for (i = kEndPosModelIndex / 2; i < kDicLogSizeMax; i++) if (p->dictSize <= ((UInt32)1 << i)) break; p->distTableSize = i * 2; @@ -2168,7 +2770,7 @@ typedef struct ISeqOutStream vt; Byte *data; SizeT rem; - Bool overflow; + BoolInt overflow; } CLzmaEnc_SeqOutStreamBuf; static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size) @@ -2200,7 +2802,7 @@ const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp) } -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize) { CLzmaEnc *p = (CLzmaEnc *)pp; @@ -2220,11 +2822,15 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, if (reInit) LzmaEnc_Init(p); LzmaEnc_InitPrices(p); + nowPos64 = p->nowPos64; RangeEnc_Init(&p->rc); p->rc.outStream = &outStream.vt; - res = LzmaEnc_CodeOneBlock(p, True, desiredPackSize, *unpackSize); + if (desiredPackSize == 0) + return SZ_ERROR_OUTPUT_EOF; + + res = LzmaEnc_CodeOneBlock(p, desiredPackSize, *unpackSize); *unpackSize = (UInt32)(p->nowPos64 - nowPos64); *destLen -= outStream.rem; @@ -2247,7 +2853,7 @@ static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) for (;;) { - res = LzmaEnc_CodeOneBlock(p, False, 0, 0); + res = LzmaEnc_CodeOneBlock(p, 0, 0); if (res != SZ_OK || p->finished) break; if (progress) diff --git a/7zip/C/MtCoder.c b/7zip/C/MtCoder.c index 32bd0fae8..95359857b 100644 --- a/7zip/C/MtCoder.c +++ b/7zip/C/MtCoder.c @@ -1,85 +1,28 @@ /* MtCoder.c -- Multi-thread Coder -2017-07-17 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "MtCoder.h" -static void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) -{ - unsigned i; +#ifndef _7ZIP_ST - p->progress = progress; - p->res = SZ_OK; - p->totalInSize = 0; - p->totalOutSize = 0; - - for (i = 0; i < MTCODER__THREADS_MAX; i++) +SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) +{ + CMtProgressThunk *thunk = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt); + UInt64 inSize2 = 0; + UInt64 outSize2 = 0; + if (inSize != (UInt64)(Int64)-1) { - CMtProgressSizes *pair = &p->sizes[i]; - pair->inSize = 0; - pair->outSize = 0; + inSize2 = inSize - thunk->inSize; + thunk->inSize = inSize; } -} - - -static void MtProgress_Reinit(CMtProgress *p, unsigned index) -{ - CMtProgressSizes *pair = &p->sizes[index]; - pair->inSize = 0; - pair->outSize = 0; -} - - -#define UPDATE_PROGRESS(size, prev, total) \ - if (size != (UInt64)(Int64)-1) { total += size - prev; prev = size; } - -SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize) -{ - SRes res; - CMtProgressSizes *pair; - - CriticalSection_Enter(&p->cs); - - pair = &p->sizes[index]; - UPDATE_PROGRESS(inSize, pair->inSize, p->totalInSize) - UPDATE_PROGRESS(outSize, pair->outSize, p->totalOutSize) - if (p->res == SZ_OK && p->progress) + if (outSize != (UInt64)(Int64)-1) { - if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) - p->res = SZ_ERROR_PROGRESS; + outSize2 = outSize - thunk->outSize; + thunk->outSize = outSize; } - res = p->res; - - CriticalSection_Leave(&p->cs); - - return res; -} - - -static SRes MtProgress_GetError(CMtProgress *p) -{ - SRes res; - CriticalSection_Enter(&p->cs); - res = p->res; - CriticalSection_Leave(&p->cs); - return res; -} - - -static void MtProgress_SetError(CMtProgress *p, SRes res) -{ - CriticalSection_Enter(&p->cs); - if (p->res == SZ_OK) - p->res = res; - CriticalSection_Leave(&p->cs); -} - - -static SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) -{ - CMtProgressThunk *p = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt); - return MtProgress_Set(p->mtProgress, p->index, inSize, outSize); + return MtProgress_ProgressAdd(thunk->mtProgress, inSize2, outSize2); } @@ -176,7 +119,7 @@ static SRes ThreadFunc2(CMtCoderThread *t) unsigned bi; SRes res; SRes res2; - Bool finished; + BoolInt finished; unsigned bufIndex; size_t size; const Byte *inData; @@ -298,7 +241,7 @@ static SRes ThreadFunc2(CMtCoderThread *t) res = mtc->mtCallback->Code(mtc->mtCallbackObject, t->index, bufIndex, mtc->inStream ? t->inBuf : inData, size, finished); - MtProgress_Reinit(&mtc->mtProgress, t->index); + // MtProgress_Reinit(&mtc->mtProgress, t->index); if (res != SZ_OK) MtProgress_SetError(&mtc->mtProgress, res); @@ -351,7 +294,7 @@ static SRes ThreadFunc2(CMtCoderThread *t) if (++wi >= mtc->numBlocksMax) wi = 0; { - Bool isReady; + BoolInt isReady; CriticalSection_Enter(&mtc->cs); @@ -604,7 +547,7 @@ SRes MtCoder_Code(CMtCoder *p) { const CMtCoderBlock *block = &p->blocks[bi]; unsigned bufIndex = block->bufIndex; - Bool finished = block->finished; + BoolInt finished = block->finished; if (res == SZ_OK && block->res != SZ_OK) res = block->res; @@ -654,3 +597,5 @@ SRes MtCoder_Code(CMtCoder *p) MtCoder_Free(p); return res; } + +#endif diff --git a/7zip/C/MtCoder.h b/7zip/C/MtCoder.h index 624aff24d..5a5f4d11b 100644 --- a/7zip/C/MtCoder.h +++ b/7zip/C/MtCoder.h @@ -1,10 +1,10 @@ /* MtCoder.h -- Multi-thread Coder -2017-06-18 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __MT_CODER_H #define __MT_CODER_H -#include "Threads.h" +#include "MtDec.h" EXTERN_C_BEGIN @@ -24,33 +24,20 @@ EXTERN_C_BEGIN #endif -typedef struct -{ - UInt64 inSize; - UInt64 outSize; -} CMtProgressSizes; - - -typedef struct -{ - ICompressProgress *progress; - SRes res; - UInt64 totalInSize; - UInt64 totalOutSize; - CCriticalSection cs; - CMtProgressSizes sizes[MTCODER__THREADS_MAX]; -} CMtProgress; +#ifndef _7ZIP_ST typedef struct { ICompressProgress vt; CMtProgress *mtProgress; - unsigned index; + UInt64 inSize; + UInt64 outSize; } CMtProgressThunk; void MtProgressThunk_CreateVTable(CMtProgressThunk *p); +#define MtProgressThunk_Init(p) { (p)->inSize = 0; (p)->outSize = 0; } struct _CMtCoder; @@ -80,7 +67,7 @@ typedef struct { SRes res; unsigned bufIndex; - Bool finished; + BoolInt finished; } CMtCoderBlock; @@ -110,7 +97,7 @@ typedef struct _CMtCoder CAutoResetEvent readEvent; CSemaphore blocksSemaphore; - Bool stopReading; + BoolInt stopReading; SRes readRes; #ifdef MTCODER__USE_WRITE_THREAD @@ -146,6 +133,9 @@ void MtCoder_Destruct(CMtCoder *p); SRes MtCoder_Code(CMtCoder *p); +#endif + + EXTERN_C_END #endif diff --git a/7zip/C/MtDec.c b/7zip/C/MtDec.c new file mode 100644 index 000000000..7803bf2a9 --- /dev/null +++ b/7zip/C/MtDec.c @@ -0,0 +1,1138 @@ +/* MtDec.c -- Multi-thread Decoder +2019-02-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +// #include + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) + +#include "MtDec.h" + +#ifndef _7ZIP_ST + +void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) +{ + p->progress = progress; + p->res = SZ_OK; + p->totalInSize = 0; + p->totalOutSize = 0; +} + + +SRes MtProgress_Progress_ST(CMtProgress *p) +{ + if (p->res == SZ_OK && p->progress) + if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) + p->res = SZ_ERROR_PROGRESS; + return p->res; +} + + +SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize) +{ + SRes res; + CriticalSection_Enter(&p->cs); + + p->totalInSize += inSize; + p->totalOutSize += outSize; + if (p->res == SZ_OK && p->progress) + if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) + p->res = SZ_ERROR_PROGRESS; + res = p->res; + + CriticalSection_Leave(&p->cs); + return res; +} + + +SRes MtProgress_GetError(CMtProgress *p) +{ + SRes res; + CriticalSection_Enter(&p->cs); + res = p->res; + CriticalSection_Leave(&p->cs); + return res; +} + + +void MtProgress_SetError(CMtProgress *p, SRes res) +{ + CriticalSection_Enter(&p->cs); + if (p->res == SZ_OK) + p->res = res; + CriticalSection_Leave(&p->cs); +} + + +#define RINOK_THREAD(x) RINOK(x) + + +static WRes ArEvent_OptCreate_And_Reset(CEvent *p) +{ + if (Event_IsCreated(p)) + return Event_Reset(p); + return AutoResetEvent_CreateNotSignaled(p); +} + + +struct __CMtDecBufLink +{ + struct __CMtDecBufLink *next; + void *pad[3]; +}; + +typedef struct __CMtDecBufLink CMtDecBufLink; + +#define MTDEC__LINK_DATA_OFFSET sizeof(CMtDecBufLink) +#define MTDEC__DATA_PTR_FROM_LINK(link) ((Byte *)(link) + MTDEC__LINK_DATA_OFFSET) + + + +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp); + + +static WRes MtDecThread_CreateEvents(CMtDecThread *t) +{ + WRes wres = ArEvent_OptCreate_And_Reset(&t->canWrite); + if (wres == 0) + { + wres = ArEvent_OptCreate_And_Reset(&t->canRead); + if (wres == 0) + return SZ_OK; + } + return wres; +} + + +static SRes MtDecThread_CreateAndStart(CMtDecThread *t) +{ + WRes wres = MtDecThread_CreateEvents(t); + // wres = 17; // for test + if (wres == 0) + { + if (Thread_WasCreated(&t->thread)) + return SZ_OK; + wres = Thread_Create(&t->thread, ThreadFunc, t); + if (wres == 0) + return SZ_OK; + } + return MY_SRes_HRESULT_FROM_WRes(wres); +} + + +void MtDecThread_FreeInBufs(CMtDecThread *t) +{ + if (t->inBuf) + { + void *link = t->inBuf; + t->inBuf = NULL; + do + { + void *next = ((CMtDecBufLink *)link)->next; + ISzAlloc_Free(t->mtDec->alloc, link); + link = next; + } + while (link); + } +} + + +static void MtDecThread_CloseThread(CMtDecThread *t) +{ + if (Thread_WasCreated(&t->thread)) + { + Event_Set(&t->canWrite); /* we can disable it. There are no threads waiting canWrite in normal cases */ + Event_Set(&t->canRead); + Thread_Wait(&t->thread); + Thread_Close(&t->thread); + } + + Event_Close(&t->canRead); + Event_Close(&t->canWrite); +} + +static void MtDec_CloseThreads(CMtDec *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + MtDecThread_CloseThread(&p->threads[i]); +} + +static void MtDecThread_Destruct(CMtDecThread *t) +{ + MtDecThread_CloseThread(t); + MtDecThread_FreeInBufs(t); +} + + + +static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) +{ + size_t size = *processedSize; + *processedSize = 0; + while (size != 0) + { + size_t cur = size; + SRes res = ISeqInStream_Read(stream, data, &cur); + *processedSize += cur; + data += cur; + size -= cur; + RINOK(res); + if (cur == 0) + return SZ_OK; + } + return SZ_OK; +} + + +static SRes MtDec_GetError_Spec(CMtDec *p, UInt64 interruptIndex, BoolInt *wasInterrupted) +{ + SRes res; + CriticalSection_Enter(&p->mtProgress.cs); + *wasInterrupted = (p->needInterrupt && interruptIndex > p->interruptIndex); + res = p->mtProgress.res; + CriticalSection_Leave(&p->mtProgress.cs); + return res; +} + +static SRes MtDec_Progress_GetError_Spec(CMtDec *p, UInt64 inSize, UInt64 outSize, UInt64 interruptIndex, BoolInt *wasInterrupted) +{ + SRes res; + CriticalSection_Enter(&p->mtProgress.cs); + + p->mtProgress.totalInSize += inSize; + p->mtProgress.totalOutSize += outSize; + if (p->mtProgress.res == SZ_OK && p->mtProgress.progress) + if (ICompressProgress_Progress(p->mtProgress.progress, p->mtProgress.totalInSize, p->mtProgress.totalOutSize) != SZ_OK) + p->mtProgress.res = SZ_ERROR_PROGRESS; + + *wasInterrupted = (p->needInterrupt && interruptIndex > p->interruptIndex); + res = p->mtProgress.res; + + CriticalSection_Leave(&p->mtProgress.cs); + + return res; +} + +static void MtDec_Interrupt(CMtDec *p, UInt64 interruptIndex) +{ + CriticalSection_Enter(&p->mtProgress.cs); + if (!p->needInterrupt || interruptIndex < p->interruptIndex) + { + p->interruptIndex = interruptIndex; + p->needInterrupt = True; + } + CriticalSection_Leave(&p->mtProgress.cs); +} + +Byte *MtDec_GetCrossBuff(CMtDec *p) +{ + Byte *cr = p->crossBlock; + if (!cr) + { + cr = (Byte *)ISzAlloc_Alloc(p->alloc, MTDEC__LINK_DATA_OFFSET + p->inBufSize); + if (!cr) + return NULL; + p->crossBlock = cr; + } + return MTDEC__DATA_PTR_FROM_LINK(cr); +} + + +/* + ThreadFunc2() returns: + 0 - in all normal cases (even for stream error or memory allocation error) + (!= 0) - WRes error return by system threading function +*/ + +// #define MTDEC_ProgessStep (1 << 22) +#define MTDEC_ProgessStep (1 << 0) + +static WRes ThreadFunc2(CMtDecThread *t) +{ + CMtDec *p = t->mtDec; + + PRF_STR_INT("ThreadFunc2", t->index); + + // SetThreadAffinityMask(GetCurrentThread(), 1 << t->index); + + for (;;) + { + SRes res, codeRes; + BoolInt wasInterrupted, isAllocError, overflow, finish; + SRes threadingErrorSRes; + BoolInt needCode, needWrite, needContinue; + + size_t inDataSize_Start; + UInt64 inDataSize; + // UInt64 inDataSize_Full; + + UInt64 blockIndex; + + UInt64 inPrev = 0; + UInt64 outPrev = 0; + UInt64 inCodePos; + UInt64 outCodePos; + + Byte *afterEndData = NULL; + size_t afterEndData_Size = 0; + + BoolInt canCreateNewThread = False; + // CMtDecCallbackInfo parse; + CMtDecThread *nextThread; + + PRF_STR_INT("Event_Wait(&t->canRead)", t->index); + + RINOK_THREAD(Event_Wait(&t->canRead)); + if (p->exitThread) + return 0; + + PRF_STR_INT("after Event_Wait(&t->canRead)", t->index); + + // if (t->index == 3) return 19; // for test + + blockIndex = p->blockIndex++; + + // PRF(printf("\ncanRead\n")) + + res = MtDec_Progress_GetError_Spec(p, 0, 0, blockIndex, &wasInterrupted); + + finish = p->readWasFinished; + needCode = False; + needWrite = False; + isAllocError = False; + overflow = False; + + inDataSize_Start = 0; + inDataSize = 0; + // inDataSize_Full = 0; + + if (res == SZ_OK && !wasInterrupted) + { + // if (p->inStream) + { + CMtDecBufLink *prev = NULL; + CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf; + size_t crossSize = p->crossEnd - p->crossStart; + + PRF(printf("\ncrossSize = %d\n", crossSize)); + + for (;;) + { + if (!link) + { + link = (CMtDecBufLink *)ISzAlloc_Alloc(p->alloc, MTDEC__LINK_DATA_OFFSET + p->inBufSize); + if (!link) + { + finish = True; + // p->allocError_for_Read_BlockIndex = blockIndex; + isAllocError = True; + break; + } + link->next = NULL; + if (prev) + { + // static unsigned g_num = 0; + // printf("\n%6d : %x", ++g_num, (unsigned)(size_t)((Byte *)link - (Byte *)prev)); + prev->next = link; + } + else + t->inBuf = (void *)link; + } + + { + Byte *data = MTDEC__DATA_PTR_FROM_LINK(link); + Byte *parseData = data; + size_t size; + + if (crossSize != 0) + { + inDataSize = crossSize; + // inDataSize_Full = inDataSize; + inDataSize_Start = crossSize; + size = crossSize; + parseData = MTDEC__DATA_PTR_FROM_LINK(p->crossBlock) + p->crossStart; + PRF(printf("\ncross : crossStart = %7d crossEnd = %7d finish = %1d", + (int)p->crossStart, (int)p->crossEnd, (int)finish)); + } + else + { + size = p->inBufSize; + + res = FullRead(p->inStream, data, &size); + + // size = 10; // test + + inDataSize += size; + // inDataSize_Full = inDataSize; + if (!prev) + inDataSize_Start = size; + + p->readProcessed += size; + finish = (size != p->inBufSize); + if (finish) + p->readWasFinished = True; + + // res = E_INVALIDARG; // test + + if (res != SZ_OK) + { + // PRF(printf("\nRead error = %d\n", res)) + // we want to decode all data before error + p->readRes = res; + // p->readError_BlockIndex = blockIndex; + p->readWasFinished = True; + finish = True; + res = SZ_OK; + // break; + } + + if (inDataSize - inPrev >= MTDEC_ProgessStep) + { + res = MtDec_Progress_GetError_Spec(p, 0, 0, blockIndex, &wasInterrupted); + if (res != SZ_OK || wasInterrupted) + break; + inPrev = inDataSize; + } + } + + { + CMtDecCallbackInfo parse; + + parse.startCall = (prev == NULL); + parse.src = parseData; + parse.srcSize = size; + parse.srcFinished = finish; + parse.canCreateNewThread = True; + + // PRF(printf("\nParse size = %d\n", (unsigned)size)) + + p->mtCallback->Parse(p->mtCallbackObject, t->index, &parse); + + needWrite = True; + canCreateNewThread = parse.canCreateNewThread; + + // printf("\n\n%12I64u %12I64u", (UInt64)p->mtProgress.totalInSize, (UInt64)p->mtProgress.totalOutSize); + + if ( + // parseRes != SZ_OK || + // inDataSize - (size - parse.srcSize) > p->inBlockMax + // || + parse.state == MTDEC_PARSE_OVERFLOW + // || wasInterrupted + ) + { + // Overflow or Parse error - switch from MT decoding to ST decoding + finish = True; + overflow = True; + + { + PRF(printf("\n Overflow")); + // PRF(printf("\nisBlockFinished = %d", (unsigned)parse.blockWasFinished)); + PRF(printf("\n inDataSize = %d", (unsigned)inDataSize)); + } + + if (crossSize != 0) + memcpy(data, parseData, size); + p->crossStart = 0; + p->crossEnd = 0; + break; + } + + if (crossSize != 0) + { + memcpy(data, parseData, parse.srcSize); + p->crossStart += parse.srcSize; + } + + if (parse.state != MTDEC_PARSE_CONTINUE || finish) + { + // we don't need to parse in current thread anymore + + if (parse.state == MTDEC_PARSE_END) + finish = True; + + needCode = True; + // p->crossFinished = finish; + + if (parse.srcSize == size) + { + // full parsed - no cross transfer + p->crossStart = 0; + p->crossEnd = 0; + break; + } + + if (parse.state == MTDEC_PARSE_END) + { + p->crossStart = 0; + p->crossEnd = 0; + + if (crossSize != 0) + memcpy(data + parse.srcSize, parseData + parse.srcSize, size - parse.srcSize); // we need all data + afterEndData_Size = size - parse.srcSize; + afterEndData = parseData + parse.srcSize; + + // we reduce data size to required bytes (parsed only) + inDataSize -= (size - parse.srcSize); + if (!prev) + inDataSize_Start = parse.srcSize; + break; + } + + { + // partial parsed - need cross transfer + if (crossSize != 0) + inDataSize = parse.srcSize; // it's only parsed now + else + { + // partial parsed - is not in initial cross block - we need to copy new data to cross block + Byte *cr = MtDec_GetCrossBuff(p); + if (!cr) + { + { + PRF(printf("\ncross alloc error error\n")); + // res = SZ_ERROR_MEM; + finish = True; + // p->allocError_for_Read_BlockIndex = blockIndex; + isAllocError = True; + break; + } + } + + { + size_t crSize = size - parse.srcSize; + inDataSize -= crSize; + p->crossEnd = crSize; + p->crossStart = 0; + memcpy(cr, parseData + parse.srcSize, crSize); + } + } + + // inDataSize_Full = inDataSize; + if (!prev) + inDataSize_Start = parse.srcSize; // it's partial size (parsed only) + + finish = False; + break; + } + } + + if (parse.srcSize != size) + { + res = SZ_ERROR_FAIL; + PRF(printf("\nfinished error SZ_ERROR_FAIL = %d\n", res)); + break; + } + } + } + + prev = link; + link = link->next; + + if (crossSize != 0) + { + crossSize = 0; + p->crossStart = 0; + p->crossEnd = 0; + } + } + } + + if (res == SZ_OK) + res = MtDec_GetError_Spec(p, blockIndex, &wasInterrupted); + } + + codeRes = SZ_OK; + + if (res == SZ_OK && needCode && !wasInterrupted) + { + codeRes = p->mtCallback->PreCode(p->mtCallbackObject, t->index); + if (codeRes != SZ_OK) + { + needCode = False; + finish = True; + // SZ_ERROR_MEM is expected error here. + // if (codeRes == SZ_ERROR_MEM) - we will try single-thread decoding later. + // if (codeRes != SZ_ERROR_MEM) - we can stop decoding or try single-thread decoding. + } + } + + if (res != SZ_OK || wasInterrupted) + finish = True; + + nextThread = NULL; + threadingErrorSRes = SZ_OK; + + if (!finish) + { + if (p->numStartedThreads < p->numStartedThreads_Limit && canCreateNewThread) + { + SRes res2 = MtDecThread_CreateAndStart(&p->threads[p->numStartedThreads]); + if (res2 == SZ_OK) + { + // if (p->numStartedThreads % 1000 == 0) PRF(printf("\n numStartedThreads=%d\n", p->numStartedThreads)); + p->numStartedThreads++; + } + else + { + PRF(printf("\nERROR: numStartedThreads=%d\n", p->numStartedThreads)); + if (p->numStartedThreads == 1) + { + // if only one thread is possible, we leave muti-threading code + finish = True; + needCode = False; + threadingErrorSRes = res2; + } + else + p->numStartedThreads_Limit = p->numStartedThreads; + } + } + + if (!finish) + { + unsigned nextIndex = t->index + 1; + nextThread = &p->threads[nextIndex >= p->numStartedThreads ? 0 : nextIndex]; + RINOK_THREAD(Event_Set(&nextThread->canRead)) + // We have started executing for new iteration (with next thread) + // And that next thread now is responsible for possible exit from decoding (threading_code) + } + } + + // each call of Event_Set(&nextThread->canRead) must be followed by call of Event_Set(&nextThread->canWrite) + // if ( !finish ) we must call Event_Set(&nextThread->canWrite) in any case + // if ( finish ) we switch to single-thread mode and there are 2 ways at the end of current iteration (current block): + // - if (needContinue) after Write(&needContinue), we restore decoding with new iteration + // - otherwise we stop decoding and exit from ThreadFunc2() + + // Don't change (finish) variable in the further code + + + // ---------- CODE ---------- + + inPrev = 0; + outPrev = 0; + inCodePos = 0; + outCodePos = 0; + + if (res == SZ_OK && needCode && codeRes == SZ_OK) + { + BoolInt isStartBlock = True; + CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf; + + for (;;) + { + size_t inSize; + int stop; + + if (isStartBlock) + inSize = inDataSize_Start; + else + { + UInt64 rem = inDataSize - inCodePos; + inSize = p->inBufSize; + if (inSize > rem) + inSize = (size_t)rem; + } + + inCodePos += inSize; + stop = True; + + codeRes = p->mtCallback->Code(p->mtCallbackObject, t->index, + (const Byte *)MTDEC__DATA_PTR_FROM_LINK(link), inSize, + (inCodePos == inDataSize), // srcFinished + &inCodePos, &outCodePos, &stop); + + if (codeRes != SZ_OK) + { + PRF(printf("\nCode Interrupt error = %x\n", codeRes)); + // we interrupt only later blocks + MtDec_Interrupt(p, blockIndex); + break; + } + + if (stop || inCodePos == inDataSize) + break; + + { + const UInt64 inDelta = inCodePos - inPrev; + const UInt64 outDelta = outCodePos - outPrev; + if (inDelta >= MTDEC_ProgessStep || outDelta >= MTDEC_ProgessStep) + { + // Sleep(1); + res = MtDec_Progress_GetError_Spec(p, inDelta, outDelta, blockIndex, &wasInterrupted); + if (res != SZ_OK || wasInterrupted) + break; + inPrev = inCodePos; + outPrev = outCodePos; + } + } + + link = link->next; + isStartBlock = False; + } + } + + + // ---------- WRITE ---------- + + RINOK_THREAD(Event_Wait(&t->canWrite)); + + { + BoolInt isErrorMode = False; + BoolInt canRecode = True; + BoolInt needWriteToStream = needWrite; + + if (p->exitThread) return 0; // it's never executed in normal cases + + if (p->wasInterrupted) + wasInterrupted = True; + else + { + if (codeRes != SZ_OK) // || !needCode // check it !!! + { + p->wasInterrupted = True; + p->codeRes = codeRes; + if (codeRes == SZ_ERROR_MEM) + isAllocError = True; + } + + if (threadingErrorSRes) + { + p->wasInterrupted = True; + p->threadingErrorSRes = threadingErrorSRes; + needWriteToStream = False; + } + if (isAllocError) + { + p->wasInterrupted = True; + p->isAllocError = True; + needWriteToStream = False; + } + if (overflow) + { + p->wasInterrupted = True; + p->overflow = True; + needWriteToStream = False; + } + } + + if (needCode) + { + if (wasInterrupted) + { + inCodePos = 0; + outCodePos = 0; + } + { + const UInt64 inDelta = inCodePos - inPrev; + const UInt64 outDelta = outCodePos - outPrev; + // if (inDelta != 0 || outDelta != 0) + res = MtProgress_ProgressAdd(&p->mtProgress, inDelta, outDelta); + } + } + + needContinue = (!finish); + + // if (res == SZ_OK && needWrite && !wasInterrupted) + if (needWrite) + { + // p->inProcessed += inCodePos; + + res = p->mtCallback->Write(p->mtCallbackObject, t->index, + res == SZ_OK && needWriteToStream && !wasInterrupted, // needWrite + afterEndData, afterEndData_Size, + &needContinue, + &canRecode); + + // res= E_INVALIDARG; // for test + + PRF(printf("\nAfter Write needContinue = %d\n", (unsigned)needContinue)); + PRF(printf("\nprocessed = %d\n", (unsigned)p->inProcessed)); + + if (res != SZ_OK) + { + PRF(printf("\nWrite error = %d\n", res)); + isErrorMode = True; + p->wasInterrupted = True; + } + if (res != SZ_OK + || (!needContinue && !finish)) + { + PRF(printf("\nWrite Interrupt error = %x\n", res)); + MtDec_Interrupt(p, blockIndex); + } + } + + if (canRecode) + if (!needCode + || res != SZ_OK + || p->wasInterrupted + || codeRes != SZ_OK + || wasInterrupted + || p->numFilledThreads != 0 + || isErrorMode) + { + if (p->numFilledThreads == 0) + p->filledThreadStart = t->index; + if (inDataSize != 0 || !finish) + { + t->inDataSize_Start = inDataSize_Start; + t->inDataSize = inDataSize; + p->numFilledThreads++; + } + PRF(printf("\np->numFilledThreads = %d\n", p->numFilledThreads)); + PRF(printf("p->filledThreadStart = %d\n", p->filledThreadStart)); + } + + if (!finish) + { + RINOK_THREAD(Event_Set(&nextThread->canWrite)); + } + else + { + if (needContinue) + { + // we restore decoding with new iteration + RINOK_THREAD(Event_Set(&p->threads[0].canWrite)); + } + else + { + // we exit from decoding + if (t->index == 0) + return SZ_OK; + p->exitThread = True; + } + RINOK_THREAD(Event_Set(&p->threads[0].canRead)); + } + } + } +} + +#ifdef _WIN32 +#define USE_ALLOCA +#endif + +#ifdef USE_ALLOCA +#ifdef _WIN32 +#include +#else +#include +#endif +#endif + + +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc1(void *pp) +{ + WRes res; + + CMtDecThread *t = (CMtDecThread *)pp; + CMtDec *p; + + // fprintf(stdout, "\n%d = %p\n", t->index, &t); + + res = ThreadFunc2(t); + p = t->mtDec; + if (res == 0) + return p->exitThreadWRes; + { + // it's unexpected situation for some threading function error + if (p->exitThreadWRes == 0) + p->exitThreadWRes = res; + PRF(printf("\nthread exit error = %d\n", res)); + p->exitThread = True; + Event_Set(&p->threads[0].canRead); + Event_Set(&p->threads[0].canWrite); + MtProgress_SetError(&p->mtProgress, MY_SRes_HRESULT_FROM_WRes(res)); + } + return res; +} + +static MY_NO_INLINE THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp) +{ + CMtDecThread *t = (CMtDecThread *)pp; + + // fprintf(stderr, "\n%d = %p - before", t->index, &t); + #ifdef USE_ALLOCA + t->allocaPtr = alloca(t->index * 128); + #endif + return ThreadFunc1(pp); +} + + +int MtDec_PrepareRead(CMtDec *p) +{ + if (p->crossBlock && p->crossStart == p->crossEnd) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + if (i > p->numStartedThreads + || p->numFilledThreads <= + (i >= p->filledThreadStart ? + i - p->filledThreadStart : + i + p->numStartedThreads - p->filledThreadStart)) + MtDecThread_FreeInBufs(&p->threads[i]); + } + + return (p->numFilledThreads != 0) || (p->crossStart != p->crossEnd); +} + + +const Byte *MtDec_Read(CMtDec *p, size_t *inLim) +{ + while (p->numFilledThreads != 0) + { + CMtDecThread *t = &p->threads[p->filledThreadStart]; + + if (*inLim != 0) + { + { + void *link = t->inBuf; + void *next = ((CMtDecBufLink *)link)->next; + ISzAlloc_Free(p->alloc, link); + t->inBuf = next; + } + + if (t->inDataSize == 0) + { + MtDecThread_FreeInBufs(t); + if (--p->numFilledThreads == 0) + break; + if (++p->filledThreadStart == p->numStartedThreads) + p->filledThreadStart = 0; + t = &p->threads[p->filledThreadStart]; + } + } + + { + size_t lim = t->inDataSize_Start; + if (lim != 0) + t->inDataSize_Start = 0; + else + { + UInt64 rem = t->inDataSize; + lim = p->inBufSize; + if (lim > rem) + lim = (size_t)rem; + } + t->inDataSize -= lim; + *inLim = lim; + return (const Byte *)MTDEC__DATA_PTR_FROM_LINK(t->inBuf); + } + } + + { + size_t crossSize = p->crossEnd - p->crossStart; + if (crossSize != 0) + { + const Byte *data = MTDEC__DATA_PTR_FROM_LINK(p->crossBlock) + p->crossStart; + *inLim = crossSize; + p->crossStart = 0; + p->crossEnd = 0; + return data; + } + *inLim = 0; + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + return NULL; + } +} + + +void MtDec_Construct(CMtDec *p) +{ + unsigned i; + + p->inBufSize = (size_t)1 << 18; + + p->numThreadsMax = 0; + + p->inStream = NULL; + + // p->inData = NULL; + // p->inDataSize = 0; + + p->crossBlock = NULL; + p->crossStart = 0; + p->crossEnd = 0; + + p->numFilledThreads = 0; + + p->progress = NULL; + p->alloc = NULL; + + p->mtCallback = NULL; + p->mtCallbackObject = NULL; + + p->allocatedBufsSize = 0; + + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CMtDecThread *t = &p->threads[i]; + t->mtDec = p; + t->index = i; + t->inBuf = NULL; + Event_Construct(&t->canRead); + Event_Construct(&t->canWrite); + Thread_Construct(&t->thread); + } + + // Event_Construct(&p->finishedEvent); + + CriticalSection_Init(&p->mtProgress.cs); +} + + +static void MtDec_Free(CMtDec *p) +{ + unsigned i; + + p->exitThread = True; + + for (i = 0; i < MTDEC__THREADS_MAX; i++) + MtDecThread_Destruct(&p->threads[i]); + + // Event_Close(&p->finishedEvent); + + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } +} + + +void MtDec_Destruct(CMtDec *p) +{ + MtDec_Free(p); + + CriticalSection_Delete(&p->mtProgress.cs); +} + + +SRes MtDec_Code(CMtDec *p) +{ + unsigned i; + + p->inProcessed = 0; + + p->blockIndex = 1; // it must be larger than not_defined index (0) + p->isAllocError = False; + p->overflow = False; + p->threadingErrorSRes = SZ_OK; + + p->needContinue = True; + + p->readWasFinished = False; + p->needInterrupt = False; + p->interruptIndex = (UInt64)(Int64)-1; + + p->readProcessed = 0; + p->readRes = SZ_OK; + p->codeRes = SZ_OK; + p->wasInterrupted = False; + + p->crossStart = 0; + p->crossEnd = 0; + + p->filledThreadStart = 0; + p->numFilledThreads = 0; + + { + unsigned numThreads = p->numThreadsMax; + if (numThreads > MTDEC__THREADS_MAX) + numThreads = MTDEC__THREADS_MAX; + p->numStartedThreads_Limit = numThreads; + p->numStartedThreads = 0; + } + + if (p->inBufSize != p->allocatedBufsSize) + { + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CMtDecThread *t = &p->threads[i]; + if (t->inBuf) + MtDecThread_FreeInBufs(t); + } + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + + p->allocatedBufsSize = p->inBufSize; + } + + MtProgress_Init(&p->mtProgress, p->progress); + + // RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent)); + p->exitThread = False; + p->exitThreadWRes = 0; + + { + WRes wres; + WRes sres; + CMtDecThread *nextThread = &p->threads[p->numStartedThreads++]; + // wres = MtDecThread_CreateAndStart(nextThread); + wres = MtDecThread_CreateEvents(nextThread); + if (wres == 0) { wres = Event_Set(&nextThread->canWrite); + if (wres == 0) { wres = Event_Set(&nextThread->canRead); + if (wres == 0) { wres = ThreadFunc(nextThread); + if (wres != 0) + { + p->needContinue = False; + MtDec_CloseThreads(p); + }}}} + + // wres = 17; // for test + // wres = Event_Wait(&p->finishedEvent); + + sres = MY_SRes_HRESULT_FROM_WRes(wres); + + if (sres != 0) + p->threadingErrorSRes = sres; + + if ( + // wres == 0 + // wres != 0 + // || p->mtc.codeRes == SZ_ERROR_MEM + p->isAllocError + || p->threadingErrorSRes != SZ_OK + || p->overflow) + { + // p->needContinue = True; + } + else + p->needContinue = False; + + if (p->needContinue) + return SZ_OK; + + // if (sres != SZ_OK) + return sres; + // return E_FAIL; + } +} + +#endif diff --git a/7zip/C/MtDec.h b/7zip/C/MtDec.h new file mode 100644 index 000000000..9b5776672 --- /dev/null +++ b/7zip/C/MtDec.h @@ -0,0 +1,201 @@ +/* MtDec.h -- Multi-thread Decoder +2018-07-04 : Igor Pavlov : Public domain */ + +#ifndef __MT_DEC_H +#define __MT_DEC_H + +#include "7zTypes.h" + +#ifndef _7ZIP_ST +#include "Threads.h" +#endif + +EXTERN_C_BEGIN + +#ifndef _7ZIP_ST + +#ifndef _7ZIP_ST + #define MTDEC__THREADS_MAX 32 +#else + #define MTDEC__THREADS_MAX 1 +#endif + + +typedef struct +{ + ICompressProgress *progress; + SRes res; + UInt64 totalInSize; + UInt64 totalOutSize; + CCriticalSection cs; +} CMtProgress; + +void MtProgress_Init(CMtProgress *p, ICompressProgress *progress); +SRes MtProgress_Progress_ST(CMtProgress *p); +SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize); +SRes MtProgress_GetError(CMtProgress *p); +void MtProgress_SetError(CMtProgress *p, SRes res); + +struct _CMtDec; + +typedef struct +{ + struct _CMtDec *mtDec; + unsigned index; + void *inBuf; + + size_t inDataSize_Start; // size of input data in start block + UInt64 inDataSize; // total size of input data in all blocks + + CThread thread; + CAutoResetEvent canRead; + CAutoResetEvent canWrite; + void *allocaPtr; +} CMtDecThread; + +void MtDecThread_FreeInBufs(CMtDecThread *t); + + +typedef enum +{ + MTDEC_PARSE_CONTINUE, // continue this block with more input data + MTDEC_PARSE_OVERFLOW, // MT buffers overflow, need switch to single-thread + MTDEC_PARSE_NEW, // new block + MTDEC_PARSE_END // end of block threading. But we still can return to threading after Write(&needContinue) +} EMtDecParseState; + +typedef struct +{ + // in + int startCall; + const Byte *src; + size_t srcSize; + // in : (srcSize == 0) is allowed + // out : it's allowed to return less that actually was used ? + int srcFinished; + + // out + EMtDecParseState state; + BoolInt canCreateNewThread; + UInt64 outPos; // check it (size_t) +} CMtDecCallbackInfo; + + +typedef struct +{ + void (*Parse)(void *p, unsigned coderIndex, CMtDecCallbackInfo *ci); + + // PreCode() and Code(): + // (SRes_return_result != SZ_OK) means stop decoding, no need another blocks + SRes (*PreCode)(void *p, unsigned coderIndex); + SRes (*Code)(void *p, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop); + // stop - means stop another Code calls + + + /* Write() must be called, if Parse() was called + set (needWrite) if + { + && (was not interrupted by progress) + && (was not interrupted in previous block) + } + + out: + if (*needContinue), decoder still need to continue decoding with new iteration, + even after MTDEC_PARSE_END + if (*canRecode), we didn't flush current block data, so we still can decode current block later. + */ + SRes (*Write)(void *p, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, + // int srcFinished, + BoolInt *needContinue, + BoolInt *canRecode); +} IMtDecCallback; + + + +typedef struct _CMtDec +{ + /* input variables */ + + size_t inBufSize; /* size of input block */ + unsigned numThreadsMax; + // size_t inBlockMax; + unsigned numThreadsMax_2; + + ISeqInStream *inStream; + // const Byte *inData; + // size_t inDataSize; + + ICompressProgress *progress; + ISzAllocPtr alloc; + + IMtDecCallback *mtCallback; + void *mtCallbackObject; + + + /* internal variables */ + + size_t allocatedBufsSize; + + BoolInt exitThread; + WRes exitThreadWRes; + + UInt64 blockIndex; + BoolInt isAllocError; + BoolInt overflow; + SRes threadingErrorSRes; + + BoolInt needContinue; + + // CAutoResetEvent finishedEvent; + + SRes readRes; + SRes codeRes; + + BoolInt wasInterrupted; + + unsigned numStartedThreads_Limit; + unsigned numStartedThreads; + + Byte *crossBlock; + size_t crossStart; + size_t crossEnd; + UInt64 readProcessed; + BoolInt readWasFinished; + UInt64 inProcessed; + + unsigned filledThreadStart; + unsigned numFilledThreads; + + #ifndef _7ZIP_ST + BoolInt needInterrupt; + UInt64 interruptIndex; + CMtProgress mtProgress; + CMtDecThread threads[MTDEC__THREADS_MAX]; + #endif +} CMtDec; + + +void MtDec_Construct(CMtDec *p); +void MtDec_Destruct(CMtDec *p); + +/* +MtDec_Code() returns: + SZ_OK - in most cases + MY_SRes_HRESULT_FROM_WRes(WRes_error) - in case of unexpected error in threading function +*/ + +SRes MtDec_Code(CMtDec *p); +Byte *MtDec_GetCrossBuff(CMtDec *p); + +int MtDec_PrepareRead(CMtDec *p); +const Byte *MtDec_Read(CMtDec *p, size_t *inLim); + +#endif + +EXTERN_C_END + +#endif diff --git a/7zip/C/Ppmd7.c b/7zip/C/Ppmd7.c index 0951a7c74..470aadccf 100644 --- a/7zip/C/Ppmd7.c +++ b/7zip/C/Ppmd7.c @@ -1,5 +1,5 @@ /* Ppmd7.c -- PPMdH codec -2017-04-03 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -95,7 +95,7 @@ void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc) p->Base = 0; } -Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc) +BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc) { if (!p->Base || p->Size != size) { @@ -342,7 +342,7 @@ void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder) p->DummySee.Count = 64; /* unused */ } -static CTX_PTR CreateSuccessors(CPpmd7 *p, Bool skip) +static CTX_PTR CreateSuccessors(CPpmd7 *p, BoolInt skip) { CPpmd_State upState; CTX_PTR c = p->MinContext; diff --git a/7zip/C/Ppmd7.h b/7zip/C/Ppmd7.h index 45002d95b..610539a04 100644 --- a/7zip/C/Ppmd7.h +++ b/7zip/C/Ppmd7.h @@ -1,5 +1,5 @@ /* Ppmd7.h -- PPMdH compression codec -2017-04-03 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ /* This code supports virtual RangeDecoder and includes the implementation @@ -60,7 +60,7 @@ typedef struct } CPpmd7; void Ppmd7_Construct(CPpmd7 *p); -Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc); +BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc); void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc); void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder); #define Ppmd7_WasAllocated(p) ((p)->Base != NULL) @@ -115,7 +115,7 @@ typedef struct } CPpmd7z_RangeDec; void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p); -Bool Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p); +BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p); #define Ppmd7z_RangeDec_IsFinishedOK(p) ((p)->Code == 0) int Ppmd7_DecodeSymbol(CPpmd7 *p, const IPpmd7_RangeDec *rc); diff --git a/7zip/C/Ppmd7Dec.c b/7zip/C/Ppmd7Dec.c index 7953bb07c..311e9f9dd 100644 --- a/7zip/C/Ppmd7Dec.c +++ b/7zip/C/Ppmd7Dec.c @@ -1,5 +1,5 @@ /* Ppmd7Dec.c -- PPMdH Decoder -2017-04-03 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -8,7 +8,7 @@ This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #define kTopValue (1 << 24) -Bool Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p) +BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p) { unsigned i; p->Code = 0; diff --git a/7zip/C/Ppmd8.c b/7zip/C/Ppmd8.c index 15c76bb5e..58141633a 100644 --- a/7zip/C/Ppmd8.c +++ b/7zip/C/Ppmd8.c @@ -1,5 +1,5 @@ /* Ppmd8.c -- PPMdI codec -2017-04-03 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.I (2002): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -93,7 +93,7 @@ void Ppmd8_Free(CPpmd8 *p, ISzAllocPtr alloc) p->Base = 0; } -Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc) +BoolInt Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc) { if (!p->Base || p->Size != size) { @@ -605,7 +605,7 @@ static void RestoreModel(CPpmd8 *p, CTX_PTR c1 } } -static CTX_PTR CreateSuccessors(CPpmd8 *p, Bool skip, CPpmd_State *s1, CTX_PTR c) +static CTX_PTR CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, CTX_PTR c) { CPpmd_State upState; Byte flags; diff --git a/7zip/C/Ppmd8.h b/7zip/C/Ppmd8.h index 3b15920e5..51c497dcf 100644 --- a/7zip/C/Ppmd8.h +++ b/7zip/C/Ppmd8.h @@ -1,5 +1,5 @@ /* Ppmd8.h -- PPMdI codec -2017-04-03 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on: PPMd var.I (2002): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -86,7 +86,7 @@ typedef struct } CPpmd8; void Ppmd8_Construct(CPpmd8 *p); -Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc); +BoolInt Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc); void Ppmd8_Free(CPpmd8 *p, ISzAllocPtr alloc); void Ppmd8_Init(CPpmd8 *p, unsigned maxOrder, unsigned restoreMethod); #define Ppmd8_WasAllocated(p) ((p)->Base != NULL) @@ -121,7 +121,7 @@ CPpmd_See *Ppmd8_MakeEscFreq(CPpmd8 *p, unsigned numMasked, UInt32 *scale); /* ---------- Decode ---------- */ -Bool Ppmd8_RangeDec_Init(CPpmd8 *p); +BoolInt Ppmd8_RangeDec_Init(CPpmd8 *p); #define Ppmd8_RangeDec_IsFinishedOK(p) ((p)->Code == 0) int Ppmd8_DecodeSymbol(CPpmd8 *p); /* returns: -1 as EndMarker, -2 as DataError */ diff --git a/7zip/C/Ppmd8Dec.c b/7zip/C/Ppmd8Dec.c index dc79f0fd2..a18ec677b 100644 --- a/7zip/C/Ppmd8Dec.c +++ b/7zip/C/Ppmd8Dec.c @@ -1,5 +1,5 @@ /* Ppmd8Dec.c -- PPMdI Decoder -2017-04-03 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on: PPMd var.I (2002): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -11,7 +11,7 @@ This code is based on: #define kTop (1 << 24) #define kBot (1 << 15) -Bool Ppmd8_RangeDec_Init(CPpmd8 *p) +BoolInt Ppmd8_RangeDec_Init(CPpmd8 *p) { unsigned i; p->Low = 0; diff --git a/7zip/C/Util/7z/7zMain.c b/7zip/C/Util/7z/7zMain.c index cb60a9ba2..6ccc83052 100644 --- a/7zip/C/Util/7z/7zMain.c +++ b/7zip/C/Util/7z/7zMain.c @@ -1,5 +1,5 @@ /* 7zMain.c - Test application for 7z Decoder -2017-08-26 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -176,7 +176,7 @@ static SRes Utf16_To_Char(CBuf *buf, const UInt16 *s char defaultChar = '_'; BOOL defUsed; unsigned numChars = 0; - numChars = WideCharToMultiByte(codePage, 0, s, len, (char *)buf->data, size, &defaultChar, &defUsed); + numChars = WideCharToMultiByte(codePage, 0, (LPCWSTR)s, len, (char *)buf->data, size, &defaultChar, &defUsed); if (numChars == 0 || numChars >= size) return SZ_ERROR_FAIL; buf->data[numChars] = 0; @@ -202,7 +202,7 @@ static WRes MyCreateDir(const UInt16 *name) { #ifdef USE_WINDOWS_FILE - return CreateDirectoryW(name, NULL) ? 0 : GetLastError(); + return CreateDirectoryW((LPCWSTR)name, NULL) ? 0 : GetLastError(); #else @@ -227,7 +227,7 @@ static WRes MyCreateDir(const UInt16 *name) static WRes OutFile_OpenUtf16(CSzFile *p, const UInt16 *name) { #ifdef USE_WINDOWS_FILE - return OutFile_OpenW(p, name); + return OutFile_OpenW(p, (LPCWSTR)name); #else CBuf buf; WRes res; @@ -354,7 +354,7 @@ static void PrintError(char *s) PrintLF(); } -static void GetAttribString(UInt32 wa, Bool isDir, char *s) +static void GetAttribString(UInt32 wa, BoolInt isDir, char *s) { #ifdef USE_WINDOWS_FILE s[0] = (char)(((wa & FILE_ATTRIBUTE_DIRECTORY) != 0 || isDir) ? 'D' : '.'); @@ -430,7 +430,7 @@ int MY_CDECL main(int numargs, char *args[]) res = SZ_OK; { - lookStream.buf = ISzAlloc_Alloc(&allocImp, kInputBufSize); + lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); if (!lookStream.buf) res = SZ_ERROR_MEM; else @@ -606,6 +606,31 @@ int MY_CDECL main(int numargs, char *args[]) res = SZ_ERROR_FAIL; break; } + + #ifdef USE_WINDOWS_FILE + { + FILETIME mtime, ctime; + FILETIME *mtimePtr = NULL; + FILETIME *ctimePtr = NULL; + + if (SzBitWithVals_Check(&db.MTime, i)) + { + const CNtfsFileTime *t = &db.MTime.Vals[i]; + mtime.dwLowDateTime = (DWORD)(t->Low); + mtime.dwHighDateTime = (DWORD)(t->High); + mtimePtr = &mtime; + } + if (SzBitWithVals_Check(&db.CTime, i)) + { + const CNtfsFileTime *t = &db.CTime.Vals[i]; + ctime.dwLowDateTime = (DWORD)(t->Low); + ctime.dwHighDateTime = (DWORD)(t->High); + ctimePtr = &ctime; + } + if (mtimePtr || ctimePtr) + SetFileTime(outFile.handle, ctimePtr, NULL, mtimePtr); + } + #endif if (File_Close(&outFile)) { @@ -622,7 +647,7 @@ int MY_CDECL main(int numargs, char *args[]) We remove posix bits, if we detect posix mode field */ if ((attrib & 0xF0000000) != 0) attrib &= 0x7FFF; - SetFileAttributesW(destPath, attrib); + SetFileAttributesW((LPCWSTR)destPath, attrib); } #endif } diff --git a/7zip/C/Util/7zipInstall/7zipInstall.c b/7zip/C/Util/7zipInstall/7zipInstall.c index db09a32d7..2022e2f36 100644 --- a/7zip/C/Util/7zipInstall/7zipInstall.c +++ b/7zip/C/Util/7zipInstall/7zipInstall.c @@ -1,5 +1,5 @@ /* 7zipInstall.c - 7-Zip Installer -2017-08-28 : Igor Pavlov : Public domain */ +2019-02-19 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -89,9 +89,9 @@ static LPCWSTR const k_Reg_CLSID_7zip_Inproc = L"CLSID\\" k_7zip_CLSID L"\\Inpro #define g_AllUsers True -static Bool g_Install_was_Pressed; -static Bool g_Finished; -static Bool g_SilentMode; +static BoolInt g_Install_was_Pressed; +static BoolInt g_Finished; +static BoolInt g_SilentMode; static HWND g_HWND; static HWND g_Path_HWND; @@ -121,13 +121,51 @@ static void PrintErrorMessage(const char *s) } +typedef DWORD (WINAPI * Func_GetFileVersionInfoSizeW)(LPCWSTR lptstrFilename, LPDWORD lpdwHandle); +typedef BOOL (WINAPI * Func_GetFileVersionInfoW)(LPCWSTR lptstrFilename, DWORD dwHandle, DWORD dwLen, LPVOID lpData); +typedef BOOL (WINAPI * Func_VerQueryValueW)(const LPVOID pBlock, LPWSTR lpSubBlock, LPVOID * lplpBuffer, PUINT puLen); + +static HMODULE g_version_dll_hModule; + static DWORD GetFileVersion(LPCWSTR s) { DWORD size = 0; - BYTE *vi = NULL; + void *vi = NULL; DWORD version = 0; + + Func_GetFileVersionInfoSizeW my_GetFileVersionInfoSizeW; + Func_GetFileVersionInfoW my_GetFileVersionInfoW; + Func_VerQueryValueW my_VerQueryValueW; + + if (!g_version_dll_hModule) + { + wchar_t buf[MAX_PATH + 100]; + { + unsigned len = GetSystemDirectoryW(buf, MAX_PATH + 2); + if (len == 0 || len > MAX_PATH) + return 0; + } + { + unsigned pos = (unsigned)lstrlenW(buf); + if (buf[pos - 1] != '\\') + buf[pos++] = '\\'; + lstrcpyW(buf + pos, L"version.dll"); + } + g_version_dll_hModule = LoadLibraryW(buf); + if (!g_version_dll_hModule) + return 0; + } + + my_GetFileVersionInfoSizeW = (Func_GetFileVersionInfoSizeW)GetProcAddress(g_version_dll_hModule, "GetFileVersionInfoSizeW"); + my_GetFileVersionInfoW = (Func_GetFileVersionInfoW)GetProcAddress(g_version_dll_hModule, "GetFileVersionInfoW"); + my_VerQueryValueW = (Func_VerQueryValueW)GetProcAddress(g_version_dll_hModule, "VerQueryValueW"); + + if (!my_GetFileVersionInfoSizeW + || !my_GetFileVersionInfoW + || !my_VerQueryValueW) + return 0; - size = GetFileVersionInfoSizeW(s, NULL); + size = my_GetFileVersionInfoSizeW(s, NULL); if (size == 0) return 0; @@ -135,11 +173,11 @@ static DWORD GetFileVersion(LPCWSTR s) if (!vi) return 0; - if (GetFileVersionInfoW(s, 0, size, vi)) + if (my_GetFileVersionInfoW(s, 0, size, vi)) { VS_FIXEDFILEINFO *fi = NULL; UINT fiLen = 0; - if (VerQueryValueW(vi, L"\\", (LPVOID *)&fi, &fiLen)) + if (my_VerQueryValueW(vi, L"\\", (LPVOID *)&fi, &fiLen)) version = fi->dwFileVersionMS; } @@ -270,7 +308,7 @@ static int MyRegistry_QueryString2(HKEY hKey, LPCWSTR keyName, LPCWSTR valName, if (res != ERROR_SUCCESS) return False; { - Bool res2 = MyRegistry_QueryString(key, valName, dest); + BoolInt res2 = MyRegistry_QueryString(key, valName, dest); RegCloseKey(key); return res2; } @@ -343,7 +381,7 @@ static LONG MyRegistry_CreateKeyAndVal_32(HKEY parentKey, LPCWSTR keyName, LPCWS #define kSignatureSearchLimit (1 << 22) -static Bool FindSignature(CSzFile *stream, UInt64 *resPos) +static BoolInt FindSignature(CSzFile *stream, UInt64 *resPos) { Byte buf[kBufSize]; size_t numPrevBytes = 0; @@ -431,7 +469,7 @@ int CALLBACK BrowseCallbackProc(HWND hwnd, UINT uMsg, LPARAM lp, LPARAM data) return 0; } -static Bool MyBrowseForFolder(HWND owner, LPCWSTR title, UINT ulFlags, +static BoolInt MyBrowseForFolder(HWND owner, LPCWSTR title, UINT ulFlags, LPCWSTR initialFolder, LPWSTR resultPath) { WCHAR displayName[MAX_PATH]; @@ -694,7 +732,7 @@ static void SetShellProgramsGroup(HWND hwndOwner) for (; i < 3; i++) { - Bool isOK = True; + BoolInt isOK = True; WCHAR link[MAX_PATH + 40]; WCHAR destPath[MAX_PATH + 40]; @@ -874,7 +912,7 @@ static void WriteShellEx() static const wchar_t *GetCmdParam(const wchar_t *s) { - Bool quoteMode = False; + BoolInt quoteMode = False; for (;; s++) { wchar_t c = *s; @@ -988,7 +1026,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, if (path[0] == 0) { HKEY key = 0; - Bool ok = False; + BoolInt ok = False; LONG res = RegOpenKeyExW(HKEY_CURRENT_USER, k_Reg_Software_7zip, 0, KEY_READ | k_Reg_WOW_Flag, &key); if (res == ERROR_SUCCESS) { @@ -1101,9 +1139,9 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, } -static Bool GetErrorMessage(DWORD errorCode, WCHAR *message) +static BoolInt GetErrorMessage(DWORD errorCode, WCHAR *message) { - LPVOID msgBuf; + LPWSTR msgBuf; if (FormatMessageW( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM @@ -1201,7 +1239,7 @@ if (res == SZ_OK) if (res == SZ_OK) { - lookStream.buf = ISzAlloc_Alloc(&allocImp, kInputBufSize); + lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); if (!lookStream.buf) res = SZ_ERROR_MEM; else @@ -1272,7 +1310,7 @@ if (res == SZ_OK) temp = path + pathLen; - SzArEx_GetFileNameUtf16(&db, i, temp); + SzArEx_GetFileNameUtf16(&db, i, (UInt16 *)temp); if (!g_SilentMode) SetWindowTextW(g_InfoLine_HWND, temp); @@ -1313,7 +1351,7 @@ if (res == SZ_OK) } { - // Bool skipFile = False; + // BoolInt skipFile = False; wcscpy(origPath, path); diff --git a/7zip/C/Util/7zipInstall/makefile b/7zip/C/Util/7zipInstall/makefile index e84d552d9..ab8893a9b 100644 --- a/7zip/C/Util/7zipInstall/makefile +++ b/7zip/C/Util/7zipInstall/makefile @@ -1,6 +1,5 @@ PROG = 7zipInstall.exe - -LIBS = $(LIBS) version.lib +MY_FIXED = 1 !IFDEF _64BIT_INSTALLER CFLAGS = $(CFLAGS) -D_64BIT_INSTALLER diff --git a/7zip/C/Util/7zipUninstall/7zipUninstall.c b/7zip/C/Util/7zipUninstall/7zipUninstall.c index 0bf27cc26..16761170c 100644 --- a/7zip/C/Util/7zipUninstall/7zipUninstall.c +++ b/7zip/C/Util/7zipUninstall/7zipUninstall.c @@ -1,5 +1,5 @@ /* 7zipUninstall.c - 7-Zip Uninstaller -2017-04-04 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -70,9 +70,9 @@ static const WCHAR * const k_Reg_CLSID_7zip_Inproc = L"CLSID\\" k_7zip_CLSID L"\ #define g_AllUsers True -static Bool g_Install_was_Pressed; -static Bool g_Finished; -static Bool g_SilentMode; +static BoolInt g_Install_was_Pressed; +static BoolInt g_Finished; +static BoolInt g_SilentMode; static HWND g_HWND; static HWND g_Path_HWND; @@ -94,7 +94,7 @@ static const WCHAR * const kUninstallExe = L"Uninstall.exe"; #define MAKE_CHAR_UPPER(c) ((((c) >= 'a' && (c) <= 'z') ? (c) -= 0x20 : (c))) -static Bool AreStringsEqual_NoCase(const wchar_t *s1, const wchar_t *s2) +static BoolInt AreStringsEqual_NoCase(const wchar_t *s1, const wchar_t *s2) { for (;;) { @@ -107,7 +107,7 @@ static Bool AreStringsEqual_NoCase(const wchar_t *s1, const wchar_t *s2) } } -static Bool IsString1PrefixedByString2_NoCase(const wchar_t *s1, const wchar_t *s2) +static BoolInt IsString1PrefixedByString2_NoCase(const wchar_t *s1, const wchar_t *s2) { for (;;) { @@ -149,7 +149,7 @@ static int MyRegistry_QueryString2(HKEY hKey, LPCWSTR keyName, LPCWSTR valName, if (res != ERROR_SUCCESS) return False; { - Bool res2 = MyRegistry_QueryString(key, valName, dest); + BoolInt res2 = MyRegistry_QueryString(key, valName, dest); RegCloseKey(key); return res2; } @@ -180,7 +180,7 @@ static int MyRegistry_QueryString2_32(HKEY hKey, LPCWSTR keyName, LPCWSTR valNam if (res != ERROR_SUCCESS) return False; { - Bool res2 = MyRegistry_QueryString(key, valName, dest); + BoolInt res2 = MyRegistry_QueryString(key, valName, dest); RegCloseKey(key); return res2; } @@ -283,7 +283,7 @@ static void SetShellProgramsGroup(HWND hwndOwner) for (; i < 3; i++) { - // Bool isOK = True; + // BoolInt isOK = True; WCHAR link[MAX_PATH + 40]; WCHAR destPath[MAX_PATH + 40]; @@ -300,7 +300,7 @@ static void SetShellProgramsGroup(HWND hwndOwner) { const size_t baseLen = wcslen(link); unsigned k; - Bool needDelete = False; + BoolInt needDelete = False; for (k = 0; k < 2; k++) { @@ -347,7 +347,7 @@ static const WCHAR * const k_AppPaths_7zFm = L"Software\\Microsoft\\Windows\\Cur static const WCHAR * const k_Uninstall_7zip = k_REG_Uninstall L"7-Zip"; -static Bool AreEqual_Path_PrefixName(const wchar_t *s, const wchar_t *prefix, const wchar_t *name) +static BoolInt AreEqual_Path_PrefixName(const wchar_t *s, const wchar_t *prefix, const wchar_t *name) { if (!IsString1PrefixedByString2_NoCase(s, prefix)) return False; @@ -444,7 +444,7 @@ static void WriteCLSID() static const wchar_t *GetCmdParam(const wchar_t *s) { - Bool quoteMode = False; + BoolInt quoteMode = False; for (;; s++) { wchar_t c = *s; @@ -469,7 +469,7 @@ static void RemoveQuotes(wchar_t *s) } } -static Bool DoesFileOrDirExist() +static BoolInt DoesFileOrDirExist() { return (GetFileAttributesW(path) != INVALID_FILE_ATTRIBUTES); } @@ -491,7 +491,7 @@ static BOOL RemoveFileAfterReboot() #define IS_LIMIT_CHAR(c) (c == 0 || c == ' ') -static Bool IsThereSpace(const wchar_t *s) +static BoolInt IsThereSpace(const wchar_t *s) { for (;;) { @@ -505,7 +505,7 @@ static Bool IsThereSpace(const wchar_t *s) static void AddPathParam(wchar_t *dest, const wchar_t *src) { - Bool needQuote = IsThereSpace(src); + BoolInt needQuote = IsThereSpace(src); if (needQuote) wcscat(dest, L"\""); wcscat(dest, src); @@ -515,9 +515,9 @@ static void AddPathParam(wchar_t *dest, const wchar_t *src) -static Bool GetErrorMessage(DWORD errorCode, WCHAR *message) +static BoolInt GetErrorMessage(DWORD errorCode, WCHAR *message) { - LPVOID msgBuf; + LPWSTR msgBuf; if (FormatMessageW( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM @@ -546,7 +546,7 @@ static BOOL RemoveDir() #define k_Lang L"Lang" // NUM_LANG_TXT_FILES files are placed before en.ttt -#define NUM_LANG_TXT_FILES 87 +#define NUM_LANG_TXT_FILES 88 #ifdef _64BIT_INSTALLER #define NUM_EXTRA_FILES_64BIT 1 @@ -558,7 +558,7 @@ static BOOL RemoveDir() static const char * const k_Names = "af an ar ast az ba be bg bn br ca co cs cy da de el eo es et eu ext" - " fa fi fr fur fy ga gl gu he hi hr hu hy id io is it ja ka kaa kk ko ku ku-ckb ky" + " fa fi fr fur fy ga gl gu he hi hr hu hy id io is it ja ka kaa kab kk ko ku ku-ckb ky" " lij lt lv mk mn mng mng2 mr ms nb ne nl nn pa-in pl ps pt pt-br ro ru" " sa si sk sl sq sr-spc sr-spl sv ta th tr tt ug uk uz va vi yo zh-cn zh-tw" " en.ttt" @@ -586,7 +586,7 @@ static int Install() SRes res = SZ_OK; WRes winRes = 0; - // Bool needReboot = False; + // BoolInt needReboot = False; const size_t pathLen = wcslen(path); if (!g_SilentMode) @@ -805,7 +805,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, lpCmdLine, int nCmdShow) { const wchar_t *cmdParams; - Bool useTemp = True; + BoolInt useTemp = True; UNUSED_VAR(hPrevInstance) UNUSED_VAR(lpCmdLine) @@ -1005,7 +1005,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, if (path[0] == 0) { HKEY key = 0; - Bool ok = False; + BoolInt ok = False; LONG res = RegOpenKeyExW(HKEY_CURRENT_USER, k_Reg_Software_7zip, 0, KEY_READ | k_Reg_WOW_Flag, &key); if (res == ERROR_SUCCESS) { diff --git a/7zip/C/Util/7zipUninstall/makefile b/7zip/C/Util/7zipUninstall/makefile index 6c7a22eec..60c2fe20b 100644 --- a/7zip/C/Util/7zipUninstall/makefile +++ b/7zip/C/Util/7zipUninstall/makefile @@ -1,4 +1,5 @@ PROG = 7zipUninstall.exe +MY_FIXED = 1 !IFDEF _64BIT_INSTALLER CFLAGS = $(CFLAGS) -D_64BIT_INSTALLER diff --git a/7zip/C/Util/Lzma/LzmaUtil.c b/7zip/C/Util/Lzma/LzmaUtil.c index 33db3e16e..739bc0fde 100644 --- a/7zip/C/Util/Lzma/LzmaUtil.c +++ b/7zip/C/Util/Lzma/LzmaUtil.c @@ -1,5 +1,5 @@ /* LzmaUtil.c -- Test application for LZMA compression -2017-04-27 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "../../Precomp.h" @@ -177,7 +177,7 @@ static int main2(int numArgs, const char *args[], char *rs) char c; int res; int encodeMode; - Bool useOutFile = False; + BoolInt useOutFile = False; FileSeqInStream_CreateVTable(&inStream); File_Construct(&inStream.file); diff --git a/7zip/C/Util/SfxSetup/SfxSetup.c b/7zip/C/Util/SfxSetup/SfxSetup.c index 8275dfeac..ef19aeac5 100644 --- a/7zip/C/Util/SfxSetup/SfxSetup.c +++ b/7zip/C/Util/SfxSetup/SfxSetup.c @@ -1,5 +1,5 @@ /* SfxSetup.c - 7z SFX Setup -2017-04-04 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -127,7 +127,7 @@ static WRes MyCreateDir(const WCHAR *name) #define kSignatureSearchLimit (1 << 22) -static Bool FindSignature(CSzFile *stream, UInt64 *resPos) +static BoolInt FindSignature(CSzFile *stream, UInt64 *resPos) { Byte buf[kBufferSize]; size_t numPrevBytes = 0; @@ -163,7 +163,7 @@ static Bool FindSignature(CSzFile *stream, UInt64 *resPos) } } -static Bool DoesFileOrDirExist(const WCHAR *path) +static BoolInt DoesFileOrDirExist(const WCHAR *path) { WIN32_FIND_DATAW fd; HANDLE handle; @@ -254,7 +254,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, DWORD winRes; const wchar_t *cmdLineParams; const char *errorMessage = NULL; - Bool useShellExecute = True; + BoolInt useShellExecute = True; DWORD exitCode = 0; LoadSecurityDlls(); @@ -287,7 +287,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, cmdLineParams = GetCommandLineW(); #ifndef UNDER_CE { - Bool quoteMode = False; + BoolInt quoteMode = False; for (;; cmdLineParams++) { wchar_t c = *cmdLineParams; @@ -379,7 +379,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, if (res == SZ_OK) { - lookStream.buf = ISzAlloc_Alloc(&allocImp, kInputBufSize); + lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); if (!lookStream.buf) res = SZ_ERROR_MEM; else @@ -420,7 +420,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, temp = path + pathLen; - SzArEx_GetFileNameUtf16(&db, i, temp); + SzArEx_GetFileNameUtf16(&db, i, (UInt16 *)temp); { res = SzArEx_Extract(&db, &lookStream.vt, i, &blockIndex, &outBuffer, &outBufferSize, @@ -527,7 +527,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, { WCHAR *temp = path + pathLen; UInt32 j; - SzArEx_GetFileNameUtf16(&db, executeFileIndex, temp); + SzArEx_GetFileNameUtf16(&db, executeFileIndex, (UInt16 *)temp); for (j = 0; temp[j] != 0; j++) if (temp[j] == '/') temp[j] = CHAR_PATH_SEPARATOR; diff --git a/7zip/C/Util/SfxSetup/makefile b/7zip/C/Util/SfxSetup/makefile index 76cd91124..544da67df 100644 --- a/7zip/C/Util/SfxSetup/makefile +++ b/7zip/C/Util/SfxSetup/makefile @@ -1,4 +1,5 @@ PROG = 7zS2.sfx +MY_FIXED = 1 C_OBJS = \ $O\7zAlloc.obj \ diff --git a/7zip/C/Util/SfxSetup/makefile_con b/7zip/C/Util/SfxSetup/makefile_con index ed9174d3b..d0f835254 100644 --- a/7zip/C/Util/SfxSetup/makefile_con +++ b/7zip/C/Util/SfxSetup/makefile_con @@ -1,4 +1,5 @@ PROG = 7zS2con.sfx +MY_FIXED = 1 CFLAGS = $(CFLAGS) -D_CONSOLE C_OBJS = \ diff --git a/7zip/C/Xz.h b/7zip/C/Xz.h index 285fb5531..544ee18f2 100644 --- a/7zip/C/Xz.h +++ b/7zip/C/Xz.h @@ -1,5 +1,5 @@ /* Xz.h - Xz interface -2017-07-27 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __XZ_H #define __XZ_H @@ -53,7 +53,7 @@ typedef struct #define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0) SRes XzBlock_Parse(CXzBlock *p, const Byte *header); -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes); +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes); /* ---------- xz stream ---------- */ @@ -139,6 +139,9 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICom UInt64 Xzs_GetNumBlocks(const CXzs *p); UInt64 Xzs_GetUnpackSize(const CXzs *p); + +// ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + typedef enum { CODER_STATUS_NOT_SPECIFIED, /* use main error code instead */ @@ -147,22 +150,31 @@ typedef enum CODER_STATUS_NEEDS_MORE_INPUT /* you must provide more input bytes */ } ECoderStatus; + +// ECoderFinishMode values are identical to ELzmaFinishMode + typedef enum { CODER_FINISH_ANY, /* finish at any point */ CODER_FINISH_END /* block must be finished at the end */ } ECoderFinishMode; + typedef struct _IStateCoder { void *p; void (*Free)(void *p, ISzAllocPtr alloc); SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAllocPtr alloc); void (*Init)(void *p); - SRes (*Code)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished); + SRes (*Code2)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status); + SizeT (*Filter)(void *p, Byte *data, SizeT size); } IStateCoder; + + #define MIXCODER_NUM_FILTERS_MAX 4 typedef struct @@ -170,20 +182,23 @@ typedef struct ISzAllocPtr alloc; Byte *buf; unsigned numCoders; + + Byte *outBuf; + size_t outBufSize; + size_t outWritten; // is equal to lzmaDecoder.dicPos (in outBuf mode) + BoolInt wasFinished; + SRes res; + ECoderStatus status; + // BoolInt SingleBufMode; + int finished[MIXCODER_NUM_FILTERS_MAX - 1]; size_t pos[MIXCODER_NUM_FILTERS_MAX - 1]; size_t size[MIXCODER_NUM_FILTERS_MAX - 1]; UInt64 ids[MIXCODER_NUM_FILTERS_MAX]; + SRes results[MIXCODER_NUM_FILTERS_MAX]; IStateCoder coders[MIXCODER_NUM_FILTERS_MAX]; } CMixCoder; -void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc); -void MixCoder_Free(CMixCoder *p); -void MixCoder_Init(CMixCoder *p); -SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId); -SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode, ECoderStatus *status); typedef enum { @@ -197,6 +212,7 @@ typedef enum XZ_STATE_BLOCK_FOOTER } EXzState; + typedef struct { EXzState state; @@ -210,7 +226,7 @@ typedef struct UInt64 packSize; UInt64 unpackSize; - UInt64 numBlocks; + UInt64 numBlocks; // number of finished blocks in current stream UInt64 indexSize; UInt64 indexPos; UInt64 padSize; @@ -225,16 +241,64 @@ typedef struct CXzCheck check; CSha256 sha; + BoolInt parseMode; + BoolInt headerParsedOk; + BoolInt decodeToStreamSignature; unsigned decodeOnlyOneBlock; + Byte *outBuf; + size_t outBufSize; + size_t outDataWritten; // the size of data in (outBuf) that were fully unpacked + Byte shaDigest[SHA256_DIGEST_SIZE]; Byte buf[XZ_BLOCK_HEADER_SIZE_MAX]; } CXzUnpacker; +/* alloc : aligned for cache line allocation is better */ void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc); void XzUnpacker_Init(CXzUnpacker *p); +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize); void XzUnpacker_Free(CXzUnpacker *p); +/* + XzUnpacker + The sequence for decoding functions: + { + XzUnpacker_Construct() + [Decoding_Calls] + XzUnpacker_Free() + } + + [Decoding_Calls] + + There are 3 types of interfaces for [Decoding_Calls] calls: + + Interface-1 : Partial output buffers: + { + XzUnpacker_Init() + for() + XzUnpacker_Code(); + } + + Interface-2 : Direct output buffer: + Use it, if you know exact size of decoded data, and you need + whole xz unpacked data in one output buffer. + xz unpacker doesn't allocate additional buffer for lzma2 dictionary in that mode. + { + XzUnpacker_Init() + XzUnpacker_SetOutBufMode(); // to set output buffer and size + for() + XzUnpacker_Code(); // (dest = NULL) in XzUnpacker_Code() + } + + Interface-3 : Direct output buffer : One call full decoding + It unpacks whole input buffer to output buffer in one call. + It uses Interface-2 internally. + { + XzUnpacker_CodeFull() + } +*/ + /* finishMode: It has meaning only if the decoding reaches output limit (*destLen). @@ -264,15 +328,23 @@ void XzUnpacker_Free(CXzUnpacker *p); SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, - ECoderStatus *status); + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status); -Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p); +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status); + +BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p); /* -Call XzUnpacker_GetExtraSize after XzUnpacker_Code function to detect real size of -xz stream in two cases: -XzUnpacker_Code() returns: +XzUnpacker_GetExtraSize() returns then number of uncofirmed bytes, + if it's in (XZ_STATE_STREAM_HEADER) state or in (XZ_STATE_STREAM_PADDING) state. +These bytes can be some bytes after xz archive, or +it can be start of new xz stream. + +Call XzUnpacker_GetExtraSize() after XzUnpacker_Code() function to detect real size of +xz stream in two cases, if XzUnpacker_Code() returns: res == SZ_OK && status == CODER_STATUS_NEEDS_MORE_INPUT res == SZ_ERROR_NO_ARCHIVE */ @@ -293,10 +365,96 @@ UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p); */ void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p); -Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p); +BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p); #define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags)) + + +/* ---------- Multi Threading Decoding ---------- */ + + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + BoolInt ignoreErrors; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t memUseMax; + #endif +} CXzDecMtProps; + +void XzDecMtProps_Init(CXzDecMtProps *p); + + +typedef void * CXzDecMtHandle; + +/* + alloc : XzDecMt uses CAlignOffsetAlloc for addresses allocated by (alloc). + allocMid : for big allocations, aligned allocation is better +*/ + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void XzDecMt_Destroy(CXzDecMtHandle p); + + +typedef struct +{ + Byte UnpackSize_Defined; + Byte NumStreams_Defined; + Byte NumBlocks_Defined; + + Byte DataAfterEnd; + Byte DecodingTruncated; // Decoding was Truncated, we need only partial output data + + UInt64 InSize; // pack size processed + UInt64 OutSize; + + UInt64 NumStreams; + UInt64 NumBlocks; + + SRes DecodeRes; + SRes ReadRes; + SRes ProgressRes; + SRes CombinedRes; + SRes CombinedRes_Type; + +} CXzStatInfo; + +void XzStatInfo_Clear(CXzStatInfo *p); + +/* +XzDecMt_Decode() +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_NO_ARCHIVE - is not xz archive + SZ_ERROR_ARCHIVE - Headers error + SZ_ERROR_DATA - Data Error + SZ_ERROR_CRC - CRC Error + SZ_ERROR_INPUT_EOF - it needs more input data + SZ_ERROR_WRITE - ISeqOutStream error + (SZ_ERROR_READ) - ISeqInStream errors + (SZ_ERROR_PROGRESS) - ICompressProgress errors + // SZ_ERROR_THREAD - error in multi-threading functions + MY_SRes_HRESULT_FROM_WRes(WRes_error) - error in multi-threading function +*/ + +SRes XzDecMt_Decode(CXzDecMtHandle p, + const CXzDecMtProps *props, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - xz stream(s) must be finished + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, + int *isMT, // 0 means that ST (Single-Thread) version was used + ICompressProgress *progress); + EXTERN_C_END #endif diff --git a/7zip/C/XzDec.c b/7zip/C/XzDec.c index 4c5c93aec..395e83f67 100644 --- a/7zip/C/XzDec.c +++ b/7zip/C/XzDec.c @@ -1,14 +1,33 @@ /* XzDec.c -- Xz Decode -2018-01-21 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" +// #include + +// #define XZ_DUMP + /* #define XZ_DUMP */ #ifdef XZ_DUMP #include #endif +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) + #include #include @@ -19,9 +38,11 @@ #include "Delta.h" #include "Lzma2Dec.h" +// #define USE_SUBBLOCK + #ifdef USE_SUBBLOCK #include "Bcj3Dec.c" -#include "SbDec.c" +#include "SbDec.h" #endif #include "Xz.h" @@ -56,8 +77,9 @@ typedef struct size_t bufConv; size_t bufTotal; - UInt32 methodId; int encodeMode; + + UInt32 methodId; UInt32 delta; UInt32 ip; UInt32 x86State; @@ -121,69 +143,84 @@ static void BraState_Init(void *pp) Delta_Init(p->deltaState); } -#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: p->bufConv = isa ## _Convert(p->buf, p->bufTotal, p->ip, p->encodeMode); break; -static SRes BraState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) +#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: size = isa ## _Convert(data, size, p->ip, p->encodeMode); break; + +static SizeT BraState_Filter(void *pp, Byte *data, SizeT size) { CBraState *p = ((CBraState *)pp); - SizeT destLenOrig = *destLen; - SizeT srcLenOrig = *srcLen; + switch (p->methodId) + { + case XZ_ID_Delta: + if (p->encodeMode) + Delta_Encode(p->deltaState, p->delta, data, size); + else + Delta_Decode(p->deltaState, p->delta, data, size); + break; + case XZ_ID_X86: + size = x86_Convert(data, size, p->ip, &p->x86State, p->encodeMode); + break; + CASE_BRA_CONV(PPC) + CASE_BRA_CONV(IA64) + CASE_BRA_CONV(ARM) + CASE_BRA_CONV(ARMT) + CASE_BRA_CONV(SPARC) + } + p->ip += (UInt32)size; + return size; +} + + +static SRes BraState_Code2(void *pp, + Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, int srcWasFinished, + ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) +{ + CBraState *p = ((CBraState *)pp); + SizeT destRem = *destLen; + SizeT srcRem = *srcLen; UNUSED_VAR(finishMode); + *destLen = 0; *srcLen = 0; - *wasFinished = 0; - while (destLenOrig > 0) + // *wasFinished = False; + *status = CODER_STATUS_NOT_FINISHED; + + while (destRem > 0) { if (p->bufPos != p->bufConv) { - size_t curSize = p->bufConv - p->bufPos; - if (curSize > destLenOrig) - curSize = destLenOrig; - memcpy(dest, p->buf + p->bufPos, curSize); - p->bufPos += curSize; - *destLen += curSize; - dest += curSize; - destLenOrig -= curSize; + size_t size = p->bufConv - p->bufPos; + if (size > destRem) + size = destRem; + memcpy(dest, p->buf + p->bufPos, size); + p->bufPos += size; + *destLen += size; + dest += size; + destRem -= size; continue; } + p->bufTotal -= p->bufPos; memmove(p->buf, p->buf + p->bufPos, p->bufTotal); p->bufPos = 0; p->bufConv = 0; { - size_t curSize = BRA_BUF_SIZE - p->bufTotal; - if (curSize > srcLenOrig) - curSize = srcLenOrig; - memcpy(p->buf + p->bufTotal, src, curSize); - *srcLen += curSize; - src += curSize; - srcLenOrig -= curSize; - p->bufTotal += curSize; + size_t size = BRA_BUF_SIZE - p->bufTotal; + if (size > srcRem) + size = srcRem; + memcpy(p->buf + p->bufTotal, src, size); + *srcLen += size; + src += size; + srcRem -= size; + p->bufTotal += size; } if (p->bufTotal == 0) break; - switch (p->methodId) - { - case XZ_ID_Delta: - if (p->encodeMode) - Delta_Encode(p->deltaState, p->delta, p->buf, p->bufTotal); - else - Delta_Decode(p->deltaState, p->delta, p->buf, p->bufTotal); - p->bufConv = p->bufTotal; - break; - case XZ_ID_X86: - p->bufConv = x86_Convert(p->buf, p->bufTotal, p->ip, &p->x86State, p->encodeMode); - break; - CASE_BRA_CONV(PPC) - CASE_BRA_CONV(IA64) - CASE_BRA_CONV(ARM) - CASE_BRA_CONV(ARMT) - CASE_BRA_CONV(SPARC) - default: - return SZ_ERROR_UNSUPPORTED; - } - p->ip += (UInt32)p->bufConv; + + p->bufConv = BraState_Filter(pp, p->buf, p->bufTotal); if (p->bufConv == 0) { @@ -192,36 +229,42 @@ static SRes BraState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, p->bufConv = p->bufTotal; } } - if (p->bufTotal == p->bufPos && srcLenOrig == 0 && srcWasFinished) - *wasFinished = 1; + + if (p->bufTotal == p->bufPos && srcRem == 0 && srcWasFinished) + { + *status = CODER_STATUS_FINISHED_WITH_MARK; + // *wasFinished = 1; + } + return SZ_OK; } + SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc) { CBraState *decoder; - if (id != XZ_ID_Delta && - id != XZ_ID_X86 && - id != XZ_ID_PPC && - id != XZ_ID_IA64 && - id != XZ_ID_ARM && - id != XZ_ID_ARMT && - id != XZ_ID_SPARC) + if (id < XZ_ID_Delta || id > XZ_ID_SPARC) return SZ_ERROR_UNSUPPORTED; - p->p = 0; - decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState)); + decoder = (CBraState *)p->p; if (!decoder) - return SZ_ERROR_MEM; + { + decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = BraState_Free; + p->SetProps = BraState_SetProps; + p->Init = BraState_Init; + p->Code2 = BraState_Code2; + p->Filter = BraState_Filter; + } decoder->methodId = (UInt32)id; decoder->encodeMode = encodeMode; - p->p = decoder; - p->Free = BraState_Free; - p->SetProps = BraState_SetProps; - p->Init = BraState_Init; - p->Code = BraState_Code; return SZ_OK; } + + /* ---------- SbState ---------- */ #ifdef USE_SUBBLOCK @@ -246,8 +289,10 @@ static void SbState_Init(void *pp) SbDec_Init((CSbDec *)pp); } -static SRes SbState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) +static SRes SbState_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) { CSbDec *p = (CSbDec *)pp; SRes res; @@ -260,33 +305,53 @@ static SRes SbState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, res = SbDec_Decode((CSbDec *)pp); *destLen -= p->destLen; *srcLen -= p->srcLen; - *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ + // *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ + *status = (*destLen == 0 && *srcLen == 0) ? + CODER_STATUS_FINISHED_WITH_MARK : + CODER_STATUS_NOT_FINISHED; return res; } -SRes SbState_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) +static SRes SbState_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) { - CSbDec *decoder; - p->p = 0; - decoder = ISzAlloc_Alloc(alloc, sizeof(CSbDec)); + CSbDec *decoder = (CSbDec *)p->p; if (!decoder) - return SZ_ERROR_MEM; - p->p = decoder; - p->Free = SbState_Free; - p->SetProps = SbState_SetProps; - p->Init = SbState_Init; - p->Code = SbState_Code; + { + decoder = (CSbDec *)ISzAlloc_Alloc(alloc, sizeof(CSbDec)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = SbState_Free; + p->SetProps = SbState_SetProps; + p->Init = SbState_Init; + p->Code2 = SbState_Code2; + p->Filter = NULL; + } SbDec_Construct(decoder); SbDec_SetAlloc(decoder, alloc); return SZ_OK; } + #endif -/* ---------- Lzma2State ---------- */ + + +/* ---------- Lzma2 ---------- */ + +typedef struct +{ + CLzma2Dec decoder; + BoolInt outBufMode; +} CLzma2Dec_Spec; + static void Lzma2State_Free(void *pp, ISzAllocPtr alloc) { - Lzma2Dec_Free((CLzma2Dec *)pp, alloc); + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + Lzma2Dec_FreeProbs(&p->decoder, alloc); + else + Lzma2Dec_Free(&p->decoder, alloc); ISzAlloc_Free(alloc, pp); } @@ -294,60 +359,123 @@ static SRes Lzma2State_SetProps(void *pp, const Byte *props, size_t propSize, IS { if (propSize != 1) return SZ_ERROR_UNSUPPORTED; - return Lzma2Dec_Allocate((CLzma2Dec *)pp, props[0], alloc); + { + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + return Lzma2Dec_AllocateProbs(&p->decoder, props[0], alloc); + else + return Lzma2Dec_Allocate(&p->decoder, props[0], alloc); + } } static void Lzma2State_Init(void *pp) { - Lzma2Dec_Init((CLzma2Dec *)pp); + Lzma2Dec_Init(&((CLzma2Dec_Spec *)pp)->decoder); } -static SRes Lzma2State_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) + +/* + if (outBufMode), then (dest) is not used. Use NULL. + Data is unpacked to (spec->decoder.decoder.dic) output buffer. +*/ + +static SRes Lzma2State_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status) { - ELzmaStatus status; + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)pp; + ELzmaStatus status2; /* ELzmaFinishMode fm = (finishMode == LZMA_FINISH_ANY) ? LZMA_FINISH_ANY : LZMA_FINISH_END; */ - SRes res = Lzma2Dec_DecodeToBuf((CLzma2Dec *)pp, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status); + SRes res; UNUSED_VAR(srcWasFinished); - *wasFinished = (status == LZMA_STATUS_FINISHED_WITH_MARK); + if (spec->outBufMode) + { + SizeT dicPos = spec->decoder.decoder.dicPos; + SizeT dicLimit = dicPos + *destLen; + res = Lzma2Dec_DecodeToDic(&spec->decoder, dicLimit, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + *destLen = spec->decoder.decoder.dicPos - dicPos; + } + else + res = Lzma2Dec_DecodeToBuf(&spec->decoder, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + // *wasFinished = (status2 == LZMA_STATUS_FINISHED_WITH_MARK); + // ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + *status = (ECoderStatus)status2; return res; } -static SRes Lzma2State_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) + +static SRes Lzma2State_SetFromMethod(IStateCoder *p, Byte *outBuf, size_t outBufSize, ISzAllocPtr alloc) { - CLzma2Dec *decoder = (CLzma2Dec *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Dec)); - p->p = decoder; - if (!decoder) - return SZ_ERROR_MEM; - p->Free = Lzma2State_Free; - p->SetProps = Lzma2State_SetProps; - p->Init = Lzma2State_Init; - p->Code = Lzma2State_Code; - Lzma2Dec_Construct(decoder); + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if (!spec) + { + spec = (CLzma2Dec_Spec *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Dec_Spec)); + if (!spec) + return SZ_ERROR_MEM; + p->p = spec; + p->Free = Lzma2State_Free; + p->SetProps = Lzma2State_SetProps; + p->Init = Lzma2State_Init; + p->Code2 = Lzma2State_Code2; + p->Filter = NULL; + Lzma2Dec_Construct(&spec->decoder); + } + spec->outBufMode = False; + if (outBuf) + { + spec->outBufMode = True; + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } + return SZ_OK; +} + + +static SRes Lzma2State_ResetOutBuf(IStateCoder *p, Byte *outBuf, size_t outBufSize) +{ + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if ((spec->outBufMode && !outBuf) || (!spec->outBufMode && outBuf)) + return SZ_ERROR_FAIL; + if (outBuf) + { + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } return SZ_OK; } -void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc) + +static void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc) { unsigned i; p->alloc = alloc; p->buf = NULL; p->numCoders = 0; + + p->outBufSize = 0; + p->outBuf = NULL; + // p->SingleBufMode = False; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) p->coders[i].p = NULL; } -void MixCoder_Free(CMixCoder *p) + +static void MixCoder_Free(CMixCoder *p) { unsigned i; - for (i = 0; i < p->numCoders; i++) + p->numCoders = 0; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) { IStateCoder *sc = &p->coders[i]; - if (p->alloc && sc->p) + if (sc->p) + { sc->Free(sc->p, p->alloc); + sc->p = NULL; + } } - p->numCoders = 0; if (p->buf) { ISzAlloc_Free(p->alloc, p->buf); @@ -355,7 +483,7 @@ void MixCoder_Free(CMixCoder *p) } } -void MixCoder_Init(CMixCoder *p) +static void MixCoder_Init(CMixCoder *p) { unsigned i; for (i = 0; i < MIXCODER_NUM_FILTERS_MAX - 1; i++) @@ -368,16 +496,22 @@ void MixCoder_Init(CMixCoder *p) { IStateCoder *coder = &p->coders[i]; coder->Init(coder->p); + p->results[i] = SZ_OK; } + p->outWritten = 0; + p->wasFinished = False; + p->res = SZ_OK; + p->status = CODER_STATUS_NOT_SPECIFIED; } -SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId) + +static SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) { IStateCoder *sc = &p->coders[coderIndex]; p->ids[coderIndex] = methodId; switch (methodId) { - case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, p->alloc); + case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, outBuf, outBufSize, p->alloc); #ifdef USE_SUBBLOCK case XZ_ID_Subblock: return SbState_SetFromMethod(sc, p->alloc); #endif @@ -387,32 +521,128 @@ SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId) return BraState_SetFromMethod(sc, methodId, 0, p->alloc); } -SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, + +static SRes MixCoder_ResetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) +{ + IStateCoder *sc = &p->coders[coderIndex]; + switch (methodId) + { + case XZ_ID_LZMA2: return Lzma2State_ResetOutBuf(sc, outBuf, outBufSize); + } + return SZ_ERROR_UNSUPPORTED; +} + + + +/* + if (destFinish) - then unpack data block is finished at (*destLen) position, + and we can return data that were not processed by filter + +output (status) can be : + CODER_STATUS_NOT_FINISHED + CODER_STATUS_FINISHED_WITH_MARK + CODER_STATUS_NEEDS_MORE_INPUT - not implemented still +*/ + +static SRes MixCoder_Code(CMixCoder *p, + Byte *dest, SizeT *destLen, int destFinish, const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode, ECoderStatus *status) + ECoderFinishMode finishMode) { SizeT destLenOrig = *destLen; SizeT srcLenOrig = *srcLen; *destLen = 0; *srcLen = 0; - *status = CODER_STATUS_NOT_FINISHED; - if (!p->buf) + if (p->wasFinished) + return p->res; + + p->status = CODER_STATUS_NOT_FINISHED; + + // if (p->SingleBufMode) + if (p->outBuf) { - p->buf = (Byte *)ISzAlloc_Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); - if (!p->buf) - return SZ_ERROR_MEM; + SRes res; + SizeT destLen2, srcLen2; + int wasFinished; + + PRF_STR("------- MixCoder Single ----------"); + + srcLen2 = srcLenOrig; + destLen2 = destLenOrig; + + { + IStateCoder *coder = &p->coders[0]; + res = coder->Code2(coder->p, NULL, &destLen2, src, &srcLen2, srcWasFinished, finishMode, + // &wasFinished, + &p->status); + wasFinished = (p->status == CODER_STATUS_FINISHED_WITH_MARK); + } + + p->res = res; + + /* + if (wasFinished) + p->status = CODER_STATUS_FINISHED_WITH_MARK; + else + { + if (res == SZ_OK) + if (destLen2 != destLenOrig) + p->status = CODER_STATUS_NEEDS_MORE_INPUT; + } + */ + + + *srcLen = srcLen2; + src += srcLen2; + p->outWritten += destLen2; + + if (res != SZ_OK || srcWasFinished || wasFinished) + p->wasFinished = True; + + if (p->numCoders == 1) + *destLen = destLen2; + else if (p->wasFinished) + { + unsigned i; + size_t processed = p->outWritten; + + for (i = 1; i < p->numCoders; i++) + { + IStateCoder *coder = &p->coders[i]; + processed = coder->Filter(coder->p, p->outBuf, processed); + if (wasFinished || (destFinish && p->outWritten == destLenOrig)) + processed = p->outWritten; + PRF_STR_INT("filter", i); + } + *destLen = processed; + } + return res; } + PRF_STR("standard mix"); + if (p->numCoders != 1) + { + if (!p->buf) + { + p->buf = (Byte *)ISzAlloc_Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); + if (!p->buf) + return SZ_ERROR_MEM; + } + finishMode = CODER_FINISH_ANY; + } for (;;) { - Bool processed = False; - Bool allFinished = True; + BoolInt processed = False; + BoolInt allFinished = True; + SRes resMain = SZ_OK; unsigned i; + + p->status = CODER_STATUS_NOT_FINISHED; /* if (p->numCoders == 1 && *destLen == destLenOrig && finishMode == LZMA_FINISH_ANY) break; @@ -422,78 +652,101 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, { SRes res; IStateCoder *coder = &p->coders[i]; - Byte *destCur; - SizeT destLenCur, srcLenCur; - const Byte *srcCur; - int srcFinishedCur; + Byte *dest2; + SizeT destLen2, srcLen2; // destLen2_Orig; + const Byte *src2; + int srcFinished2; int encodingWasFinished; + ECoderStatus status2; if (i == 0) { - srcCur = src; - srcLenCur = srcLenOrig - *srcLen; - srcFinishedCur = srcWasFinished; + src2 = src; + srcLen2 = srcLenOrig - *srcLen; + srcFinished2 = srcWasFinished; } else { size_t k = i - 1; - srcCur = p->buf + (CODER_BUF_SIZE * k) + p->pos[k]; - srcLenCur = p->size[k] - p->pos[k]; - srcFinishedCur = p->finished[k]; + src2 = p->buf + (CODER_BUF_SIZE * k) + p->pos[k]; + srcLen2 = p->size[k] - p->pos[k]; + srcFinished2 = p->finished[k]; } if (i == p->numCoders - 1) { - destCur = dest; - destLenCur = destLenOrig - *destLen; + dest2 = dest; + destLen2 = destLenOrig - *destLen; } else { if (p->pos[i] != p->size[i]) continue; - destCur = p->buf + (CODER_BUF_SIZE * i); - destLenCur = CODER_BUF_SIZE; + dest2 = p->buf + (CODER_BUF_SIZE * i); + destLen2 = CODER_BUF_SIZE; } - res = coder->Code(coder->p, destCur, &destLenCur, srcCur, &srcLenCur, srcFinishedCur, finishMode, &encodingWasFinished); + // destLen2_Orig = destLen2; + + if (p->results[i] != SZ_OK) + { + if (resMain == SZ_OK) + resMain = p->results[i]; + continue; + } + + res = coder->Code2(coder->p, + dest2, &destLen2, + src2, &srcLen2, srcFinished2, + finishMode, + // &encodingWasFinished, + &status2); + + if (res != SZ_OK) + { + p->results[i] = res; + if (resMain == SZ_OK) + resMain = res; + } + encodingWasFinished = (status2 == CODER_STATUS_FINISHED_WITH_MARK); + if (!encodingWasFinished) + { allFinished = False; + if (p->numCoders == 1 && res == SZ_OK) + p->status = status2; + } if (i == 0) { - *srcLen += srcLenCur; - src += srcLenCur; + *srcLen += srcLen2; + src += srcLen2; } else - { - p->pos[(size_t)i - 1] += srcLenCur; - } + p->pos[(size_t)i - 1] += srcLen2; if (i == p->numCoders - 1) { - *destLen += destLenCur; - dest += destLenCur; + *destLen += destLen2; + dest += destLen2; } else { - p->size[i] = destLenCur; + p->size[i] = destLen2; p->pos[i] = 0; p->finished[i] = encodingWasFinished; } - if (res != SZ_OK) - return res; - - if (destLenCur != 0 || srcLenCur != 0) + if (destLen2 != 0 || srcLen2 != 0) processed = True; } if (!processed) { if (allFinished) - *status = CODER_STATUS_FINISHED_WITH_MARK; - return SZ_OK; + p->status = CODER_STATUS_FINISHED_WITH_MARK; + return resMain; } } } @@ -508,7 +761,7 @@ SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf) return XzFlags_IsSupported(*p) ? SZ_OK : SZ_ERROR_UNSUPPORTED; } -static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf) +static BoolInt Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf) { return indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) && GetUi32(buf) == CrcCalc(buf + 4, 6) @@ -522,6 +775,33 @@ static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *b if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } +static BoolInt XzBlock_AreSupportedFilters(const CXzBlock *p) +{ + unsigned numFilters = XzBlock_GetNumFilters(p) - 1; + unsigned i; + { + const CXzFilter *f = &p->filters[numFilters]; + if (f->id != XZ_ID_LZMA2 || f->propsSize != 1 || f->props[0] > 40) + return False; + } + + for (i = 0; i < numFilters; i++) + { + const CXzFilter *f = &p->filters[i]; + if (f->id == XZ_ID_Delta) + { + if (f->propsSize != 1) + return False; + } + else if (f->id < XZ_ID_Delta + || f->id > XZ_ID_SPARC + || (f->propsSize != 0 && f->propsSize != 4)) + return False; + } + return True; +} + + SRes XzBlock_Parse(CXzBlock *p, const Byte *header) { unsigned pos; @@ -580,31 +860,47 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header) return SZ_OK; } -SRes XzDec_Init(CMixCoder *p, const CXzBlock *block) + + + +static SRes XzDecMix_Init(CMixCoder *p, const CXzBlock *block, Byte *outBuf, size_t outBufSize) { unsigned i; - Bool needReInit = True; + BoolInt needReInit = True; unsigned numFilters = XzBlock_GetNumFilters(block); - - if (numFilters == p->numCoders) + + if (numFilters == p->numCoders && ((p->outBuf && outBuf) || (!p->outBuf && !outBuf))) { + needReInit = False; for (i = 0; i < numFilters; i++) if (p->ids[i] != block->filters[numFilters - 1 - i].id) + { + needReInit = True; break; - needReInit = (i != numFilters); + } } + + // p->SingleBufMode = (outBuf != NULL); + p->outBuf = outBuf; + p->outBufSize = outBufSize; + + // p->SingleBufMode = False; + // outBuf = NULL; if (needReInit) { MixCoder_Free(p); - p->numCoders = numFilters; for (i = 0; i < numFilters; i++) { - const CXzFilter *f = &block->filters[numFilters - 1 - i]; - RINOK(MixCoder_SetFromMethod(p, i, f->id)); + RINOK(MixCoder_SetFromMethod(p, i, block->filters[numFilters - 1 - i].id, outBuf, outBufSize)); } + p->numCoders = numFilters; } - + else + { + RINOK(MixCoder_ResetFromMethod(p, 0, block->filters[numFilters - 1].id, outBuf, outBufSize)); + } + for (i = 0; i < numFilters; i++) { const CXzFilter *f = &block->filters[numFilters - 1 - i]; @@ -616,6 +912,8 @@ SRes XzDec_Init(CMixCoder *p, const CXzBlock *block) return SZ_OK; } + + void XzUnpacker_Init(CXzUnpacker *p) { p->state = XZ_STATE_STREAM_HEADER; @@ -625,14 +923,32 @@ void XzUnpacker_Init(CXzUnpacker *p) p->numTotalBlocks = 0; p->padSize = 0; p->decodeOnlyOneBlock = 0; + + p->parseMode = False; + p->decodeToStreamSignature = False; + + // p->outBuf = NULL; + // p->outBufSize = 0; + p->outDataWritten = 0; +} + + +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize) +{ + p->outBuf = outBuf; + p->outBufSize = outBufSize; } + void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc) { MixCoder_Construct(&p->decoder, alloc); + p->outBuf = NULL; + p->outBufSize = 0; XzUnpacker_Init(p); } + void XzUnpacker_Free(CXzUnpacker *p) { MixCoder_Free(&p->decoder); @@ -650,14 +966,28 @@ void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p) } +static void XzUnpacker_UpdateIndex(CXzUnpacker *p, UInt64 packSize, UInt64 unpackSize) +{ + Byte temp[32]; + unsigned num = Xz_WriteVarInt(temp, packSize); + num += Xz_WriteVarInt(temp + num, unpackSize); + Sha256_Update(&p->sha, temp, num); + p->indexSize += num; + p->numBlocks++; +} + + + SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, ECoderStatus *status) + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status) { SizeT destLenOrig = *destLen; SizeT srcLenOrig = *srcLen; *destLen = 0; *srcLen = 0; *status = CODER_STATUS_NOT_SPECIFIED; + for (;;) { SizeT srcRem; @@ -669,12 +999,17 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, SRes res; ECoderFinishMode finishMode2 = finishMode; + BoolInt srcFinished2 = srcFinished; + BoolInt destFinish = False; if (p->block.packSize != (UInt64)(Int64)-1) { UInt64 rem = p->block.packSize - p->packSize; - if (srcLen2 > rem) + if (srcLen2 >= rem) + { + srcFinished2 = True; srcLen2 = (SizeT)rem; + } if (rem == 0 && p->block.unpackSize == p->unpackSize) return SZ_ERROR_DATA; } @@ -684,6 +1019,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, UInt64 rem = p->block.unpackSize - p->unpackSize; if (destLen2 >= rem) { + destFinish = True; finishMode2 = CODER_FINISH_END; destLen2 = (SizeT)rem; } @@ -697,15 +1033,23 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } */ - res = MixCoder_Code(&p->decoder, dest, &destLen2, src, &srcLen2, False, finishMode2, status); - XzCheck_Update(&p->check, dest, destLen2); + { + res = MixCoder_Code(&p->decoder, + (p->outBuf ? NULL : dest), &destLen2, destFinish, + src, &srcLen2, srcFinished2, + finishMode2); + + *status = p->decoder.status; + XzCheck_Update(&p->check, (p->outBuf ? p->outBuf + p->outDataWritten : dest), destLen2); + if (!p->outBuf) + dest += destLen2; + p->outDataWritten += destLen2; + } (*srcLen) += srcLen2; src += srcLen2; p->packSize += srcLen2; - (*destLen) += destLen2; - dest += destLen2; p->unpackSize += destLen2; RINOK(res); @@ -715,27 +1059,26 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, if (p->block.packSize == p->packSize && *status == CODER_STATUS_NEEDS_MORE_INPUT) { + PRF_STR("CODER_STATUS_NEEDS_MORE_INPUT"); *status = CODER_STATUS_NOT_SPECIFIED; return SZ_ERROR_DATA; } - // if (srcLen2 == 0 && destLen2 == 0) return SZ_OK; } { - Byte temp[32]; - unsigned num = Xz_WriteVarInt(temp, XzUnpacker_GetPackSizeForIndex(p)); - num += Xz_WriteVarInt(temp + num, p->unpackSize); - Sha256_Update(&p->sha, temp, num); - p->indexSize += num; - p->numBlocks++; + XzUnpacker_UpdateIndex(p, XzUnpacker_GetPackSizeForIndex(p), p->unpackSize); p->state = XZ_STATE_BLOCK_FOOTER; p->pos = 0; p->alignPos = 0; - - if (p->block.unpackSize != (UInt64)(Int64)-1) - if (p->block.unpackSize != p->unpackSize) - return SZ_ERROR_DATA; + *status = CODER_STATUS_NOT_SPECIFIED; + + if ((p->block.packSize != (UInt64)(Int64)-1 && p->block.packSize != p->packSize) + || (p->block.unpackSize != (UInt64)(Int64)-1 && p->block.unpackSize != p->unpackSize)) + { + PRF_STR("ERROR: block.size mismatch"); + return SZ_ERROR_DATA; + } } // continue; } @@ -757,6 +1100,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, { if (p->pos < XZ_SIG_SIZE && *src != XZ_SIG[p->pos]) return SZ_ERROR_NO_ARCHIVE; + if (p->decodeToStreamSignature) + return SZ_OK; p->buf[p->pos++] = *src++; (*srcLen)++; } @@ -809,12 +1154,19 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, else { RINOK(XzBlock_Parse(&p->block, p->buf)); + if (!XzBlock_AreSupportedFilters(&p->block)) + return SZ_ERROR_UNSUPPORTED; p->numTotalBlocks++; p->state = XZ_STATE_BLOCK; p->packSize = 0; p->unpackSize = 0; XzCheck_Init(&p->check, XzFlags_GetCheckType(p->streamFlags)); - RINOK(XzDec_Init(&p->decoder, &p->block)); + if (p->parseMode) + { + p->headerParsedOk = True; + return SZ_OK; + } + RINOK(XzDecMix_Init(&p->decoder, &p->block, p->outBuf, p->outBufSize)); } break; } @@ -980,12 +1332,26 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } -Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p) +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status) +{ + XzUnpacker_Init(p); + XzUnpacker_SetOutBuf(p, dest, *destLen); + + return XzUnpacker_Code(p, + NULL, destLen, + src, srcLen, True, + finishMode, status); +} + + +BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p) { return (p->state == XZ_STATE_BLOCK_HEADER) && (p->pos == 0); } -Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p) +BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p) { return (p->state == XZ_STATE_STREAM_PADDING) && (((UInt32)p->padSize & 3) == 0); } @@ -994,8 +1360,1407 @@ UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p) { UInt64 num = 0; if (p->state == XZ_STATE_STREAM_PADDING) - num += p->padSize; + num = p->padSize; else if (p->state == XZ_STATE_STREAM_HEADER) - num += p->padSize + p->pos; + num = p->padSize + p->pos; return num; } + + + + + + + + + + + + + + + + + + + + + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +void XzDecMtProps_Init(CXzDecMtProps *p) +{ + p->inBufSize_ST = 1 << 18; + p->outStep_ST = 1 << 20; + p->ignoreErrors = False; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->memUseMax = sizeof(size_t) << 28; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CXzDecMtThread ---------- */ + +typedef struct +{ + Byte *outBuf; + size_t outBufSize; + size_t outPreSize; + size_t inPreSize; + size_t inPreHeaderSize; + size_t blockPackSize_for_Index; // including block header and checksum. + size_t blockPackTotal; // including stream header, block header and checksum. + size_t inCodeSize; + size_t outCodeSize; + ECoderStatus status; + SRes codeRes; + BoolInt skipMode; + // BoolInt finishedWithMark; + EMtDecParseState parseState; + BoolInt parsing_Truncated; + BoolInt atBlockHeader; + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + BoolInt dec_created; + CXzUnpacker dec; + + Byte mtPad[1 << 7]; +} CXzDecMtThread; + +#endif + + +/* ---------- CXzDecMt ---------- */ + +typedef struct +{ + CAlignOffsetAlloc alignOffsetAlloc; + ISzAllocPtr allocMid; + + CXzDecMtProps props; + size_t unpackBlockMaxSize; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + // CXzStatInfo *stat; + + BoolInt finishMode; + BoolInt outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + UInt64 readProcessed; + BoolInt readWasFinished; + SRes readRes; + SRes writeRes; + + Byte *outBuf; + size_t outBufSize; + Byte *inBuf; + size_t inBufSize; + + CXzUnpacker dec; + + ECoderStatus status; + SRes codeRes; + + #ifndef _7ZIP_ST + BoolInt mainDecoderWasCalled; + // int statErrorDefined; + int finishedDecoderIndex; + + // global values that are used in Parse stage + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + // UInt64 numBadBlocks; + SRes mainErrorCode; + + BoolInt isBlockHeaderState_Parse; + BoolInt isBlockHeaderState_Write; + UInt64 outProcessed_Parse; + BoolInt parsing_Truncated; + + BoolInt mtc_WasConstructed; + CMtDec mtc; + CXzDecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CXzDecMt; + + + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CXzDecMt *p = (CXzDecMt *)ISzAlloc_Alloc(alloc, sizeof(CXzDecMt)); + if (!p) + return NULL; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.baseAlloc = alloc; + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + + p->allocMid = allocMid; + + p->outBuf = NULL; + p->outBufSize = 0; + p->inBuf = NULL; + p->inBufSize = 0; + + XzUnpacker_Construct(&p->dec, &p->alignOffsetAlloc.vt); + + p->unpackBlockMaxSize = 0; + + XzDecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + coder->dec_created = False; + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void XzDecMt_FreeOutBufs(CXzDecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + if (coder->outBuf) + { + ISzAlloc_Free(p->allocMid, coder->outBuf); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + p->unpackBlockMaxSize = 0; +} + +#endif + + + +static void XzDecMt_FreeSt(CXzDecMt *p) +{ + XzUnpacker_Free(&p->dec); + + if (p->outBuf) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBuf = NULL; + } + p->outBufSize = 0; + + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void XzDecMt_Destroy(CXzDecMtHandle pp) +{ + CXzDecMt *p = (CXzDecMt *)pp; + + XzDecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + XzUnpacker_Free(&t->dec); + t->dec_created = False; + } + } + } + XzDecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void XzDecMt_Callback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CXzDecMt *me = (CXzDecMt *)obj; + CXzDecMtThread *coder = &me->coders[coderIndex]; + size_t srcSize = cc->srcSize; + + cc->srcSize = 0; + cc->outPos = 0; + cc->state = MTDEC_PARSE_CONTINUE; + + cc->canCreateNewThread = True; + + if (cc->startCall) + { + coder->outPreSize = 0; + coder->inPreSize = 0; + coder->inPreHeaderSize = 0; + coder->parseState = MTDEC_PARSE_CONTINUE; + coder->parsing_Truncated = False; + coder->skipMode = False; + coder->codeRes = SZ_OK; + coder->status = CODER_STATUS_NOT_SPECIFIED; + coder->inCodeSize = 0; + coder->outCodeSize = 0; + + coder->numStreams = me->numStreams; + coder->numTotalBlocks = me->numTotalBlocks; + coder->numBlocks = me->numBlocks; + + if (!coder->dec_created) + { + XzUnpacker_Construct(&coder->dec, &me->alignOffsetAlloc.vt); + coder->dec_created = True; + } + + XzUnpacker_Init(&coder->dec); + + if (me->isBlockHeaderState_Parse) + { + coder->dec.streamFlags = me->streamFlags; + coder->atBlockHeader = True; + XzUnpacker_PrepareToRandomBlockDecoding(&coder->dec); + } + else + { + coder->atBlockHeader = False; + me->isBlockHeaderState_Parse = True; + } + + coder->dec.numStartedStreams = me->numStreams; + coder->dec.numTotalBlocks = me->numTotalBlocks; + coder->dec.numBlocks = me->numBlocks; + } + + while (!coder->skipMode) + { + ECoderStatus status; + SRes res; + size_t srcSize2 = srcSize; + size_t destSize = (size_t)0 - 1; + + coder->dec.parseMode = True; + coder->dec.headerParsedOk = False; + + PRF_STR_INT("Parse", srcSize2); + + res = XzUnpacker_Code(&coder->dec, + NULL, &destSize, + cc->src, &srcSize2, cc->srcFinished, + CODER_FINISH_END, &status); + + // PRF(printf(" res = %d, srcSize2 = %d", res, (unsigned)srcSize2)); + + coder->codeRes = res; + coder->status = status; + cc->srcSize += srcSize2; + srcSize -= srcSize2; + coder->inPreHeaderSize += srcSize2; + coder->inPreSize = coder->inPreHeaderSize; + + if (res != SZ_OK) + { + cc->state = + coder->parseState = MTDEC_PARSE_END; + /* + if (res == SZ_ERROR_MEM) + return res; + return SZ_OK; + */ + return; // res; + } + + if (coder->dec.headerParsedOk) + { + const CXzBlock *block = &coder->dec.block; + if (XzBlock_HasUnpackSize(block) + // && block->unpackSize <= me->props.outBlockMax + && XzBlock_HasPackSize(block)) + { + { + if (block->unpackSize * 2 * me->mtc.numStartedThreads > me->props.memUseMax) + { + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + } + { + UInt64 packSize = block->packSize; + UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); + UInt32 checkSize = XzFlags_GetCheckSize(coder->dec.streamFlags); + UInt64 blockPackSum = coder->inPreSize + packSizeAligned + checkSize; + // if (blockPackSum <= me->props.inBlockMax) + // unpackBlockMaxSize + { + coder->blockPackSize_for_Index = (size_t)(coder->dec.blockHeaderSize + packSize + checkSize); + coder->blockPackTotal = (size_t)blockPackSum; + coder->outPreSize = (size_t)block->unpackSize; + coder->streamFlags = coder->dec.streamFlags; + me->streamFlags = coder->dec.streamFlags; + coder->skipMode = True; + break; + } + } + } + } + else + // if (coder->inPreSize <= me->props.inBlockMax) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = + coder->parseState = MTDEC_PARSE_END; + return; // SZ_OK; + } + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + + // ---------- skipMode ---------- + { + UInt64 rem = coder->blockPackTotal - coder->inPreSize; + size_t cur = srcSize; + if (cur > rem) + cur = (size_t)rem; + cc->srcSize += cur; + coder->inPreSize += cur; + srcSize -= cur; + + if (coder->inPreSize == coder->blockPackTotal) + { + if (srcSize == 0) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = MTDEC_PARSE_END; + } + else if ((cc->src)[cc->srcSize] == 0) // we check control byte of next block + cc->state = MTDEC_PARSE_END; + else + { + cc->state = MTDEC_PARSE_NEW; + + { + size_t blockMax = me->unpackBlockMaxSize; + if (blockMax < coder->outPreSize) + blockMax = coder->outPreSize; + { + UInt64 required = (UInt64)blockMax * (me->mtc.numStartedThreads + 1) * 2; + if (me->props.memUseMax < required) + cc->canCreateNewThread = False; + } + } + + if (me->outSize_Defined) + { + // next block can be zero size + const UInt64 rem2 = me->outSize - me->outProcessed_Parse; + if (rem2 < coder->outPreSize) + { + coder->parsing_Truncated = True; + cc->state = MTDEC_PARSE_END; + } + me->outProcessed_Parse += coder->outPreSize; + } + } + } + else if (cc->srcFinished) + cc->state = MTDEC_PARSE_END; + else + return; // SZ_OK; + + coder->parseState = cc->state; + cc->outPos = coder->outPreSize; + + me->numStreams = coder->dec.numStartedStreams; + me->numTotalBlocks = coder->dec.numTotalBlocks; + me->numBlocks = coder->dec.numBlocks + 1; + return; // SZ_OK; + } +} + + +static SRes XzDecMt_Callback_PreCode(void *pp, unsigned coderIndex) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + Byte *dest; + + if (!coder->dec.headerParsedOk) + return SZ_OK; + + dest = coder->outBuf; + + if (!dest || coder->outBufSize < coder->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + { + size_t outPreSize = coder->outPreSize; + if (outPreSize == 0) + outPreSize = 1; + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, outPreSize); + } + if (!dest) + return SZ_ERROR_MEM; + coder->outBuf = dest; + coder->outBufSize = coder->outPreSize; + + if (coder->outBufSize > me->unpackBlockMaxSize) + me->unpackBlockMaxSize = coder->outBufSize; + } + + // return SZ_ERROR_MEM; + + XzUnpacker_SetOutBuf(&coder->dec, coder->outBuf, coder->outBufSize); + + { + SRes res = XzDecMix_Init(&coder->dec.decoder, &coder->dec.block, coder->outBuf, coder->outBufSize); + // res = SZ_ERROR_UNSUPPORTED; // to test + coder->codeRes = res; + if (res != SZ_OK) + { + // if (res == SZ_ERROR_MEM) return res; + if (me->props.ignoreErrors && res != SZ_ERROR_MEM) + return S_OK; + return res; + } + } + + return SZ_OK; +} + + +static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + *stop = True; + + if (coder->inCodeSize < coder->inPreHeaderSize) + { + UInt64 rem = coder->inPreHeaderSize - coder->inCodeSize; + size_t step = srcSize; + if (step > rem) + step = (size_t)rem; + src += step; + srcSize -= step; + coder->inCodeSize += step; + if (coder->inCodeSize < coder->inPreHeaderSize) + { + *stop = False; + return SZ_OK; + } + } + + if (!coder->dec.headerParsedOk) + return SZ_OK; + if (!coder->outBuf) + return SZ_OK; + + if (coder->codeRes == SZ_OK) + { + ECoderStatus status; + SRes res; + size_t srcProcessed = srcSize; + size_t outSizeCur = coder->outPreSize - coder->dec.outDataWritten; + + // PRF(printf("\nCallback_Code: Code %d %d\n", (unsigned)srcSize, (unsigned)outSizeCur)); + + res = XzUnpacker_Code(&coder->dec, + NULL, &outSizeCur, + src, &srcProcessed, srcFinished, + // coder->finishedWithMark ? CODER_FINISH_END : CODER_FINISH_ANY, + CODER_FINISH_END, + &status); + + // PRF(printf(" res = %d, srcSize2 = %d, outSizeCur = %d", res, (unsigned)srcProcessed, (unsigned)outSizeCur)); + + coder->codeRes = res; + coder->status = status; + coder->inCodeSize += srcProcessed; + coder->outCodeSize = coder->dec.outDataWritten; + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + + if (res == SZ_OK) + { + if (srcProcessed == srcSize) + *stop = False; + return SZ_OK; + } + } + + if (me->props.ignoreErrors && coder->codeRes != SZ_ERROR_MEM) + { + *inCodePos = coder->inPreSize; + *outCodePos = coder->outPreSize; + return S_OK; + } + return coder->codeRes; +} + + +#define XZDECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, + // int srcFinished, + BoolInt *needContinue, + BoolInt *canRecode) +{ + CXzDecMt *me = (CXzDecMt *)pp; + const CXzDecMtThread *coder = &me->coders[coderIndex]; + + // PRF(printf("\nWrite processed = %d srcSize = %d\n", (unsigned)me->mtc.inProcessed, (unsigned)srcSize)); + + *needContinue = False; + *canRecode = True; + + if (!needWriteToStream) + return SZ_OK; + + if (!coder->dec.headerParsedOk || !coder->outBuf) + { + if (me->finishedDecoderIndex < 0) + me->finishedDecoderIndex = coderIndex; + return SZ_OK; + } + + if (me->finishedDecoderIndex >= 0) + return SZ_OK; + + me->mtc.inProcessed += coder->inCodeSize; + + *canRecode = False; + + { + SRes res; + size_t size = coder->outCodeSize; + Byte *data = coder->outBuf; + + // we use in me->dec: sha, numBlocks, indexSize + + if (!me->isBlockHeaderState_Write) + { + XzUnpacker_PrepareToRandomBlockDecoding(&me->dec); + me->dec.decodeOnlyOneBlock = False; + me->dec.numStartedStreams = coder->dec.numStartedStreams; + me->dec.streamFlags = coder->streamFlags; + + me->isBlockHeaderState_Write = True; + } + + me->dec.numTotalBlocks = coder->dec.numTotalBlocks; + XzUnpacker_UpdateIndex(&me->dec, coder->blockPackSize_for_Index, coder->outPreSize); + + if (coder->outPreSize != size) + { + if (me->props.ignoreErrors) + { + memset(data + size, 0, coder->outPreSize - size); + size = coder->outPreSize; + } + // me->numBadBlocks++; + if (me->mainErrorCode == SZ_OK) + { + if ((int)coder->status == LZMA_STATUS_NEEDS_MORE_INPUT) + me->mainErrorCode = SZ_ERROR_INPUT_EOF; + else + me->mainErrorCode = SZ_ERROR_DATA; + } + } + + if (me->writeRes != SZ_OK) + return me->writeRes; + + res = SZ_OK; + { + if (me->outSize_Defined) + { + const UInt64 rem = me->outSize - me->outProcessed; + if (size > rem) + size = (SizeT)rem; + } + + for (;;) + { + size_t cur = size; + size_t written; + if (cur > XZDECMT_STREAM_WRITE_STEP) + cur = XZDECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + // PRF(printf("\nWritten ask = %d written = %d\n", (unsigned)cur, (unsigned)written)); + + me->outProcessed += written; + if (written != cur) + { + me->writeRes = SZ_ERROR_WRITE; + res = me->writeRes; + break; + } + data += cur; + size -= cur; + // PRF_STR_INT("Written size =", size); + if (size == 0) + break; + res = MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0); + if (res != SZ_OK) + break; + } + } + + if (coder->codeRes != SZ_OK) + if (!me->props.ignoreErrors) + { + me->finishedDecoderIndex = coderIndex; + return res; + } + + RINOK(res); + + if (coder->inPreSize != coder->inCodeSize + || coder->blockPackTotal != coder->inCodeSize) + { + me->finishedDecoderIndex = coderIndex; + return SZ_OK; + } + + if (coder->parseState != MTDEC_PARSE_END) + { + *needContinue = True; + return SZ_OK; + } + } + + // (coder->state == MTDEC_PARSE_END) means that there are no other working threads + // so we can use mtc variables without lock + + PRF_STR_INT("Write MTDEC_PARSE_END", me->mtc.inProcessed); + + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + { + CXzUnpacker *dec = &me->dec; + + PRF_STR_INT("PostSingle", srcSize); + + { + size_t srcProcessed = srcSize; + ECoderStatus status; + size_t outSizeCur = 0; + SRes res; + + // dec->decodeOnlyOneBlock = False; + dec->decodeToStreamSignature = True; + + me->mainDecoderWasCalled = True; + + if (coder->parsing_Truncated) + { + me->parsing_Truncated = True; + return SZ_OK; + } + + res = XzUnpacker_Code(dec, + NULL, &outSizeCur, + src, &srcProcessed, + me->mtc.readWasFinished, // srcFinished + CODER_FINISH_END, // CODER_FINISH_ANY, + &status); + + me->status = status; + me->codeRes = res; + + me->mtc.inProcessed += srcProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + if (res != SZ_OK) + { + return S_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + { + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + memcpy(crossBuf, src + srcProcessed, srcSize - srcProcessed); + } + me->mtc.crossStart = 0; + me->mtc.crossEnd = srcSize - srcProcessed; + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT) + { + return E_FAIL; + } + + if (me->mtc.readWasFinished) + { + return SZ_OK; + } + } + + { + size_t inPos; + size_t inLim; + const Byte *inData; + UInt64 inProgressPrev = me->mtc.inProcessed; + + // XzDecMt_Prepare_InBuf_ST(p); + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + + inPos = 0; + inLim = 0; + // outProcessed = 0; + + inData = crossBuf; + + for (;;) + { + SizeT inProcessed; + SizeT outProcessed; + ECoderStatus status; + SRes res; + + if (inPos == inLim) + { + if (!me->mtc.readWasFinished) + { + inPos = 0; + inLim = me->mtc.inBufSize; + me->mtc.readRes = ISeqInStream_Read(me->inStream, (void *)inData, &inLim); + me->mtc.readProcessed += inLim; + if (inLim == 0 || me->mtc.readRes != SZ_OK) + me->mtc.readWasFinished = True; + } + } + + inProcessed = inLim - inPos; + outProcessed = 0; + + res = XzUnpacker_Code(dec, + NULL, &outProcessed, + inData + inPos, &inProcessed, + (inProcessed == 0), // srcFinished + CODER_FINISH_END, &status); + + me->codeRes = res; + me->status = status; + inPos += inProcessed; + me->mtc.inProcessed += inProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + if (res != SZ_OK) + { + return S_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->mtc.crossStart = inPos; + me->mtc.crossEnd = inLim; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT) + return E_FAIL; + + if (me->mtc.progress) + { + UInt64 inDelta = me->mtc.inProcessed - inProgressPrev; + if (inDelta >= (1 << 22)) + { + RINOK(MtProgress_Progress_ST(&me->mtc.mtProgress)); + inProgressPrev = me->mtc.inProcessed; + } + } + if (me->mtc.readWasFinished) + return SZ_OK; + } + } + } +} + + +#endif + + + +void XzStatInfo_Clear(CXzStatInfo *p) +{ + p->InSize = 0; + p->OutSize = 0; + + p->NumStreams = 0; + p->NumBlocks = 0; + + p->UnpackSize_Defined = False; + + p->NumStreams_Defined = False; + p->NumBlocks_Defined = False; + + // p->IsArc = False; + // p->UnexpectedEnd = False; + // p->Unsupported = False; + // p->HeadersError = False; + // p->DataError = False; + // p->CrcError = False; + + p->DataAfterEnd = False; + p->DecodingTruncated = False; + + p->DecodeRes = SZ_OK; + p->ReadRes = SZ_OK; + p->ProgressRes = SZ_OK; + + p->CombinedRes = SZ_OK; + p->CombinedRes_Type = SZ_OK; +} + + + + +static SRes XzDecMt_Decode_ST(CXzDecMt *p + #ifndef _7ZIP_ST + , BoolInt tMode + #endif + , CXzStatInfo *stat) +{ + size_t outPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CXzUnpacker *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + XzDecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + if (!p->outBuf || p->outBufSize != p->props.outStep_ST) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBufSize = 0; + p->outBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.outStep_ST); + if (!p->outBuf) + return SZ_ERROR_MEM; + p->outBufSize = p->props.outStep_ST; + } + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + dec = &p->dec; + dec->decodeToStreamSignature = False; + // dec->decodeOnlyOneBlock = False; + + XzUnpacker_SetOutBuf(dec, NULL, 0); + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + outPos = 0; + + for (;;) + { + SizeT outSize; + BoolInt finished; + ECoderFinishMode finishMode; + SizeT inProcessed; + ECoderStatus status; + SRes res; + + SizeT outProcessed; + + + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + p->readProcessed += inLim; + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + outSize = p->props.outStep_ST - outPos; + + finishMode = CODER_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (outSize >= rem) + { + outSize = (SizeT)rem; + if (p->finishMode) + finishMode = CODER_FINISH_END; + } + } + + inProcessed = inLim - inPos; + outProcessed = outSize; + + res = XzUnpacker_Code(dec, p->outBuf + outPos, &outProcessed, + inData + inPos, &inProcessed, + (inPos == inLim), // srcFinished + finishMode, &status); + + p->codeRes = res; + p->status = status; + + inPos += inProcessed; + outPos += outProcessed; + p->inProcessed += inProcessed; + p->outProcessed += outProcessed; + + finished = ((inProcessed == 0 && outProcessed == 0) || res != SZ_OK); + + if (finished || outProcessed >= outSize) + if (outPos != 0) + { + size_t written = ISeqOutStream_Write(p->outStream, p->outBuf, outPos); + p->outProcessed += written; + if (written != outPos) + { + stat->CombinedRes_Type = SZ_ERROR_WRITE; + return SZ_ERROR_WRITE; + } + outPos = 0; + } + + if (p->progress && res == SZ_OK) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + res = ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed); + if (res != SZ_OK) + { + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + stat->ProgressRes = res; + return res; + } + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + + if (finished) + return res; + } +} + +static SRes XzStatInfo_SetStat(const CXzUnpacker *dec, + int finishMode, + UInt64 readProcessed, UInt64 inProcessed, + SRes res, ECoderStatus status, + BoolInt decodingTruncated, + CXzStatInfo *stat) +{ + UInt64 extraSize; + + stat->DecodingTruncated = (Byte)(decodingTruncated ? 1 : 0); + stat->InSize = inProcessed; + stat->NumStreams = dec->numStartedStreams; + stat->NumBlocks = dec->numTotalBlocks; + + stat->UnpackSize_Defined = True; + stat->NumStreams_Defined = True; + stat->NumBlocks_Defined = True; + + extraSize = XzUnpacker_GetExtraSize(dec); + + if (res == SZ_OK) + { + if (status == CODER_STATUS_NEEDS_MORE_INPUT) + { + // CODER_STATUS_NEEDS_MORE_INPUT is expected status for correct xz streams + extraSize = 0; + if (!XzUnpacker_IsStreamWasFinished(dec)) + res = SZ_ERROR_INPUT_EOF; + } + else if (!decodingTruncated || finishMode) // (status == CODER_STATUS_NOT_FINISHED) + res = SZ_ERROR_DATA; + } + else if (res == SZ_ERROR_NO_ARCHIVE) + { + /* + SZ_ERROR_NO_ARCHIVE is possible for 2 states: + XZ_STATE_STREAM_HEADER - if bad signature or bad CRC + XZ_STATE_STREAM_PADDING - if non-zero padding data + extraSize / inProcessed don't include "bad" byte + */ + if (inProcessed != extraSize) // if good streams before error + if (extraSize != 0 || readProcessed != inProcessed) + { + stat->DataAfterEnd = True; + // there is some good xz stream before. So we set SZ_OK + res = SZ_OK; + } + } + + stat->DecodeRes = res; + + stat->InSize -= extraSize; + return res; +} + + +SRes XzDecMt_Decode(CXzDecMtHandle pp, + const CXzDecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, + int *isMT, + ICompressProgress *progress) +{ + CXzDecMt *p = (CXzDecMt *)pp; + #ifndef _7ZIP_ST + BoolInt tMode; + #endif + + XzStatInfo_Clear(stat); + + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + // p->stat = stat; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + + p->finishMode = finishMode; + + // p->outSize = 457; p->outSize_Defined = True; p->finishMode = False; // for test + + p->writeRes = SZ_OK; + p->outProcessed = 0; + p->inProcessed = 0; + p->readProcessed = 0; + p->readWasFinished = False; + + p->codeRes = 0; + p->status = CODER_STATUS_NOT_SPECIFIED; + + XzUnpacker_Init(&p->dec); + + *isMT = False; + + /* + p->outBuf = NULL; + p->outBufSize = 0; + if (!outStream) + { + p->outBuf = outBuf; + p->outBufSize = *outBufSize; + *outBufSize = 0; + } + */ + + + #ifndef _7ZIP_ST + + p->isBlockHeaderState_Parse = False; + p->isBlockHeaderState_Write = False; + // p->numBadBlocks = 0; + p->mainErrorCode = SZ_OK; + p->mainDecoderWasCalled = False; + + tMode = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback vt; + + // we just free ST buffers here + // but we still keep state variables, that was set in XzUnpacker_Init() + XzDecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + p->parsing_Truncated = False; + + p->numStreams = 0; + p->numTotalBlocks = 0; + p->numBlocks = 0; + p->finishedDecoderIndex = -1; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.inBufSize = p->props.inBufSize_MT; + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = XzDecMt_Callback_Parse; + vt.PreCode = XzDecMt_Callback_PreCode; + vt.Code = XzDecMt_Callback_Code; + vt.Write = XzDecMt_Callback_Write; + + { + BoolInt needContinue; + + SRes res = MtDec_Code(&p->mtc); + + stat->InSize = p->mtc.inProcessed; + + p->inProcessed = p->mtc.inProcessed; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->readProcessed = p->mtc.readProcessed; + + tMode = True; + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + { + res = p->mtc.mtProgress.res; + stat->ProgressRes = res; + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + } + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + SRes codeRes; + BoolInt truncated = False; + ECoderStatus status; + CXzUnpacker *dec; + + stat->OutSize = p->outProcessed; + + if (p->finishedDecoderIndex >= 0) + { + CXzDecMtThread *coder = &p->coders[(unsigned)p->finishedDecoderIndex]; + codeRes = coder->codeRes; + dec = &coder->dec; + status = coder->status; + } + else if (p->mainDecoderWasCalled) + { + codeRes = p->codeRes; + dec = &p->dec; + status = p->status; + truncated = p->parsing_Truncated; + } + else + return E_FAIL; + + XzStatInfo_SetStat(dec, p->finishMode, + p->mtc.readProcessed, p->mtc.inProcessed, + codeRes, status, + truncated, + stat); + + if (res == SZ_OK) + { + if (p->writeRes != SZ_OK) + { + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; + } + else if (p->mtc.readRes != SZ_OK && p->mtc.inProcessed == p->mtc.readProcessed) + { + res = p->mtc.readRes; + stat->ReadRes = res; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + else if (p->mainErrorCode != SZ_OK) + { + res = p->mainErrorCode; + } + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = XzDecMt_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + , stat + ); + + XzStatInfo_SetStat(&p->dec, + p->finishMode, + p->readProcessed, p->inProcessed, + p->codeRes, p->status, + False, // truncated + stat); + + if (res == SZ_OK) + { + /* + if (p->writeRes != SZ_OK) + { + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; + } + else + */ + if (p->readRes != SZ_OK && p->inProcessed == p->readProcessed) + { + res = p->readRes; + stat->ReadRes = res; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + #ifndef _7ZIP_ST + else if (p->mainErrorCode != SZ_OK) + res = p->mainErrorCode; + #endif + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } +} diff --git a/7zip/C/XzEnc.c b/7zip/C/XzEnc.c index 1e1a4bf0d..d0a8b4489 100644 --- a/7zip/C/XzEnc.c +++ b/7zip/C/XzEnc.c @@ -1,5 +1,5 @@ /* XzEnc.c -- Xz Encode -2017-08-25 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -136,7 +136,7 @@ static void XzEncIndex_Free(CXzEncIndex *p, ISzAllocPtr alloc) static SRes XzEncIndex_ReAlloc(CXzEncIndex *p, size_t newSize, ISzAllocPtr alloc) { - Byte *blocks = ISzAlloc_Alloc(alloc, newSize); + Byte *blocks = (Byte *)ISzAlloc_Alloc(alloc, newSize); if (!blocks) return SZ_ERROR_MEM; if (p->size != 0) @@ -329,7 +329,7 @@ static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props, ISzAllocPt { if (!p->buf) { - p->buf = ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE); + p->buf = (Byte *)ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE); if (!p->buf) return SZ_ERROR_MEM; } @@ -362,13 +362,16 @@ static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size) } { SizeT srcLen = p->endPos - p->curPos; - int wasFinished; + ECoderStatus status; SRes res; *size = sizeOriginal; - res = p->StateCoder.Code(p->StateCoder.p, data, size, p->buf + p->curPos, &srcLen, - p->srcWasFinished, CODER_FINISH_ANY, &wasFinished); + res = p->StateCoder.Code2(p->StateCoder.p, + (Byte *)data, size, + p->buf + p->curPos, &srcLen, + p->srcWasFinished, CODER_FINISH_ANY, + &status); p->curPos += srcLen; - if (*size != 0 || srcLen == 0 || res != 0) + if (*size != 0 || srcLen == 0 || res != SZ_OK) return res; } } @@ -811,7 +814,7 @@ static SRes Xz_CompressBlock( SRes res; Byte *outBuf = NULL; size_t outSize = 0; - Bool useStream = (fp || inStream); + BoolInt useStream = (fp || inStream); // useStream = True; if (!useStream) @@ -937,7 +940,7 @@ typedef struct #ifndef _7ZIP_ST unsigned checkType; ISeqOutStream *outStream; - Bool mtCoder_WasConstructed; + BoolInt mtCoder_WasConstructed; CMtCoder mtCoder; CXzEncBlockInfo EncBlocks[MTCODER__BLOCKS_MAX]; #endif @@ -1061,7 +1064,7 @@ static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBuf if (!dest) { - dest = ISzAlloc_Alloc(me->alloc, me->outBufSize); + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); if (!dest) return SZ_ERROR_MEM; me->outBufs[outBufIndex] = dest; @@ -1069,7 +1072,7 @@ static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBuf MtProgressThunk_CreateVTable(&progressThunk); progressThunk.mtProgress = &me->mtCoder.mtProgress; - progressThunk.index = coderIndex; + MtProgressThunk_Init(&progressThunk); { CXzEncBlockInfo blockSizes; @@ -1230,7 +1233,7 @@ SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStr if (!p->outBufs[0] || t2 != p->outBufSize) { XzEnc_FreeOutBufs(p); - p->outBufs[0] = ISzAlloc_Alloc(p->alloc, t2); + p->outBufs[0] = (Byte *)ISzAlloc_Alloc(p->alloc, t2); if (!p->outBufs[0]) return SZ_ERROR_MEM; p->outBufSize = t2; @@ -1251,6 +1254,8 @@ SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStr && props->reduceSize >= progress2.inOffset) rem = props->reduceSize - progress2.inOffset; */ + + blockSizes.headerSize = 0; // for GCC RINOK(Xz_CompressBlock( &p->lzmaf_Items[0], diff --git a/7zip/C/XzIn.c b/7zip/C/XzIn.c index 357549e32..ff48e2dd4 100644 --- a/7zip/C/XzIn.c +++ b/7zip/C/XzIn.c @@ -1,5 +1,5 @@ /* XzIn.c - Xz input -2017-05-11 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -28,7 +28,7 @@ SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream) { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes) +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes) { Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; unsigned headerSize; @@ -103,7 +103,7 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPt { size_t i; p->numBlocks = numBlocks; - p->blocks = ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); + p->blocks = (CXzBlockSizes *)ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); if (!p->blocks) return SZ_ERROR_MEM; for (i = 0; i < numBlocks; i++) @@ -131,7 +131,7 @@ static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, size = (size_t)indexSize; if (size != indexSize) return SZ_ERROR_UNSUPPORTED; - buf = ISzAlloc_Alloc(alloc, size); + buf = (Byte *)ISzAlloc_Alloc(alloc, size); if (!buf) return SZ_ERROR_MEM; res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED); diff --git a/7zip/CPP/7zip/Aes.mak b/7zip/CPP/7zip/Aes.mak index fbd70aae2..20f1a72fa 100644 --- a/7zip/CPP/7zip/Aes.mak +++ b/7zip/CPP/7zip/Aes.mak @@ -1,7 +1,7 @@ C_OBJS = $(C_OBJS) \ $O\Aes.obj -!IF "$(CPU)" != "IA64" && "$(CPU)" != "MIPS" && "$(CPU)" != "ARM" && "$(CPU)" != "ARM64" +!IF "$(PLATFORM)" != "ia64" && "$(PLATFORM)" != "mips" && "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" ASM_OBJS = $(ASM_OBJS) \ $O\AesOpt.obj !ENDIF diff --git a/7zip/CPP/7zip/Archive/7z/7zCompressionMode.h b/7zip/CPP/7zip/Archive/7z/7zCompressionMode.h index 8105ff041..608293d6d 100644 --- a/7zip/CPP/7zip/Archive/7z/7zCompressionMode.h +++ b/7zip/CPP/7zip/Archive/7z/7zCompressionMode.h @@ -13,7 +13,9 @@ struct CMethodFull: public CMethodProps { CMethodId Id; UInt32 NumStreams; + int CodecIndex; + CMethodFull(): CodecIndex(-1) {} bool IsSimpleCoder() const { return NumStreams == 1; } }; diff --git a/7zip/CPP/7zip/Archive/7z/7zDecode.cpp b/7zip/CPP/7zip/Archive/7z/7zDecode.cpp index d26874793..9df531e0e 100644 --- a/7zip/CPP/7zip/Archive/7z/7zDecode.cpp +++ b/7zip/CPP/7zip/Archive/7z/7zDecode.cpp @@ -236,8 +236,8 @@ HRESULT CDecoder::Decode( _7Z_DECODER_CRYPRO_VARS_DECL - #if !defined(_7ZIP_ST) && !defined(_SFX) - , bool mtMode, UInt32 numThreads + #if !defined(_7ZIP_ST) + , bool mtMode, UInt32 numThreads, UInt64 memUsage #endif ) { @@ -312,7 +312,7 @@ HRESULT CDecoder::Decode( #endif CCreatedCoder cod; - RINOK(CreateCoder( + RINOK(CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS coderInfo.MethodID, false, cod)); @@ -355,11 +355,41 @@ HRESULT CDecoder::Decode( unsigned i; + #if !defined(_7ZIP_ST) + bool mt_wasUsed = false; + #endif + for (i = 0; i < folderInfo.Coders.Size(); i++) { const CCoderInfo &coderInfo = folderInfo.Coders[i]; IUnknown *decoder = _mixer->GetCoder(i).GetUnknown(); + #if !defined(_7ZIP_ST) + if (!mt_wasUsed) + { + if (mtMode) + { + CMyComPtr setCoderMt; + decoder->QueryInterface(IID_ICompressSetCoderMt, (void **)&setCoderMt); + if (setCoderMt) + { + mt_wasUsed = true; + RINOK(setCoderMt->SetNumberOfThreads(numThreads)); + } + } + // if (memUsage != 0) + { + CMyComPtr setMemLimit; + decoder->QueryInterface(IID_ICompressSetMemLimit, (void **)&setMemLimit); + if (setMemLimit) + { + mt_wasUsed = true; + RINOK(setMemLimit->SetMemLimit(memUsage)); + } + } + } + #endif + { CMyComPtr setDecoderProperties; decoder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties); @@ -376,18 +406,6 @@ HRESULT CDecoder::Decode( } } - #if !defined(_7ZIP_ST) && !defined(_SFX) - if (mtMode) - { - CMyComPtr setCoderMt; - decoder->QueryInterface(IID_ICompressSetCoderMt, (void **)&setCoderMt); - if (setCoderMt) - { - RINOK(setCoderMt->SetNumberOfThreads(numThreads)); - } - } - #endif - #ifndef _NO_CRYPTO { CMyComPtr cryptoSetPassword; diff --git a/7zip/CPP/7zip/Archive/7z/7zDecode.h b/7zip/CPP/7zip/Archive/7z/7zDecode.h index 62a380388..eeb146e38 100644 --- a/7zip/CPP/7zip/Archive/7z/7zDecode.h +++ b/7zip/CPP/7zip/Archive/7z/7zDecode.h @@ -59,8 +59,8 @@ class CDecoder _7Z_DECODER_CRYPRO_VARS_DECL - #if !defined(_7ZIP_ST) && !defined(_SFX) - , bool mtMode, UInt32 numThreads + #if !defined(_7ZIP_ST) + , bool mtMode, UInt32 numThreads, UInt64 memUsage #endif ); }; diff --git a/7zip/CPP/7zip/Archive/7z/7zEncode.cpp b/7zip/CPP/7zip/Archive/7z/7zEncode.cpp index 8700d721b..7d8270f93 100644 --- a/7zip/CPP/7zip/Archive/7z/7zEncode.cpp +++ b/7zip/CPP/7zip/Archive/7z/7zEncode.cpp @@ -154,9 +154,18 @@ HRESULT CEncoder::CreateMixerCoder( CCreatedCoder cod; - RINOK(CreateCoder( + if (methodFull.CodecIndex >= 0) + { + RINOK(CreateCoder_Index( + EXTERNAL_CODECS_LOC_VARS + methodFull.CodecIndex, true, cod)); + } + else + { + RINOK(CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodFull.Id, true, cod)); + } if (cod.NumStreams != methodFull.NumStreams) return E_FAIL; diff --git a/7zip/CPP/7zip/Archive/7z/7zExtract.cpp b/7zip/CPP/7zip/Archive/7z/7zExtract.cpp index 8fea5aa56..9ffe2fdcf 100644 --- a/7zip/CPP/7zip/Archive/7z/7zExtract.cpp +++ b/7zip/CPP/7zip/Archive/7z/7zExtract.cpp @@ -152,6 +152,12 @@ STDMETHODIMP CFolderOutStream::Write(const void *data, UInt32 size, UInt32 *proc if (_fileIsOpen) { UInt32 cur = (size < _rem ? size : (UInt32)_rem); + if (_calcCrc) + { + const UInt32 k_Step = (UInt32)1 << 20; + if (cur > k_Step) + cur = k_Step; + } HRESULT result = S_OK; if (_stream) result = _stream->Write(data, cur, &cur); @@ -363,8 +369,8 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, , dataAfterEnd_Error _7Z_DECODER_CRYPRO_VARS - #if !defined(_7ZIP_ST) && !defined(_SFX) - , true, _numThreads + #if !defined(_7ZIP_ST) + , true, _numThreads, _memUsage #endif ); diff --git a/7zip/CPP/7zip/Archive/7z/7zHandler.cpp b/7zip/CPP/7zip/Archive/7z/7zHandler.cpp index 2642e6911..bbb892e76 100644 --- a/7zip/CPP/7zip/Archive/7z/7zHandler.cpp +++ b/7zip/CPP/7zip/Archive/7z/7zHandler.cpp @@ -40,7 +40,6 @@ CHandler::CHandler() _crcSize = 4; #ifdef __7Z_SET_PROPERTIES - _numThreads = NSystem::GetNumberOfProcessors(); _useMultiThreadMixer = true; #endif @@ -237,6 +236,13 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) prop = v; break; } + + case kpidReadOnly: + { + if (!_db.CanUpdate()) + prop = true; + break; + } } prop.Detach(value); return S_OK; @@ -714,8 +720,8 @@ STDMETHODIMP CHandler::Close() STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) { COM_TRY_BEGIN - const UInt32 numProcessors = NSystem::GetNumberOfProcessors(); - _numThreads = numProcessors; + + InitCommon(); _useMultiThreadMixer = true; for (UInt32 i = 0; i < numProps; i++) @@ -734,13 +740,15 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR RINOK(PROPVARIANT_to_bool(value, _useMultiThreadMixer)); continue; } - if (name.IsPrefixedBy_Ascii_NoCase("mt")) { - RINOK(ParseMtProp(name.Ptr(2), value, numProcessors, _numThreads)); - continue; + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + { + RINOK(hres); + continue; + } } - else - return E_INVALIDARG; + return E_INVALIDARG; } } return S_OK; diff --git a/7zip/CPP/7zip/Archive/7z/7zHandler.h b/7zip/CPP/7zip/Archive/7z/7zHandler.h index 89e3275f0..99942eb0e 100644 --- a/7zip/CPP/7zip/Archive/7z/7zHandler.h +++ b/7zip/CPP/7zip/Archive/7z/7zHandler.h @@ -8,16 +8,6 @@ #include "../../Common/CreateCoder.h" -#ifndef EXTRACT_ONLY -#include "../Common/HandlerOut.h" -#endif - -#include "7zCompressionMode.h" -#include "7zIn.h" - -namespace NArchive { -namespace N7z { - #ifndef __7Z_SET_PROPERTIES #ifdef EXTRACT_ONLY @@ -30,6 +20,16 @@ namespace N7z { #endif +// #ifdef __7Z_SET_PROPERTIES +#include "../Common/HandlerOut.h" +// #endif + +#include "7zCompressionMode.h" +#include "7zIn.h" + +namespace NArchive { +namespace N7z { + #ifndef EXTRACT_ONLY @@ -38,8 +38,6 @@ class COutHandler: public CMultiMethodProps HRESULT SetSolidFromString(const UString &s); HRESULT SetSolidFromPROPVARIANT(const PROPVARIANT &value); public: - bool _removeSfxBlock; - UInt64 _numSolidFiles; UInt64 _numSolidBytes; bool _numSolidBytesDefined; @@ -58,6 +56,8 @@ class COutHandler: public CMultiMethodProps bool _useMultiThreadMixer; + bool _removeSfxBlock; + // bool _volumeMode; void InitSolidFiles() { _numSolidFiles = (UInt64)(Int64)(-1); } @@ -70,9 +70,10 @@ class COutHandler: public CMultiMethodProps _numSolidBytesDefined = false; } + void InitProps7z(); void InitProps(); - COutHandler() { InitProps(); } + COutHandler() { InitProps7z(); } HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); }; @@ -82,16 +83,23 @@ class COutHandler: public CMultiMethodProps class CHandler: public IInArchive, public IArchiveGetRawProps, + #ifdef __7Z_SET_PROPERTIES public ISetProperties, #endif + #ifndef EXTRACT_ONLY public IOutArchive, #endif + PUBLIC_ISetCompressCodecsInfo - public CMyUnknownImp + + public CMyUnknownImp, + #ifndef EXTRACT_ONLY - , public COutHandler + public COutHandler + #else + public CCommonMethodProps #endif { public: @@ -135,7 +143,6 @@ class CHandler: #ifdef EXTRACT_ONLY #ifdef __7Z_SET_PROPERTIES - UInt32 _numThreads; bool _useMultiThreadMixer; #endif diff --git a/7zip/CPP/7zip/Archive/7z/7zHandlerOut.cpp b/7zip/CPP/7zip/Archive/7z/7zHandlerOut.cpp index c4fabed78..b6be06a82 100644 --- a/7zip/CPP/7zip/Archive/7z/7zHandlerOut.cpp +++ b/7zip/CPP/7zip/Archive/7z/7zHandlerOut.cpp @@ -13,6 +13,8 @@ #include "7zOut.h" #include "7zUpdate.h" +#ifndef EXTRACT_ONLY + using namespace NWindows; namespace NArchive { @@ -41,9 +43,11 @@ STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type) HRESULT CHandler::PropsMethod_To_FullMethod(CMethodFull &dest, const COneMethodInfo &m) { - if (!FindMethod( + dest.CodecIndex = FindMethod_Index( EXTERNAL_CODECS_VARS - m.MethodName, dest.Id, dest.NumStreams)) + m.MethodName, true, + dest.Id, dest.NumStreams); + if (dest.CodecIndex < 0) return E_INVALIDARG; (CProps &)dest = (CProps &)m; return S_OK; @@ -263,6 +267,9 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt db = &_db; #endif + if (db && !db->CanUpdate()) + return E_NOTIMPL; + /* CMyComPtr getRawProps; updateCallback->QueryInterface(IID_IArchiveGetRawProps, (void **)&getRawProps); @@ -699,10 +706,8 @@ static HRESULT ParseBond(UString &srcString, UInt32 &coder, UInt32 &stream) return S_OK; } -void COutHandler::InitProps() +void COutHandler::InitProps7z() { - CMultiMethodProps::Init(); - _removeSfxBlock = false; _compressHeaders = true; _encryptHeadersSpecified = false; @@ -722,6 +727,14 @@ void COutHandler::InitProps() _useTypeSorting = false; } +void COutHandler::InitProps() +{ + CMultiMethodProps::Init(); + InitProps7z(); +} + + + HRESULT COutHandler::SetSolidFromString(const UString &s) { UString s2 = s; @@ -762,6 +775,10 @@ HRESULT COutHandler::SetSolidFromString(const UString &s) } _numSolidBytes = (v << numBits); _numSolidBytesDefined = true; + /* + if (_numSolidBytes == 0) + _numSolidFiles = 1; + */ } } return S_OK; @@ -810,7 +827,7 @@ HRESULT COutHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &val return E_INVALIDARG; return SetSolidFromString(name); } - + UInt32 number; int index = ParseStringToUInt32(name, number); // UString realName = name.Ptr(index); @@ -921,3 +938,5 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR } }} + +#endif diff --git a/7zip/CPP/7zip/Archive/7z/7zIn.cpp b/7zip/CPP/7zip/Archive/7z/7zIn.cpp index 3db5f515c..ab5b5de4e 100644 --- a/7zip/CPP/7zip/Archive/7z/7zIn.cpp +++ b/7zip/CPP/7zip/Archive/7z/7zIn.cpp @@ -1115,11 +1115,11 @@ HRESULT CInArchive::ReadAndDecodePackedStreams( , dataAfterEnd_Error _7Z_DECODER_CRYPRO_VARS - #if !defined(_7ZIP_ST) && !defined(_SFX) + #if !defined(_7ZIP_ST) , false // mtMode , 1 // numThreads + , 0 // memUsage #endif - ); RINOK(result); @@ -1465,21 +1465,24 @@ void CDbEx::FillLinks() } if (indexInFolder != 0) + { folderIndex++; - /* - if (indexInFolder != 0) - ThrowIncorrect(); - */ + // 18.06 + ThereIsHeaderError = true; + // ThrowIncorrect(); + } for (;;) { if (folderIndex >= NumFolders) return; FolderStartFileIndex[folderIndex] = i; - /* if (NumUnpackStreamsVector[folderIndex] != 0) - ThrowIncorrect();; - */ + { + // 18.06 + ThereIsHeaderError = true; + // ThrowIncorrect(); + } folderIndex++; } } diff --git a/7zip/CPP/7zip/Archive/7z/7zIn.h b/7zip/CPP/7zip/Archive/7z/7zIn.h index 2d5fbb79e..6a61d314d 100644 --- a/7zip/CPP/7zip/Archive/7z/7zIn.h +++ b/7zip/CPP/7zip/Archive/7z/7zIn.h @@ -257,6 +257,16 @@ struct CDbEx: public CDatabase PhySize = 0; } + bool CanUpdate() const + { + if (ThereIsHeaderError + || UnexpectedEnd + || StartHeaderWasRecovered + || UnsupportedFeatureError) + return false; + return true; + } + void FillLinks(); UInt64 GetFolderStreamPos(CNum folderIndex, unsigned indexInFolder) const diff --git a/7zip/CPP/7zip/Archive/7z/7zUpdate.cpp b/7zip/CPP/7zip/Archive/7z/7zUpdate.cpp index 53ea3a553..6705fc009 100644 --- a/7zip/CPP/7zip/Archive/7z/7zUpdate.cpp +++ b/7zip/CPP/7zip/Archive/7z/7zUpdate.cpp @@ -124,13 +124,13 @@ static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode) #define ELF_DATA_2LSB 1 #define ELF_DATA_2MSB 2 -static UInt16 Get16(const Byte *p, Bool be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); } -static UInt32 Get32(const Byte *p, Bool be) { if (be) return GetBe32(p); return GetUi32(p); } -// static UInt64 Get64(const Byte *p, Bool be) { if (be) return GetBe64(p); return GetUi64(p); } +static UInt16 Get16(const Byte *p, BoolInt be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); } +static UInt32 Get32(const Byte *p, BoolInt be) { if (be) return GetBe32(p); return GetUi32(p); } +// static UInt64 Get64(const Byte *p, BoolInt be) { if (be) return GetBe64(p); return GetUi64(p); } static int Parse_ELF(const Byte *buf, size_t size, CFilterMode *filterMode) { - Bool /* is32, */ be; + BoolInt /* is32, */ be; UInt32 filterId; if (size < 512 || buf[6] != 1) /* ver */ @@ -200,7 +200,7 @@ static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode if (size < 512) return 0; - Bool /* mode64, */ be; + BoolInt /* mode64, */ be; switch (GetUi32(buf)) { case MACH_SIG_BE_32: /* mode64 = False; */ be = True; break; @@ -239,7 +239,7 @@ static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode #define RIFF_SIG 0x46464952 -static Bool Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) +static BoolInt Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) { UInt32 subChunkSize, pos; if (size < 0x2C) @@ -285,7 +285,7 @@ static Bool Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) return False; } -static Bool ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode) +static BoolInt ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode) { filterMode->Id = 0; filterMode->Delta = 0; @@ -894,7 +894,7 @@ HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMo // RINOK(Callback->SetOperationResult2(index, NUpdate::NOperationResult::kOK)); if (result == S_OK) { - Bool parseRes = ParseFile(Buffer, size, &filterModeTemp); + BoolInt parseRes = ParseFile(Buffer, size, &filterModeTemp); if (parseRes && filterModeTemp.Delta == 0) { filterModeTemp.SetDelta(); @@ -1507,7 +1507,8 @@ void CThreadDecoder::Execute() _7Z_DECODER_CRYPRO_VARS #ifndef _7ZIP_ST - , MtMode, NumThreads + , MtMode, NumThreads, + 0 // MemUsage #endif ); @@ -1647,6 +1648,9 @@ HRESULT Update( for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++) { + if (fi >= db->Files.Size()) + return E_FAIL; + const CFileItem &file = db->Files[fi]; if (file.HasStream) { @@ -1696,13 +1700,14 @@ HRESULT Update( UInt64 inSizeForReduce = 0; { + bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0); FOR_VECTOR (i, updateItems) { const CUpdateItem &ui = updateItems[i]; if (ui.NewData) { complexity += ui.Size; - if (numSolidFiles != 1) + if (isSolid) inSizeForReduce += ui.Size; else if (inSizeForReduce < ui.Size) inSizeForReduce = ui.Size; @@ -2142,8 +2147,8 @@ HRESULT Update( #ifndef _7ZIP_ST , false // mtMode , 1 // numThreads + , 0 // memUsage #endif - ); RINOK(res); @@ -2293,7 +2298,8 @@ HRESULT Update( continue; CRecordVector refItems; refItems.ClearAndSetSize(numFiles); - bool sortByType = (options.UseTypeSorting && numSolidFiles > 1); + // bool sortByType = (options.UseTypeSorting && isSoid); // numSolidFiles > 1 + bool sortByType = options.UseTypeSorting; unsigned i; diff --git a/7zip/CPP/7zip/Archive/ArHandler.cpp b/7zip/CPP/7zip/Archive/ArHandler.cpp index 816a84e44..09a622017 100644 --- a/7zip/CPP/7zip/Archive/ArHandler.cpp +++ b/7zip/CPP/7zip/Archive/ArHandler.cpp @@ -170,8 +170,8 @@ static bool OctalToNumber32(const char *s, unsigned size, UInt32 &res) res = 0; char sz[32]; size = RemoveTailSpaces(sz, s, size); - if (size == 0) - return true; // some items doesn't contaion any numbers + if (size == 0 || strcmp(sz, "-1") == 0) + return true; // some items don't contain any numbers const char *end; UInt64 res64 = ConvertOctStringToUInt64(sz, &end); if ((unsigned)(end - sz) != size) @@ -185,8 +185,8 @@ static bool DecimalToNumber(const char *s, unsigned size, UInt64 &res) res = 0; char sz[32]; size = RemoveTailSpaces(sz, s, size); - if (size == 0) - return true; // some items doesn't contaion any numbers + if (size == 0 || strcmp(sz, "-1") == 0) + return true; // some items don't contain any numbers const char *end; res = ConvertStringToUInt64(sz, &end); return ((unsigned)(end - sz) == size); diff --git a/7zip/CPP/7zip/Archive/Bz2Handler.cpp b/7zip/CPP/7zip/Archive/Bz2Handler.cpp index 98428430e..b0c2f7508 100644 --- a/7zip/CPP/7zip/Archive/Bz2Handler.cpp +++ b/7zip/CPP/7zip/Archive/Bz2Handler.cpp @@ -366,7 +366,13 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt return E_INVALIDARG; size = prop.uhVal.QuadPart; } - return UpdateArchive(size, outStream, _props, updateCallback); + + CMethodProps props2 = _props; + #ifndef _7ZIP_ST + props2.AddProp_NumThreads(_props._numThreads); + #endif + + return UpdateArchive(size, outStream, props2, updateCallback); } if (indexInArchive != 0) diff --git a/7zip/CPP/7zip/Archive/Cab/CabHandler.cpp b/7zip/CPP/7zip/Archive/Cab/CabHandler.cpp index c62efbd81..4395ae1a8 100644 --- a/7zip/CPP/7zip/Archive/Cab/CabHandler.cpp +++ b/7zip/CPP/7zip/Archive/Cab/CabHandler.cpp @@ -549,9 +549,9 @@ class CFolderOutStream: Byte *TempBuf; UInt32 TempBufSize; + UInt32 TempBufWritten; unsigned NumIdenticalFiles; bool TempBufMode; - UInt32 m_BufStartFolderOffset; unsigned m_StartIndex; unsigned m_CurrentIndex; @@ -575,7 +575,6 @@ class CFolderOutStream: HRESULT OpenFile(); HRESULT CloseFileWithResOp(Int32 resOp); HRESULT CloseFile(); - HRESULT Write2(const void *data, UInt32 size, UInt32 *processedSize, bool isOK); public: HRESULT WriteEmptyFiles(); @@ -672,11 +671,11 @@ HRESULT CFolderOutStream::OpenFile() FreeTempBuf(); TempBuf = (Byte *)MyAlloc(item.Size); TempBufSize = item.Size; - if (TempBuf == NULL) + if (!TempBuf) return E_OUTOFMEMORY; } TempBufMode = true; - m_BufStartFolderOffset = item.Offset; + TempBufWritten = 0; } else if (numExtractItems == 1) { @@ -725,8 +724,9 @@ HRESULT CFolderOutStream::WriteEmptyFiles() } -HRESULT CFolderOutStream::Write2(const void *data, UInt32 size, UInt32 *processedSize, bool isOK) +HRESULT CFolderOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize) { + // (data == NULL) means Error_Data for solid folder flushing COM_TRY_BEGIN UInt32 realProcessed = 0; @@ -741,21 +741,34 @@ HRESULT CFolderOutStream::Write2(const void *data, UInt32 size, UInt32 *processe HRESULT res = S_OK; if (numBytesToWrite != 0) { - if (!isOK) + if (!data) m_IsOk = false; + if (m_RealOutStream) { UInt32 processedSizeLocal = 0; - res = m_RealOutStream->Write((const Byte *)data, numBytesToWrite, &processedSizeLocal); + // 18.01 : we don't want ZEROs instead of missing data + if (data) + res = m_RealOutStream->Write((const Byte *)data, numBytesToWrite, &processedSizeLocal); + else + processedSizeLocal = numBytesToWrite; numBytesToWrite = processedSizeLocal; } + if (TempBufMode && TempBuf) - memcpy(TempBuf + (m_PosInFolder - m_BufStartFolderOffset), data, numBytesToWrite); + { + if (data) + { + memcpy(TempBuf + TempBufWritten, data, numBytesToWrite); + TempBufWritten += numBytesToWrite; + } + } } realProcessed += numBytesToWrite; if (processedSize) *processedSize = realProcessed; - data = (const void *)((const Byte *)data + numBytesToWrite); + if (data) + data = (const void *)((const Byte *)data + numBytesToWrite); size -= numBytesToWrite; m_RemainFileSize -= numBytesToWrite; m_PosInFolder += numBytesToWrite; @@ -773,7 +786,7 @@ HRESULT CFolderOutStream::Write2(const void *data, UInt32 size, UInt32 *processe m_FileIsOpen = true; m_CurrentIndex++; if (result == S_OK && m_RealOutStream && TempBuf) - result = WriteStream(m_RealOutStream, TempBuf, (size_t)(m_PosInFolder - m_BufStartFolderOffset)); + result = WriteStream(m_RealOutStream, TempBuf, TempBufWritten); if (!TempBuf && TempBufMode && m_RealOutStream) { @@ -822,7 +835,8 @@ HRESULT CFolderOutStream::Write2(const void *data, UInt32 size, UInt32 *processe realProcessed += numBytesToWrite; if (processedSize) *processedSize = realProcessed; - data = (const void *)((const Byte *)data + numBytesToWrite); + if (data) + data = (const void *)((const Byte *)data + numBytesToWrite); size -= numBytesToWrite; m_PosInFolder += numBytesToWrite; } @@ -843,12 +857,6 @@ HRESULT CFolderOutStream::Write2(const void *data, UInt32 size, UInt32 *processe } -STDMETHODIMP CFolderOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize) -{ - return Write2(data, size, processedSize, true); -} - - HRESULT CFolderOutStream::FlushCorrupted(unsigned folderIndex) { if (!NeedMoreWrite()) @@ -862,19 +870,16 @@ HRESULT CFolderOutStream::FlushCorrupted(unsigned folderIndex) return S_OK; } - const unsigned kBufSize = (1 << 12); - Byte buf[kBufSize]; - for (unsigned i = 0; i < kBufSize; i++) - buf[i] = 0; - for (;;) { if (!NeedMoreWrite()) return S_OK; UInt64 remain = GetRemain(); - UInt32 size = (remain < kBufSize ? (UInt32)remain : (UInt32)kBufSize); + UInt32 size = (UInt32)1 << 20; + if (size > remain) + size = (UInt32)remain; UInt32 processedSizeLocal = 0; - RINOK(Write2(buf, size, &processedSizeLocal, false)); + RINOK(Write(NULL, size, &processedSizeLocal)); } } diff --git a/7zip/CPP/7zip/Archive/Common/CoderMixer2.cpp b/7zip/CPP/7zip/Archive/Common/CoderMixer2.cpp index 7834c6054..c7002121d 100644 --- a/7zip/CPP/7zip/Archive/Common/CoderMixer2.cpp +++ b/7zip/CPP/7zip/Archive/Common/CoderMixer2.cpp @@ -91,6 +91,7 @@ HRESULT CCoder::CheckDataAfterEnd(bool &dataAfterEnd_Error /* , bool &InternalPa { CMyComPtr getInStreamProcessedSize2; Coder2.QueryInterface(IID_ICompressGetInStreamProcessedSize2, (void **)&getInStreamProcessedSize2); + if (getInStreamProcessedSize2) FOR_VECTOR (i, PackSizePointers) { if (!PackSizePointers[i]) diff --git a/7zip/CPP/7zip/Archive/Common/HandlerOut.cpp b/7zip/CPP/7zip/Archive/Common/HandlerOut.cpp index ea320e665..77a35c74c 100644 --- a/7zip/CPP/7zip/Archive/Common/HandlerOut.cpp +++ b/7zip/CPP/7zip/Archive/Common/HandlerOut.cpp @@ -2,18 +2,92 @@ #include "StdAfx.h" -#ifndef _7ZIP_ST -#include "../../../Windows/System.h" -#endif +#include "../../../Common/StringToInt.h" #include "../Common/ParseProperties.h" #include "HandlerOut.h" -using namespace NWindows; - namespace NArchive { +bool ParseSizeString(const wchar_t *s, const PROPVARIANT &prop, UInt64 percentsBase, UInt64 &res) +{ + if (*s == 0) + { + switch (prop.vt) + { + case VT_UI4: res = prop.ulVal; return true; + case VT_UI8: res = prop.uhVal.QuadPart; return true; + case VT_BSTR: + s = prop.bstrVal; + break; + default: return false; + } + } + else if (prop.vt != VT_EMPTY) + return false; + + const wchar_t *end; + UInt64 v = ConvertStringToUInt64(s, &end); + if (s == end) + return false; + wchar_t c = *end; + if (c == 0) + { + res = v; + return true; + } + if (end[1] != 0) + return false; + + if (c == '%') + { + res = percentsBase / 100 * v; + return true; + } + + unsigned numBits; + switch (MyCharLower_Ascii(c)) + { + case 'b': numBits = 0; break; + case 'k': numBits = 10; break; + case 'm': numBits = 20; break; + case 'g': numBits = 30; break; + case 't': numBits = 40; break; + default: return false; + } + UInt64 val2 = v << numBits; + if ((val2 >> numBits) != v) + return false; + res = val2; + return true; +} + +bool CCommonMethodProps::SetCommonProperty(const UString &name, const PROPVARIANT &value, HRESULT &hres) +{ + hres = S_OK; + + if (name.IsPrefixedBy_Ascii_NoCase("mt")) + { + #ifndef _7ZIP_ST + hres = ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads); + #endif + return true; + } + + if (name.IsPrefixedBy_Ascii_NoCase("memuse")) + { + if (!ParseSizeString(name.Ptr(6), value, _memAvail, _memUsage)) + hres = E_INVALIDARG; + return true; + } + + return false; +} + + +#ifndef EXTRACT_ONLY + static void SetMethodProp32(COneMethodInfo &m, PROPID propID, UInt32 value) { if (m.FindProp(propID) < 0) @@ -34,21 +108,23 @@ void CMultiMethodProps::SetMethodThreadsTo(COneMethodInfo &oneMethodInfo, UInt32 } #endif -void CMultiMethodProps::Init() +void CMultiMethodProps::InitMulti() { - #ifndef _7ZIP_ST - _numProcessors = _numThreads = NSystem::GetNumberOfProcessors(); - #endif - _level = (UInt32)(Int32)-1; _analysisLevel = -1; - - _autoFilter = true; _crcSize = 4; - _filterMethod.Clear(); + _autoFilter = true; +} + +void CMultiMethodProps::Init() +{ + InitCommon(); + InitMulti(); _methods.Clear(); + _filterMethod.Clear(); } + HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) { UString name = nameSpec; @@ -78,20 +154,18 @@ HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIAN _crcSize = 4; return ParsePropToUInt32(name, value, _crcSize); } + + { + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + return hres; + } UInt32 number; unsigned index = ParseStringToUInt32(name, number); UString realName = name.Ptr(index); if (index == 0) { - if (name.IsPrefixedBy_Ascii_NoCase("mt")) - { - #ifndef _7ZIP_ST - RINOK(ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads)); - #endif - - return S_OK; - } if (name.IsEqualTo("f")) { HRESULT res = PROPVARIANT_to_bool(value, _autoFilter); @@ -110,20 +184,20 @@ HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIAN return _methods[number].ParseMethodFromPROPVARIANT(realName, value); } + + void CSingleMethodProps::Init() { + InitCommon(); + InitSingle(); Clear(); - _level = (UInt32)(Int32)-1; - - #ifndef _7ZIP_ST - _numProcessors = _numThreads = NWindows::NSystem::GetNumberOfProcessors(); - AddProp_NumThreads(_numThreads); - #endif } + HRESULT CSingleMethodProps::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) { Init(); + for (UInt32 i = 0; i < numProps; i++) { UString name = names[i]; @@ -137,20 +211,22 @@ HRESULT CSingleMethodProps::SetProperties(const wchar_t * const *names, const PR RINOK(ParsePropToUInt32(name.Ptr(1), value, a)); _level = a; AddProp_Level(a); + continue; } - else if (name.IsPrefixedBy_Ascii_NoCase("mt")) - { - #ifndef _7ZIP_ST - RINOK(ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads)); - AddProp_NumThreads(_numThreads); - #endif - } - else { - RINOK(ParseMethodFromPROPVARIANT(names[i], value)); + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + { + RINOK(hres) + continue; + } } + RINOK(ParseMethodFromPROPVARIANT(names[i], value)); } + return S_OK; } +#endif + } diff --git a/7zip/CPP/7zip/Archive/Common/HandlerOut.h b/7zip/CPP/7zip/Archive/Common/HandlerOut.h index e24686da9..bbb4336e7 100644 --- a/7zip/CPP/7zip/Archive/Common/HandlerOut.h +++ b/7zip/CPP/7zip/Archive/Common/HandlerOut.h @@ -3,20 +3,57 @@ #ifndef __HANDLER_OUT_H #define __HANDLER_OUT_H +#include "../../../Windows/System.h" + #include "../../Common/MethodProps.h" namespace NArchive { -class CMultiMethodProps +bool ParseSizeString(const wchar_t *name, const PROPVARIANT &prop, UInt64 percentsBase, UInt64 &res); + +class CCommonMethodProps { - UInt32 _level; - int _analysisLevel; +protected: + void InitCommon() + { + #ifndef _7ZIP_ST + _numProcessors = _numThreads = NWindows::NSystem::GetNumberOfProcessors(); + #endif + + UInt64 memAvail = (UInt64)(sizeof(size_t)) << 28; + _memAvail = memAvail; + _memUsage = memAvail; + if (NWindows::NSystem::GetRamSize(memAvail)) + { + _memAvail = memAvail; + _memUsage = memAvail / 32 * 17; + } + } + public: #ifndef _7ZIP_ST UInt32 _numThreads; UInt32 _numProcessors; #endif + UInt64 _memUsage; + UInt64 _memAvail; + + bool SetCommonProperty(const UString &name, const PROPVARIANT &value, HRESULT &hres); + + CCommonMethodProps() { InitCommon(); } +}; + + +#ifndef EXTRACT_ONLY + +class CMultiMethodProps: public CCommonMethodProps +{ + UInt32 _level; + int _analysisLevel; + + void InitMulti(); +public: UInt32 _crcSize; CObjectVector _methods; COneMethodInfo _filterMethod; @@ -43,27 +80,31 @@ class CMultiMethodProps int GetAnalysisLevel() const { return _analysisLevel; } void Init(); + CMultiMethodProps() { InitMulti(); } - CMultiMethodProps() { Init(); } HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); }; -class CSingleMethodProps: public COneMethodInfo + +class CSingleMethodProps: public COneMethodInfo, public CCommonMethodProps { UInt32 _level; - -public: - #ifndef _7ZIP_ST - UInt32 _numThreads; - UInt32 _numProcessors; - #endif + void InitSingle() + { + _level = (UInt32)(Int32)-1; + } + +public: void Init(); - CSingleMethodProps() { Init(); } + CSingleMethodProps() { InitSingle(); } + int GetLevel() const { return _level == (UInt32)(Int32)-1 ? 5 : (int)_level; } HRESULT SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); }; +#endif + } #endif diff --git a/7zip/CPP/7zip/Archive/DmgHandler.cpp b/7zip/CPP/7zip/Archive/DmgHandler.cpp index 608ad4c41..d9fe6016a 100644 --- a/7zip/CPP/7zip/Archive/DmgHandler.cpp +++ b/7zip/CPP/7zip/Archive/DmgHandler.cpp @@ -219,6 +219,7 @@ class CHandler: bool _masterCrcError; bool _headersError; + UInt32 _dataStartOffset; UInt64 _startPos; UInt64 _phySize; @@ -333,6 +334,7 @@ static const Byte kProps[] = kpidCRC, kpidComment, kpidMethod + // kpidOffset }; IMP_IInArchive_Props @@ -631,17 +633,40 @@ bool CHandler::ParseBlob(const CByteBuffer &data) HRESULT CHandler::Open2(IInStream *stream) { + /* + - usual dmg contains Koly Header at the end: + - rare case dmg contains Koly Header at the start. + */ + + _dataStartOffset = 0; RINOK(stream->Seek(0, STREAM_SEEK_CUR, &_startPos)); + UInt64 fileSize = 0; + RINOK(stream->Seek(0, STREAM_SEEK_END, &fileSize)); + RINOK(stream->Seek(_startPos, STREAM_SEEK_SET, NULL)); + Byte buf[HEADER_SIZE]; RINOK(ReadStream_FALSE(stream, buf, HEADER_SIZE)); UInt64 headerPos; + bool startKolyMode = false; + if (IsKoly(buf)) + { + // it can be normal koly-at-the-end or koly-at-the-start headerPos = _startPos; + if (_startPos <= (1 << 8)) + { + // we want to support startKolyMode, even if there is + // some data before dmg file, like 128 bytes MacBin header + _dataStartOffset = HEADER_SIZE; + startKolyMode = true; + } + } else { - RINOK(stream->Seek(0, STREAM_SEEK_END, &headerPos)); + // we check only koly-at-the-end + headerPos = fileSize; if (headerPos < HEADER_SIZE) return S_FALSE; headerPos -= HEADER_SIZE; @@ -667,24 +692,35 @@ HRESULT CHandler::Open2(IInStream *stream) // CChecksum dataForkChecksum; // dataForkChecksum.Parse(buf + 0x50); - _startPos = 0; - UInt64 top = 0; - if (!dataForkPair.UpdateTop(headerPos, top)) return S_FALSE; - if (!xmlPair.UpdateTop(headerPos, top)) return S_FALSE; - if (!rsrcPair.UpdateTop(headerPos, top)) return S_FALSE; + UInt64 limit = startKolyMode ? fileSize : headerPos; + + if (!dataForkPair.UpdateTop(limit, top)) return S_FALSE; + if (!xmlPair.UpdateTop(limit, top)) return S_FALSE; + if (!rsrcPair.UpdateTop(limit, top)) return S_FALSE; /* Some old dmg files contain garbage data in blobPair field. So we need to ignore such garbage case; And we still need to detect offset of start of archive for "parser" mode. */ - bool useBlob = blobPair.UpdateTop(headerPos, top); + bool useBlob = blobPair.UpdateTop(limit, top); - _startPos = 0; - _phySize = headerPos + HEADER_SIZE; - if (top != headerPos) + if (startKolyMode) + _phySize = top; + else { + _phySize = headerPos + HEADER_SIZE; + _startPos = 0; + if (top != headerPos) + { + /* + if expected absolute offset is not equal to real header offset, + 2 cases are possible: + - additional (unknown) headers + - archive with offset. + So we try to read XML with absolute offset to select from these two ways. + */ CForkPair xmlPair2 = xmlPair; const char *sz = "= rsrcPair.Len || mainDataSize >= rsrcPair.Len - || footerOffset + footerSize != rsrcPair.Len + || footerOffset < mainDataSize || footerOffset != headSize + mainDataSize) return S_FALSE; + + const UInt32 footerEnd = footerOffset + footerSize; + if (footerEnd != rsrcPair.Len) + { + // there is rare case dmg example, where there are 4 additional bytes + UInt64 rem = rsrcPair.Len - footerOffset; + if (rem < footerSize + || rem - footerSize != 4 + || Get32(p + footerEnd) != 0) + return S_FALSE; + } + if (footerSize < 16) return S_FALSE; if (memcmp(p, p + footerOffset, 16) != 0) @@ -1041,6 +1091,14 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val break; } + /* + case kpidOffset: + { + prop = item.StartPos; + break; + } + */ + case kpidMethod: { CMethods m; @@ -1384,7 +1442,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, break; } - RINOK(_inStream->Seek(_startPos + item.StartPos + block.PackPos, STREAM_SEEK_SET, NULL)); + RINOK(_inStream->Seek(_startPos + _dataStartOffset + item.StartPos + block.PackPos, STREAM_SEEK_SET, NULL)); streamSpec->Init(block.PackSize); bool realMethod = true; outStreamSpec->Init(block.UnpSize); @@ -1781,7 +1839,7 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) spec->Stream = _inStream; spec->Size = spec->File->Size; - RINOK(spec->InitAndSeek(_startPos)); + RINOK(spec->InitAndSeek(_startPos + _dataStartOffset)); *stream = specStream.Detach(); return S_OK; diff --git a/7zip/CPP/7zip/Archive/ElfHandler.cpp b/7zip/CPP/7zip/Archive/ElfHandler.cpp index 626747741..c5ce279ec 100644 --- a/7zip/CPP/7zip/Archive/ElfHandler.cpp +++ b/7zip/CPP/7zip/Archive/ElfHandler.cpp @@ -567,6 +567,7 @@ static const char * const g_Machines[] = static const CUInt32PCharPair g_MachinePairs[] = { + { 243, "RISC-V" }, { 47787, "Xilinx MicroBlaze" } // { 0x9026, "Alpha" } }; diff --git a/7zip/CPP/7zip/Archive/ExtHandler.cpp b/7zip/CPP/7zip/Archive/ExtHandler.cpp index d8417fe51..db65fbea8 100644 --- a/7zip/CPP/7zip/Archive/ExtHandler.cpp +++ b/7zip/CPP/7zip/Archive/ExtHandler.cpp @@ -1596,6 +1596,17 @@ STDMETHODIMP CHandler::Close() } +static void ChangeSeparatorsInName(char *s, unsigned num) +{ + for (unsigned i = 0; i < num; i++) + { + char c = s[i]; + if (c == CHAR_PATH_SEPARATOR || c == '/') + s[i] = '_'; + } +} + + void CHandler::GetPath(unsigned index, AString &s) const { s.Empty(); @@ -1612,6 +1623,8 @@ void CHandler::GetPath(unsigned index, AString &s) const if (!s.IsEmpty()) s.InsertAtFront(CHAR_PATH_SEPARATOR); s.Insert(0, item.Name); + // 18.06 + ChangeSeparatorsInName(s.GetBuf(), item.Name.Len()); if (item.ParentNode == k_INODE_ROOT) return; diff --git a/7zip/CPP/7zip/Archive/FatHandler.cpp b/7zip/CPP/7zip/Archive/FatHandler.cpp index 9c37c062f..bf6053e0d 100644 --- a/7zip/CPP/7zip/Archive/FatHandler.cpp +++ b/7zip/CPP/7zip/Archive/FatHandler.cpp @@ -161,7 +161,8 @@ bool CHeader::Parse(const Byte *p) return false; // we also support images that contain 0 in offset field. - bool isOkOffset = (codeOffset == 0 || (p[0] == 0xEB && p[1] == 0)); + bool isOkOffset = (codeOffset == 0) + || (codeOffset == (p[0] == 0xEB ? 2 : 3)); UInt16 numRootDirEntries = Get16(p + 17); if (numRootDirEntries == 0) @@ -732,7 +733,13 @@ HRESULT CDatabase::Open() RINOK(OpenProgressFat()); if ((Fat[0] & 0xFF) != Header.MediaType) - return S_FALSE; + { + // that case can mean error in FAT, + // but xdf file: (MediaType == 0xF0 && Fat[0] == 0xFF9) + // 19.00: so we use non-strict check + if ((Fat[0] & 0xFF) < 0xF0) + return S_FALSE; + } RINOK(ReadDir(-1, Header.RootCluster, 0)); diff --git a/7zip/CPP/7zip/Archive/HfsHandler.cpp b/7zip/CPP/7zip/Archive/HfsHandler.cpp index d02f07257..ca1d7bda4 100644 --- a/7zip/CPP/7zip/Archive/HfsHandler.cpp +++ b/7zip/CPP/7zip/Archive/HfsHandler.cpp @@ -317,11 +317,15 @@ class CDatabase // bool CaseSensetive; UString ResFileName; + UInt64 SpecOffset; UInt64 PhySize; + UInt64 PhySize2; void Clear() { + SpecOffset = 0; PhySize = 0; + PhySize2 = 0; HeadersError = false; ThereAreAltStreams = false; // CaseSensetive = false; @@ -415,7 +419,13 @@ void CDatabase::GetItemPath(unsigned index, NWindows::NCOM::CPropVariant &path) const wchar_t *src = (const wchar_t *)*s; wchar_t *dest = p + len; for (unsigned j = 0; j < curLen; j++) - dest[j] = src[j]; + { + wchar_t c = src[j]; + // 18.06 + if (c == CHAR_PATH_SEPARATOR || c == '/') + c = '_'; + dest[j] = c; + } if (len == 0) break; @@ -444,7 +454,7 @@ HRESULT CDatabase::ReadFile(const CFork &fork, CByteBuffer &buf, IInStream *inSt e.NumBlocks > fork.NumBlocks - curBlock || e.NumBlocks > Header.NumBlocks - e.Pos) return S_FALSE; - RINOK(inStream->Seek((UInt64)e.Pos << Header.BlockSizeLog, STREAM_SEEK_SET, NULL)); + RINOK(inStream->Seek(SpecOffset + ((UInt64)e.Pos << Header.BlockSizeLog), STREAM_SEEK_SET, NULL)); RINOK(ReadStream_FALSE(inStream, (Byte *)buf + ((size_t)curBlock << Header.BlockSizeLog), (size_t)e.NumBlocks << Header.BlockSizeLog)); @@ -1154,13 +1164,36 @@ HRESULT CDatabase::LoadCatalog(const CFork &fork, const CObjectVector 5) + return k_IsArc_Res_NO; + } + return k_IsArc_Res_YES; +} +} HRESULT CDatabase::Open2(IInStream *inStream, IArchiveOpenCallback *progress) { Clear(); - static const unsigned kHeaderSize = kHeaderPadSize + 512; - Byte buf[kHeaderSize]; - RINOK(ReadStream_FALSE(inStream, buf, kHeaderSize)); + Byte buf[kHfsHeaderSize]; + RINOK(ReadStream_FALSE(inStream, buf, kHfsHeaderSize)); { for (unsigned i = 0; i < kHeaderPadSize; i++) if (buf[i] != 0) @@ -1171,6 +1204,67 @@ HRESULT CDatabase::Open2(IInStream *inStream, IArchiveOpenCallback *progress) h.Header[0] = p[0]; h.Header[1] = p[1]; + + if (p[0] == 'B' && p[1] == 'D') + { + /* + It's header for old HFS format. + We don't support old HFS format, but we support + special HFS volume that contains embedded HFS+ volume + */ + + if (p[0x7C] != 'H' || p[0x7C + 1] != '+') + return S_FALSE; + + /* + h.CTime = Get32(p + 0x2); + h.MTime = Get32(p + 0x6); + + h.NumFiles = Get32(p + 0x54); + h.NumFolders = Get32(p + 0x58); + + if (h.NumFolders > ((UInt32)1 << 29) || + h.NumFiles > ((UInt32)1 << 30)) + return S_FALSE; + if (progress) + { + UInt64 numFiles = (UInt64)h.NumFiles + h.NumFolders + 1; + RINOK(progress->SetTotal(&numFiles, NULL)); + } + h.NumFreeBlocks = Get16(p + 0x22); + */ + + UInt32 blockSize = Get32(p + 0x14); + + { + unsigned i; + for (i = 9; ((UInt32)1 << i) != blockSize; i++) + if (i == 31) + return S_FALSE; + h.BlockSizeLog = i; + } + + h.NumBlocks = Get16(p + 0x12); + /* + we suppose that it has the follwing layout + { + start block with header + [h.NumBlocks] + end block with header + } + */ + PhySize2 = ((UInt64)h.NumBlocks + 2) << h.BlockSizeLog; + + UInt32 startBlock = Get16(p + 0x7C + 2); + UInt32 blockCount = Get16(p + 0x7C + 4); + SpecOffset = (UInt64)(1 + startBlock) << h.BlockSizeLog; + UInt64 phy = SpecOffset + ((UInt64)blockCount << h.BlockSizeLog); + if (PhySize2 < phy) + PhySize2 = phy; + RINOK(inStream->Seek(SpecOffset, STREAM_SEEK_SET, NULL)); + RINOK(ReadStream_FALSE(inStream, buf, kHfsHeaderSize)); + } + if (p[0] != 'H' || (p[1] != '+' && p[1] != 'X')) return S_FALSE; h.Version = Get16(p + 2); @@ -1330,7 +1424,14 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) { case kpidExtension: prop = Header.IsHfsX() ? "hfsx" : "hfs"; break; case kpidMethod: prop = Header.IsHfsX() ? "HFSX" : "HFS+"; break; - case kpidPhySize: prop = PhySize; break; + case kpidPhySize: + { + UInt64 v = SpecOffset + PhySize; + if (v < PhySize2) + v = PhySize2; + prop = v; + break; + } case kpidClusterSize: prop = (UInt32)1 << Header.BlockSizeLog; break; case kpidFreeSpace: prop = (UInt64)Header.GetFreeSize(); break; case kpidMTime: HfsTimeToProp(Header.MTime, prop); break; @@ -1754,7 +1855,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (fork.Size == pos) break; const CExtent &e = fork.Extents[extentIndex]; - RINOK(_stream->Seek((UInt64)e.Pos << Header.BlockSizeLog, STREAM_SEEK_SET, NULL)); + RINOK(_stream->Seek(SpecOffset + ((UInt64)e.Pos << Header.BlockSizeLog), STREAM_SEEK_SET, NULL)); UInt64 extentRem = (UInt64)e.NumBlocks << Header.BlockSizeLog; while (extentRem != 0) { @@ -1865,6 +1966,7 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) } static const Byte k_Signature[] = { + 2, 'B', 'D', 4, 'H', '+', 0, 4, 4, 'H', 'X', 0, 5 }; @@ -1873,6 +1975,6 @@ REGISTER_ARC_I( k_Signature, kHeaderPadSize, NArcInfoFlags::kMultiSignature, - NULL) + IsArc_HFS) }} diff --git a/7zip/CPP/7zip/Archive/IArchive.h b/7zip/CPP/7zip/Archive/IArchive.h index 0028d7624..7d7256c93 100644 --- a/7zip/CPP/7zip/Archive/IArchive.h +++ b/7zip/CPP/7zip/Archive/IArchive.h @@ -488,6 +488,16 @@ ARCHIVE_INTERFACE(IOutArchive, 0xA0) }; +/* +ISetProperties::SetProperties() + PROPVARIANT values[i].vt: + VT_EMPTY + VT_BOOL + VT_UI4 - if 32-bit number + VT_UI8 - if 64-bit number + VT_BSTR +*/ + ARCHIVE_INTERFACE(ISetProperties, 0x03) { STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) PURE; diff --git a/7zip/CPP/7zip/Archive/LzmaHandler.cpp b/7zip/CPP/7zip/Archive/LzmaHandler.cpp index afdc2bfae..9eac3ca12 100644 --- a/7zip/CPP/7zip/Archive/LzmaHandler.cpp +++ b/7zip/CPP/7zip/Archive/LzmaHandler.cpp @@ -131,13 +131,7 @@ HRESULT CDecoder::Code(const CHeader &header, ISequentialOutStream *outStream, if (header.FilterID > 1) return E_NOTIMPL; - { - CMyComPtr setDecoderProperties; - _lzmaDecoder.QueryInterface(IID_ICompressSetDecoderProperties2, &setDecoderProperties); - if (!setDecoderProperties) - return E_NOTIMPL; - RINOK(setDecoderProperties->SetDecoderProperties2(header.LzmaProps, 5)); - } + RINOK(_lzmaDecoderSpec->SetDecoderProperties2(header.LzmaProps, 5)); bool filteredMode = (header.FilterID == 1); @@ -357,24 +351,54 @@ API_FUNC_static_IsArc IsArc_Lzma86(const Byte *p, size_t size) } } + + STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *, IArchiveOpenCallback *) { Close(); - const UInt32 kBufSize = 1 + 5 + 8 + 2; + const unsigned headerSize = GetHeaderSize(); + const UInt32 kBufSize = 1 << 7; Byte buf[kBufSize]; - - RINOK(ReadStream_FALSE(inStream, buf, kBufSize)); - + size_t processedSize = kBufSize; + RINOK(ReadStream(inStream, buf, &processedSize)); + if (processedSize < headerSize + 2) + return S_FALSE; if (!_header.Parse(buf, _lzma86)) return S_FALSE; - const Byte *start = buf + GetHeaderSize(); + const Byte *start = buf + headerSize; if (start[0] != 0 /* || (start[1] & 0x80) != 0 */ ) // empty stream with EOS is not 0x80 return S_FALSE; - + RINOK(inStream->Seek(0, STREAM_SEEK_END, &_packSize)); - if (_packSize >= 24 && _header.Size == 0 && _header.FilterID == 0 && _header.LzmaProps[0] == 0) + + SizeT srcLen = processedSize - headerSize; + + if (srcLen > 10 + && _header.Size == 0 + // && _header.FilterID == 0 + && _header.LzmaProps[0] == 0 + ) return S_FALSE; + + CDecoder state; + const UInt32 outLimit = 1 << 11; + Byte outBuf[outLimit]; + + SizeT outSize = outLimit; + if (outSize > _header.Size) + outSize = (SizeT)_header.Size; + SizeT destLen = outSize; + ELzmaStatus status; + + SRes res = LzmaDecode(outBuf, &destLen, start, &srcLen, + _header.LzmaProps, 5, LZMA_FINISH_ANY, + &status, &g_Alloc); + + if (res != SZ_OK) + if (res != SZ_ERROR_INPUT_EOF) + return S_FALSE; + _isArc = true; _stream = inStream; _seqStream = inStream; diff --git a/7zip/CPP/7zip/Archive/Nsis/NsisIn.cpp b/7zip/CPP/7zip/Archive/Nsis/NsisIn.cpp index bbdab45ab..f4f9ab04e 100644 --- a/7zip/CPP/7zip/Archive/Nsis/NsisIn.cpp +++ b/7zip/CPP/7zip/Archive/Nsis/NsisIn.cpp @@ -2445,23 +2445,24 @@ void CInArchive::DetectNsisType(const CBlockHeader &bh, const Byte *p) bool strongPark = false; bool strongNsis = false; + if (NumStringChars > 2) { const Byte *strData = _data + _stringsPos; if (IsUnicode) { - UInt32 num = NumStringChars; + UInt32 num = NumStringChars - 2; for (UInt32 i = 0; i < num; i++) { if (Get16(strData + i * 2) == 0) { unsigned c2 = Get16(strData + 2 + i * 2); + // it can be TXT/RTF with marker char (1 or 2). so we must check next char // if (c2 <= NS_3_CODE_SKIP && c2 != NS_3_CODE_SHELL) if (c2 == NS_3_CODE_VAR) { - // it can be TXT/RTF string with marker char (1 or 2). so we must next char - // const wchar_t *p2 = (const wchar_t *)(strData + i * 2 + 2); - // p2 = p2; - if ((Get16(strData + 3 + i * 2) & 0x8000) != 0) + // 18.06: fixed: is it correct ? + // if ((Get16(strData + 3 + i * 2) & 0x8000) != 0) + if ((Get16(strData + 4 + i * 2) & 0x8080) == 0x8080) { NsisType = k_NsisType_Nsis3; strongNsis = true; @@ -2478,7 +2479,7 @@ void CInArchive::DetectNsisType(const CBlockHeader &bh, const Byte *p) } else { - UInt32 num = NumStringChars; + UInt32 num = NumStringChars - 2; for (UInt32 i = 0; i < num; i++) { if (strData[i] == 0) diff --git a/7zip/CPP/7zip/Archive/NtfsHandler.cpp b/7zip/CPP/7zip/Archive/NtfsHandler.cpp index 0eebf005e..21372033e 100644 --- a/7zip/CPP/7zip/Archive/NtfsHandler.cpp +++ b/7zip/CPP/7zip/Archive/NtfsHandler.cpp @@ -1475,6 +1475,21 @@ void CDatabase::ClearAndClose() InStream.Release(); } + +static void CopyName(wchar_t *dest, const wchar_t *src) +{ + for (;;) + { + wchar_t c = *src++; + // 18.06 + if (c == '\\' || c == '/') + c = '_'; + *dest++ = c; + if (c == 0) + return; + } +} + void CDatabase::GetItemPath(unsigned index, NCOM::CPropVariant &path) const { const CItem *item = &Items[index]; @@ -1492,7 +1507,7 @@ void CDatabase::GetItemPath(unsigned index, NCOM::CPropVariant &path) const wchar_t *s = path.AllocBstr(data.Name.Len() + 1); s[0] = L':'; if (!data.Name.IsEmpty()) - MyStringCopy(s + 1, data.Name.GetRawPtr()); + CopyName(s + 1, data.Name.GetRawPtr()); return; } @@ -1541,7 +1556,7 @@ void CDatabase::GetItemPath(unsigned index, NCOM::CPropVariant &path) const if (!name.IsEmpty()) { size -= name.Len(); - MyStringCopy(s + size, name.GetRawPtr()); + CopyName(s + size, name.GetRawPtr()); } s[--size] = ':'; needColon = true; @@ -1551,7 +1566,7 @@ void CDatabase::GetItemPath(unsigned index, NCOM::CPropVariant &path) const const UString2 &name = rec.FileNames[item->NameIndex].Name; unsigned len = name.Len(); if (len != 0) - MyStringCopy(s + size - len, name.GetRawPtr()); + CopyName(s + size - len, name.GetRawPtr()); if (needColon) s[size] = ':'; size -= len; @@ -1575,7 +1590,7 @@ void CDatabase::GetItemPath(unsigned index, NCOM::CPropVariant &path) const if (len != 0) { size -= len; - MyStringCopy(s + size, name.GetRawPtr()); + CopyName(s + size, name.GetRawPtr()); } s[size + len] = WCHAR_PATH_SEPARATOR; continue; diff --git a/7zip/CPP/7zip/Archive/PeHandler.cpp b/7zip/CPP/7zip/Archive/PeHandler.cpp index cb0836cc8..675293ba3 100644 --- a/7zip/CPP/7zip/Archive/PeHandler.cpp +++ b/7zip/CPP/7zip/Archive/PeHandler.cpp @@ -41,9 +41,8 @@ static const UInt32 k_Signature32 = 0x00004550; static HRESULT CalcCheckSum(ISequentialInStream *stream, UInt32 size, UInt32 excludePos, UInt32 &res) { - const UInt32 kBufSizeMax = (UInt32)1 << 16; - UInt32 bufSize = MyMin(kBufSizeMax, size); - bufSize += (bufSize & 1); + const UInt32 kBufSizeMax = (UInt32)1 << 15; + UInt32 bufSize = kBufSizeMax; CByteBuffer buffer(bufSize); Byte *buf = buffer; UInt32 sum = 0; @@ -58,9 +57,6 @@ static HRESULT CalcCheckSum(ISequentialInStream *stream, UInt32 size, UInt32 exc size_t processed = rem; RINOK(ReadStream(stream, buf, &processed)); - if ((processed & 1) != 0) - buf[processed] = 0; - for (unsigned j = 0; j < 4; j++) { UInt32 e = excludePos + j; @@ -72,11 +68,30 @@ static HRESULT CalcCheckSum(ISequentialInStream *stream, UInt32 size, UInt32 exc } } - for (size_t i = 0; i < processed; i += 2) + const unsigned kStep = (1 << 4); + { + for (size_t i = processed; (i & (kStep - 1)) != 0; i++) + buf[i] = 0; + } { - sum += Get16(buf + i); - sum = (sum + (sum >> 16)) & 0xFFFF; + const Byte *buf2 = buf; + const Byte *bufLimit = buf + processed; + UInt64 sum2 = 0; + for (; buf2 < bufLimit; buf2 += kStep) + { + UInt64 sum3 = (UInt64)Get32(buf2) + + Get32(buf2 + 4) + + Get32(buf2 + 8) + + Get32(buf2 + 12); + sum2 += sum3; + } + sum2 = (UInt32)(sum2) + (UInt64)(sum2 >> 32); + UInt32 sum3 = ((UInt32)sum2 + (UInt32)(sum2 >> 32)); + sum += (sum3 & 0xFFFF) + (sum3 >> 16); + sum = (sum & 0xFFFF) + (sum >> 16); + sum = (sum & 0xFFFF) + (sum >> 16); } + pos += (UInt32)processed; if (rem != processed) break; @@ -2201,6 +2216,11 @@ bool CHeader::ParseCoff(const Byte *p) return false; if (OptHeaderSize != 0 && OptHeaderSize < k_OptHeader32_Size_MIN) return false; + + // 18.04: we reduce false detections + if (NumSections == 0 && OptHeaderSize == 0) + return false; + for (unsigned i = 0; i < ARRAY_SIZE(g_MachinePairs); i++) if (Machine == g_MachinePairs[i].Value) return true; diff --git a/7zip/CPP/7zip/Archive/Rar/Rar5Handler.cpp b/7zip/CPP/7zip/Archive/Rar/Rar5Handler.cpp index c42254269..320771d5b 100644 --- a/7zip/CPP/7zip/Archive/Rar/Rar5Handler.cpp +++ b/7zip/CPP/7zip/Archive/Rar/Rar5Handler.cpp @@ -7,6 +7,7 @@ #include "../../../Common/ComTry.h" #include "../../../Common/IntToString.h" +#include "../../../Common/MyBuffer2.h" #include "../../../Common/UTFConvert.h" #include "../../../Windows/PropVariantUtils.h" @@ -104,46 +105,14 @@ static const char * const g_LinkTypes[] = static const char g_ExtraTimeFlags[] = { 'u', 'M', 'C', 'A', 'n' }; -template -struct CAlignedBuffer -{ - Byte *_buf; - Byte *_bufBase; - size_t _size; - - CAlignedBuffer(): _buf(NULL), _bufBase(NULL), _size(0) {} - ~CAlignedBuffer() { ::MyFree(_bufBase); } -public: - operator Byte *() { return _buf; } - operator const Byte *() const { return _buf; } - - void AllocAtLeast(size_t size) - { - if (_buf && _size >= size) - return; - ::MyFree(_bufBase); - _buf = NULL; - _size = 0; - _bufBase = (Byte *)::MyAlloc(size + alignMask); - - if (_bufBase) - { - _size = size; - // _buf = (Byte *)(((uintptr_t)_bufBase + alignMask) & ~(uintptr_t)alignMask); - _buf = (Byte *)(((ptrdiff_t)_bufBase + alignMask) & ~(ptrdiff_t)alignMask); - } - } -}; - static unsigned ReadVarInt(const Byte *p, size_t maxSize, UInt64 *val) { *val = 0; - for (unsigned i = 0; i < maxSize;) + for (unsigned i = 0; i < maxSize && i < 10;) { Byte b = p[i]; - if (i < 10) - *val |= (UInt64)(b & 0x7F) << (7 * i); + *val |= (UInt64)(b & 0x7F) << (7 * i); i++; if ((b & 0x80) == 0) return i; @@ -579,7 +548,7 @@ STDMETHODIMP COutStreamWithHash::Write(const void *data, UInt32 size, UInt32 *pr class CInArchive { - CAlignedBuffer _buf; + CAlignedBuffer _buf; size_t _bufSize; size_t _bufPos; ISequentialInStream *_stream; @@ -587,7 +556,7 @@ class CInArchive NCrypto::NRar5::CDecoder *m_CryptoDecoderSpec; CMyComPtr m_CryptoDecoder; - + CLASS_NO_COPY(CInArchive) HRESULT ReadStream_Check(void *data, size_t size); @@ -611,6 +580,8 @@ class CInArchive UInt64 DataSize; }; + CInArchive() {} + HRESULT ReadBlockHeader(CHeader &h); bool ReadFileHeader(const CHeader &header, CItem &item); void AddToSeekValue(UInt64 addValue) @@ -1001,7 +972,7 @@ bool CInArchive::ReadFileHeader(const CHeader &header, CItem &item) struct CLinkFile { unsigned Index; - unsigned NumLinks; + unsigned NumLinks; // the number of links to Data CByteBuffer Data; HRESULT Res; bool crcOK; @@ -1016,7 +987,7 @@ struct CUnpacker CMyComPtr copyCoder; CMyComPtr LzCoders[2]; - bool NeedClearSolid[2]; + bool SolidAllowed; CFilterCoder *filterStreamSpec; CMyComPtr filterStream; @@ -1033,7 +1004,7 @@ struct CUnpacker CLinkFile *linkFile; - CUnpacker(): linkFile(NULL) { NeedClearSolid[0] = NeedClearSolid[1] = true; } + CUnpacker(): linkFile(NULL) { SolidAllowed = false; } HRESULT Create(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, bool isSolid, bool &wrongPassword); @@ -1086,7 +1057,7 @@ HRESULT CUnpacker::Create(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, bool if (!lzCoder) { const UInt32 methodID = 0x40305; - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS methodID, false, lzCoder)); + RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS methodID, false, lzCoder)); if (!lzCoder) return E_NOTIMPL; } @@ -1143,7 +1114,9 @@ HRESULT CUnpacker::Code(const CItem &item, const CItem &lastItem, UInt64 packSiz if (method > kLzMethodMax) return E_NOTIMPL; - if (linkFile && !lastItem.Is_UnknownSize()) + bool needBuf = (linkFile && linkFile->NumLinks != 0); + + if (needBuf && !lastItem.Is_UnknownSize()) { size_t dataSize = (size_t)lastItem.Size; if (dataSize != lastItem.Size) @@ -1168,15 +1141,15 @@ HRESULT CUnpacker::Code(const CItem &item, const CItem &lastItem, UInt64 packSiz ICompressCoder *commonCoder = (method == 0) ? copyCoder : LzCoders[item.IsService() ? 1 : 0]; outStreamSpec->SetStream(realOutStream); - outStreamSpec->Init(lastItem, (linkFile ? (Byte *)linkFile->Data : NULL)); - - NeedClearSolid[item.IsService() ? 1 : 0] = false; + outStreamSpec->Init(lastItem, (needBuf ? (Byte *)linkFile->Data : NULL)); HRESULT res = S_OK; if (packSize != 0 || lastItem.Is_UnknownSize() || lastItem.Size != 0) { res = commonCoder->Code(inStream, outStream, &packSize, lastItem.Is_UnknownSize() ? NULL : &lastItem.Size, progress); + if (!item.IsService()) + SolidAllowed = true; } else { @@ -1211,7 +1184,9 @@ HRESULT CUnpacker::Code(const CItem &item, const CItem &lastItem, UInt64 packSiz { linkFile->Res = res; linkFile->crcOK = isCrcOK; - if (!lastItem.Is_UnknownSize() && processedSize != lastItem.Size) + if (needBuf + && !lastItem.Is_UnknownSize() + && processedSize != lastItem.Size) linkFile->Data.ChangeSize_KeepData((size_t)processedSize, (size_t)processedSize); } @@ -1363,7 +1338,9 @@ static const Byte kProps[] = kpidCharacts, kpidSymLink, kpidHardLink, - kpidCopyLink + kpidCopyLink, + + kpidVolumeIndex }; @@ -1794,6 +1771,18 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidSplitBefore: prop = item.IsSplitBefore(); break; case kpidSplitAfter: prop = lastItem.IsSplitAfter(); break; + + case kpidVolumeIndex: + { + if (item.VolIndex < _arcs.Size()) + { + const CInArcInfo &arcInfo = _arcs[item.VolIndex].Info; + if (arcInfo.IsVolume()) + prop = (UInt64)arcInfo.GetVolIndex(); + } + break; + } + case kpidCRC: { const CItem *item2 = (lastItem.IsSplitAfter() ? &item : &lastItem); @@ -2565,6 +2554,15 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, const Byte kStatus_Skip = 1 << 1; const Byte kStatus_Link = 1 << 2; + /* + In original RAR: + 1) service streams are not allowed to be solid, + and solid flag must be ignored for service streams. + 2) If RAR creates new solid block and first file in solid block is Link file, + then it can clear solid flag for Link file and + clear solid flag for first non-Link file after Link file. + */ + CObjectVector linkFiles; { @@ -2590,13 +2588,15 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (ref.Link >= 0) { - if (!testMode) + // 18.06 fixed: we use links for Test mode too + // if (!testMode) { if ((unsigned)ref.Link < index) { const CRefItem &linkRef = _refs[(unsigned)ref.Link]; const CItem &linkItem = _items[linkRef.Item]; - if (linkItem.IsSolid() && linkItem.Size <= k_CopyLinkFile_MaxSize) + if (linkItem.IsSolid()) + if (testMode || linkItem.Size <= k_CopyLinkFile_MaxSize) { if (extractStatuses[(unsigned)ref.Link] == 0) { @@ -2651,19 +2651,25 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, { unsigned solidLimit = 0; - FOR_VECTOR(i, _refs) + FOR_VECTOR (i, _refs) { if ((extractStatuses[i] & kStatus_Link) == 0) continue; + + // We use CLinkFile for testMode too. + // So we can show errors for copy files. + // if (!testMode) + { + CLinkFile &linkFile = linkFiles.AddNew(); + linkFile.Index = i; + } + const CItem &item = _items[_refs[i].Item]; /* if (item.IsService()) continue; */ - CLinkFile &linkFile = linkFiles.AddNew(); - linkFile.Index = i; - if (item.IsSolid()) { unsigned j = i; @@ -2694,6 +2700,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, solidLimit = i + 1; } + if (!testMode) for (UInt32 t = 0; t < numItems; t++) { unsigned index = allFilesMode ? t : indices[t]; @@ -2735,7 +2742,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, // bool needClearSolid = true; - FOR_VECTOR(i, _refs) + FOR_VECTOR (i, _refs) { if (extractStatuses[i] == 0) continue; @@ -2748,15 +2755,19 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, CMyComPtr realOutStream; + // isExtract means that we don't skip that item. So we need read data. + + bool isExtract = ((extractStatuses[i] & kStatus_Extract) != 0); Int32 askMode = - ((extractStatuses[i] & kStatus_Extract) != 0) ? (testMode ? + isExtract ? (testMode ? NExtract::NAskMode::kTest : NExtract::NAskMode::kExtract) : NExtract::NAskMode::kSkip; unpacker.linkFile = NULL; - if (((extractStatuses[i] & kStatus_Link) != 0)) + // if (!testMode) + if ((extractStatuses[i] & kStatus_Link) != 0) { int bufIndex = FindLinkBuf(linkFiles, i); if (bufIndex < 0) @@ -2778,13 +2789,12 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(extractCallback->GetStream(index, &realOutStream, askMode)); - bool isSolid; + bool isSolid = false; + if (!item->IsService()) { - bool &needClearSolid = unpacker.NeedClearSolid[item->IsService() ? 1 : 0]; - isSolid = (item->IsSolid() && !needClearSolid); - if (item->IsService()) - isSolid = false; - needClearSolid = !item->IsSolid(); + if (item->IsSolid()) + isSolid = unpacker.SolidAllowed; + unpacker.SolidAllowed = isSolid; } if (item->IsDir()) @@ -2813,18 +2823,30 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, curUnpackSize = 0; curPackSize = GetPackSize(index2); } - else if ((unsigned)index2 < index) - bufIndex = FindLinkBuf(linkFiles, index2); + else + { + if ((unsigned)index2 < index) + bufIndex = FindLinkBuf(linkFiles, index2); + } } + bool needCallback = true; + if (!realOutStream) { if (testMode) { if (item->NeedUse_as_CopyLink_or_HardLink()) { + Int32 opRes = NExtract::NOperationResult::kOK; + if (bufIndex >= 0) + { + const CLinkFile &linkFile = linkFiles[bufIndex]; + opRes = DecoderRes_to_OpRes(linkFile.Res, linkFile.crcOK); + } + RINOK(extractCallback->PrepareOperation(askMode)); - RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)); + RINOK(extractCallback->SetOperationResult(opRes)); continue; } } @@ -2833,10 +2855,10 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (item->IsService()) continue; - if (item->NeedUse_as_HardLink()) - continue; + needCallback = false; - bool needDecode = false; + if (!item->NeedUse_as_HardLink()) + if (index2 < 0) for (unsigned n = i + 1; n < _refs.Size(); n++) { @@ -2847,41 +2869,56 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, break; if (extractStatuses[i] != 0) { - needDecode = true; + needCallback = true; break; } } - if (!needDecode) - continue; - askMode = NExtract::NAskMode::kSkip; } } - RINOK(extractCallback->PrepareOperation(askMode)); + if (needCallback) + { + RINOK(extractCallback->PrepareOperation(askMode)); + } if (bufIndex >= 0) { CLinkFile &linkFile = linkFiles[bufIndex]; - if (linkFile.NumLinks == 0) - return E_FAIL; - if (realOutStream) + + if (isExtract) + { + if (linkFile.NumLinks == 0) + return E_FAIL; + + if (needCallback) + if (realOutStream) + { + RINOK(CopyData_with_Progress(linkFile.Data, linkFile.Data.Size(), realOutStream, progress)); + } + + if (--linkFile.NumLinks == 0) + linkFile.Data.Free(); + } + + if (needCallback) { - RINOK(CopyData_with_Progress(linkFile.Data, linkFile.Data.Size(), realOutStream, progress)); + RINOK(extractCallback->SetOperationResult(DecoderRes_to_OpRes(linkFile.Res, linkFile.crcOK))); } - if (--linkFile.NumLinks == 0) - linkFile.Data.Free(); - RINOK(extractCallback->SetOperationResult(DecoderRes_to_OpRes(linkFile.Res, linkFile.crcOK))); continue; } + if (!needCallback) + continue; + if (item->NeedUse_as_CopyLink()) { - RINOK(extractCallback->SetOperationResult( - realOutStream ? - NExtract::NOperationResult::kUnsupportedMethod: - NExtract::NOperationResult::kOK)); + int opRes = realOutStream ? + NExtract::NOperationResult::kUnsupportedMethod: + NExtract::NOperationResult::kOK; + realOutStream.Release(); + RINOK(extractCallback->SetOperationResult(opRes)); continue; } @@ -2928,7 +2965,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, } { - FOR_VECTOR(i, linkFiles) + FOR_VECTOR (i, linkFiles) if (linkFiles[i].NumLinks != 0) return E_FAIL; } diff --git a/7zip/CPP/7zip/Archive/Rar/Rar5Handler.h b/7zip/CPP/7zip/Archive/Rar/Rar5Handler.h index d3e33d7c3..3b3b940a6 100644 --- a/7zip/CPP/7zip/Archive/Rar/Rar5Handler.h +++ b/7zip/CPP/7zip/Archive/Rar/Rar5Handler.h @@ -180,7 +180,7 @@ struct CItem AString Name; - int VolIndex; + unsigned VolIndex; int NextItem; UInt32 UnixMTime; diff --git a/7zip/CPP/7zip/Archive/Rar/RarHandler.cpp b/7zip/CPP/7zip/Archive/Rar/RarHandler.cpp index c097b15c1..dd78e3126 100644 --- a/7zip/CPP/7zip/Archive/Rar/RarHandler.cpp +++ b/7zip/CPP/7zip/Archive/Rar/RarHandler.cpp @@ -6,6 +6,7 @@ #include "../../../Common/ComTry.h" #include "../../../Common/IntToString.h" +#include "../../../Common/MyBuffer2.h" #include "../../../Common/UTFConvert.h" #include "../../../Windows/PropVariantUtils.h" @@ -136,8 +137,7 @@ class CInArchive NHeader::NBlock::CBlock m_BlockHeader; NCrypto::NRar3::CDecoder *m_RarAESSpec; CMyComPtr m_RarAES; - CByteBuffer m_DecryptedData; - Byte *m_DecryptedDataAligned; + CAlignedBuffer m_DecryptedDataAligned; UInt32 m_DecryptedDataSize; bool m_CryptoMode; UInt32 m_CryptoPos; @@ -415,6 +415,8 @@ bool CInArchive::ReadHeaderReal(const Byte *p, unsigned size, CItem &item) if (size < 8) return false; item.PackSize |= ((UInt64)Get32(p) << 32); + if (item.PackSize >= ((UInt64)1 << 63)) + return false; item.Size |= ((UInt64)Get32(p + 4) << 32); p += 8; size -= 8; @@ -551,11 +553,12 @@ HRESULT CInArchive::GetNextItem(CItem &item, ICryptoGetTextPassword *getTextPass m_RarAESSpec->SetPassword((const Byte *)buffer, len * 2); const UInt32 kDecryptedBufferSize = (1 << 12); - if (m_DecryptedData.Size() == 0) + if (m_DecryptedDataAligned.Size() == 0) { - const UInt32 kAlign = 16; - m_DecryptedData.Alloc(kDecryptedBufferSize + kAlign); - m_DecryptedDataAligned = (Byte *)((ptrdiff_t)((Byte *)m_DecryptedData + kAlign - 1) & ~(ptrdiff_t)(kAlign - 1)); + // const UInt32 kAlign = 16; + m_DecryptedDataAligned.AllocAtLeast(kDecryptedBufferSize); + if (!m_DecryptedDataAligned.IsAllocated()) + return E_OUTOFMEMORY; } RINOK(m_RarAES->Init()); size_t decryptedDataSizeT = kDecryptedBufferSize; @@ -667,7 +670,8 @@ HRESULT CInArchive::GetNextItem(CItem &item, ICryptoGetTextPassword *getTextPass { if (processed < offset + 2) error = k_ErrorType_Corrupted; - ArcInfo.VolNumber = (UInt32)Get16(m_FileHeaderData + offset); + else + ArcInfo.VolNumber = (UInt32)Get16(m_FileHeaderData + offset); } ArcInfo.EndOfArchive_was_Read = true; @@ -777,7 +781,9 @@ static const Byte kProps[] = kpidCRC, kpidHostOS, kpidMethod, - kpidUnpackVer + kpidUnpackVer, + + kpidVolumeIndex }; static const Byte kArcProps[] = @@ -995,6 +1001,12 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidCommented: prop = item.IsCommented(); break; case kpidSplitBefore: prop = item.IsSplitBefore(); break; case kpidSplitAfter: prop = _items[refItem.ItemIndex + refItem.NumItems - 1].IsSplitAfter(); break; + + case kpidVolumeIndex: + if (_arcInfo.Is_VolNumber_Defined()) + prop = (UInt32)(_arcInfo.VolNumber + refItem.VolumeIndex); + break; + case kpidCRC: { prop = ((lastItem.IsSplitAfter()) ? item.FileCRC : lastItem.FileCRC); @@ -1691,7 +1703,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, methodID += 2; else methodID += 3; - RINOK(CreateCoder(EXTERNAL_CODECS_VARS methodID, false, mi.Coder)); + RINOK(CreateCoder_Id(EXTERNAL_CODECS_VARS methodID, false, mi.Coder)); } if (mi.Coder == 0) diff --git a/7zip/CPP/7zip/Archive/Rar/RarHandler.h b/7zip/CPP/7zip/Archive/Rar/RarHandler.h index 901406d49..e444bd77d 100644 --- a/7zip/CPP/7zip/Archive/Rar/RarHandler.h +++ b/7zip/CPP/7zip/Archive/Rar/RarHandler.h @@ -26,7 +26,7 @@ struct CInArcInfo UInt32 DataCRC; bool EndOfArchive_was_Read; - CInArcInfo(): EndFlags(0), EndOfArchive_was_Read(false) {} + CInArcInfo(): EndFlags(0), EndOfArchive_was_Read(false), VolNumber(0) {} UInt64 GetPhySize() const { return EndPos - StartPos; } diff --git a/7zip/CPP/7zip/Archive/SquashfsHandler.cpp b/7zip/CPP/7zip/Archive/SquashfsHandler.cpp index bc9ff8b0e..89de660f2 100644 --- a/7zip/CPP/7zip/Archive/SquashfsHandler.cpp +++ b/7zip/CPP/7zip/Archive/SquashfsHandler.cpp @@ -2,9 +2,9 @@ #include "StdAfx.h" -#include "../../../C/7zCrc.h" #include "../../../C/Alloc.h" #include "../../../C/CpuArch.h" +#include "../../../C/LzmaDec.h" #include "../../../C/Xz.h" #include "../../Common/ComTry.h" @@ -25,7 +25,7 @@ #include "../Compress/CopyCoder.h" #include "../Compress/ZlibDecoder.h" -#include "../Compress/LzmaDecoder.h" +// #include "../Compress/LzmaDecoder.h" namespace NArchive { namespace NSquashfs { @@ -867,8 +867,8 @@ class CHandler: CBufPtrSeqOutStream *_outStreamSpec; CMyComPtr _outStream; - NCompress::NLzma::CDecoder *_lzmaDecoderSpec; - CMyComPtr _lzmaDecoder; + // NCompress::NLzma::CDecoder *_lzmaDecoderSpec; + // CMyComPtr _lzmaDecoder; NCompress::NZlib::CDecoder *_zlibDecoderSpec; CMyComPtr _zlibDecoder; @@ -1156,12 +1156,13 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool if (inSize != _zlibDecoderSpec->GetInputProcessedSize()) return S_FALSE; } + /* else if (method == kMethod_LZMA) { if (!_lzmaDecoder) { _lzmaDecoderSpec = new NCompress::NLzma::CDecoder(); - _lzmaDecoderSpec->FinishStream = true; + // _lzmaDecoderSpec->FinishStream = true; _lzmaDecoder = _lzmaDecoderSpec; } const UInt32 kPropsSize = LZMA_PROPS_SIZE + 8; @@ -1188,6 +1189,7 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool if (inSize != propsSize + _lzmaDecoderSpec->GetInputProcessedSize()) return S_FALSE; } + */ else { if (_inputBuffer.Size() < inSize) @@ -1201,21 +1203,62 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool if (!dest) return E_OUTOFMEMORY; } + SizeT destLen = outSizeMax, srcLen = inSize; + if (method == kMethod_LZO) { RINOK(LzoDecode(dest, &destLen, _inputBuffer, &srcLen)); } + else if (method == kMethod_LZMA) + { + Byte props[5]; + const Byte *src = _inputBuffer; + + if (_noPropsLZMA) + { + props[0] = 0x5D; + SetUi32(&props[1], _h.BlockSize); + } + else + { + const UInt32 kPropsSize = LZMA_PROPS_SIZE + 8; + if (inSize < kPropsSize) + return S_FALSE; + memcpy(props, src, LZMA_PROPS_SIZE); + UInt64 outSize = GetUi64(src + LZMA_PROPS_SIZE); + if (outSize > outSizeMax) + return S_FALSE; + destLen = (SizeT)outSize; + src += kPropsSize; + inSize -= kPropsSize; + srcLen = inSize; + } + + ELzmaStatus status; + SRes res = LzmaDecode(dest, &destLen, + src, &srcLen, + props, LZMA_PROPS_SIZE, + LZMA_FINISH_END, + &status, &g_Alloc); + if (res != 0) + return SResToHRESULT(res); + if (status != LZMA_STATUS_FINISHED_WITH_MARK) + return S_FALSE; + } else { ECoderStatus status; - XzUnpacker_Init(&_xz); - SRes res = XzUnpacker_Code(&_xz, dest, &destLen, _inputBuffer, &srcLen, CODER_FINISH_END, &status); + SRes res = XzUnpacker_CodeFull(&_xz, + dest, &destLen, + _inputBuffer, &srcLen, + CODER_FINISH_END, &status); if (res != 0) return SResToHRESULT(res); if (status != CODER_STATUS_NEEDS_MORE_INPUT || !XzUnpacker_IsStreamWasFinished(&_xz)) return S_FALSE; } + if (inSize != srcLen) return S_FALSE; if (outBuf) diff --git a/7zip/CPP/7zip/Archive/SwfHandler.cpp b/7zip/CPP/7zip/Archive/SwfHandler.cpp index 16a6e6f96..ebb6acfdd 100644 --- a/7zip/CPP/7zip/Archive/SwfHandler.cpp +++ b/7zip/CPP/7zip/Archive/SwfHandler.cpp @@ -46,7 +46,7 @@ static const Byte SWF_COMPRESSED_LZMA = 'Z'; static const Byte SWF_MIN_COMPRESSED_ZLIB_VER = 6; static const Byte SWF_MIN_COMPRESSED_LZMA_VER = 13; -static const Byte kVerLim = 20; +static const Byte kVerLim = 64; API_FUNC_static_IsArc IsArc_Swf(const Byte *p, size_t size) { diff --git a/7zip/CPP/7zip/Archive/Tar/TarIn.cpp b/7zip/CPP/7zip/Archive/Tar/TarIn.cpp index 760bfa929..327616585 100644 --- a/7zip/CPP/7zip/Archive/Tar/TarIn.cpp +++ b/7zip/CPP/7zip/Archive/Tar/TarIn.cpp @@ -94,7 +94,9 @@ static bool ParseInt64(const char *p, Int64 &val) static bool ParseInt64_MTime(const char *p, Int64 &val) { - // rare case tar contains spaces instead of MTime + // rare case tar : ZEROs in Docker-Windows TARs + // rare case tar : spaces + if (GetUi32(p) != 0) for (unsigned i = 0; i < 12; i++) if (p[i] != ' ') return ParseInt64(p, val); @@ -441,14 +443,14 @@ HRESULT ReadItem(ISequentialInStream *stream, bool &filled, CItemEx &item, EErro case 'X': { // pax Extended Header - if (item.Name.IsPrefixedBy("PaxHeader/")) + if (item.Name.IsPrefixedBy("PaxHeader/") + || item.Name.Find("PaxHeaders.4467/") >= 0) { RINOK(ReadDataToString(stream, item, pax, error)); if (error != k_ErrorType_OK) return S_OK; continue; } - break; } case NFileHeader::NLinkFlag::kDumpDir: @@ -486,7 +488,11 @@ HRESULT ReadItem(ISequentialInStream *stream, bool &filled, CItemEx &item, EErro if (ParsePaxLongName(pax, name)) item.Name = name; else - error = k_ErrorType_Warning; + { + // no "path" property is allowed in pax4467 + // error = k_ErrorType_Warning; + } + pax.Empty(); } return S_OK; diff --git a/7zip/CPP/7zip/Archive/UefiHandler.cpp b/7zip/CPP/7zip/Archive/UefiHandler.cpp index f49b62c8a..5892d5684 100644 --- a/7zip/CPP/7zip/Archive/UefiHandler.cpp +++ b/7zip/CPP/7zip/Archive/UefiHandler.cpp @@ -791,20 +791,24 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val void CHandler::AddCommentString(const char *name, UInt32 pos) { UString s; - const Byte *buf = _bufs[0]; if (pos < _h.HeaderSize) return; - for (UInt32 i = pos;; i += 2) + if (pos >= _h.OffsetToCapsuleBody) + return; + UInt32 limit = (_h.OffsetToCapsuleBody - pos) & ~(UInt32)1; + const Byte *buf = _bufs[0] + pos; + for (UInt32 i = 0;;) { - if (s.Len() > (1 << 16) || i >= _h.OffsetToCapsuleBody) + if (s.Len() > (1 << 16) || i >= limit) return; wchar_t c = Get16(buf + i); + i += 2; if (c == 0) { - i += 2; - if (i >= _h.OffsetToCapsuleBody) + if (i >= limit) return; c = Get16(buf + i); + i += 2; if (c == 0) break; s.Add_LF(); diff --git a/7zip/CPP/7zip/Archive/Wim/WimHandlerOut.cpp b/7zip/CPP/7zip/Archive/Wim/WimHandlerOut.cpp index 99fd46ef5..18740c703 100644 --- a/7zip/CPP/7zip/Archive/Wim/WimHandlerOut.cpp +++ b/7zip/CPP/7zip/Archive/Wim/WimHandlerOut.cpp @@ -671,7 +671,7 @@ void CHeader::SetDefaultFields(bool useLZX) ChunkSize = kChunkSize; ChunkSizeBits = kChunkSizeBits; } - g_RandomGenerator.Generate(Guid, 16); + MY_RAND_GEN(Guid, 16); PartNumber = 1; NumParts = 1; NumImages = 1; diff --git a/7zip/CPP/7zip/Archive/Wim/WimIn.cpp b/7zip/CPP/7zip/Archive/Wim/WimIn.cpp index b934478e4..3c7120844 100644 --- a/7zip/CPP/7zip/Archive/Wim/WimIn.cpp +++ b/7zip/CPP/7zip/Archive/Wim/WimIn.cpp @@ -111,7 +111,9 @@ HRESULT CUnpacker::UnpackChunk( } else if (method == NMethod::kLZX) { - lzxDecoderSpec->SetExternalWindow(unpackBuf.Data, chunkSizeBits); + res = lzxDecoderSpec->SetExternalWindow(unpackBuf.Data, chunkSizeBits); + if (res != S_OK) + return E_NOTIMPL; lzxDecoderSpec->KeepHistoryForNext = false; lzxDecoderSpec->SetKeepHistory(false); res = lzxDecoderSpec->Code(packBuf.Data, inSize, (UInt32)outSize); @@ -563,7 +565,13 @@ void CDatabase::GetItemPath(unsigned index1, bool showImageNumber, NWindows::NCO wchar_t *dest = s + size; meta += 2; for (unsigned i = 0; i < len; i++) - dest[i] = Get16(meta + i * 2); + { + wchar_t c = Get16(meta + i * 2); + // 18.06 + if (c == CHAR_PATH_SEPARATOR || c == '/') + c = '_'; + dest[i] = c; + } } if (index < 0) return; @@ -866,7 +874,11 @@ HRESULT CDatabase::ParseImageDirs(CByteBuffer &buf, int parent) if (DirProcessed == DirSize - 8 && Get64(p + DirSize - 8) != 0) return S_OK; - return S_FALSE; + // 18.06: we support cases, when some old dism can capture images + // where DirProcessed much smaller than DirSize + HeadersError = true; + return S_OK; + // return S_FALSE; } diff --git a/7zip/CPP/7zip/Archive/Wim/WimIn.h b/7zip/CPP/7zip/Archive/Wim/WimIn.h index 6a3872128..9e835b01f 100644 --- a/7zip/CPP/7zip/Archive/Wim/WimIn.h +++ b/7zip/CPP/7zip/Archive/Wim/WimIn.h @@ -468,7 +468,7 @@ class CDatabase int ExludedItem; // -1 : if there are no exclude items CUIntVector VirtualRoots; // we use them for old 1.10 WIM archives - bool ThereIsError() const { return RefCountError; } + bool ThereIsError() const { return RefCountError || HeadersError; } unsigned GetNumUserItemsInImage(unsigned imageIndex) const { @@ -544,7 +544,10 @@ class CDatabase HeadersError = false; } - CDatabase(): RefCountError(false) {} + CDatabase(): + RefCountError(false), + HeadersError(false) + {} void GetShortName(unsigned index, NWindows::NCOM::CPropVariant &res) const; void GetItemName(unsigned index1, NWindows::NCOM::CPropVariant &res) const; diff --git a/7zip/CPP/7zip/Archive/XzHandler.cpp b/7zip/CPP/7zip/Archive/XzHandler.cpp index e3779a417..801619fba 100644 --- a/7zip/CPP/7zip/Archive/XzHandler.cpp +++ b/7zip/CPP/7zip/Archive/XzHandler.cpp @@ -24,9 +24,7 @@ #include "IArchive.h" -#ifndef EXTRACT_ONLY #include "Common/HandlerOut.h" -#endif using namespace NWindows; @@ -49,14 +47,22 @@ class CHandler: public IInArchive, public IArchiveOpenSeq, public IInArchiveGetStream, + public ISetProperties, + #ifndef EXTRACT_ONLY public IOutArchive, - public ISetProperties, - public CMultiMethodProps, #endif - public CMyUnknownImp + + public CMyUnknownImp, + + #ifndef EXTRACT_ONLY + public CMultiMethodProps + #else + public CCommonMethodProps + #endif { - NCompress::NXz::CStatInfo _stat; + CXzStatInfo _stat; + SRes MainDecodeSRes; bool _isArc; bool _needSeekToStart; @@ -71,34 +77,48 @@ class CHandler: UInt64 _numSolidBytes; - HRESULT SetSolidFromString(const UString &s); - HRESULT SetSolidFromPROPVARIANT(const PROPVARIANT &value); - HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); - - void InitSolid() + void InitXz() { + _filterId = 0; _numSolidBytes = XZ_PROPS__BLOCK_SIZE__AUTO; } + #endif + void Init() { - InitSolid(); - _filterId = 0; - CMultiMethodProps::Init(); + #ifndef EXTRACT_ONLY + InitXz(); + CMultiMethodProps::Init(); + #else + CCommonMethodProps::InitCommon(); + #endif } - #endif + HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); HRESULT Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCallback *callback); - HRESULT Decode2(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, - NCompress::NXz::CDecoder &decoder, ICompressProgressInfo *progress) + HRESULT Decode(NCompress::NXz::CDecoder &decoder, + ISequentialInStream *seqInStream, + ISequentialOutStream *outStream, + ICompressProgressInfo *progress) { + #ifndef _7ZIP_ST + decoder._numThreads = _numThreads; + #endif + decoder._memUsage = _memUsage; + + MainDecodeSRes = SZ_OK; + RINOK(decoder.Decode(seqInStream, outStream, NULL, // *outSizeLimit true, // finishStream progress)); - _stat = decoder; + + _stat = decoder.Stat; + MainDecodeSRes = decoder.MainDecodeSRes; + _phySize_Defined = true; return S_OK; } @@ -107,9 +127,9 @@ class CHandler: MY_QUERYINTERFACE_BEGIN2(IInArchive) MY_QUERYINTERFACE_ENTRY(IArchiveOpenSeq) MY_QUERYINTERFACE_ENTRY(IInArchiveGetStream) + MY_QUERYINTERFACE_ENTRY(ISetProperties) #ifndef EXTRACT_ONLY MY_QUERYINTERFACE_ENTRY(IOutArchive) - MY_QUERYINTERFACE_ENTRY(ISetProperties) #endif MY_QUERYINTERFACE_END MY_ADDREF_RELEASE @@ -117,10 +137,10 @@ class CHandler: INTERFACE_IInArchive(;) STDMETHOD(OpenSeq)(ISequentialInStream *stream); STDMETHOD(GetStream)(UInt32 index, ISequentialInStream **stream); + STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); #ifndef EXTRACT_ONLY INTERFACE_IOutArchive(;) - STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); #endif size_t _blocksArraySize; @@ -146,7 +166,7 @@ CHandler::CHandler(): _blocksArraySize(0) { #ifndef EXTRACT_ONLY - Init(); + InitXz(); #endif } @@ -307,7 +327,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) NCOM::CPropVariant prop; switch (propID) { - case kpidPhySize: if (_phySize_Defined) prop = _stat.PhySize; break; + case kpidPhySize: if (_phySize_Defined) prop = _stat.InSize; break; case kpidNumStreams: if (_stat.NumStreams_Defined) prop = _stat.NumStreams; break; case kpidNumBlocks: if (_stat.NumBlocks_Defined) prop = _stat.NumBlocks; break; case kpidUnpackSize: if (_stat.UnpackSize_Defined) prop = _stat.OutSize; break; @@ -330,19 +350,22 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) case kpidErrorFlags: { UInt32 v = 0; + SRes sres = MainDecodeSRes; // _stat.DecodeRes2; // if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; - if (_stat.UnexpectedEnd) v |= kpv_ErrorFlags_UnexpectedEnd; + if (/*_stat.UnexpectedEnd */ sres == SZ_ERROR_INPUT_EOF) v |= kpv_ErrorFlags_UnexpectedEnd; if (_stat.DataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd; - if (_stat.HeadersError) v |= kpv_ErrorFlags_HeadersError; - if (_stat.Unsupported) v |= kpv_ErrorFlags_UnsupportedMethod; - if (_stat.DataError) v |= kpv_ErrorFlags_DataError; - if (_stat.CrcError) v |= kpv_ErrorFlags_CrcError; - prop = v; + if (/* _stat.HeadersError */ sres == SZ_ERROR_ARCHIVE) v |= kpv_ErrorFlags_HeadersError; + if (/* _stat.Unsupported */ sres == SZ_ERROR_UNSUPPORTED) v |= kpv_ErrorFlags_UnsupportedMethod; + if (/* _stat.DataError */ sres == SZ_ERROR_DATA) v |= kpv_ErrorFlags_DataError; + if (/* _stat.CrcError */ sres == SZ_ERROR_CRC) v |= kpv_ErrorFlags_CrcError; + if (v != 0) + prop = v; break; } case kpidMainSubfile: { + // debug only, comment it: // if (_blocks) prop = (UInt32)0; break; } @@ -365,7 +388,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32, PROPID propID, PROPVARIANT *value) switch (propID) { case kpidSize: if (_stat.UnpackSize_Defined) prop = _stat.OutSize; break; - case kpidPackSize: if (_phySize_Defined) prop = _stat.PhySize; break; + case kpidPackSize: if (_phySize_Defined) prop = _stat.InSize; break; case kpidMethod: if (!_methodsString.IsEmpty()) prop = _methodsString; break; } prop.Detach(value); @@ -465,7 +488,7 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal { CXzBlock block; - Bool isIndex; + BoolInt isIndex; UInt32 headerSizeRes; SRes res2 = XzBlock_ReadHeader(&block, &inStreamWrap.vt, &isIndex, &headerSizeRes); if (res2 == SZ_OK && !isIndex) @@ -483,10 +506,10 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal } } - RINOK(inStream->Seek(0, STREAM_SEEK_END, &_stat.PhySize)); + RINOK(inStream->Seek(0, STREAM_SEEK_END, &_stat.InSize)); if (callback) { - RINOK(callback->SetTotal(NULL, &_stat.PhySize)); + RINOK(callback->SetTotal(NULL, &_stat.InSize)); } CSeekInStreamWrap inStreamImp; @@ -621,7 +644,7 @@ STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream) STDMETHODIMP CHandler::Close() { - _stat.Clear(); + XzStatInfo_Clear(&_stat); _isArc = false; _needSeekToStart = false; @@ -637,6 +660,8 @@ STDMETHODIMP CHandler::Close() _blocksArraySize = 0; _maxBlocksSize = 0; + MainDecodeSRes = SZ_OK; + return S_OK; } @@ -743,6 +768,8 @@ static HRESULT DecodeBlock(CXzUnpackerCPP2 &xzu, xzu.p.streamFlags = (UInt16)streamFlags; XzUnpacker_PrepareToRandomBlockDecoding(&xzu.p); + XzUnpacker_SetOutBuf(&xzu.p, dest, unpackSize); + const UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); UInt64 packRem = packSizeAligned; @@ -771,8 +798,11 @@ static HRESULT DecodeBlock(CXzUnpackerCPP2 &xzu, ECoderStatus status; SRes res = XzUnpacker_Code(&xzu.p, - dest + outPos, &outLen, + // dest + outPos, + NULL, + &outLen, xzu.InBuf + inPos, &inLen, + (inLen == 0), // srcFinished CODER_FINISH_END, &status); // return E_OUTOFMEMORY; @@ -790,7 +820,7 @@ static HRESULT DecodeBlock(CXzUnpackerCPP2 &xzu, packRem -= inLen; - Bool blockFinished = XzUnpacker_IsBlockFinished(&xzu.p); + BoolInt blockFinished = XzUnpacker_IsBlockFinished(&xzu.p); if ((inLen == 0 && outLen == 0) || blockFinished) { @@ -890,15 +920,16 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) return E_INVALIDARG; if (!_stat.UnpackSize_Defined + || _maxBlocksSize == 0 // 18.02 || _maxBlocksSize > kMaxBlockSize_for_GetStream || _maxBlocksSize != (size_t)_maxBlocksSize) return S_FALSE; - UInt64 physSize = (UInt64)(sizeof(size_t)) << 29; - bool ramSize_Defined = NSystem::GetRamSize(physSize); - if (ramSize_Defined) + UInt64 memSize; + if (!NSystem::GetRamSize(memSize)) + memSize = (UInt64)(sizeof(size_t)) << 28; { - if (_maxBlocksSize > physSize / 4) + if (_maxBlocksSize > memSize / 4) return S_FALSE; } @@ -917,6 +948,31 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) } +static Int32 Get_Extract_OperationResult(const NCompress::NXz::CDecoder &decoder) +{ + Int32 opRes; + SRes sres = decoder.MainDecodeSRes; // decoder.Stat.DecodeRes2; + if (sres == SZ_ERROR_NO_ARCHIVE) // (!IsArc) + opRes = NExtract::NOperationResult::kIsNotArc; + else if (sres == SZ_ERROR_INPUT_EOF) // (UnexpectedEnd) + opRes = NExtract::NOperationResult::kUnexpectedEnd; + else if (decoder.Stat.DataAfterEnd) + opRes = NExtract::NOperationResult::kDataAfterEnd; + else if (sres == SZ_ERROR_CRC) // (CrcError) + opRes = NExtract::NOperationResult::kCRCError; + else if (sres == SZ_ERROR_UNSUPPORTED) // (Unsupported) + opRes = NExtract::NOperationResult::kUnsupportedMethod; + else if (sres == SZ_ERROR_ARCHIVE) // (HeadersError) + opRes = NExtract::NOperationResult::kDataError; + else if (sres == SZ_ERROR_DATA) // (DataError) + opRes = NExtract::NOperationResult::kDataError; + else if (sres != SZ_OK) + opRes = NExtract::NOperationResult::kDataError; + else + opRes = NExtract::NOperationResult::kOK; + return opRes; +} + @@ -930,7 +986,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, return E_INVALIDARG; if (_phySize_Defined) - extractCallback->SetTotal(_stat.PhySize); + extractCallback->SetTotal(_stat.InSize); UInt64 currentTotalPacked = 0; RINOK(extractCallback->SetCompleted(¤tTotalPacked)); @@ -959,9 +1015,18 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, else _needSeekToStart = true; + NCompress::NXz::CDecoder decoder; - RINOK(Decode2(_seqStream, realOutStream, decoder, lpsRef)); - Int32 opRes = decoder.Get_Extract_OperationResult(); + + HRESULT hres = Decode(decoder, _seqStream, realOutStream, lpsRef); + + if (!decoder.MainDecodeSRes_wasUsed) + return hres == S_OK ? E_FAIL : hres; + + Int32 opRes = Get_Extract_OperationResult(decoder); + if (opRes == NExtract::NOperationResult::kOK + && hres != S_OK) + opRes = NExtract::NOperationResult::kDataError; realOutStream.Release(); return extractCallback->SetOperationResult(opRes); @@ -1112,7 +1177,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (_stream) { if (_phySize_Defined) - RINOK(updateCallback->SetTotal(_stat.PhySize)); + RINOK(updateCallback->SetTotal(_stat.InSize)); RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); } @@ -1125,78 +1190,63 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt COM_TRY_END } +#endif + -HRESULT CHandler::SetSolidFromString(const UString &s) +HRESULT CHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) { - UString s2 = s; - s2.MakeLower_Ascii(); + UString name = nameSpec; + name.MakeLower_Ascii(); + if (name.IsEmpty()) + return E_INVALIDARG; + + #ifndef EXTRACT_ONLY + if (name[0] == L's') { - const wchar_t *start = ((const wchar_t *)s2); - const wchar_t *end; - UInt64 v = ConvertStringToUInt64(start, &end); - if (start == end) - return E_INVALIDARG; - if ((unsigned)(end - start) + 1 != s2.Len()) - return E_INVALIDARG; - wchar_t c = *end; + const wchar_t *s = name.Ptr(1); + if (*s == 0) { - unsigned numBits; - switch (c) + bool useStr = false; + bool isSolid; + switch (value.vt) { - case 'b': numBits = 0; break; - case 'k': numBits = 10; break; - case 'm': numBits = 20; break; - case 'g': numBits = 30; break; - case 't': numBits = 40; break; + case VT_EMPTY: isSolid = true; break; + case VT_BOOL: isSolid = (value.boolVal != VARIANT_FALSE); break; + case VT_BSTR: + if (!StringToBool(value.bstrVal, isSolid)) + useStr = true; + break; default: return E_INVALIDARG; } - _numSolidBytes = (v << numBits); + if (!useStr) + { + _numSolidBytes = (isSolid ? XZ_PROPS__BLOCK_SIZE__SOLID : XZ_PROPS__BLOCK_SIZE__AUTO); + return S_OK; + } } + return ParseSizeString(s, value, + 0, // percentsBase + _numSolidBytes) ? S_OK: E_INVALIDARG; } - return S_OK; -} - -HRESULT CHandler::SetSolidFromPROPVARIANT(const PROPVARIANT &value) -{ - bool isSolid; - switch (value.vt) - { - case VT_EMPTY: isSolid = true; break; - case VT_BOOL: isSolid = (value.boolVal != VARIANT_FALSE); break; - case VT_BSTR: - if (StringToBool(value.bstrVal, isSolid)) - break; - return SetSolidFromString(value.bstrVal); - default: return E_INVALIDARG; - } - _numSolidBytes = (isSolid ? XZ_PROPS__BLOCK_SIZE__SOLID : XZ_PROPS__BLOCK_SIZE__AUTO); - return S_OK; -} + return CMultiMethodProps::SetProperty(name, value); + #else -HRESULT CHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) -{ - UString name = nameSpec; - name.MakeLower_Ascii(); - if (name.IsEmpty()) - return E_INVALIDARG; - - if (name[0] == L's') { - name.Delete(0); - if (name.IsEmpty()) - return SetSolidFromPROPVARIANT(value); - if (value.vt != VT_EMPTY) - return E_INVALIDARG; - return SetSolidFromString(name); + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + return hres; } + + return E_INVALIDARG; - return CMultiMethodProps::SetProperty(name, value); + #endif } + STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) { COM_TRY_BEGIN @@ -1208,6 +1258,8 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR RINOK(SetProperty(names[i], values[i])); } + #ifndef EXTRACT_ONLY + if (!_filterMethod.MethodName.IsEmpty()) { unsigned k; @@ -1238,12 +1290,13 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR return E_INVALIDARG; } + #endif + return S_OK; COM_TRY_END } -#endif REGISTER_ARC_IO( "xz", "xz txz", "* .tar", 0xC, diff --git a/7zip/CPP/7zip/Archive/Zip/ZipAddCommon.cpp b/7zip/CPP/7zip/Archive/Zip/ZipAddCommon.cpp index 0baa254d3..1ee7e22f9 100644 --- a/7zip/CPP/7zip/Archive/Zip/ZipAddCommon.cpp +++ b/7zip/CPP/7zip/Archive/Zip/ZipAddCommon.cpp @@ -384,7 +384,7 @@ HRESULT CAddCommon::Compress( methodId = kMethodId_ZipBase + method; break; } - RINOK(CreateCoder( + RINOK(CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, true, _compressEncoder)); if (!_compressEncoder) diff --git a/7zip/CPP/7zip/Archive/Zip/ZipHandler.cpp b/7zip/CPP/7zip/Archive/Zip/ZipHandler.cpp index 927b3749f..a4794f512 100644 --- a/7zip/CPP/7zip/Archive/Zip/ZipHandler.cpp +++ b/7zip/CPP/7zip/Archive/Zip/ZipHandler.cpp @@ -112,7 +112,8 @@ static const CUInt32PCharPair g_HeaderCharacts[] = { 3, "Descriptor" }, // { 5, "Patched" }, { 6, kMethod_StrongCrypto }, - { 11, "UTF8" } + { 11, "UTF8" }, + { 14, "Alt" } }; struct CIdToNamePair @@ -169,6 +170,7 @@ static const Byte kProps[] = kpidUnpackVer, kpidVolumeIndex, kpidOffset + // kpidIsAltStream }; static const Byte kArcProps[] = @@ -307,6 +309,8 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) prop = true; break; } + + // case kpidIsAltStream: prop = true; break; } prop.Detach(value); COM_TRY_END @@ -333,6 +337,21 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val UString res; item.GetUnicodeString(res, item.Name, false, _forceCodePage, _specifiedCodePage); NItemName::ReplaceToOsSlashes_Remove_TailSlash(res); + /* + if (item.ParentOfAltStream >= 0) + { + const CItemEx &prevItem = m_Items[item.ParentOfAltStream]; + UString prevName; + prevItem.GetUnicodeString(prevName, prevItem.Name, false, _forceCodePage, _specifiedCodePage); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(prevName); + if (res.IsPrefixedBy(prevName)) + if (IsString1PrefixedByString2(res.Ptr(prevName.Len()), k_SpecName_NTFS_STREAM)) + { + res.Delete(prevName.Len(), (unsigned)strlen(k_SpecName_NTFS_STREAM)); + res.Insert(prevName.Len(), L":"); + } + } + */ prop = res; break; } @@ -580,8 +599,12 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidHostOS: { - const Byte hostOS = item.GetHostOS(); - TYPE_TO_PROP(kHostOS, hostOS, prop); + if (item.FromCentral) + { + // 18.06: now we use HostOS only from Central::MadeByVersion + const Byte hostOS = item.MadeByVersion.HostOS; + TYPE_TO_PROP(kHostOS, hostOS, prop); + } break; } @@ -596,6 +619,19 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidOffset: prop = item.LocalHeaderPos; break; + + /* + case kpidIsAltStream: + prop = (bool)(item.ParentOfAltStream >= 0); // item.IsAltStream(); + break; + + case kpidName: + if (item.ParentOfAltStream >= 0) + { + // extract name of stream here + } + break; + */ } prop.Detach(value); @@ -604,6 +640,85 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } + +/* +STDMETHODIMP CHandler::GetNumRawProps(UInt32 *numProps) +{ + *numProps = 0; + return S_OK; +} + +STDMETHODIMP CHandler::GetRawPropInfo(UInt32 index, BSTR *name, PROPID *propID) +{ + UNUSED_VAR(index); + *propID = 0; + *name = 0; + return S_OK; +} + +STDMETHODIMP CHandler::GetParent(UInt32 index, UInt32 *parent, UInt32 *parentType) +{ + *parentType = NParentType::kDir; + *parent = (UInt32)(Int32)-1; + if (index >= m_Items.Size()) + return S_OK; + const CItemEx &item = m_Items[index]; + + if (item.ParentOfAltStream >= 0) + { + *parentType = NParentType::kAltStream; + *parent = item.ParentOfAltStream; + } + return S_OK; +} + +STDMETHODIMP CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data, UInt32 *dataSize, UInt32 *propType) +{ + UNUSED_VAR(index); + UNUSED_VAR(propID); + *data = NULL; + *dataSize = 0; + *propType = 0; + return S_OK; +} + + +void CHandler::MarkAltStreams(CObjectVector &items) +{ + int prevIndex = -1; + UString prevName; + UString name; + + for (unsigned i = 0; i < items.Size(); i++) + { + CItemEx &item = m_Items[i]; + if (item.IsAltStream()) + { + if (prevIndex == -1) + continue; + if (prevName.IsEmpty()) + { + const CItemEx &prevItem = m_Items[prevIndex]; + prevItem.GetUnicodeString(prevName, prevItem.Name, false, _forceCodePage, _specifiedCodePage); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(prevName); + } + name.Empty(); + item.GetUnicodeString(name, item.Name, false, _forceCodePage, _specifiedCodePage); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(name); + + if (name.IsPrefixedBy(prevName)) + if (IsString1PrefixedByString2(name.Ptr(prevName.Len()), k_SpecName_NTFS_STREAM)) + item.ParentOfAltStream = prevIndex; + } + else + { + prevIndex = i; + prevName.Empty(); + } + } +} +*/ + STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *maxCheckStartPosition, IArchiveOpenCallback *callback) { @@ -617,6 +732,7 @@ STDMETHODIMP CHandler::Open(IInStream *inStream, m_Items.Clear(); m_Archive.ClearRefs(); // we don't want to clear error flags } + // MarkAltStreams(m_Items); return res; } catch(...) { Close(); throw; } @@ -738,7 +854,7 @@ class CZipDecoder IArchiveExtractCallback *extractCallback, ICompressProgressInfo *compressProgress, #ifndef _7ZIP_ST - UInt32 numThreads, + UInt32 numThreads, UInt64 memUsage, #endif Int32 &res); }; @@ -767,7 +883,7 @@ HRESULT CZipDecoder::Decode( IArchiveExtractCallback *extractCallback, ICompressProgressInfo *compressProgress, #ifndef _7ZIP_ST - UInt32 numThreads, + UInt32 numThreads, UInt64 memUsage, #endif Int32 &res) { @@ -962,7 +1078,7 @@ HRESULT CZipDecoder::Decode( szMethodID = kMethodId_ZipBase + (Byte)id; } - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS szMethodID, false, mi.Coder)); + RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS szMethodID, false, mi.Coder)); if (!mi.Coder) { @@ -974,16 +1090,7 @@ HRESULT CZipDecoder::Decode( } ICompressCoder *coder = methodItems[m].Coder; - - { - CMyComPtr setDecoderProperties; - coder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties); - if (setDecoderProperties) - { - Byte properties = (Byte)item.Flags; - RINOK(setDecoderProperties->SetDecoderProperties2(&properties, 1)); - } - } + #ifndef _7ZIP_ST { @@ -994,7 +1101,27 @@ HRESULT CZipDecoder::Decode( RINOK(setCoderMt->SetNumberOfThreads(numThreads)); } } + // if (memUsage != 0) + { + CMyComPtr setMemLimit; + coder->QueryInterface(IID_ICompressSetMemLimit, (void **)&setMemLimit); + if (setMemLimit) + { + RINOK(setMemLimit->SetMemLimit(memUsage)); + } + } #endif + + { + CMyComPtr setDecoderProperties; + coder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties); + if (setDecoderProperties) + { + Byte properties = (Byte)item.Flags; + RINOK(setDecoderProperties->SetDecoderProperties2(&properties, 1)); + } + } + CMyComPtr inStreamNew; @@ -1319,7 +1446,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, m_Archive, item, realOutStream, extractCallback, progress, #ifndef _7ZIP_ST - _props._numThreads, + _props._numThreads, _props._memUsage, #endif res); diff --git a/7zip/CPP/7zip/Archive/Zip/ZipHandler.h b/7zip/CPP/7zip/Archive/Zip/ZipHandler.h index 53e6a4604..bee57c00d 100644 --- a/7zip/CPP/7zip/Archive/Zip/ZipHandler.h +++ b/7zip/CPP/7zip/Archive/Zip/ZipHandler.h @@ -25,6 +25,7 @@ extern const char * const kMethodNames2[kNumMethodNames2]; class CHandler: public IInArchive, + // public IArchiveGetRawProps, public IOutArchive, public ISetProperties, PUBLIC_ISetCompressCodecsInfo @@ -32,6 +33,7 @@ class CHandler: { public: MY_QUERYINTERFACE_BEGIN2(IInArchive) + // MY_QUERYINTERFACE_ENTRY(IArchiveGetRawProps) MY_QUERYINTERFACE_ENTRY(IOutArchive) MY_QUERYINTERFACE_ENTRY(ISetProperties) QUERY_ENTRY_ISetCompressCodecsInfo @@ -39,6 +41,7 @@ class CHandler: MY_ADDREF_RELEASE INTERFACE_IInArchive(;) + // INTERFACE_IArchiveGetRawProps(;) INTERFACE_IOutArchive(;) STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); @@ -75,6 +78,10 @@ class CHandler: _forceCodePage = false; _specifiedCodePage = CP_OEMCP; } + + // void MarkAltStreams(CObjectVector &items); + + HRESULT GetOutProperty(IArchiveUpdateCallback *callback, UInt32 callbackIndex, Int32 arcIndex, PROPID propID, PROPVARIANT *value); }; }} diff --git a/7zip/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp b/7zip/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp index 5acbb6d41..c21b5605a 100644 --- a/7zip/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp +++ b/7zip/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp @@ -376,7 +376,8 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt { CMethodId methodId; UInt32 numStreams; - if (!FindMethod(EXTERNAL_CODECS_VARS methodName, methodId, numStreams)) + if (FindMethod_Index(EXTERNAL_CODECS_VARS methodName, true, + methodId, numStreams) < 0) return E_NOTIMPL; if (numStreams != 1) return E_NOTIMPL; diff --git a/7zip/CPP/7zip/Archive/Zip/ZipIn.cpp b/7zip/CPP/7zip/Archive/Zip/ZipIn.cpp index 9e9918dc9..509753c23 100644 --- a/7zip/CPP/7zip/Archive/Zip/ZipIn.cpp +++ b/7zip/CPP/7zip/Archive/Zip/ZipIn.cpp @@ -155,6 +155,7 @@ void CInArchive::Close() HeadersError = false; HeadersWarning = false; ExtraMinorError = false; + UnexpectedEnd = false; LocalsWereRead = false; LocalsCenterMerged = false; @@ -1729,6 +1730,9 @@ HRESULT CInArchive::FindCd(bool checkOffsetMode) HRESULT CInArchive::TryReadCd(CObjectVector &items, const CCdInfo &cdInfo, UInt64 cdOffset, UInt64 cdSize) { items.Clear(); + + // _startLocalFromCd_Disk = (UInt32)(Int32)-1; + // _startLocalFromCd_Offset = (UInt64)(Int64)-1; RINOK(SeekToVol(IsMultiVol ? cdInfo.CdDisk : -1, cdOffset)); @@ -1752,6 +1756,17 @@ HRESULT CInArchive::TryReadCd(CObjectVector &items, const CCdInfo &cdIn { CItemEx cdItem; RINOK(ReadCdItem(cdItem)); + + /* + if (cdItem.Disk < _startLocalFromCd_Disk || + cdItem.Disk == _startLocalFromCd_Disk && + cdItem.LocalHeaderPos < _startLocalFromCd_Offset) + { + _startLocalFromCd_Disk = cdItem.Disk; + _startLocalFromCd_Offset = cdItem.LocalHeaderPos; + } + */ + items.Add(cdItem); } if (Callback && (items.Size() & 0xFFF) == 0) @@ -1993,6 +2008,13 @@ HRESULT CVols::ParseArcName(IArchiveOpenVolumeCallback *volCallback) } else if (ext.IsEqualTo_Ascii_NoCase("exe")) { + /* possible cases: + - exe with zip inside + - sfx: a.exe, a.z02, a.z03,... , a.zip + a.exe is start volume. + - zip renamed to exe + */ + StartIsExe = true; BaseName = name; StartVolIndex = 0; @@ -2000,7 +2022,22 @@ HRESULT CVols::ParseArcName(IArchiveOpenVolumeCallback *volCallback) We can open arc.zip, if it was requesed to open arc.exe. But it's possible that arc.exe and arc.zip are not parts of same archive. So we can disable such operation */ - return S_FALSE; // don't open arc.zip instead of arc.exe + + // 18.04: we still want to open zip renamed to exe. + /* + { + UString volName = name; + volName += IsUpperCase ? "Z01" : "z01"; + { + CMyComPtr stream; + HRESULT res2 = volCallback->GetStream(volName, &stream); + if (res2 == S_OK) + DisableVolsSearch = true; + } + } + */ + DisableVolsSearch = true; + return S_OK; } else if (ext[0] == 'z' || ext[0] == 'Z') { @@ -2040,6 +2077,9 @@ HRESULT CVols::ParseArcName(IArchiveOpenVolumeCallback *volCallback) HRESULT CInArchive::ReadVols2(IArchiveOpenVolumeCallback *volCallback, unsigned start, int lastDisk, int zipDisk, unsigned numMissingVolsMax, unsigned &numMissingVols) { + if (Vols.DisableVolsSearch) + return S_OK; + numMissingVols = 0; for (unsigned i = start;; i++) @@ -2090,6 +2130,8 @@ HRESULT CInArchive::ReadVols2(IArchiveOpenVolumeCallback *volCallback, } if (res == S_FALSE || !stream) { + if (i == 1 && Vols.StartIsExe) + return S_OK; if (Vols.MissingName.IsEmpty()) Vols.MissingName = volName; numMissingVols++; @@ -2482,6 +2524,8 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) { ArcInfo.CdWasRead = true; ArcInfo.FirstItemRelatOffset = items[0].LocalHeaderPos; + + // ArcInfo.FirstItemRelatOffset = _startLocalFromCd_Offset; } } } @@ -2508,6 +2552,10 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) items.Clear(); localsWereRead = true; + HeadersError = false; + HeadersWarning = false; + ExtraMinorError = false; + // we can use any mode: with buffer and without buffer // without buffer : skips packed data : fast for big files : slow for small files // with buffer : reads packed data : slow for big files : fast for small files diff --git a/7zip/CPP/7zip/Archive/Zip/ZipIn.h b/7zip/CPP/7zip/Archive/Zip/ZipIn.h index e67bce423..f46f1f07a 100644 --- a/7zip/CPP/7zip/Archive/Zip/ZipIn.h +++ b/7zip/CPP/7zip/Archive/Zip/ZipIn.h @@ -162,6 +162,7 @@ struct CVols bool NeedSeek; + bool DisableVolsSearch; bool StartIsExe; // is .exe bool StartIsZ; // is .zip or .zNN bool StartIsZip; // is .zip @@ -201,6 +202,7 @@ struct CVols StreamIndex = -1; NeedSeek = false; + DisableVolsSearch = false; StartIsExe = false; StartIsZ = false; StartIsZip = false; @@ -248,6 +250,9 @@ class CInArchive UInt64 _streamPos; UInt64 _cnt; + // UInt32 _startLocalFromCd_Disk; + // UInt64 _startLocalFromCd_Offset; + size_t GetAvail() const { return _bufCached - _bufPos; } void InitBuf() { _bufPos = 0; _bufCached = 0; } @@ -381,6 +386,9 @@ class CInArchive UInt64 GetEmbeddedStubSize() const { + // it's possible that first item in CD doesn refers to first local item + // so FirstItemRelatOffset is not first local item + if (ArcInfo.CdWasRead) return ArcInfo.FirstItemRelatOffset; if (IsMultiVol) diff --git a/7zip/CPP/7zip/Archive/Zip/ZipItem.h b/7zip/CPP/7zip/Archive/Zip/ZipItem.h index 78e3e01c4..e5769711d 100644 --- a/7zip/CPP/7zip/Archive/Zip/ZipItem.h +++ b/7zip/CPP/7zip/Archive/Zip/ZipItem.h @@ -214,6 +214,12 @@ class CLocalItem public: UInt16 Flags; UInt16 Method; + + /* + Zip specification doesn't mention that ExtractVersion field uses HostOS subfield. + 18.06: 7-Zip now doesn't use ExtractVersion::HostOS to detect codePage + */ + CVersion ExtractVersion; UInt64 Size; @@ -309,7 +315,8 @@ class CItem: public CLocalItem UInt32 GetWinAttrib() const; bool GetPosixAttrib(UInt32 &attrib) const; - Byte GetHostOS() const { return FromCentral ? MadeByVersion.HostOS : ExtractVersion.HostOS; } + // 18.06: 0 instead of ExtractVersion.HostOS for local item + Byte GetHostOS() const { return FromCentral ? MadeByVersion.HostOS : (Byte)0; } void GetUnicodeString(UString &res, const AString &s, bool isComment, bool useSpecifiedCodePage, UINT codePage) const; @@ -326,7 +333,10 @@ class CItem: public CLocalItem UINT GetCodePage() const { - Byte hostOS = GetHostOS(); + // 18.06: now we use HostOS only from Central::MadeByVersion + if (!FromCentral) + return CP_OEMCP; + Byte hostOS = MadeByVersion.HostOS; return (UINT)(( hostOS == NFileHeader::NHostOS::kFAT || hostOS == NFileHeader::NHostOS::kNTFS diff --git a/7zip/CPP/7zip/Archive/Zip/ZipUpdate.cpp b/7zip/CPP/7zip/Archive/Zip/ZipUpdate.cpp index 2289203b8..e65c2b8bf 100644 --- a/7zip/CPP/7zip/Archive/Zip/ZipUpdate.cpp +++ b/7zip/CPP/7zip/Archive/Zip/ZipUpdate.cpp @@ -40,7 +40,10 @@ static const Byte kHostOS = #endif static const Byte kMadeByHostOS = kHostOS; -static const Byte kExtractHostOS = kHostOS; + +// 18.06: now we always write zero to high byte of ExtractVersion field. +// Previous versions of p7zip wrote (NFileHeader::NHostOS::kUnix) there, that is not correct +static const Byte kExtractHostOS = 0; static const Byte kMethodForDirectory = NFileHeader::NCompressionMethod::kStore; diff --git a/7zip/CPP/7zip/Bundles/Alone/Alone.dsp b/7zip/CPP/7zip/Bundles/Alone/Alone.dsp index f959e4a14..96a0f0548 100644 --- a/7zip/CPP/7zip/Bundles/Alone/Alone.dsp +++ b/7zip/CPP/7zip/Bundles/Alone/Alone.dsp @@ -1112,18 +1112,6 @@ SOURCE=..\..\Compress\ShrinkDecoder.cpp SOURCE=..\..\Compress\ShrinkDecoder.h # End Source File # End Group -# Begin Group "Z" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Compress\ZDecoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\ZDecoder.h -# End Source File -# End Group # Begin Group "BWT" # PROP Default_Filter "" @@ -1648,10 +1636,6 @@ SOURCE=..\..\Archive\SplitHandler.cpp SOURCE=..\..\Archive\XzHandler.cpp # End Source File -# Begin Source File - -SOURCE=..\..\Archive\ZHandler.cpp -# End Source File # End Group # Begin Group "UI Common" @@ -2512,6 +2496,15 @@ SOURCE=..\..\..\..\C\Delta.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\DllSecur.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\HuffEnc.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -2632,6 +2625,34 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Enc.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -2744,6 +2765,34 @@ SOURCE=..\..\..\..\C\MtCoder.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd.h # End Source File # Begin Source File diff --git a/7zip/CPP/7zip/Bundles/Alone/makefile b/7zip/CPP/7zip/Bundles/Alone/makefile index 8e9d59a64..27bdbb946 100644 --- a/7zip/CPP/7zip/Bundles/Alone/makefile +++ b/7zip/CPP/7zip/Bundles/Alone/makefile @@ -1,11 +1,4 @@ PROG = 7za.exe -MY_CONSOLE = 1 -CFLAGS = $(CFLAGS) - -!IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -D_7ZIP_LARGE_PAGES -DSUPPORT_DEVICE_FILE -!ENDIF - COMMON_OBJS = \ $O\CommandLineParser.obj \ @@ -75,7 +68,6 @@ AR_OBJS = \ $O\LzmaHandler.obj \ $O\SplitHandler.obj \ $O\XzHandler.obj \ - $O\ZHandler.obj \ AR_COMMON_OBJS = \ $O\CoderMixer2.obj \ @@ -169,7 +161,6 @@ COMPRESS_OBJS = \ $O\ShrinkDecoder.obj \ $O\XzDecoder.obj \ $O\XzEncoder.obj \ - $O\ZDecoder.obj \ CRYPTO_OBJS = \ $O\7zAes.obj \ @@ -198,10 +189,12 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Ppmd7Enc.obj \ @@ -222,5 +215,6 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../Crc64.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/7zip/CPP/7zip/Bundles/Alone/resource.rc b/7zip/CPP/7zip/Bundles/Alone/resource.rc index fc9063c12..c85acaa9c 100644 --- a/7zip/CPP/7zip/Bundles/Alone/resource.rc +++ b/7zip/CPP/7zip/Bundles/Alone/resource.rc @@ -1,3 +1,7 @@ #include "../../MyVersionInfo.rc" MY_VERSION_INFO_APP("7-Zip Standalone Console", "7za") + +#ifndef UNDER_CE +1 24 MOVEABLE PURE "../../UI/Console/Console.manifest" +#endif diff --git a/7zip/CPP/7zip/Bundles/Alone7z/Alone.dsp b/7zip/CPP/7zip/Bundles/Alone7z/Alone.dsp index 334b66352..bb7ca4c9c 100644 --- a/7zip/CPP/7zip/Bundles/Alone7z/Alone.dsp +++ b/7zip/CPP/7zip/Bundles/Alone7z/Alone.dsp @@ -44,7 +44,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /Gr /MT /W3 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /FAc /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gr /MT /W3 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /FAc /Yu"StdAfx.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -69,7 +69,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c -# ADD CPP /nologo /Gr /MDd /W3 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /Gr /MDd /W3 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" BSC32=bscmake.exe @@ -94,7 +94,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /MD /W3 /GX /O2 /D "NDEBUG" /D "WIN32" /D "_CONSOLE" /D "EXCLUDE_COM" /D "NO_REGISTRY" /Yu"StdAfx.h" /FD /c -# ADD CPP /nologo /Gr /MD /W4 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "UNICODE" /D "_UNICODE" /D "WIN32" /D "_CONSOLE" /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gr /MD /W4 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "UNICODE" /D "_UNICODE" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -121,7 +121,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "_DEBUG" /D "WIN32" /D "_CONSOLE" /D "EXCLUDE_COM" /D "NO_REGISTRY" /D "_MBCS" /Yu"StdAfx.h" /FD /GZ /c -# ADD CPP /nologo /Gr /MDd /W4 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_UNICODE" /D "UNICODE" /D "WIN32" /D "_CONSOLE" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /Gr /MDd /W4 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_UNICODE" /D "UNICODE" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" BSC32=bscmake.exe @@ -482,6 +482,14 @@ SOURCE=..\..\..\Windows\FileName.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\FileSystem.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\FileSystem.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Handle.h # End Source File # Begin Source File @@ -1638,6 +1646,15 @@ SOURCE=..\..\..\..\C\Delta.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\DllSecur.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\IStream.h # End Source File # Begin Source File @@ -1696,6 +1713,34 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Enc.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -1789,6 +1834,34 @@ SOURCE=..\..\..\..\C\MtCoder.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/7zip/CPP/7zip/Bundles/Alone7z/makefile b/7zip/CPP/7zip/Bundles/Alone7z/makefile index d83358180..10be3cfba 100644 --- a/7zip/CPP/7zip/Bundles/Alone7z/makefile +++ b/7zip/CPP/7zip/Bundles/Alone7z/makefile @@ -1,10 +1,6 @@ PROG = 7zr.exe -MY_CONSOLE = 1 -CFLAGS = $(CFLAGS) -DPROG_VARIANT_R - -!IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -!ENDIF +CFLAGS = $(CFLAGS) \ + -DPROG_VARIANT_R \ COMMON_OBJS = \ $O\CommandLineParser.obj \ @@ -33,6 +29,7 @@ WIN_OBJS = \ $O\FileIO.obj \ $O\FileLink.obj \ $O\FileName.obj \ + $O\FileSystem.obj \ $O\MemoryLock.obj \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ @@ -133,10 +130,12 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Sha256.obj \ $O\Sort.obj \ $O\Threads.obj \ @@ -150,5 +149,6 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../Crc64.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/7zip/CPP/7zip/Bundles/Alone7z/resource.rc b/7zip/CPP/7zip/Bundles/Alone7z/resource.rc index b3fb079a6..593785007 100644 --- a/7zip/CPP/7zip/Bundles/Alone7z/resource.rc +++ b/7zip/CPP/7zip/Bundles/Alone7z/resource.rc @@ -1,3 +1,7 @@ #include "../../../../C/7zVersion.rc" MY_VERSION_INFO_APP("7-Zip Reduced Standalone Console", "7zr") + +#ifndef UNDER_CE +1 24 MOVEABLE PURE "../../UI/Console/Console.manifest" +#endif diff --git a/7zip/CPP/7zip/Bundles/Fm/FM.dsp b/7zip/CPP/7zip/Bundles/Fm/FM.dsp index 7ebf37ef6..03fbb50ef 100644 --- a/7zip/CPP/7zip/Bundles/Fm/FM.dsp +++ b/7zip/CPP/7zip/Bundles/Fm/FM.dsp @@ -677,6 +677,14 @@ SOURCE=..\..\UI\FileManager\DialogSize.h # End Source File # Begin Source File +SOURCE=..\..\UI\FileManager\EditDialog.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\UI\FileManager\EditDialog.h +# End Source File +# Begin Source File + SOURCE=..\..\UI\FileManager\LinkDialog.cpp # End Source File # Begin Source File @@ -977,6 +985,15 @@ SOURCE=..\..\..\..\C\CpuArch.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\DllSecur.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzFind.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -1004,6 +1021,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Enc.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -1040,6 +1066,15 @@ SOURCE=..\..\..\..\C\MtCoder.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sha256.c !IF "$(CFG)" == "FM - Win32 Release" @@ -1549,14 +1584,6 @@ SOURCE=..\..\..\Common\Wildcard.h # PROP Default_Filter "" # Begin Source File -SOURCE=..\..\UI\Common\ArchiveCommandLine.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\UI\Common\ArchiveCommandLine.h -# End Source File -# Begin Source File - SOURCE=..\..\UI\Common\ArchiveExtractCallback.cpp # End Source File # Begin Source File diff --git a/7zip/CPP/7zip/Bundles/Fm/makefile b/7zip/CPP/7zip/Bundles/Fm/makefile index 8ae5f4f65..8da34edb8 100644 --- a/7zip/CPP/7zip/Bundles/Fm/makefile +++ b/7zip/CPP/7zip/Bundles/Fm/makefile @@ -1,14 +1,4 @@ PROG = 7zFM.exe -CFLAGS = $(CFLAGS) \ - -DLANG \ - -DNEW_FOLDER_INTERFACE \ - -!IFDEF UNDER_CE -LIBS = $(LIBS) ceshell.lib Commctrl.lib -!ELSE -LIBS = $(LIBS) comctl32.lib htmlhelp.lib comdlg32.lib Mpr.lib Gdi32.lib -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -DSUPPORT_DEVICE_FILE -!ENDIF !include "../Format7zF/Arc.mak" @@ -48,7 +38,6 @@ WIN_CTRL_OBJS = \ $O\FileStreams.obj \ UI_COMMON_OBJS = \ - $O\ArchiveCommandLine.obj \ $O\ArchiveExtractCallback.obj \ $O\ArchiveName.obj \ $O\ArchiveOpenCallback.obj \ diff --git a/7zip/CPP/7zip/Bundles/Format7z/makefile b/7zip/CPP/7zip/Bundles/Format7z/makefile index 1fb1d4512..1233d4fcb 100644 --- a/7zip/CPP/7zip/Bundles/Format7z/makefile +++ b/7zip/CPP/7zip/Bundles/Format7z/makefile @@ -123,10 +123,12 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Ppmd7Enc.obj \ @@ -136,5 +138,6 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/7zip/CPP/7zip/Bundles/Format7zExtract/makefile b/7zip/CPP/7zip/Bundles/Format7zExtract/makefile index 82265b089..e22faf36d 100644 --- a/7zip/CPP/7zip/Bundles/Format7zExtract/makefile +++ b/7zip/CPP/7zip/Bundles/Format7zExtract/makefile @@ -42,6 +42,7 @@ AR_OBJS = \ AR_COMMON_OBJS = \ $O\CoderMixer2.obj \ + $O\HandlerOut.obj \ $O\ItemNameUtils.obj \ $O\OutStreamWithCRC.obj \ $O\ParseProperties.obj \ @@ -98,7 +99,9 @@ C_OBJS = \ $O\CpuArch.obj \ $O\Delta.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Sha256.obj \ @@ -106,5 +109,6 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/7zip/CPP/7zip/Bundles/Format7zExtractR/makefile b/7zip/CPP/7zip/Bundles/Format7zExtractR/makefile index 724477e94..756c8ae69 100644 --- a/7zip/CPP/7zip/Bundles/Format7zExtractR/makefile +++ b/7zip/CPP/7zip/Bundles/Format7zExtractR/makefile @@ -85,9 +85,12 @@ C_OBJS = \ $O\CpuArch.obj \ $O\Delta.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/7zip/CPP/7zip/Bundles/Format7zF/Arc.mak b/7zip/CPP/7zip/Bundles/Format7zF/Arc.mak index 28dce218b..fd24d5a82 100644 --- a/7zip/CPP/7zip/Bundles/Format7zF/Arc.mak +++ b/7zip/CPP/7zip/Bundles/Format7zF/Arc.mak @@ -263,10 +263,12 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Ppmd7Enc.obj \ @@ -285,3 +287,4 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../Crc64.mak" +!include "../../LzmaDec.mak" diff --git a/7zip/CPP/7zip/Bundles/Format7zF/Format7z.dsp b/7zip/CPP/7zip/Bundles/Format7zF/Format7z.dsp index 7b1b233ec..3e202fe72 100644 --- a/7zip/CPP/7zip/Bundles/Format7zF/Format7z.dsp +++ b/7zip/CPP/7zip/Bundles/Format7zF/Format7z.dsp @@ -1805,6 +1805,26 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Enc.c !IF "$(CFG)" == "7z - Win32 Release" @@ -1874,6 +1894,26 @@ SOURCE=..\..\..\..\C\MtCoder.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd.h # End Source File # Begin Source File diff --git a/7zip/CPP/7zip/Bundles/Format7zR/makefile b/7zip/CPP/7zip/Bundles/Format7zR/makefile index 916dc568c..5c05abd17 100644 --- a/7zip/CPP/7zip/Bundles/Format7zR/makefile +++ b/7zip/CPP/7zip/Bundles/Format7zR/makefile @@ -102,12 +102,15 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/7zip/CPP/7zip/Bundles/LzmaCon/makefile b/7zip/CPP/7zip/Bundles/LzmaCon/makefile index 54919f733..000ca4469 100644 --- a/7zip/CPP/7zip/Bundles/LzmaCon/makefile +++ b/7zip/CPP/7zip/Bundles/LzmaCon/makefile @@ -1,6 +1,5 @@ PROG = lzma.exe MY_CONSOLE = 1 -CFLAGS = $(CFLAGS) CURRENT_OBJS = \ $O\LzmaAlone.obj \ @@ -55,5 +54,6 @@ C_OBJS = \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/7zip/CPP/7zip/Bundles/SFXCon/SFXCon.dsp b/7zip/CPP/7zip/Bundles/SFXCon/SFXCon.dsp index b21b9e841..0f032d81a 100644 --- a/7zip/CPP/7zip/Bundles/SFXCon/SFXCon.dsp +++ b/7zip/CPP/7zip/Bundles/SFXCon/SFXCon.dsp @@ -117,6 +117,10 @@ SOURCE=..\..\Archive\Common\CoderMixer2.h # End Source File # Begin Source File +SOURCE=..\..\Archive\Common\HandlerOut.h +# End Source File +# Begin Source File + SOURCE=..\..\Archive\Common\ItemNameUtils.cpp # End Source File # Begin Source File @@ -293,6 +297,10 @@ SOURCE=..\..\Compress\Lzma2Decoder.cpp # End Source File # Begin Source File +SOURCE=..\..\Compress\Lzma2Decoder.h +# End Source File +# Begin Source File + SOURCE=..\..\Compress\Lzma2Register.cpp # End Source File # Begin Source File @@ -411,6 +419,14 @@ SOURCE=..\..\..\Windows\Synchronization.cpp SOURCE=..\..\..\Windows\Synchronization.h # End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.h +# End Source File # End Group # Begin Group "Common" @@ -822,6 +838,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzmaDec.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -831,6 +856,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd7.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/7zip/CPP/7zip/Bundles/SFXCon/makefile b/7zip/CPP/7zip/Bundles/SFXCon/makefile index 4c201c00a..077dbc0aa 100644 --- a/7zip/CPP/7zip/Bundles/SFXCon/makefile +++ b/7zip/CPP/7zip/Bundles/SFXCon/makefile @@ -1,10 +1,11 @@ PROG = 7zCon.sfx MY_CONSOLE = 1 +MY_FIXED = 1 + CFLAGS = $(CFLAGS) \ -DEXTRACT_ONLY \ -DNO_READ_FROM_CODER \ -D_SFX \ - -D_CONSOLE \ CURRENT_OBJS = \ $O\SfxCon.obj \ @@ -41,6 +42,7 @@ WIN_OBJS = \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ $O\Synchronization.obj \ + $O\System.obj \ 7ZIP_COMMON_OBJS = \ $O\CreateCoder.obj \ @@ -117,7 +119,9 @@ C_OBJS = \ $O\Delta.obj \ $O\DllSecur.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Sha256.obj \ @@ -125,5 +129,6 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/7zip/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp b/7zip/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp index 787c1bdef..754cc0e0c 100644 --- a/7zip/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp +++ b/7zip/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp @@ -449,6 +449,14 @@ SOURCE=..\..\..\Windows\Synchronization.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\System.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Window.cpp # End Source File # Begin Source File @@ -469,6 +477,14 @@ SOURCE=..\..\Common\CreateCoder.h # End Source File # Begin Source File +SOURCE=..\..\Common\CWrappers.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Common\CWrappers.h +# End Source File +# Begin Source File + SOURCE=..\..\Common\FileStreams.cpp # End Source File # Begin Source File @@ -724,6 +740,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzmaDec.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -733,6 +758,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/7zip/CPP/7zip/Bundles/SFXSetup/makefile b/7zip/CPP/7zip/Bundles/SFXSetup/makefile index 5849cf369..9fddbc078 100644 --- a/7zip/CPP/7zip/Bundles/SFXSetup/makefile +++ b/7zip/CPP/7zip/Bundles/SFXSetup/makefile @@ -1,4 +1,6 @@ PROG = 7zS.sfx +MY_FIXED = 1 + CFLAGS = $(CFLAGS) \ -DNO_REGISTRY \ -DEXTRACT_ONLY \ @@ -33,6 +35,7 @@ WIN_OBJS = \ $O\PropVariant.obj \ $O\ResourceString.obj \ $O\Synchronization.obj \ + $O\System.obj \ $O\Window.obj \ WIN_CTRL_OBJS = \ @@ -40,6 +43,7 @@ WIN_CTRL_OBJS = \ 7ZIP_COMMON_OBJS = \ $O\CreateCoder.obj \ + $O\CWrappers.obj \ $O\FileStreams.obj \ $O\InBuffer.obj \ $O\FilterCoder.obj \ @@ -102,9 +106,12 @@ C_OBJS = \ $O\Delta.obj \ $O\DllSecur.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/7zip/CPP/7zip/Bundles/SFXWin/SFXWin.dsp b/7zip/CPP/7zip/Bundles/SFXWin/SFXWin.dsp index 301a6c5c7..14492ca18 100644 --- a/7zip/CPP/7zip/Bundles/SFXWin/SFXWin.dsp +++ b/7zip/CPP/7zip/Bundles/SFXWin/SFXWin.dsp @@ -633,6 +633,14 @@ SOURCE=..\..\..\Windows\Synchronization.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\System.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Window.cpp # End Source File # Begin Source File @@ -906,6 +914,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzmaDec.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -915,6 +932,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd7.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/7zip/CPP/7zip/Bundles/SFXWin/makefile b/7zip/CPP/7zip/Bundles/SFXWin/makefile index dc48ae88a..b3b4546f3 100644 --- a/7zip/CPP/7zip/Bundles/SFXWin/makefile +++ b/7zip/CPP/7zip/Bundles/SFXWin/makefile @@ -1,4 +1,6 @@ PROG = 7z.sfx +MY_FIXED = 1 + CFLAGS = $(CFLAGS) \ -DNO_REGISTRY \ -DEXTRACT_ONLY \ @@ -41,6 +43,7 @@ WIN_OBJS = \ $O\ResourceString.obj \ $O\Shell.obj \ $O\Synchronization.obj \ + $O\System.obj \ $O\Window.obj \ WIN_CTRL_OBJS = \ @@ -135,7 +138,9 @@ C_OBJS = \ $O\Delta.obj \ $O\DllSecur.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Sha256.obj \ @@ -143,5 +148,6 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/7zip/CPP/7zip/Common/CreateCoder.cpp b/7zip/CPP/7zip/Common/CreateCoder.cpp index 75074ad89..bf525dc8c 100644 --- a/7zip/CPP/7zip/Common/CreateCoder.cpp +++ b/7zip/CPP/7zip/Common/CreateCoder.cpp @@ -148,20 +148,23 @@ HRESULT CExternalCodecs::Load() #endif -bool FindMethod( +int FindMethod_Index( DECL_EXTERNAL_CODECS_LOC_VARS const AString &name, - CMethodId &methodId, UInt32 &numStreams) + bool encode, + CMethodId &methodId, + UInt32 &numStreams) { unsigned i; for (i = 0; i < g_NumCodecs; i++) { const CCodecInfo &codec = *g_Codecs[i]; - if (StringsAreEqualNoCase_Ascii(name, codec.Name)) + if ((encode ? codec.CreateEncoder : codec.CreateDecoder) + && StringsAreEqualNoCase_Ascii(name, codec.Name)) { methodId = codec.Id; numStreams = codec.NumStreams; - return true; + return i; } } @@ -173,19 +176,51 @@ bool FindMethod( for (i = 0; i < __externalCodecs->Codecs.Size(); i++) { const CCodecInfoEx &codec = __externalCodecs->Codecs[i]; - if (StringsAreEqualNoCase_Ascii(name, codec.Name)) + if ((encode ? codec.EncoderIsAssigned : codec.DecoderIsAssigned) + && StringsAreEqualNoCase_Ascii(name, codec.Name)) { methodId = codec.Id; numStreams = codec.NumStreams; - return true; + return g_NumCodecs + i; } } #endif - return false; + return -1; +} + + +static int FindMethod_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + CMethodId methodId, bool encode) +{ + unsigned i; + for (i = 0; i < g_NumCodecs; i++) + { + const CCodecInfo &codec = *g_Codecs[i]; + if (codec.Id == methodId && (encode ? codec.CreateEncoder : codec.CreateDecoder)) + return i; + } + + #ifdef EXTERNAL_CODECS + + CHECK_GLOBAL_CODECS + + if (__externalCodecs) + for (i = 0; i < __externalCodecs->Codecs.Size(); i++) + { + const CCodecInfoEx &codec = __externalCodecs->Codecs[i]; + if (codec.Id == methodId && (encode ? codec.EncoderIsAssigned : codec.DecoderIsAssigned)) + return g_NumCodecs + i; + } + + #endif + + return -1; } + bool FindMethod( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, @@ -280,9 +315,11 @@ void GetHashMethods( #endif } -HRESULT CreateCoder( + + +HRESULT CreateCoder_Index( DECL_EXTERNAL_CODECS_LOC_VARS - CMethodId methodId, bool encode, + unsigned i, bool encode, CMyComPtr &filter, CCreatedCoder &cod) { @@ -290,11 +327,10 @@ HRESULT CreateCoder( cod.IsFilter = false; cod.NumStreams = 1; - unsigned i; - for (i = 0; i < g_NumCodecs; i++) + if (i < g_NumCodecs) { const CCodecInfo &codec = *g_Codecs[i]; - if (codec.Id == methodId) + // if (codec.Id == methodId) { if (encode) { @@ -325,11 +361,12 @@ HRESULT CreateCoder( if (__externalCodecs) { + i -= g_NumCodecs; cod.IsExternal = true; - for (i = 0; i < __externalCodecs->Codecs.Size(); i++) + if (i < __externalCodecs->Codecs.Size()) { const CCodecInfoEx &codec = __externalCodecs->Codecs[i]; - if (codec.Id == methodId) + // if (codec.Id == methodId) { if (encode) { @@ -371,13 +408,50 @@ HRESULT CreateCoder( return S_OK; } -HRESULT CreateCoder( + +HRESULT CreateCoder_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + unsigned index, bool encode, + CCreatedCoder &cod) +{ + CMyComPtr filter; + HRESULT res = CreateCoder_Index( + EXTERNAL_CODECS_LOC_VARS + index, encode, + filter, cod); + + if (filter) + { + cod.IsFilter = true; + CFilterCoder *coderSpec = new CFilterCoder(encode); + cod.Coder = coderSpec; + coderSpec->Filter = filter; + } + + return res; +} + + +HRESULT CreateCoder_Id( + DECL_EXTERNAL_CODECS_LOC_VARS + CMethodId methodId, bool encode, + CMyComPtr &filter, + CCreatedCoder &cod) +{ + int index = FindMethod_Index(EXTERNAL_CODECS_LOC_VARS methodId, encode); + if (index < 0) + return S_OK; + return CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS index, encode, filter, cod); +} + + +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CCreatedCoder &cod) { CMyComPtr filter; - HRESULT res = CreateCoder( + HRESULT res = CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, encode, filter, cod); @@ -393,13 +467,14 @@ HRESULT CreateCoder( return res; } -HRESULT CreateCoder( + +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CMyComPtr &coder) { CCreatedCoder cod; - HRESULT res = CreateCoder( + HRESULT res = CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, encode, cod); @@ -413,7 +488,7 @@ HRESULT CreateFilter( CMyComPtr &filter) { CCreatedCoder cod; - return CreateCoder( + return CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, encode, filter, cod); diff --git a/7zip/CPP/7zip/Common/CreateCoder.h b/7zip/CPP/7zip/Common/CreateCoder.h index f06064b6a..20d0ef338 100644 --- a/7zip/CPP/7zip/Common/CreateCoder.h +++ b/7zip/CPP/7zip/Common/CreateCoder.h @@ -116,13 +116,12 @@ extern CExternalCodecs g_ExternalCodecs; #endif - - - -bool FindMethod( +int FindMethod_Index( DECL_EXTERNAL_CODECS_LOC_VARS const AString &name, - CMethodId &methodId, UInt32 &numStreams); + bool encode, + CMethodId &methodId, + UInt32 &numStreams); bool FindMethod( DECL_EXTERNAL_CODECS_LOC_VARS @@ -152,18 +151,29 @@ struct CCreatedCoder }; -HRESULT CreateCoder( +HRESULT CreateCoder_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + unsigned codecIndex, bool encode, + CMyComPtr &filter, + CCreatedCoder &cod); + +HRESULT CreateCoder_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + unsigned index, bool encode, + CCreatedCoder &cod); + +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CMyComPtr &filter, CCreatedCoder &cod); -HRESULT CreateCoder( +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CCreatedCoder &cod); -HRESULT CreateCoder( +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CMyComPtr &coder); diff --git a/7zip/CPP/7zip/Common/FilterCoder.cpp b/7zip/CPP/7zip/Common/FilterCoder.cpp index 4f3ae4e7d..f4c00273c 100644 --- a/7zip/CPP/7zip/Common/FilterCoder.cpp +++ b/7zip/CPP/7zip/Common/FilterCoder.cpp @@ -7,6 +7,23 @@ #include "FilterCoder.h" #include "StreamUtils.h" +#ifdef _WIN32 + #define alignedMidBuffer_Alloc g_MidAlloc +#else + #define alignedMidBuffer_Alloc g_AlignedAlloc +#endif + +CAlignedMidBuffer::~CAlignedMidBuffer() +{ + ISzAlloc_Free(&alignedMidBuffer_Alloc, _buf); +} + +void CAlignedMidBuffer::AllocAligned(size_t size) +{ + ISzAlloc_Free(&alignedMidBuffer_Alloc, _buf); + _buf = (Byte *)ISzAlloc_Alloc(&alignedMidBuffer_Alloc, size); +} + /* AES filters need 16-bytes alignment for HARDWARE-AES instructions. So we call IFilter::Filter(, size), where (size != 16 * N) only for last data block. @@ -36,7 +53,7 @@ HRESULT CFilterCoder::Alloc() size = kMinSize; if (!_buf || _bufSize != size) { - AllocAlignedMask(size, 16 - 1); + AllocAligned(size); if (!_buf) return E_OUTOFMEMORY; _bufSize = size; diff --git a/7zip/CPP/7zip/Common/FilterCoder.h b/7zip/CPP/7zip/Common/FilterCoder.h index 85963f591..6668fdb31 100644 --- a/7zip/CPP/7zip/Common/FilterCoder.h +++ b/7zip/CPP/7zip/Common/FilterCoder.h @@ -19,41 +19,11 @@ struct CAlignedMidBuffer { - #ifdef _WIN32 - Byte *_buf; CAlignedMidBuffer(): _buf(NULL) {} - ~CAlignedMidBuffer() { ::MidFree(_buf); } - - void AllocAlignedMask(size_t size, size_t) - { - ::MidFree(_buf); - _buf = (Byte *)::MidAlloc(size); - } - - #else - - Byte *_bufBase; - Byte *_buf; - - CAlignedMidBuffer(): _bufBase(NULL), _buf(NULL) {} - ~CAlignedMidBuffer() { ::MidFree(_bufBase); } - - void AllocAlignedMask(size_t size, size_t alignMask) - { - ::MidFree(_bufBase); - _buf = NULL; - _bufBase = (Byte *)::MidAlloc(size + alignMask); - - if (_bufBase) - { - // _buf = (Byte *)(((uintptr_t)_bufBase + alignMask) & ~(uintptr_t)alignMask); - _buf = (Byte *)(((ptrdiff_t)_bufBase + alignMask) & ~(ptrdiff_t)alignMask); - } - } - - #endif + ~CAlignedMidBuffer(); + void AllocAligned(size_t size); }; class CFilterCoder: diff --git a/7zip/CPP/7zip/Common/MethodProps.cpp b/7zip/CPP/7zip/Common/MethodProps.cpp index 9e82a1153..8a6ebcafb 100644 --- a/7zip/CPP/7zip/Common/MethodProps.cpp +++ b/7zip/CPP/7zip/Common/MethodProps.cpp @@ -253,6 +253,9 @@ struct CNameToPropID const char *Name; }; + +// the following are related to NCoderPropID::EEnum values + static const CNameToPropID g_NameToPropID[] = { { VT_UI4, "" }, @@ -275,7 +278,8 @@ static const CNameToPropID g_NameToPropID[] = { VT_UI8, "expect" }, { VT_UI4, "b" }, { VT_UI4, "check" }, - { VT_BSTR, "filter" } + { VT_BSTR, "filter" }, + { VT_UI8, "memuse" } }; static int FindPropIdExact(const UString &name) @@ -293,6 +297,13 @@ static bool ConvertProperty(const PROPVARIANT &srcProp, VARTYPE varType, NCOM::C destProp = srcProp; return true; } + + if (varType == VT_UI8 && srcProp.vt == VT_UI4) + { + destProp = (UInt64)srcProp.ulVal; + return true; + } + if (varType == VT_BOOL) { bool res; diff --git a/7zip/CPP/7zip/Compress/DeflateDecoder.cpp b/7zip/CPP/7zip/Compress/DeflateDecoder.cpp index 88ab1f91e..fceef8570 100644 --- a/7zip/CPP/7zip/Compress/DeflateDecoder.cpp +++ b/7zip/CPP/7zip/Compress/DeflateDecoder.cpp @@ -446,6 +446,14 @@ void CCoder::SetOutStreamSizeResume(const UInt64 *outSize) STDMETHODIMP CCoder::SetOutStreamSize(const UInt64 *outSize) { + /* + 18.06: + We want to support GetInputProcessedSize() before CCoder::Read() + So we call m_InBitStream.Init() even before buffer allocations + m_InBitStream.Init() just sets variables to default values + But later we will call m_InBitStream.Init() again with real buffer pointers + */ + m_InBitStream.Init(); _needInitInStream = true; SetOutStreamSizeResume(outSize); return S_OK; diff --git a/7zip/CPP/7zip/Compress/DeflateEncoder.cpp b/7zip/CPP/7zip/Compress/DeflateEncoder.cpp index 7cf59cc06..233edb5f7 100644 --- a/7zip/CPP/7zip/Compress/DeflateEncoder.cpp +++ b/7zip/CPP/7zip/Compress/DeflateEncoder.cpp @@ -969,6 +969,10 @@ HRESULT CCoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *ou } } while (Inline_MatchFinder_GetNumAvailableBytes(&_lzInWindow) != 0); + + if (_seqInStream.Res != S_OK) + return _seqInStream.Res; + if (_lzInWindow.result != SZ_OK) return SResToHRESULT(_lzInWindow.result); return m_OutStream.Flush(); diff --git a/7zip/CPP/7zip/Compress/Lzma2Decoder.cpp b/7zip/CPP/7zip/Compress/Lzma2Decoder.cpp index 98af203de..653fe2de0 100644 --- a/7zip/CPP/7zip/Compress/Lzma2Decoder.cpp +++ b/7zip/CPP/7zip/Compress/Lzma2Decoder.cpp @@ -2,81 +2,48 @@ #include "StdAfx.h" +// #include + #include "../../../C/Alloc.h" +// #include "../../../C/CpuTicks.h" #include "../Common/StreamUtils.h" #include "Lzma2Decoder.h" -static HRESULT SResToHRESULT(SRes res) -{ - switch (res) - { - case SZ_OK: return S_OK; - case SZ_ERROR_MEM: return E_OUTOFMEMORY; - case SZ_ERROR_PARAM: return E_INVALIDARG; - case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL; - case SZ_ERROR_DATA: return S_FALSE; - } - return E_FAIL; -} - namespace NCompress { namespace NLzma2 { CDecoder::CDecoder(): - _inBuf(NULL), - _finishMode(false), - _outSizeDefined(false), - _outStep(1 << 22), - _inBufSize(0), - _inBufSizeNew(1 << 20) -{ - Lzma2Dec_Construct(&_state); -} + _dec(NULL) + , _inProcessed(0) + , _prop(0xFF) + , _finishMode(false) + , _inBufSize(1 << 20) + , _outStep(1 << 20) + #ifndef _7ZIP_ST + , _tryMt(1) + , _numThreads(1) + , _memUsage((UInt64)(sizeof(size_t)) << 28) + #endif +{} CDecoder::~CDecoder() { - Lzma2Dec_Free(&_state, &g_Alloc); - MidFree(_inBuf); + if (_dec) + Lzma2DecMt_Destroy(_dec); } -STDMETHODIMP CDecoder::SetInBufSize(UInt32 , UInt32 size) { _inBufSizeNew = size; return S_OK; } +STDMETHODIMP CDecoder::SetInBufSize(UInt32 , UInt32 size) { _inBufSize = size; return S_OK; } STDMETHODIMP CDecoder::SetOutBufSize(UInt32 , UInt32 size) { _outStep = size; return S_OK; } STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *prop, UInt32 size) { if (size != 1) return E_NOTIMPL; - - RINOK(SResToHRESULT(Lzma2Dec_Allocate(&_state, prop[0], &g_Alloc))); - - if (!_inBuf || _inBufSize != _inBufSizeNew) - { - MidFree(_inBuf); - _inBufSize = 0; - _inBuf = (Byte *)MidAlloc(_inBufSizeNew); - if (!_inBuf) - return E_OUTOFMEMORY; - _inBufSize = _inBufSizeNew; - } - - return S_OK; -} - - -STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) -{ - _outSizeDefined = (outSize != NULL); - _outSize = 0; - if (_outSizeDefined) - _outSize = *outSize; - _inPos = _inLim = 0; - _inProcessed = 0; - _outProcessed = 0; - - Lzma2Dec_Init(&_state); - + if (prop[0] > 40) + return E_NOTIMPL; + _prop = prop[0]; return S_OK; } @@ -88,116 +55,189 @@ STDMETHODIMP CDecoder::SetFinishMode(UInt32 finishMode) } -STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value) + +#ifndef _7ZIP_ST + +static UInt64 Get_ExpectedBlockSize_From_Dict(UInt32 dictSize) { - *value = _inProcessed; - return S_OK; + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + UInt64 blockSize = (UInt64)dictSize << 2; + if (blockSize < kMinSize) blockSize = kMinSize; + if (blockSize > kMaxSize) blockSize = kMaxSize; + if (blockSize < dictSize) blockSize = dictSize; + blockSize += (kMinSize - 1); + blockSize &= ~(UInt64)(kMinSize - 1); + return blockSize; } +#define LZMA2_DIC_SIZE_FROM_PROP_FULL(p) ((p) == 40 ? 0xFFFFFFFF : (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))) + +#endif + +#define RET_IF_WRAP_ERROR_CONFIRMED(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK && sRes == sResErrorCode) return wrapRes; + +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) { - if (!_inBuf) - return S_FALSE; + _inProcessed = 0; - SetOutStreamSize(outSize); + if (!_dec) + { + _dec = Lzma2DecMt_Create( + // &g_AlignedAlloc, + &g_Alloc, + &g_MidAlloc); + if (!_dec) + return E_OUTOFMEMORY; + } - SizeT wrPos = _state.decoder.dicPos; - HRESULT readRes = S_OK; + CLzma2DecMtProps props; + Lzma2DecMtProps_Init(&props); - for (;;) - { - if (_inPos == _inLim && readRes == S_OK) - { - _inPos = _inLim = 0; - readRes = inStream->Read(_inBuf, _inBufSize, &_inLim); - } + props.inBufSize_ST = _inBufSize; + props.outStep_ST = _outStep; - const SizeT dicPos = _state.decoder.dicPos; - SizeT size; - { - SizeT next = _state.decoder.dicBufSize; - if (next - wrPos > _outStep) - next = wrPos + _outStep; - size = next - dicPos; - } + #ifndef _7ZIP_ST + { + props.numThreads = 1; + UInt32 numThreads = _numThreads; - ELzmaFinishMode finishMode = LZMA_FINISH_ANY; - if (_outSizeDefined) + if (_tryMt && numThreads >= 1) { - const UInt64 rem = _outSize - _outProcessed; - if (size >= rem) + UInt64 useLimit = _memUsage; + UInt32 dictSize = LZMA2_DIC_SIZE_FROM_PROP_FULL(_prop); + UInt64 expectedBlockSize64 = Get_ExpectedBlockSize_From_Dict(dictSize); + size_t expectedBlockSize = (size_t)expectedBlockSize64; + size_t inBlockMax = expectedBlockSize + expectedBlockSize / 16; + if (expectedBlockSize == expectedBlockSize64 && inBlockMax >= expectedBlockSize) { - size = (SizeT)rem; - if (_finishMode) - finishMode = LZMA_FINISH_END; + props.outBlockMax = expectedBlockSize; + props.inBlockMax = inBlockMax; + const size_t kOverheadSize = props.inBufSize_MT + (1 << 16); + UInt64 okThreads = useLimit / (props.outBlockMax + props.inBlockMax + kOverheadSize); + if (numThreads > okThreads) + numThreads = (UInt32)okThreads; + if (numThreads == 0) + numThreads = 1; + props.numThreads = numThreads; } } + } + #endif - SizeT inProcessed = _inLim - _inPos; - ELzmaStatus status; - - SRes res = Lzma2Dec_DecodeToDic(&_state, dicPos + size, _inBuf + _inPos, &inProcessed, finishMode, &status); + CSeqInStreamWrap inWrap; + CSeqOutStreamWrap outWrap; + CCompressProgressWrap progressWrap; - - _inPos += (UInt32)inProcessed; - _inProcessed += inProcessed; - const SizeT outProcessed = _state.decoder.dicPos - dicPos; - _outProcessed += outProcessed; + inWrap.Init(inStream); + outWrap.Init(outStream); + progressWrap.Init(progress); - - bool outFinished = (_outSizeDefined && _outProcessed >= _outSize); + SRes res; - bool needStop = (res != 0 - || (inProcessed == 0 && outProcessed == 0) - || status == LZMA_STATUS_FINISHED_WITH_MARK - || (!_finishMode && outFinished)); + UInt64 inProcessed = 0; + int isMT = False; - if (needStop || outProcessed >= size) - { - HRESULT res2 = WriteStream(outStream, _state.decoder.dic + wrPos, _state.decoder.dicPos - wrPos); + #ifndef _7ZIP_ST + isMT = _tryMt; + #endif - if (_state.decoder.dicPos == _state.decoder.dicBufSize) - _state.decoder.dicPos = 0; - wrPos = _state.decoder.dicPos; + // UInt64 cpuTicks = GetCpuTicks(); - RINOK(res2); + res = Lzma2DecMt_Decode(_dec, _prop, &props, + &outWrap.vt, outSize, _finishMode, + &inWrap.vt, + &inProcessed, + &isMT, + progress ? &progressWrap.vt : NULL); - if (needStop) - { - if (res != 0) - return S_FALSE; - - if (status == LZMA_STATUS_FINISHED_WITH_MARK) - { - if (_finishMode) - { - if (inSize && *inSize != _inProcessed) - return S_FALSE; - if (_outSizeDefined && _outSize != _outProcessed) - return S_FALSE; - } - return readRes; - } - - if (!_finishMode && outFinished) - return readRes; - - return S_FALSE; - } - } - - if (progress) - { - RINOK(progress->SetRatioInfo(&_inProcessed, &_outProcessed)); - } + /* + cpuTicks = GetCpuTicks() - cpuTicks; + printf("\n ticks = %10I64u\n", cpuTicks / 1000000); + */ + + + #ifndef _7ZIP_ST + /* we reset _tryMt, only if p->props.numThreads was changed */ + if (props.numThreads > 1) + _tryMt = isMT; + #endif + + _inProcessed = inProcessed; + + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR_CONFIRMED(inWrap.Res, res, SZ_ERROR_READ) + + if (res == SZ_OK && _finishMode) + { + if (inSize && *inSize != inProcessed) + res = SZ_ERROR_DATA; + if (outSize && *outSize != outWrap.Processed) + res = SZ_ERROR_DATA; } + + return SResToHRESULT(res); +} + + +STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value) +{ + *value = _inProcessed; + return S_OK; +} + + +#ifndef _7ZIP_ST + +STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads) +{ + _numThreads = numThreads; + return S_OK; } +STDMETHODIMP CDecoder::SetMemLimit(UInt64 memUsage) +{ + _memUsage = memUsage; + return S_OK; +} + +#endif + #ifndef NO_READ_FROM_CODER +STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) +{ + CLzma2DecMtProps props; + Lzma2DecMtProps_Init(&props); + props.inBufSize_ST = _inBufSize; + props.outStep_ST = _outStep; + + _inProcessed = 0; + + if (!_dec) + { + _dec = Lzma2DecMt_Create(&g_AlignedAlloc, &g_MidAlloc); + if (!_dec) + return E_OUTOFMEMORY; + } + + _inWrap.Init(_inStream); + + SRes res = Lzma2DecMt_Init(_dec, _prop, &props, outSize, _finishMode, &_inWrap.vt); + + if (res != SZ_OK) + return SResToHRESULT(res); + return S_OK; +} + + STDMETHODIMP CDecoder::SetInStream(ISequentialInStream *inStream) { _inStream = inStream; return S_OK; } STDMETHODIMP CDecoder::ReleaseInStream() { _inStream.Release(); return S_OK; } @@ -207,62 +247,17 @@ STDMETHODIMP CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize) if (processedSize) *processedSize = 0; - ELzmaFinishMode finishMode = LZMA_FINISH_ANY; - if (_outSizeDefined) - { - const UInt64 rem = _outSize - _outProcessed; - if (size >= rem) - { - size = (UInt32)rem; - if (_finishMode) - finishMode = LZMA_FINISH_END; - } - } + size_t size2 = size; + UInt64 inProcessed = 0; - HRESULT readRes = S_OK; + SRes res = Lzma2DecMt_Read(_dec, (Byte *)data, &size2, &inProcessed); - for (;;) - { - if (_inPos == _inLim && readRes == S_OK) - { - _inPos = _inLim = 0; - readRes = _inStream->Read(_inBuf, _inBufSize, &_inLim); - } - - SizeT inProcessed = _inLim - _inPos; - SizeT outProcessed = size; - ELzmaStatus status; - - SRes res = Lzma2Dec_DecodeToBuf(&_state, (Byte *)data, &outProcessed, - _inBuf + _inPos, &inProcessed, finishMode, &status); - - - _inPos += (UInt32)inProcessed; - _inProcessed += inProcessed; - _outProcessed += outProcessed; - size -= (UInt32)outProcessed; - data = (Byte *)data + outProcessed; - if (processedSize) - *processedSize += (UInt32)outProcessed; - - if (res != 0) - return S_FALSE; - - /* - if (status == LZMA_STATUS_FINISHED_WITH_MARK) - return readRes; - - if (size == 0 && status != LZMA_STATUS_NEEDS_MORE_INPUT) - { - if (_finishMode && _outSizeDefined && _outProcessed >= _outSize) - return S_FALSE; - return readRes; - } - */ - - if (inProcessed == 0 && outProcessed == 0) - return readRes; - } + _inProcessed += inProcessed; + if (processedSize) + *processedSize = (UInt32)size2; + if (res != SZ_OK) + return SResToHRESULT(res); + return S_OK; } #endif diff --git a/7zip/CPP/7zip/Compress/Lzma2Decoder.h b/7zip/CPP/7zip/Compress/Lzma2Decoder.h index 440914e7e..e14148848 100644 --- a/7zip/CPP/7zip/Compress/Lzma2Decoder.h +++ b/7zip/CPP/7zip/Compress/Lzma2Decoder.h @@ -3,10 +3,9 @@ #ifndef __LZMA2_DECODER_H #define __LZMA2_DECODER_H -#include "../../../C/Lzma2Dec.h" +#include "../../../C/Lzma2DecMt.h" -#include "../../Common/MyCom.h" -#include "../ICoder.h" +#include "../Common/CWrappers.h" namespace NCompress { namespace NLzma2 { @@ -17,28 +16,26 @@ class CDecoder: public ICompressSetFinishMode, public ICompressGetInStreamProcessedSize, public ICompressSetBufSize, + #ifndef NO_READ_FROM_CODER public ICompressSetInStream, public ICompressSetOutStreamSize, public ISequentialInStream, #endif + + #ifndef _7ZIP_ST + public ICompressSetCoderMt, + public ICompressSetMemLimit, + #endif + public CMyUnknownImp { - Byte *_inBuf; - UInt32 _inPos; - UInt32 _inLim; - - bool _finishMode; - bool _outSizeDefined; - UInt64 _outSize; + CLzma2DecMtHandle _dec; UInt64 _inProcessed; - UInt64 _outProcessed; - - UInt32 _outStep; + Byte _prop; + int _finishMode; UInt32 _inBufSize; - UInt32 _inBufSizeNew; - - CLzma2Dec _state; + UInt32 _outStep; public: MY_QUERYINTERFACE_BEGIN2(ICompressCoder) @@ -46,11 +43,18 @@ class CDecoder: MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) MY_QUERYINTERFACE_ENTRY(ICompressGetInStreamProcessedSize) MY_QUERYINTERFACE_ENTRY(ICompressSetBufSize) + #ifndef NO_READ_FROM_CODER MY_QUERYINTERFACE_ENTRY(ICompressSetInStream) MY_QUERYINTERFACE_ENTRY(ICompressSetOutStreamSize) MY_QUERYINTERFACE_ENTRY(ISequentialInStream) #endif + + #ifndef _7ZIP_ST + MY_QUERYINTERFACE_ENTRY(ICompressSetCoderMt) + MY_QUERYINTERFACE_ENTRY(ICompressSetMemLimit) + #endif + MY_QUERYINTERFACE_END MY_ADDREF_RELEASE @@ -59,20 +63,28 @@ class CDecoder: STDMETHOD(SetDecoderProperties2)(const Byte *data, UInt32 size); STDMETHOD(SetFinishMode)(UInt32 finishMode); STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); - STDMETHOD(SetOutStreamSize)(const UInt64 *outSize); STDMETHOD(SetInBufSize)(UInt32 streamIndex, UInt32 size); STDMETHOD(SetOutBufSize)(UInt32 streamIndex, UInt32 size); - #ifndef NO_READ_FROM_CODER + #ifndef _7ZIP_ST +private: + int _tryMt; + UInt32 _numThreads; + UInt64 _memUsage; +public: + STDMETHOD(SetNumberOfThreads)(UInt32 numThreads); + STDMETHOD(SetMemLimit)(UInt64 memUsage); + #endif + #ifndef NO_READ_FROM_CODER private: CMyComPtr _inStream; + CSeqInStreamWrap _inWrap; public: - + STDMETHOD(SetOutStreamSize)(const UInt64 *outSize); STDMETHOD(SetInStream)(ISequentialInStream *inStream); STDMETHOD(ReleaseInStream)(); STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); - #endif CDecoder(); diff --git a/7zip/CPP/7zip/Compress/Lzma2Encoder.cpp b/7zip/CPP/7zip/Compress/Lzma2Encoder.cpp index a3ef4b576..caecbc09d 100644 --- a/7zip/CPP/7zip/Compress/Lzma2Encoder.cpp +++ b/7zip/CPP/7zip/Compress/Lzma2Encoder.cpp @@ -22,7 +22,7 @@ namespace NLzma2 { CEncoder::CEncoder() { _encoder = NULL; - _encoder = Lzma2Enc_Create(&g_Alloc, &g_BigAlloc); + _encoder = Lzma2Enc_Create(&g_AlignedAlloc, &g_BigAlloc); if (!_encoder) throw 1; } diff --git a/7zip/CPP/7zip/Compress/LzmaDecoder.cpp b/7zip/CPP/7zip/Compress/LzmaDecoder.cpp index 2fbe05893..83c24f1c1 100644 --- a/7zip/CPP/7zip/Compress/LzmaDecoder.cpp +++ b/7zip/CPP/7zip/Compress/LzmaDecoder.cpp @@ -30,19 +30,24 @@ CDecoder::CDecoder(): FinishStream(false), _propsWereSet(false), _outSizeDefined(false), - _outStep(1 << 22), + _outStep(1 << 20), _inBufSize(0), _inBufSizeNew(1 << 20) { _inProcessed = 0; _inPos = _inLim = 0; + /* + AlignOffsetAlloc_CreateVTable(&_alloc); + _alloc.numAlignBits = 7; + _alloc.offset = 0; + */ LzmaDec_Construct(&_state); } CDecoder::~CDecoder() { - LzmaDec_Free(&_state, &g_Alloc); + LzmaDec_Free(&_state, &g_AlignedAlloc); // &_alloc.vt MyFree(_inBuf); } @@ -66,7 +71,7 @@ HRESULT CDecoder::CreateInputBuffer() STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *prop, UInt32 size) { - RINOK(SResToHRESULT(LzmaDec_Allocate(&_state, prop, size, &g_Alloc))); + RINOK(SResToHRESULT(LzmaDec_Allocate(&_state, prop, size, &g_AlignedAlloc))) // &_alloc.vt _propsWereSet = true; return CreateInputBuffer(); } diff --git a/7zip/CPP/7zip/Compress/LzmaDecoder.h b/7zip/CPP/7zip/Compress/LzmaDecoder.h index e73925ed3..37dec0258 100644 --- a/7zip/CPP/7zip/Compress/LzmaDecoder.h +++ b/7zip/CPP/7zip/Compress/LzmaDecoder.h @@ -3,6 +3,7 @@ #ifndef __LZMA_DECODER_H #define __LZMA_DECODER_H +// #include "../../../C/Alloc.h" #include "../../../C/LzmaDec.h" #include "../../Common/MyCom.h" @@ -28,7 +29,6 @@ class CDecoder: UInt32 _inPos; UInt32 _inLim; - CLzmaDec _state; ELzmaStatus _lzmaStatus; public: @@ -45,6 +45,10 @@ class CDecoder: UInt32 _inBufSize; UInt32 _inBufSizeNew; + // CAlignOffsetAlloc _alloc; + + CLzmaDec _state; + HRESULT CreateInputBuffer(); HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress); void SetOutStreamSizeResume(const UInt64 *outSize); diff --git a/7zip/CPP/7zip/Compress/LzmaEncoder.cpp b/7zip/CPP/7zip/Compress/LzmaEncoder.cpp index c64f580af..e47f776b6 100644 --- a/7zip/CPP/7zip/Compress/LzmaEncoder.cpp +++ b/7zip/CPP/7zip/Compress/LzmaEncoder.cpp @@ -15,7 +15,7 @@ namespace NLzma { CEncoder::CEncoder() { _encoder = NULL; - _encoder = LzmaEnc_Create(&g_Alloc); + _encoder = LzmaEnc_Create(&g_AlignedAlloc); if (!_encoder) throw 1; } @@ -23,7 +23,7 @@ CEncoder::CEncoder() CEncoder::~CEncoder() { if (_encoder) - LzmaEnc_Destroy(_encoder, &g_Alloc, &g_BigAlloc); + LzmaEnc_Destroy(_encoder, &g_AlignedAlloc, &g_BigAlloc); } static inline wchar_t GetUpperChar(wchar_t c) @@ -168,7 +168,7 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream progressWrap.Init(progress); SRes res = LzmaEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt, - progress ? &progressWrap.vt : NULL, &g_Alloc, &g_BigAlloc); + progress ? &progressWrap.vt : NULL, &g_AlignedAlloc, &g_BigAlloc); _inputProcessed = inWrap.Processed; diff --git a/7zip/CPP/7zip/Compress/LzmsDecoder.cpp b/7zip/CPP/7zip/Compress/LzmsDecoder.cpp index 02a7f5dc1..5189ce16d 100644 --- a/7zip/CPP/7zip/Compress/LzmsDecoder.cpp +++ b/7zip/CPP/7zip/Compress/LzmsDecoder.cpp @@ -517,9 +517,9 @@ HRESULT CDecoder::CodeReal(const Byte *in, size_t inSize, Byte *_win, size_t out if (len > outSize - _pos) return S_FALSE; - if (dist > _pos) - return S_FALSE; size_t span = (size_t)1 << power; + if ((UInt64)dist + span > _pos) + return S_FALSE; Byte *dest = _win + _pos - span; const Byte *src = dest - dist; _pos += len; diff --git a/7zip/CPP/7zip/Compress/Rar1Decoder.cpp b/7zip/CPP/7zip/Compress/Rar1Decoder.cpp index f5f3a79df..6b55705b6 100644 --- a/7zip/CPP/7zip/Compress/Rar1Decoder.cpp +++ b/7zip/CPP/7zip/Compress/Rar1Decoder.cpp @@ -9,77 +9,85 @@ namespace NCompress { namespace NRar1 { -static const UInt32 PosL1[] = {0,0,0,2,3,5,7,11,16,20,24,32,32, 256}; -static const UInt32 PosL2[] = {0,0,0,0,5,7,9,13,18,22,26,34,36, 256}; -static const UInt32 PosHf0[] = {0,0,0,0,0,8,16,24,33,33,33,33,33, 257}; -static const UInt32 PosHf1[] = {0,0,0,0,0,0,4,44,60,76,80,80,127, 257}; -static const UInt32 PosHf2[] = {0,0,0,0,0,0,2,7,53,117,233, 257,0}; -static const UInt32 PosHf3[] = {0,0,0,0,0,0,0,2,16,218,251, 257,0}; -static const UInt32 PosHf4[] = {0,0,0,0,0,0,0,0,0,255, 257,0,0}; +static const unsigned kNumBits = 12; -static const UInt32 kHistorySize = (1 << 16); +static const Byte kShortLen1[16 * 3] = +{ + 0,0xa0,0xd0,0xe0,0xf0,0xf8,0xfc,0xfe,0xff,0xc0,0x80,0x90,0x98,0x9c,0xb0,0, + 1,3,4,4,5,6,7,8,8,4,4,5,6,6,0,0, + 1,4,4,4,5,6,7,8,8,4,4,5,6,6,4,0 +}; -/* -class CCoderReleaser +static const Byte kShortLen2[16 * 3] = { - CDecoder *m_Coder; -public: - CCoderReleaser(CDecoder *coder): m_Coder(coder) {} - ~CCoderReleaser() { m_Coder->ReleaseStreams(); } + 0,0x40,0x60,0xa0,0xd0,0xe0,0xf0,0xf8,0xfc,0xc0,0x80,0x90,0x98,0x9c,0xb0,0, + 2,3,3,3,4,4,5,6,6,4,4,5,6,6,0,0, + 2,3,3,4,4,4,5,6,6,4,4,5,6,6,4,0 }; -*/ -CDecoder::CDecoder(): m_IsSolid(false), _errorMode(false) { } +static const Byte PosL1[kNumBits + 1] = { 0,0,2,1,2,2,4,5,4,4,8,0,224 }; +static const Byte PosL2[kNumBits + 1] = { 0,0,0,5,2,2,4,5,4,4,8,2,220 }; -void CDecoder::InitStructures() -{ - for (int i = 0; i < kNumRepDists; i++) - m_RepDists[i] = 0; - m_RepDistPtr = 0; - LastLength = 0; - LastDist = 0; -} +static const Byte PosHf0[kNumBits + 1] = { 0,0,0,0,8,8,8,9,0,0,0,0,224 }; +static const Byte PosHf1[kNumBits + 1] = { 0,0,0,0,0,4,40,16,16,4,0,47,130 }; +static const Byte PosHf2[kNumBits + 1] = { 0,0,0,0,0,2,5,46,64,116,24,0,0 }; +static const Byte PosHf3[kNumBits + 1] = { 0,0,0,0,0,0,2,14,202,33,6,0,0 }; +static const Byte PosHf4[kNumBits + 1] = { 0,0,0,0,0,0,0,0,255,2,0,0,0 }; + +static const UInt32 kHistorySize = (1 << 16); + +CDecoder::CDecoder(): + _isSolid(false), + _solidAllowed(false) + { } -UInt32 CDecoder::ReadBits(int numBits) { return m_InBitStream.ReadBits(numBits); } +UInt32 CDecoder::ReadBits(unsigned numBits) { return m_InBitStream.ReadBits(numBits); } HRESULT CDecoder::CopyBlock(UInt32 distance, UInt32 len) { if (len == 0) return S_FALSE; + if (m_UnpackSize < len) + return S_FALSE; m_UnpackSize -= len; return m_OutWindowStream.CopyBlock(distance, len) ? S_OK : S_FALSE; } -UInt32 CDecoder::DecodeNum(const UInt32 *posTab) +UInt32 CDecoder::DecodeNum(const Byte *numTab) { - UInt32 startPos = 2; - UInt32 num = m_InBitStream.GetValue(12); + /* + { + // we can check that tables are correct + UInt32 sum = 0; + for (unsigned i = 0; i <= kNumBits; i++) + sum += ((UInt32)numTab[i] << (kNumBits - i)); + if (sum != (1 << kNumBits)) + throw 111; + } + */ + + UInt32 val = m_InBitStream.GetValue(kNumBits); + UInt32 sum = 0; + unsigned i = 2; + for (;;) { - UInt32 cur = (posTab[(size_t)startPos + 1] - posTab[startPos]) << (12 - startPos); - if (num < cur) + UInt32 num = numTab[i]; + UInt32 cur = num << (kNumBits - i); + if (val < cur) break; - startPos++; - num -= cur; + i++; + val -= cur; + sum += num; } - m_InBitStream.MovePos(startPos); - return((num >> (12 - startPos)) + posTab[startPos]); + m_InBitStream.MovePos(i); + return ((val >> (kNumBits - i)) + sum); } -static const Byte kShortLen1 [] = {1,3,4,4,5,6,7,8,8,4,4,5,6,6 }; -static const Byte kShortLen1a[] = {1,4,4,4,5,6,7,8,8,4,4,5,6,6,4 }; -static const Byte kShortLen2 [] = {2,3,3,3,4,4,5,6,6,4,4,5,6,6 }; -static const Byte kShortLen2a[] = {2,3,3,4,4,4,5,6,6,4,4,5,6,6,4 }; -static const UInt32 kShortXor1[] = {0,0xa0,0xd0,0xe0,0xf0,0xf8,0xfc,0xfe,0xff,0xc0,0x80,0x90,0x98,0x9c,0xb0}; -static const UInt32 kShortXor2[] = {0,0x40,0x60,0xa0,0xd0,0xe0,0xf0,0xf8,0xfc,0xc0,0x80,0x90,0x98,0x9c,0xb0}; HRESULT CDecoder::ShortLZ() { - UInt32 len, saveLen, dist; - int distancePlace; - const Byte *kShortLen; - const UInt32 *kShortXor; NumHuf = 0; if (LCount == 2) @@ -91,20 +99,14 @@ HRESULT CDecoder::ShortLZ() UInt32 bitField = m_InBitStream.GetValue(8); - if (AvrLn1 < 37) + UInt32 len, dist; { - kShortLen = Buf60 ? kShortLen1a : kShortLen1; - kShortXor = kShortXor1; + const Byte *xors = (AvrLn1 < 37) ? kShortLen1 : kShortLen2; + const Byte *lens = xors + 16 + Buf60; + for (len = 0; ((bitField ^ xors[len]) >> (8 - lens[len])) != 0; len++); + m_InBitStream.MovePos(lens[len]); } - else - { - kShortLen = Buf60 ? kShortLen2a : kShortLen2; - kShortXor = kShortXor2; - } - - for (len = 0; ((bitField ^ kShortXor[len]) & (~(0xff >> kShortLen[len]))) != 0; len++); - m_InBitStream.MovePos(kShortLen[len]); - + if (len >= 9) { if (len == 9) @@ -112,9 +114,11 @@ HRESULT CDecoder::ShortLZ() LCount++; return CopyBlock(LastDist, LastLength); } + + LCount = 0; + if (len == 14) { - LCount = 0; len = DecodeNum(PosL2) + 5; dist = 0x8000 + ReadBits(15) - 1; LastLength = len; @@ -122,19 +126,22 @@ HRESULT CDecoder::ShortLZ() return CopyBlock(dist, len); } - LCount = 0; - saveLen = len; + UInt32 saveLen = len; dist = m_RepDists[(m_RepDistPtr - (len - 9)) & 3]; - len = DecodeNum(PosL1) + 2; - if (len == 0x101 && saveLen == 10) + + len = DecodeNum(PosL1); + + if (len == 0xff && saveLen == 10) { - Buf60 ^= 1; + Buf60 ^= 16; return S_OK; } if (dist >= 256) + { len++; - if (dist >= MaxDist3 - 1) - len++; + if (dist >= MaxDist3 - 1) + len++; + } } else { @@ -142,21 +149,23 @@ HRESULT CDecoder::ShortLZ() AvrLn1 += len; AvrLn1 -= AvrLn1 >> 4; - distancePlace = DecodeNum(PosHf2) & 0xff; - dist = ChSetA[(unsigned)distancePlace]; - if (--distancePlace != -1) + unsigned distancePlace = DecodeNum(PosHf2) & 0xff; + + dist = ChSetA[distancePlace]; + + if (distancePlace != 0) { PlaceA[dist]--; - UInt32 lastDistance = ChSetA[(unsigned)distancePlace]; + UInt32 lastDistance = ChSetA[(size_t)distancePlace - 1]; PlaceA[lastDistance]++; - ChSetA[(size_t)(unsigned)distancePlace + 1] = lastDistance; - ChSetA[(unsigned)distancePlace] = dist; + ChSetA[distancePlace] = lastDistance; + ChSetA[(size_t)distancePlace - 1] = dist; } - len += 2; } m_RepDists[m_RepDistPtr++] = dist; m_RepDistPtr &= 3; + len += 2; LastLength = len; LastDist = dist; return CopyBlock(dist, len); @@ -177,12 +186,10 @@ HRESULT CDecoder::LongLZ() Nlzb = 0x90; Nhfb >>= 1; } - oldAvr2=AvrLn2; + oldAvr2 = AvrLn2; - if (AvrLn2 >= 122) - len = DecodeNum(PosL2); - else if (AvrLn2 >= 64) - len = DecodeNum(PosL1); + if (AvrLn2 >= 64) + len = DecodeNum(AvrLn2 < 122 ? PosL1 : PosL2); else { UInt32 bitField = m_InBitStream.GetValue(16); @@ -193,8 +200,8 @@ HRESULT CDecoder::LongLZ() } else { - for (len = 0; ((bitField << len) & 0x8000) == 0; len++) - ; + for (len = 0; ((bitField << len) & 0x8000) == 0; len++); + m_InBitStream.MovePos(len + 1); } } @@ -202,24 +209,26 @@ HRESULT CDecoder::LongLZ() AvrLn2 += len; AvrLn2 -= AvrLn2 >> 5; - if (AvrPlcB > 0x28ff) - distancePlace = DecodeNum(PosHf2); - else if (AvrPlcB > 0x6ff) - distancePlace = DecodeNum(PosHf1); - else - distancePlace = DecodeNum(PosHf0); + { + const Byte *tab; + if (AvrPlcB >= 0x2900) tab = PosHf2; + else if (AvrPlcB >= 0x0700) tab = PosHf1; + else tab = PosHf0; + distancePlace = DecodeNum(tab); // [0, 256] + } AvrPlcB += distancePlace; AvrPlcB -= AvrPlcB >> 8; + + distancePlace &= 0xff; for (;;) { - dist = ChSetB[distancePlace & 0xff]; + dist = ChSetB[distancePlace]; newDistancePlace = NToPlB[dist++ & 0xff]++; - if (!(dist & 0xff)) - CorrHuff(ChSetB,NToPlB); - else + if (dist & 0xff) break; + CorrHuff(ChSetB,NToPlB); } ChSetB[distancePlace] = ChSetB[newDistancePlace]; @@ -235,9 +244,8 @@ HRESULT CDecoder::LongLZ() AvrLn3++; AvrLn3 -= AvrLn3 >> 8; } - else - if (AvrLn3 > 0) - AvrLn3--; + else if (AvrLn3 > 0) + AvrLn3--; len += 3; @@ -265,57 +273,62 @@ HRESULT CDecoder::HuffDecode() UInt32 curByte, newBytePlace; UInt32 len; UInt32 dist; - int bytePlace; - - if (AvrPlc > 0x75ff) bytePlace = DecodeNum(PosHf4); - else if (AvrPlc > 0x5dff) bytePlace = DecodeNum(PosHf3); - else if (AvrPlc > 0x35ff) bytePlace = DecodeNum(PosHf2); - else if (AvrPlc > 0x0dff) bytePlace = DecodeNum(PosHf1); - else bytePlace = DecodeNum(PosHf0); + unsigned bytePlace; + { + const Byte *tab; + + if (AvrPlc >= 0x7600) tab = PosHf4; + else if (AvrPlc >= 0x5e00) tab = PosHf3; + else if (AvrPlc >= 0x3600) tab = PosHf2; + else if (AvrPlc >= 0x0e00) tab = PosHf1; + else tab = PosHf0; + + bytePlace = DecodeNum(tab); // [0, 256] + } if (StMode) { - if (--bytePlace == -1) + if (bytePlace == 0) { if (ReadBits(1)) { - NumHuf = StMode = 0; + NumHuf = 0; + StMode = false; return S_OK; } - else - { - len = (ReadBits(1)) ? 4 : 3; - dist = DecodeNum(PosHf2); - dist = (dist << 5) | ReadBits(5); - return CopyBlock(dist - 1, len); - } + len = ReadBits(1) + 3; + dist = DecodeNum(PosHf2); + dist = (dist << 5) | ReadBits(5); + if (dist == 0) + return S_FALSE; + return CopyBlock(dist - 1, len); } + bytePlace--; // bytePlace is [0, 255] } else if (NumHuf++ >= 16 && FlagsCnt == 0) - StMode = 1; + StMode = true; bytePlace &= 0xff; AvrPlc += bytePlace; AvrPlc -= AvrPlc >> 8; - Nhfb+=16; + Nhfb += 16; if (Nhfb > 0xff) { - Nhfb=0x90; + Nhfb = 0x90; Nlzb >>= 1; } - m_UnpackSize --; + m_UnpackSize--; m_OutWindowStream.PutByte((Byte)(ChSet[bytePlace] >> 8)); for (;;) { curByte = ChSet[bytePlace]; newBytePlace = NToPl[curByte++ & 0xff]++; - if ((curByte & 0xff) > 0xa1) - CorrHuff(ChSet, NToPl); - else + if ((curByte & 0xff) <= 0xa1) break; + CorrHuff(ChSet, NToPl); } ChSet[bytePlace] = ChSet[newBytePlace]; @@ -327,7 +340,10 @@ HRESULT CDecoder::HuffDecode() void CDecoder::GetFlagsBuf() { UInt32 flags, newFlagsPlace; - UInt32 flagsPlace = DecodeNum(PosHf2); + UInt32 flagsPlace = DecodeNum(PosHf2); // [0, 256] + + if (flagsPlace >= ARRAY_SIZE(ChSetC)) + return; for (;;) { @@ -343,20 +359,6 @@ void CDecoder::GetFlagsBuf() ChSetC[newFlagsPlace] = flags; } -void CDecoder::InitData() -{ - if (!m_IsSolid) - { - AvrPlcB = AvrLn1 = AvrLn2 = AvrLn3 = NumHuf = Buf60 = 0; - AvrPlc = 0x3500; - MaxDist3 = 0x2001; - Nhfb = Nlzb = 0x80; - } - FlagsCnt = 0; - FlagBuf = 0; - StMode = 0; - LCount = 0; -} void CDecoder::CorrHuff(UInt32 *CharSet,UInt32 *NumToPlace) { @@ -369,129 +371,145 @@ void CDecoder::CorrHuff(UInt32 *CharSet,UInt32 *NumToPlace) NumToPlace[i] = (7 - i) * 32; } -void CDecoder::InitHuff() -{ - for (UInt32 i = 0; i < 256; i++) - { - Place[i] = PlaceA[i] = PlaceB[i] = i; - PlaceC[i] = (~i + 1) & 0xff; - ChSet[i] = ChSetB[i] = i << 8; - ChSetA[i] = i; - ChSetC[i] = ((~i + 1) & 0xff) << 8; - } - memset(NToPl, 0, sizeof(NToPl)); - memset(NToPlB, 0, sizeof(NToPlB)); - memset(NToPlC, 0, sizeof(NToPlC)); - CorrHuff(ChSetB, NToPlB); -} + HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo * /* progress */) { - if (inSize == NULL || outSize == NULL) + if (!inSize || !outSize) return E_INVALIDARG; + if (_isSolid && !_solidAllowed) + return S_FALSE; + + _solidAllowed = false; + if (!m_OutWindowStream.Create(kHistorySize)) return E_OUTOFMEMORY; if (!m_InBitStream.Create(1 << 20)) return E_OUTOFMEMORY; - m_UnpackSize = (Int64)*outSize; + m_UnpackSize = *outSize; + m_OutWindowStream.SetStream(outStream); - m_OutWindowStream.Init(m_IsSolid); + m_OutWindowStream.Init(_isSolid); m_InBitStream.SetStream(inStream); m_InBitStream.Init(); - // CCoderReleaser coderReleaser(this); - InitData(); - if (!m_IsSolid) + // InitData + + FlagsCnt = 0; + FlagBuf = 0; + StMode = false; + LCount = 0; + + if (!_isSolid) { - _errorMode = false; - InitStructures(); - InitHuff(); - } - - if (_errorMode) - return S_FALSE; + AvrPlcB = AvrLn1 = AvrLn2 = AvrLn3 = NumHuf = Buf60 = 0; + AvrPlc = 0x3500; + MaxDist3 = 0x2001; + Nhfb = Nlzb = 0x80; + + { + // InitStructures + for (int i = 0; i < kNumRepDists; i++) + m_RepDists[i] = 0; + m_RepDistPtr = 0; + LastLength = 0; + LastDist = 0; + } + + // InitHuff + for (UInt32 i = 0; i < 256; i++) + { + Place[i] = PlaceA[i] = PlaceB[i] = i; + UInt32 c = (~i + 1) & 0xff; + PlaceC[i] = c; + ChSet[i] = ChSetB[i] = i << 8; + ChSetA[i] = i; + ChSetC[i] = c << 8; + } + memset(NToPl, 0, sizeof(NToPl)); + memset(NToPlB, 0, sizeof(NToPlB)); + memset(NToPlC, 0, sizeof(NToPlC)); + CorrHuff(ChSetB, NToPlB); + } + if (m_UnpackSize > 0) { GetFlagsBuf(); FlagsCnt = 8; } - while (m_UnpackSize > 0) + while (m_UnpackSize != 0) { - if (StMode) + if (!StMode) { - RINOK(HuffDecode()); - continue; - } - - if (--FlagsCnt < 0) - { - GetFlagsBuf(); - FlagsCnt=7; - } - - if (FlagBuf & 0x80) - { - FlagBuf <<= 1; - if (Nlzb > Nhfb) - { - RINOK(LongLZ()); - } - else - { - RINOK(HuffDecode()); - } - } - else - { - FlagBuf <<= 1; if (--FlagsCnt < 0) { GetFlagsBuf(); FlagsCnt = 7; } + if (FlagBuf & 0x80) { FlagBuf <<= 1; if (Nlzb > Nhfb) - { - RINOK(HuffDecode()); - } - else { RINOK(LongLZ()); + continue; } } else { FlagBuf <<= 1; - RINOK(ShortLZ()); + + if (--FlagsCnt < 0) + { + GetFlagsBuf(); + FlagsCnt = 7; + } + + if ((FlagBuf & 0x80) == 0) + { + FlagBuf <<= 1; + RINOK(ShortLZ()); + continue; + } + + FlagBuf <<= 1; + + if (Nlzb <= Nhfb) + { + RINOK(LongLZ()); + continue; + } } } + + RINOK(HuffDecode()); } - if (m_UnpackSize < 0) - return S_FALSE; + + _solidAllowed = true; return m_OutWindowStream.Flush(); } + STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) { try { return CodeReal(inStream, outStream, inSize, outSize, progress); } - catch(const CInBufferException &e) { _errorMode = true; return e.ErrorCode; } - catch(const CLzOutWindowException &e) { _errorMode = true; return e.ErrorCode; } - catch(...) { _errorMode = true; return S_FALSE; } + catch(const CInBufferException &e) { return e.ErrorCode; } + catch(const CLzOutWindowException &e) { return e.ErrorCode; } + catch(...) { return S_FALSE; } } STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *data, UInt32 size) { if (size < 1) return E_INVALIDARG; - m_IsSolid = ((data[0] & 1) != 0); + _isSolid = ((data[0] & 1) != 0); return S_OK; } diff --git a/7zip/CPP/7zip/Compress/Rar1Decoder.h b/7zip/CPP/7zip/Compress/Rar1Decoder.h index 01b606b3b..52907e5c4 100644 --- a/7zip/CPP/7zip/Compress/Rar1Decoder.h +++ b/7zip/CPP/7zip/Compress/Rar1Decoder.h @@ -20,49 +20,45 @@ namespace NRar1 { const UInt32 kNumRepDists = 4; -typedef NBitm::CDecoder CBitDecoder; - class CDecoder : public ICompressCoder, public ICompressSetDecoderProperties2, public CMyUnknownImp { -public: CLzOutWindow m_OutWindowStream; - CBitDecoder m_InBitStream; + NBitm::CDecoder m_InBitStream; - UInt32 m_RepDists[kNumRepDists]; - UInt32 m_RepDistPtr; + UInt64 m_UnpackSize; UInt32 LastDist; UInt32 LastLength; - Int64 m_UnpackSize; - bool m_IsSolid; - bool _errorMode; + UInt32 m_RepDistPtr; + UInt32 m_RepDists[kNumRepDists]; - UInt32 ReadBits(int numBits); - HRESULT CopyBlock(UInt32 distance, UInt32 len); + bool _isSolid; + bool _solidAllowed; - UInt32 DecodeNum(const UInt32 *posTab); + bool StMode; + int FlagsCnt; + UInt32 FlagBuf, AvrPlc, AvrPlcB, AvrLn1, AvrLn2, AvrLn3; + unsigned Buf60, NumHuf, LCount; + UInt32 Nhfb, Nlzb, MaxDist3; + + UInt32 ChSet[256], ChSetA[256], ChSetB[256], ChSetC[256]; + UInt32 Place[256], PlaceA[256], PlaceB[256], PlaceC[256]; + UInt32 NToPl[256], NToPlB[256], NToPlC[256]; + + UInt32 ReadBits(unsigned numBits); + HRESULT CopyBlock(UInt32 distance, UInt32 len); + UInt32 DecodeNum(const Byte *numTab); HRESULT ShortLZ(); HRESULT LongLZ(); HRESULT HuffDecode(); void GetFlagsBuf(); - void InitData(); - void InitHuff(); void CorrHuff(UInt32 *CharSet, UInt32 *NumToPlace); void OldUnpWriteBuf(); - UInt32 ChSet[256],ChSetA[256],ChSetB[256],ChSetC[256]; - UInt32 Place[256],PlaceA[256],PlaceB[256],PlaceC[256]; - UInt32 NToPl[256],NToPlB[256],NToPlC[256]; - UInt32 FlagBuf,AvrPlc,AvrPlcB,AvrLn1,AvrLn2,AvrLn3; - int Buf60,NumHuf,StMode,LCount,FlagsCnt; - UInt32 Nhfb,Nlzb,MaxDist3; - - void InitStructures(); - HRESULT CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); @@ -71,14 +67,6 @@ class CDecoder : MY_UNKNOWN_IMP1(ICompressSetDecoderProperties2) - /* - void ReleaseStreams() - { - m_OutWindowStream.ReleaseStream(); - m_InBitStream.ReleaseStream(); - } - */ - STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); diff --git a/7zip/CPP/7zip/Compress/Rar2Decoder.cpp b/7zip/CPP/7zip/Compress/Rar2Decoder.cpp index 1488444e4..fe458c851 100644 --- a/7zip/CPP/7zip/Compress/Rar2Decoder.cpp +++ b/7zip/CPP/7zip/Compress/Rar2Decoder.cpp @@ -80,7 +80,8 @@ static const UInt32 kHistorySize = 1 << 20; static const UInt32 kWindowReservSize = (1 << 22) + 256; CDecoder::CDecoder(): - m_IsSolid(false), + _isSolid(false), + _solidAllowed(false), m_TablesOK(false) { } @@ -227,18 +228,6 @@ bool CDecoder::ReadLastTables() return true; } -/* -class CCoderReleaser -{ - CDecoder *m_Coder; -public: - CCoderReleaser(CDecoder *coder): m_Coder(coder) {} - ~CCoderReleaser() - { - m_Coder->ReleaseStreams(); - } -}; -*/ bool CDecoder::DecodeMm(UInt32 pos) { @@ -343,9 +332,13 @@ bool CDecoder::DecodeLz(Int32 pos) HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) { - if (inSize == NULL || outSize == NULL) + if (!inSize || !outSize) return E_INVALIDARG; + if (_isSolid && !_solidAllowed) + return S_FALSE; + _solidAllowed = false; + if (!m_OutWindowStream.Create(kHistorySize)) return E_OUTOFMEMORY; if (!m_InBitStream.Create(1 << 20)) @@ -356,12 +349,12 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * UInt64 pos = 0, unPackSize = *outSize; m_OutWindowStream.SetStream(outStream); - m_OutWindowStream.Init(m_IsSolid); + m_OutWindowStream.Init(_isSolid); m_InBitStream.SetStream(inStream); m_InBitStream.Init(); // CCoderReleaser coderReleaser(this); - if (!m_IsSolid) + if (!_isSolid) { InitStructures(); if (unPackSize == 0) @@ -369,6 +362,7 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * if (m_InBitStream.GetProcessedSize() + 2 <= m_PackSize) // test it: probably incorrect; if (!ReadTables()) return S_FALSE; + _solidAllowed = true; return S_OK; } ReadTables(); @@ -415,6 +409,9 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * if (!ReadLastTables()) return S_FALSE; + + _solidAllowed = true; + return m_OutWindowStream.Flush(); } @@ -431,7 +428,7 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *data, UInt32 size) { if (size < 1) return E_INVALIDARG; - m_IsSolid = ((data[0] & 1) != 0); + _isSolid = ((data[0] & 1) != 0); return S_OK; } diff --git a/7zip/CPP/7zip/Compress/Rar2Decoder.h b/7zip/CPP/7zip/Compress/Rar2Decoder.h index 0d8142b58..f42f228d4 100644 --- a/7zip/CPP/7zip/Compress/Rar2Decoder.h +++ b/7zip/CPP/7zip/Compress/Rar2Decoder.h @@ -125,7 +125,8 @@ class CDecoder : UInt32 m_LastLength; - bool m_IsSolid; + bool _isSolid; + bool _solidAllowed; bool m_TablesOK; bool m_AudioMode; @@ -159,14 +160,6 @@ class CDecoder : MY_UNKNOWN_IMP1(ICompressSetDecoderProperties2) - /* - void ReleaseStreams() - { - m_OutWindowStream.ReleaseStream(); - m_InBitStream.ReleaseStream(); - } - */ - STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); diff --git a/7zip/CPP/7zip/Compress/Rar3Decoder.cpp b/7zip/CPP/7zip/Compress/Rar3Decoder.cpp index 4da89a65d..4456ba631 100644 --- a/7zip/CPP/7zip/Compress/Rar3Decoder.cpp +++ b/7zip/CPP/7zip/Compress/Rar3Decoder.cpp @@ -94,8 +94,8 @@ CDecoder::CDecoder(): _writtenFileSize(0), _vmData(0), _vmCode(0), - m_IsSolid(false), - _errorMode(false) + _isSolid(false), + _solidAllowed(false) { Ppmd7_Construct(&_ppmd); } @@ -148,7 +148,8 @@ void CDecoder::ExecuteFilter(unsigned tempFilterIndex, NVm::CBlockRef &outBlockR if (!_vm.Execute(filter, tempFilter, outBlockRef, filter->GlobalData)) _unsupportedFilter = true; delete tempFilter; - _tempFilters[tempFilterIndex] = 0; + _tempFilters[tempFilterIndex] = NULL; + _numEmptyTempFilters++; } HRESULT CDecoder::WriteBuf() @@ -225,6 +226,7 @@ HRESULT CDecoder::WriteBuf() void CDecoder::InitFilters() { _lastFilter = 0; + _numEmptyTempFilters = 0; unsigned i; for (i = 0; i < _tempFilters.Size(); i++) delete _tempFilters[i]; @@ -274,24 +276,27 @@ bool CDecoder::AddVmCode(UInt32 firstByte, UInt32 codeSize) filter->ExecCount++; } - unsigned numEmptyItems = 0; + if (_numEmptyTempFilters != 0) { - FOR_VECTOR (i, _tempFilters) + unsigned num = _tempFilters.Size(); + CTempFilter **tempFilters = &_tempFilters.Front(); + + unsigned w = 0; + for (unsigned i = 0; i < num; i++) { - _tempFilters[i - numEmptyItems] = _tempFilters[i]; - if (!_tempFilters[i]) - numEmptyItems++; - if (numEmptyItems != 0) - _tempFilters[i] = NULL; + CTempFilter *tf = tempFilters[i]; + if (tf) + tempFilters[w++] = tf; } + + _tempFilters.DeleteFrom(w); + _numEmptyTempFilters = 0; } - if (numEmptyItems == 0) - { - _tempFilters.Add(NULL); - numEmptyItems = 1; - } + + if (_tempFilters.Size() > MAX_UNPACK_FILTERS) + return false; CTempFilter *tempFilter = new CTempFilter; - _tempFilters[_tempFilters.Size() - numEmptyItems] = tempFilter; + _tempFilters.Add(tempFilter); tempFilter->FilterIndex = filterIndex; UInt32 blockStart = inp.ReadEncodedUInt32(); @@ -829,7 +834,7 @@ HRESULT CDecoder::CodeReal(ICompressProgressInfo *progress) _writtenFileSize = 0; _unsupportedFilter = false; - if (!m_IsSolid) + if (!_isSolid) { _lzSize = 0; _winPos = 0; @@ -842,18 +847,23 @@ HRESULT CDecoder::CodeReal(ICompressProgressInfo *progress) PpmEscChar = 2; PpmError = true; InitFilters(); - _errorMode = false; + // _errorMode = false; } + /* if (_errorMode) return S_FALSE; + */ - if (!m_IsSolid || !TablesRead) + if (!_isSolid || !TablesRead) { bool keepDecompressing; RINOK(ReadTables(keepDecompressing)); if (!keepDecompressing) + { + _solidAllowed = true; return S_OK; + } } for (;;) @@ -878,6 +888,9 @@ HRESULT CDecoder::CodeReal(ICompressProgressInfo *progress) if (!keepDecompressing) break; } + + _solidAllowed = true; + RINOK(WriteBuf()); UInt64 packSize = m_InBitStream.BitDecoder.GetProcessedSize(); RINOK(progress->SetRatioInfo(&packSize, &_writtenFileSize)); @@ -898,6 +911,10 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream if (!inSize) return E_INVALIDARG; + if (_isSolid && !_solidAllowed) + return S_FALSE; + _solidAllowed = false; + if (!_vmData) { _vmData = (Byte *)::MidAlloc(kVmDataSizeMax + kVmCodeSizeMax); @@ -926,8 +943,8 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream _unpackSize = outSize ? *outSize : (UInt64)(Int64)-1; return CodeReal(progress); } - catch(const CInBufferException &e) { _errorMode = true; return e.ErrorCode; } - catch(...) { _errorMode = true; return S_FALSE; } + catch(const CInBufferException &e) { /* _errorMode = true; */ return e.ErrorCode; } + catch(...) { /* _errorMode = true; */ return S_FALSE; } // CNewException is possible here. But probably CNewException is caused // by error in data stream. } @@ -936,7 +953,7 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *data, UInt32 size) { if (size < 1) return E_INVALIDARG; - m_IsSolid = ((data[0] & 1) != 0); + _isSolid = ((data[0] & 1) != 0); return S_OK; } diff --git a/7zip/CPP/7zip/Compress/Rar3Decoder.h b/7zip/CPP/7zip/Compress/Rar3Decoder.h index f3c1528c2..3d319deda 100644 --- a/7zip/CPP/7zip/Compress/Rar3Decoder.h +++ b/7zip/CPP/7zip/Compress/Rar3Decoder.h @@ -189,10 +189,12 @@ class CDecoder: NVm::CVm _vm; CRecordVector _filters; CRecordVector _tempFilters; + unsigned _numEmptyTempFilters; UInt32 _lastFilter; - bool m_IsSolid; - bool _errorMode; + bool _isSolid; + bool _solidAllowed; + // bool _errorMode; bool _lzMode; bool _unsupportedFilter; diff --git a/7zip/CPP/7zip/Compress/Rar5Decoder.cpp b/7zip/CPP/7zip/Compress/Rar5Decoder.cpp index e6c3e467b..d7c68e218 100644 --- a/7zip/CPP/7zip/Compress/Rar5Decoder.cpp +++ b/7zip/CPP/7zip/Compress/Rar5Decoder.cpp @@ -72,6 +72,7 @@ CDecoder::CDecoder(): _writtenFileSize(0), _dictSizeLog(0), _isSolid(false), + _solidAllowed(false), _wasInit(false), _inputBuf(NULL) { @@ -334,58 +335,63 @@ HRESULT CDecoder::ReadTables(CBitDecoder &_bitStream) { if (_progress) { - UInt64 packSize = _bitStream.GetProcessedSize(); + const UInt64 packSize = _bitStream.GetProcessedSize(); RINOK(_progress->SetRatioInfo(&packSize, &_writtenFileSize)); } _bitStream.AlignToByte(); _bitStream.Prepare(); - unsigned flags = _bitStream.ReadByteInAligned(); - unsigned checkSum = _bitStream.ReadByteInAligned(); - checkSum ^= flags; - - UInt32 blockSize; { + unsigned flags = _bitStream.ReadByteInAligned(); + unsigned checkSum = _bitStream.ReadByteInAligned(); + checkSum ^= flags; unsigned num = (flags >> 3) & 3; if (num == 3) return S_FALSE; - blockSize = _bitStream.ReadByteInAligned(); - if (num > 0) + UInt32 blockSize = _bitStream.ReadByteInAligned(); + checkSum ^= blockSize; + + if (num != 0) { - blockSize += (UInt32)_bitStream.ReadByteInAligned() << 8; + unsigned b = _bitStream.ReadByteInAligned(); + checkSum ^= b; + blockSize += (UInt32)b << 8; if (num > 1) - blockSize += (UInt32)_bitStream.ReadByteInAligned() << 16; + { + b = _bitStream.ReadByteInAligned(); + checkSum ^= b; + blockSize += (UInt32)b << 16; + } } - } - - checkSum ^= blockSize ^ (blockSize >> 8) ^ (blockSize >> 16); - if ((Byte)checkSum != 0x5A) - return S_FALSE; - - unsigned blockSizeBits7 = (flags & 7) + 1; - - if (blockSize == 0 && blockSizeBits7 != 8) - return S_FALSE; - - blockSize += (blockSizeBits7 >> 3); - blockSize--; - - _bitStream._blockEndBits7 = (Byte)(blockSizeBits7 & 7); - _bitStream._blockEnd = _bitStream.GetProcessedSize_Round() + blockSize; - - _bitStream.SetCheck2(); - - _isLastBlock = ((flags & 0x40) != 0); + + if (checkSum != 0x5A) + return S_FALSE; - if ((flags & 0x80) == 0) - { - if (!_tableWasFilled && blockSize != 0) + unsigned blockSizeBits7 = (flags & 7) + 1; + blockSize += (blockSizeBits7 >> 3); + if (blockSize == 0) return S_FALSE; - return S_OK; - } + blockSize--; + blockSizeBits7 &= 7; - _tableWasFilled = false; + _bitStream._blockEndBits7 = (Byte)blockSizeBits7; + _bitStream._blockEnd = _bitStream.GetProcessedSize_Round() + blockSize; + + _bitStream.SetCheck2(); + + _isLastBlock = ((flags & 0x40) != 0); + + if ((flags & 0x80) == 0) + { + if (!_tableWasFilled) + if (blockSize != 0 || blockSizeBits7 != 0) + return S_FALSE; + return S_OK; + } + + _tableWasFilled = false; + } { Byte lens2[kLevelTableSize]; @@ -596,6 +602,10 @@ HRESULT CDecoder::DecodeLZ() } } } + + // that check is not required, but it can help, if there is BUG in another code + if (!_tableWasFilled) + break; // return S_FALSE; } UInt32 sym = m_MainDecoder.Decode(&_bitStream); @@ -797,7 +807,10 @@ HRESULT CDecoder::CodeReal() */ if (res == S_OK) + { + _solidAllowed = true; res = res2; + } if (res == S_OK && _unpackSize_Defined && _writtenFileSize != _unpackSize) return S_FALSE; @@ -817,6 +830,10 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream { try { + if (_isSolid && !_solidAllowed) + return S_FALSE; + _solidAllowed = false; + if (_dictSizeLog >= sizeof(size_t) * 8) return E_NOTIMPL; diff --git a/7zip/CPP/7zip/Compress/Rar5Decoder.h b/7zip/CPP/7zip/Compress/Rar5Decoder.h index c37cb0e01..8174c4af7 100644 --- a/7zip/CPP/7zip/Compress/Rar5Decoder.h +++ b/7zip/CPP/7zip/Compress/Rar5Decoder.h @@ -127,7 +127,7 @@ class CBitDecoder return *_buf++; } - UInt32 GetValue(unsigned numBits) + UInt32 GetValue(unsigned numBits) const { UInt32 v = ((UInt32)_buf[0] << 16) | ((UInt32)_buf[1] << 8) | (UInt32)_buf[2]; v >>= (24 - numBits - _bitPos); @@ -218,6 +218,13 @@ class CDecoder: bool _unsupportedFilter; bool _lzError; bool _writeError; + + bool _isSolid; + bool _solidAllowed; + bool _tableWasFilled; + bool _wasInit; + + Byte _dictSizeLog; // CBitDecoder _bitStream; Byte *_window; @@ -238,11 +245,6 @@ class CDecoder: UInt64 _writtenFileSize; size_t _winSizeAllocated; - Byte _dictSizeLog; - bool _tableWasFilled; - bool _isSolid; - bool _wasInit; - UInt32 _reps[kNumReps]; UInt32 _lastLen; diff --git a/7zip/CPP/7zip/Compress/XpressDecoder.cpp b/7zip/CPP/7zip/Compress/XpressDecoder.cpp index a38d94181..864c6f8fe 100644 --- a/7zip/CPP/7zip/Compress/XpressDecoder.cpp +++ b/7zip/CPP/7zip/Compress/XpressDecoder.cpp @@ -33,10 +33,11 @@ struct CBitStream bs.Value = (bs.Value << 16) | GetUi16(in); \ in += 2; bs.BitPos += 16; } -const unsigned kNumHuffBits = 15; -const unsigned kNumLenSlots = 16; -const unsigned kNumPosSlots = 16; -const unsigned kNumSyms = 256 + kNumPosSlots * kNumLenSlots; +static const unsigned kNumHuffBits = 15; +static const unsigned kNumLenBits = 4; +static const unsigned kLenMask = (1 << kNumLenBits) - 1; +static const unsigned kNumPosSlots = 16; +static const unsigned kNumSyms = 256 + (kNumPosSlots << kNumLenBits); HRESULT Decode(const Byte *in, size_t inSize, Byte *out, size_t outSize) { @@ -83,10 +84,10 @@ HRESULT Decode(const Byte *in, size_t inSize, Byte *out, size_t outSize) else { sym -= 256; - UInt32 dist = sym / kNumLenSlots; - UInt32 len = sym & (kNumLenSlots - 1); + UInt32 dist = sym >> kNumLenBits; + UInt32 len = sym & kLenMask; - if (len == kNumLenSlots - 1) + if (len == kLenMask) { if (in > lim) return S_FALSE; @@ -99,7 +100,7 @@ HRESULT Decode(const Byte *in, size_t inSize, Byte *out, size_t outSize) in += 2; } else - len += kNumLenSlots - 1; + len += kLenMask; } bs.BitPos -= dist; @@ -108,7 +109,7 @@ HRESULT Decode(const Byte *in, size_t inSize, Byte *out, size_t outSize) BIT_STREAM_NORMALIZE - if (len > outSize - pos) + if (len + 3 > outSize - pos) return S_FALSE; if (dist > pos) return S_FALSE; diff --git a/7zip/CPP/7zip/Compress/XzDecoder.cpp b/7zip/CPP/7zip/Compress/XzDecoder.cpp index c28ac900f..7a9743118 100644 --- a/7zip/CPP/7zip/Compress/XzDecoder.cpp +++ b/7zip/CPP/7zip/Compress/XzDecoder.cpp @@ -4,248 +4,119 @@ #include "../../../C/Alloc.h" -#include "../Common/StreamUtils.h" - -#include "../Archive/IArchive.h" +#include "../Common/CWrappers.h" #include "XzDecoder.h" -using namespace NArchive; - namespace NCompress { namespace NXz { +#define RET_IF_WRAP_ERROR_CONFIRMED(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK && sRes == sResErrorCode) return wrapRes; -CXzUnpackerCPP::CXzUnpackerCPP(): InBuf(NULL), OutBuf(NULL) -{ - XzUnpacker_Construct(&p, &g_Alloc); -} - -CXzUnpackerCPP::~CXzUnpackerCPP() -{ - XzUnpacker_Free(&p); - MidFree(InBuf); - MidFree(OutBuf); -} - +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; -void CStatInfo::Clear() +static HRESULT SResToHRESULT_Code(SRes res) throw() { - InSize = 0; - OutSize = 0; - PhySize = 0; - - NumStreams = 0; - NumBlocks = 0; - - UnpackSize_Defined = false; - - NumStreams_Defined = false; - NumBlocks_Defined = false; - - IsArc = false; - UnexpectedEnd = false; - DataAfterEnd = false; - Unsupported = false; - HeadersError = false; - DataError = false; - CrcError = false; + if (res < 0) + return res; + switch (res) + { + case SZ_OK: return S_OK; + case SZ_ERROR_MEM: return E_OUTOFMEMORY; + case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL; + } + return S_FALSE; } HRESULT CDecoder::Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, const UInt64 *outSizeLimit, bool finishStream, ICompressProgressInfo *progress) { - const size_t kInBufSize = (size_t)1 << 20; - const size_t kOutBufSize = (size_t)1 << 21; - - Clear(); - DecodeRes = SZ_OK; - - XzUnpacker_Init(&xzu.p); + MainDecodeSRes = S_OK; + MainDecodeSRes_wasUsed = false; + XzStatInfo_Clear(&Stat); - if (!xzu.InBuf) + if (!xz) { - xzu.InBuf = (Byte *)MidAlloc(kInBufSize); - if (!xzu.InBuf) + xz = XzDecMt_Create(&g_Alloc, &g_MidAlloc); + if (!xz) return E_OUTOFMEMORY; } - if (!xzu.OutBuf) - { - xzu.OutBuf = (Byte *)MidAlloc(kOutBufSize); - if (!xzu.OutBuf) - return E_OUTOFMEMORY; - } - - UInt32 inSize = 0; - UInt32 inPos = 0; - SizeT outPos = 0; - HRESULT readRes = S_OK; + CXzDecMtProps props; + XzDecMtProps_Init(&props); - for (;;) + int isMT = False; + + #ifndef _7ZIP_ST { - if (inPos == inSize && readRes == S_OK) + props.numThreads = 1; + UInt32 numThreads = _numThreads; + + if (_tryMt && numThreads > 1) { - inPos = inSize = 0; - readRes = seqInStream->Read(xzu.InBuf, kInBufSize, &inSize); + size_t memUsage = (size_t)_memUsage; + if (memUsage != _memUsage) + memUsage = (size_t)0 - 1; + props.memUseMax = memUsage; + isMT = (numThreads > 1); } - SizeT inLen = inSize - inPos; - SizeT outLen = kOutBufSize - outPos; - ECoderFinishMode finishMode = CODER_FINISH_ANY; + props.numThreads = numThreads; + } + #endif - /* - // 17.01 : the code was disabled: - if (inSize == 0) - finishMode = CODER_FINISH_END; - */ + CSeqInStreamWrap inWrap; + CSeqOutStreamWrap outWrap; + CCompressProgressWrap progressWrap; - if (outSizeLimit) - { - const UInt64 rem = *outSizeLimit - OutSize; - if (outLen >= rem) - { - outLen = (SizeT)rem; - if (finishStream) - finishMode = CODER_FINISH_END; - } - } - - ECoderStatus status; + inWrap.Init(seqInStream); + outWrap.Init(outStream); + progressWrap.Init(progress); - const SizeT outLenRequested = outLen; + SRes res = XzDecMt_Decode(xz, + &props, + outSizeLimit, finishStream, + &outWrap.vt, + &inWrap.vt, + &Stat, + &isMT, + progress ? &progressWrap.vt : NULL); - SRes res = XzUnpacker_Code(&xzu.p, - xzu.OutBuf + outPos, &outLen, - xzu.InBuf + inPos, &inLen, - finishMode, &status); + MainDecodeSRes = res; - DecodeRes = res; + #ifndef _7ZIP_ST + // _tryMt = isMT; + #endif - inPos += (UInt32)inLen; - outPos += outLen; + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + RET_IF_WRAP_ERROR_CONFIRMED(inWrap.Res, res, SZ_ERROR_READ) - InSize += inLen; - OutSize += outLen; + // return E_OUTOFMEMORY; - bool finished = ((inLen == 0 && outLen == 0) || res != SZ_OK); + MainDecodeSRes_wasUsed = true; - if (outLen >= outLenRequested || finished) - { - if (outStream && outPos != 0) - { - RINOK(WriteStream(outStream, xzu.OutBuf, outPos)); - } - outPos = 0; - } - - if (progress) - { - RINOK(progress->SetRatioInfo(&InSize, &OutSize)); - } - - if (!finished) - continue; - - { - PhySize = InSize; - NumStreams = xzu.p.numStartedStreams; - if (NumStreams > 0) - IsArc = true; - NumBlocks = xzu.p.numTotalBlocks; - - UnpackSize_Defined = true; - NumStreams_Defined = true; - NumBlocks_Defined = true; - - UInt64 extraSize = XzUnpacker_GetExtraSize(&xzu.p); - - if (res == SZ_OK) - { - if (status == CODER_STATUS_NEEDS_MORE_INPUT) - { - extraSize = 0; - if (!XzUnpacker_IsStreamWasFinished(&xzu.p)) - { - // finished at padding bytes, but padding is not aligned for 4 - UnexpectedEnd = true; - res = SZ_ERROR_DATA; - } - } - else // status == CODER_STATUS_NOT_FINISHED - res = SZ_ERROR_DATA; - } - else if (res == SZ_ERROR_NO_ARCHIVE) - { - if (InSize == extraSize) - IsArc = false; - else - { - if (extraSize != 0 || inPos != inSize) - { - DataAfterEnd = true; - res = SZ_OK; - } - } - } - - DecodeRes = res; - PhySize -= extraSize; - - switch (res) - { - case SZ_OK: break; - case SZ_ERROR_NO_ARCHIVE: IsArc = false; break; - case SZ_ERROR_ARCHIVE: HeadersError = true; break; - case SZ_ERROR_UNSUPPORTED: Unsupported = true; break; - case SZ_ERROR_CRC: CrcError = true; break; - case SZ_ERROR_DATA: DataError = true; break; - default: DataError = true; break; - } - - return readRes; - } + if (res == SZ_OK && finishStream) + { + /* + if (inSize && *inSize != Stat.PhySize) + res = SZ_ERROR_DATA; + */ + if (outSizeLimit && *outSizeLimit != outWrap.Processed) + res = SZ_ERROR_DATA; } -} - -Int32 CDecoder::Get_Extract_OperationResult() const -{ - Int32 opRes; - if (!IsArc) - opRes = NExtract::NOperationResult::kIsNotArc; - else if (UnexpectedEnd) - opRes = NExtract::NOperationResult::kUnexpectedEnd; - else if (DataAfterEnd) - opRes = NExtract::NOperationResult::kDataAfterEnd; - else if (CrcError) - opRes = NExtract::NOperationResult::kCRCError; - else if (Unsupported) - opRes = NExtract::NOperationResult::kUnsupportedMethod; - else if (HeadersError) - opRes = NExtract::NOperationResult::kDataError; - else if (DataError) - opRes = NExtract::NOperationResult::kDataError; - else if (DecodeRes != SZ_OK) - opRes = NExtract::NOperationResult::kDataError; - else - opRes = NExtract::NOperationResult::kOK; - return opRes; + return SResToHRESULT_Code(res); } - HRESULT CComDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 * /* inSize */, const UInt64 *outSize, ICompressProgressInfo *progress) { - RINOK(_decoder.Decode(inStream, outStream, outSize, _finishStream, progress)); - Int32 opRes = _decoder.Get_Extract_OperationResult(); - if (opRes == NArchive::NExtract::NOperationResult::kUnsupportedMethod) - return E_NOTIMPL; - if (opRes != NArchive::NExtract::NOperationResult::kOK) - return S_FALSE; - return S_OK; + return Decode(inStream, outStream, outSize, _finishStream, progress); } STDMETHODIMP CComDecoder::SetFinishMode(UInt32 finishMode) @@ -256,8 +127,24 @@ STDMETHODIMP CComDecoder::SetFinishMode(UInt32 finishMode) STDMETHODIMP CComDecoder::GetInStreamProcessedSize(UInt64 *value) { - *value = _decoder.InSize; + *value = Stat.InSize; + return S_OK; +} + +#ifndef _7ZIP_ST + +STDMETHODIMP CComDecoder::SetNumberOfThreads(UInt32 numThreads) +{ + _numThreads = numThreads; return S_OK; } +STDMETHODIMP CComDecoder::SetMemLimit(UInt64 memUsage) +{ + _memUsage = memUsage; + return S_OK; +} + +#endif + }} diff --git a/7zip/CPP/7zip/Compress/XzDecoder.h b/7zip/CPP/7zip/Compress/XzDecoder.h index 040ed21c1..b65b46b8f 100644 --- a/7zip/CPP/7zip/Compress/XzDecoder.h +++ b/7zip/CPP/7zip/Compress/XzDecoder.h @@ -12,57 +12,36 @@ namespace NCompress { namespace NXz { -struct CXzUnpackerCPP +struct CDecoder { - Byte *InBuf; - Byte *OutBuf; - CXzUnpacker p; + CXzDecMtHandle xz; + int _tryMt; + UInt32 _numThreads; + UInt64 _memUsage; + + SRes MainDecodeSRes; // it's not HRESULT + bool MainDecodeSRes_wasUsed; + CXzStatInfo Stat; + + CDecoder(): + xz(NULL), + _tryMt(True), + _numThreads(1), + _memUsage((UInt64)(sizeof(size_t)) << 28), + MainDecodeSRes(SZ_OK), + MainDecodeSRes_wasUsed(false) + {} - CXzUnpackerCPP(); - ~CXzUnpackerCPP(); -}; - - -struct CStatInfo -{ - UInt64 InSize; - UInt64 OutSize; - UInt64 PhySize; - - UInt64 NumStreams; - UInt64 NumBlocks; - - bool UnpackSize_Defined; - - bool NumStreams_Defined; - bool NumBlocks_Defined; - - bool IsArc; - bool UnexpectedEnd; - bool DataAfterEnd; - bool Unsupported; - bool HeadersError; - bool DataError; - bool CrcError; - - CStatInfo() { Clear(); } - - void Clear(); -}; - - -struct CDecoder: public CStatInfo -{ - CXzUnpackerCPP xzu; - SRes DecodeRes; // it's not HRESULT - - CDecoder(): DecodeRes(SZ_OK) {} + ~CDecoder() + { + if (xz) + XzDecMt_Destroy(xz); + } /* Decode() can return ERROR code only if there is progress or stream error. Decode() returns S_OK in case of xz decoding error, but DecodeRes and CStatInfo contain error information */ HRESULT Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, const UInt64 *outSizeLimit, bool finishStream, ICompressProgressInfo *compressProgress); - Int32 Get_Extract_OperationResult() const; }; @@ -70,21 +49,41 @@ class CComDecoder: public ICompressCoder, public ICompressSetFinishMode, public ICompressGetInStreamProcessedSize, - public CMyUnknownImp + + #ifndef _7ZIP_ST + public ICompressSetCoderMt, + public ICompressSetMemLimit, + #endif + + public CMyUnknownImp, + public CDecoder { - CDecoder _decoder; bool _finishStream; public: - MY_UNKNOWN_IMP2( - ICompressSetFinishMode, - ICompressGetInStreamProcessedSize) + MY_QUERYINTERFACE_BEGIN2(ICompressCoder) + + MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) + MY_QUERYINTERFACE_ENTRY(ICompressGetInStreamProcessedSize) + + #ifndef _7ZIP_ST + MY_QUERYINTERFACE_ENTRY(ICompressSetCoderMt) + MY_QUERYINTERFACE_ENTRY(ICompressSetMemLimit) + #endif + MY_QUERYINTERFACE_END + MY_ADDREF_RELEASE + STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); STDMETHOD(SetFinishMode)(UInt32 finishMode); STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); + #ifndef _7ZIP_ST + STDMETHOD(SetNumberOfThreads)(UInt32 numThreads); + STDMETHOD(SetMemLimit)(UInt64 memUsage); + #endif + CComDecoder(): _finishStream(false) {} }; diff --git a/7zip/CPP/7zip/Crc.mak b/7zip/CPP/7zip/Crc.mak index 2e0b92ef8..815142db8 100644 --- a/7zip/CPP/7zip/Crc.mak +++ b/7zip/CPP/7zip/Crc.mak @@ -1,6 +1,6 @@ C_OBJS = $(C_OBJS) \ $O\7zCrc.obj -!IF "$(CPU)" == "IA64" || "$(CPU)" == "MIPS" || "$(CPU)" == "ARM" || "$(CPU)" == "ARM64" +!IF "$(PLATFORM)" == "ia64" || "$(PLATFORM)" == "mips" || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" C_OBJS = $(C_OBJS) \ !ELSE ASM_OBJS = $(ASM_OBJS) \ diff --git a/7zip/CPP/7zip/Crc64.mak b/7zip/CPP/7zip/Crc64.mak index 1228a0146..d58a48326 100644 --- a/7zip/CPP/7zip/Crc64.mak +++ b/7zip/CPP/7zip/Crc64.mak @@ -1,6 +1,6 @@ C_OBJS = $(C_OBJS) \ $O\XzCrc64.obj -!IF "$(CPU)" == "IA64" || "$(CPU)" == "MIPS" || "$(CPU)" == "ARM" || "$(CPU)" == "ARM64" +!IF "$(PLATFORM)" == "ia64" || "$(PLATFORM)" == "mips" || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" C_OBJS = $(C_OBJS) \ !ELSE ASM_OBJS = $(ASM_OBJS) \ diff --git a/7zip/CPP/7zip/Crypto/7zAes.cpp b/7zip/CPP/7zip/Crypto/7zAes.cpp index d33b562a3..2ed69badc 100644 --- a/7zip/CPP/7zip/Crypto/7zAes.cpp +++ b/7zip/CPP/7zip/Crypto/7zAes.cpp @@ -164,8 +164,8 @@ STDMETHODIMP CEncoder::ResetInitVector() { for (unsigned i = 0; i < sizeof(_iv); i++) _iv[i] = 0; - _ivSize = 8; - g_RandomGenerator.Generate(_iv, _ivSize); + _ivSize = 16; + MY_RAND_GEN(_iv, _ivSize); return S_OK; } diff --git a/7zip/CPP/7zip/Crypto/RandGen.cpp b/7zip/CPP/7zip/Crypto/RandGen.cpp index 408f73f16..f98878f5b 100644 --- a/7zip/CPP/7zip/Crypto/RandGen.cpp +++ b/7zip/CPP/7zip/Crypto/RandGen.cpp @@ -2,14 +2,44 @@ #include "StdAfx.h" +#include "RandGen.h" + +#ifndef USE_STATIC_SYSTEM_RAND + #ifndef _7ZIP_ST #include "../../Windows/Synchronization.h" #endif -#include "RandGen.h" -#ifndef _WIN32 +#ifdef _WIN32 + +#ifdef _WIN64 +#define USE_STATIC_RtlGenRandom +#endif + +#ifdef USE_STATIC_RtlGenRandom + +#include + +EXTERN_C_BEGIN +#ifndef RtlGenRandom + #define RtlGenRandom SystemFunction036 + BOOLEAN WINAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength); +#endif +EXTERN_C_END + +#else +EXTERN_C_BEGIN +typedef BOOLEAN (WINAPI * Func_RtlGenRandom)(PVOID RandomBuffer, ULONG RandomBufferLength); +EXTERN_C_END +#endif + + +#else #include +#include +#include +#include #define USE_POSIX_TIME #define USE_POSIX_TIME2 #endif @@ -21,11 +51,9 @@ #endif #endif -// This is not very good random number generator. -// Please use it only for salt. -// First generated data block depends from timer and processID. +// The seed and first generated data block depend from processID, +// theadID, timer and system random generator, if available. // Other generated data blocks depend from previous state -// Maybe it's possible to restore original timer value from generated value. #define HASH_UPD(x) Sha256_Update(&hash, (const Byte *)&x, sizeof(x)); @@ -34,25 +62,102 @@ void CRandomGenerator::Init() CSha256 hash; Sha256_Init(&hash); + unsigned numIterations = 1000; + + { + #ifndef UNDER_CE + const unsigned kNumIterations_Small = 100; + const unsigned kBufSize = 32; + Byte buf[kBufSize]; + #endif + #ifdef _WIN32 + DWORD w = ::GetCurrentProcessId(); HASH_UPD(w); w = ::GetCurrentThreadId(); HASH_UPD(w); + + #ifdef UNDER_CE + /* + if (CeGenRandom(kBufSize, buf)) + { + numIterations = kNumIterations_Small; + Sha256_Update(&hash, buf, kBufSize); + } + */ + #elif defined(USE_STATIC_RtlGenRandom) + if (RtlGenRandom(buf, kBufSize)) + { + numIterations = kNumIterations_Small; + Sha256_Update(&hash, buf, kBufSize); + } #else + { + HMODULE hModule = ::LoadLibrary(TEXT("Advapi32.dll")); + if (hModule) + { + // SystemFunction036() is real name of RtlGenRandom() function + Func_RtlGenRandom my_RtlGenRandom = (Func_RtlGenRandom)GetProcAddress(hModule, "SystemFunction036"); + if (my_RtlGenRandom) + { + if (my_RtlGenRandom(buf, kBufSize)) + { + numIterations = kNumIterations_Small; + Sha256_Update(&hash, buf, kBufSize); + } + } + ::FreeLibrary(hModule); + } + } + #endif + + #else + pid_t pid = getpid(); HASH_UPD(pid); pid = getppid(); HASH_UPD(pid); + + { + int f = open("/dev/urandom", O_RDONLY); + unsigned numBytes = kBufSize; + if (f >= 0) + { + do + { + int n = read(f, buf, numBytes); + if (n <= 0) + break; + Sha256_Update(&hash, buf, n); + numBytes -= n; + } + while (numBytes); + close(f); + if (numBytes == 0) + numIterations = kNumIterations_Small; + } + } + /* + { + int n = getrandom(buf, kBufSize, 0); + if (n > 0) + { + Sha256_Update(&hash, buf, n); + if (n == kBufSize) + numIterations = kNumIterations_Small; + } + } + */ + + #endif + } + + #ifdef _DEBUG + numIterations = 2; #endif - for (unsigned i = 0; i < - #ifdef _DEBUG - 2; - #else - 1000; - #endif - i++) + do { #ifdef _WIN32 LARGE_INTEGER v; @@ -85,6 +190,8 @@ void CRandomGenerator::Init() Sha256_Update(&hash, _buff, SHA256_DIGEST_SIZE); } } + while (--numIterations); + Sha256_Final(&hash, _buff); _needInit = false; } @@ -122,3 +229,5 @@ void CRandomGenerator::Generate(Byte *data, unsigned size) } CRandomGenerator g_RandomGenerator; + +#endif diff --git a/7zip/CPP/7zip/Crypto/RandGen.h b/7zip/CPP/7zip/Crypto/RandGen.h index cfdcd60d0..5122ec4b4 100644 --- a/7zip/CPP/7zip/Crypto/RandGen.h +++ b/7zip/CPP/7zip/Crypto/RandGen.h @@ -5,6 +5,21 @@ #include "../../../C/Sha256.h" +#ifdef _WIN64 +// #define USE_STATIC_SYSTEM_RAND +#endif + +#ifdef USE_STATIC_SYSTEM_RAND + +#ifdef _WIN32 +#include +#define MY_RAND_GEN(data, size) RtlGenRandom(data, size) +#else +#define MY_RAND_GEN(data, size) getrandom(data, size, 0) +#endif + +#else + class CRandomGenerator { Byte _buff[SHA256_DIGEST_SIZE]; @@ -18,4 +33,8 @@ class CRandomGenerator extern CRandomGenerator g_RandomGenerator; +#define MY_RAND_GEN(data, size) g_RandomGenerator.Generate(data, size) + +#endif + #endif diff --git a/7zip/CPP/7zip/Crypto/Rar5Aes.cpp b/7zip/CPP/7zip/Crypto/Rar5Aes.cpp index dc1f4ce6f..005fa6bcb 100644 --- a/7zip/CPP/7zip/Crypto/Rar5Aes.cpp +++ b/7zip/CPP/7zip/Crypto/Rar5Aes.cpp @@ -30,14 +30,13 @@ CDecoder::CDecoder(): CAesCbcDecoder(kAesKeySize) {} static unsigned ReadVarInt(const Byte *p, unsigned maxSize, UInt64 *val) { - unsigned i; *val = 0; - for (i = 0; i < maxSize;) + for (unsigned i = 0; i < maxSize && i < 10;) { Byte b = p[i]; - if (i < 10) - *val |= (UInt64)(b & 0x7F) << (7 * i++); + *val |= (UInt64)(b & 0x7F) << (7 * i); + i++; if ((b & 0x80) == 0) return i; } diff --git a/7zip/CPP/7zip/Crypto/WzAes.cpp b/7zip/CPP/7zip/Crypto/WzAes.cpp index 4572f06e1..d415ab846 100644 --- a/7zip/CPP/7zip/Crypto/WzAes.cpp +++ b/7zip/CPP/7zip/Crypto/WzAes.cpp @@ -96,7 +96,7 @@ STDMETHODIMP CBaseCoder::Init() HRESULT CEncoder::WriteHeader(ISequentialOutStream *outStream) { unsigned saltSize = _key.GetSaltSize(); - g_RandomGenerator.Generate(_key.Salt, saltSize); + MY_RAND_GEN(_key.Salt, saltSize); Init2(); RINOK(WriteStream(outStream, _key.Salt, saltSize)); return WriteStream(outStream, _key.PwdVerifComputed, kPwdVerifSize); diff --git a/7zip/CPP/7zip/Crypto/ZipCrypto.cpp b/7zip/CPP/7zip/Crypto/ZipCrypto.cpp index ae715063a..8610297a6 100644 --- a/7zip/CPP/7zip/Crypto/ZipCrypto.cpp +++ b/7zip/CPP/7zip/Crypto/ZipCrypto.cpp @@ -49,7 +49,7 @@ HRESULT CEncoder::WriteHeader_Check16(ISequentialOutStream *outStream, UInt16 cr PKZIP 2.0+ used 1 byte CRC check. It's more secure. We also use 1 byte CRC. */ - g_RandomGenerator.Generate(h, kHeaderSize - 1); + MY_RAND_GEN(h, kHeaderSize - 1); // h[kHeaderSize - 2] = (Byte)(crc); h[kHeaderSize - 1] = (Byte)(crc >> 8); diff --git a/7zip/CPP/7zip/Crypto/ZipStrong.cpp b/7zip/CPP/7zip/Crypto/ZipStrong.cpp index 2a923bf5d..22a905cb9 100644 --- a/7zip/CPP/7zip/Crypto/ZipStrong.cpp +++ b/7zip/CPP/7zip/Crypto/ZipStrong.cpp @@ -85,13 +85,14 @@ HRESULT CDecoder::ReadHeader(ISequentialInStream *inStream, UInt32 crc, UInt64 u return E_NOTIMPL; RINOK(ReadStream_FALSE(inStream, temp, 4)); _remSize = GetUi32(temp); - const UInt32 kAlign = 16; + // const UInt32 kAlign = 16; if (_remSize < 16 || _remSize > (1 << 18)) return E_NOTIMPL; - if (_remSize + kAlign > _buf.Size()) + if (_remSize > _bufAligned.Size()) { - _buf.Alloc(_remSize + kAlign); - _bufAligned = (Byte *)((ptrdiff_t)((Byte *)_buf + kAlign - 1) & ~(ptrdiff_t)(kAlign - 1)); + _bufAligned.AllocAtLeast(_remSize); + if (!(Byte *)_bufAligned) + return E_OUTOFMEMORY; } return ReadStream_FALSE(inStream, _bufAligned, _remSize); } diff --git a/7zip/CPP/7zip/Crypto/ZipStrong.h b/7zip/CPP/7zip/Crypto/ZipStrong.h index 8e3c74e24..03a0f2c7d 100644 --- a/7zip/CPP/7zip/Crypto/ZipStrong.h +++ b/7zip/CPP/7zip/Crypto/ZipStrong.h @@ -3,7 +3,7 @@ #ifndef __CRYPTO_ZIP_STRONG_H #define __CRYPTO_ZIP_STRONG_H -#include "../../Common/MyBuffer.h" +#include "../../Common/MyBuffer2.h" #include "../IPassword.h" @@ -35,8 +35,7 @@ class CBaseCoder: { protected: CKeyInfo _key; - CByteBuffer _buf; - Byte *_bufAligned; + CAlignedBuffer _bufAligned; public: STDMETHOD(Init)(); STDMETHOD(CryptoSetPassword)(const Byte *data, UInt32 size); diff --git a/7zip/CPP/7zip/GuiCommon.rc b/7zip/CPP/7zip/GuiCommon.rc index 6b26ddd85..b67409b92 100644 --- a/7zip/CPP/7zip/GuiCommon.rc +++ b/7zip/CPP/7zip/GuiCommon.rc @@ -71,6 +71,9 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US DEFPUSHBUTTON "OK", IDOK, bx2, by, bxs, bys \ PUSHBUTTON "Cancel", IDCANCEL, bx1, by, bxs, bys +#define MY_BUTTON__CLOSE \ + DEFPUSHBUTTON "&Close", IDCLOSE, bx1, by, bxs, bys + #define MY_COMBO CBS_DROPDOWNLIST | WS_VSCROLL | WS_TABSTOP #define MY_COMBO_SORTED MY_COMBO | CBS_SORT diff --git a/7zip/CPP/7zip/Guid.txt b/7zip/CPP/7zip/Guid.txt index 47da1a393..c9da7ed7b 100644 --- a/7zip/CPP/7zip/Guid.txt +++ b/7zip/CPP/7zip/Guid.txt @@ -49,6 +49,7 @@ 25 ICompressSetCoderMt 26 ICompressSetFinishMode 27 ICompressGetInStreamProcessedSize2 + 28 ICompressSetMemLimit 30 ICompressGetSubStreamSize 31 ICompressSetInStream @@ -165,6 +166,7 @@ Handler GUIDs: 0C xz 0D ppmd + C6 COFF C7 Ext C8 VMDK C9 VDI diff --git a/7zip/CPP/7zip/ICoder.h b/7zip/CPP/7zip/ICoder.h index dc9acd830..677d8cfb6 100644 --- a/7zip/CPP/7zip/ICoder.h +++ b/7zip/CPP/7zip/ICoder.h @@ -43,6 +43,7 @@ CODER_INTERFACE(ICompressCoder2, 0x18) S_OK : OK S_FALSE : data error (for decoders) E_OUTOFMEMORY : memory allocation error + E_NOTIMPL : unsupported encoding method (for decoders) another error code : some error. For example, it can be error code received from inStream or outStream function. Parameters: @@ -129,7 +130,8 @@ namespace NCoderPropID kBlockSize2, // VT_UI4 or VT_UI8 kCheckSize, // VT_UI4 : size of digest in bytes - kFilter // VT_BSTR + kFilter, // VT_BSTR + kMemUse // VT_UI8 }; } @@ -190,6 +192,12 @@ CODER_INTERFACE(ICompressGetInStreamProcessedSize2, 0x27) STDMETHOD(GetInStreamProcessedSize2)(UInt32 streamIndex, UInt64 *value) PURE; }; +CODER_INTERFACE(ICompressSetMemLimit, 0x28) +{ + STDMETHOD(SetMemLimit)(UInt64 memUsage) PURE; +}; + + CODER_INTERFACE(ICompressGetSubStreamSize, 0x30) { diff --git a/7zip/CPP/7zip/LzmaDec.mak b/7zip/CPP/7zip/LzmaDec.mak new file mode 100644 index 000000000..8d7f6f983 --- /dev/null +++ b/7zip/CPP/7zip/LzmaDec.mak @@ -0,0 +1,5 @@ +!IF "$(PLATFORM)" == "x64" +CFLAGS_C_SPEC = -D_LZMA_DEC_OPT +ASM_OBJS = $(ASM_OBJS) \ + $O\LzmaDecOpt.obj +!ENDIF diff --git a/7zip/CPP/7zip/UI/Agent/Agent.h b/7zip/CPP/7zip/UI/Agent/Agent.h index 7ba96f4a3..f7d7b5e4f 100644 --- a/7zip/CPP/7zip/UI/Agent/Agent.h +++ b/7zip/CPP/7zip/UI/Agent/Agent.h @@ -253,8 +253,8 @@ class CAgent: IInArchive *GetArchive() const { if ( _archiveLink.Arcs.IsEmpty()) return 0; return GetArc().Archive; } bool CanUpdate() const; - bool Is_Attrib_ReadOnly() const - { + bool Is_Attrib_ReadOnly() const + { return _attrib != INVALID_FILE_ATTRIBUTES && (_attrib & FILE_ATTRIBUTE_READONLY); } diff --git a/7zip/CPP/7zip/UI/Common/ArchiveCommandLine.cpp b/7zip/CPP/7zip/UI/Common/ArchiveCommandLine.cpp index 56f4fbd80..0e2a4700d 100644 --- a/7zip/CPP/7zip/UI/Common/ArchiveCommandLine.cpp +++ b/7zip/CPP/7zip/UI/Common/ArchiveCommandLine.cpp @@ -8,29 +8,42 @@ #ifndef UNDER_CE #include #endif +#else +// for isatty() +#include #endif + #include +#ifdef _7ZIP_LARGE_PAGES +#include "../../../../C/Alloc.h" +#endif + #include "../../../Common/ListFileUtils.h" #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" +#include "../../../Windows/ErrorMsg.h" #include "../../../Windows/FileDir.h" #include "../../../Windows/FileName.h" #ifdef _WIN32 #include "../../../Windows/FileMapping.h" +#include "../../../Windows/MemoryLock.h" #include "../../../Windows/Synchronization.h" #endif #include "ArchiveCommandLine.h" #include "EnumDirItems.h" -#include "SortUtils.h" #include "Update.h" #include "UpdateAction.h" extern bool g_CaseSensitive; extern bool g_PathTrailReplaceMode; +#ifdef _7ZIP_LARGE_PAGES +bool g_LargePagesMode = false; +#endif + #ifdef UNDER_CE #define MY_IS_TERMINAL(x) false; @@ -60,15 +73,6 @@ static bool StringToUInt32(const wchar_t *s, UInt32 &v) return *end == 0; } -CArcCmdLineException::CArcCmdLineException(const char *a, const wchar_t *u) -{ - (*this) += a; - if (u) - { - Add_LF(); - (*this) += u; - } -} int g_CodePage = -1; @@ -240,7 +244,7 @@ static const CSwitchForm kSwitchForms[] = { "si", NSwitchType::kString }, { "so" }, - { "slp", NSwitchType::kMinus }, + { "slp", NSwitchType::kString }, { "scs", NSwitchType::kString }, { "scc", NSwitchType::kString }, { "slt" }, @@ -282,7 +286,6 @@ static const char * const kIncorrectListFile = "Incorrect item in listfile.\nChe static const char * const kTerminalOutError = "I won't write compressed data to a terminal"; static const char * const kSameTerminalError = "I won't write data and program's messages to same stream"; static const char * const kEmptyFilePath = "Empty file path"; -static const char * const kCannotFindArchive = "Cannot find archive"; bool CArcCommand::IsFromExtractGroup() const { @@ -410,8 +413,19 @@ static void AddToCensorFromListFile( UStringVector names; if (!NFind::DoesFileExist(us2fs(fileName))) throw CArcCmdLineException(kCannotFindListFile, fileName); - if (!ReadNamesFromListFile(us2fs(fileName), names, codePage)) + DWORD lastError = 0; + if (!ReadNamesFromListFile2(us2fs(fileName), names, codePage, lastError)) + { + if (lastError != 0) + { + UString m; + m = "The file operation error for listfile"; + m.Add_LF(); + m += NError::MyFormatMessage(lastError); + throw CArcCmdLineException(m, fileName); + } throw CArcCmdLineException(kIncorrectListFile, fileName); + } if (renamePairs) { if ((names.Size() & 1) != 0) @@ -615,84 +629,6 @@ static void AddSwitchWildcardsToCensor( throw CArcCmdLineException(errorMessage, strings[i]); } -#ifdef _WIN32 - -// This code converts all short file names to long file names. - -static void ConvertToLongName(const UString &prefix, UString &name) -{ - if (name.IsEmpty() || DoesNameContainWildcard(name)) - return; - NFind::CFileInfo fi; - const FString path (us2fs(prefix + name)); - #ifndef UNDER_CE - if (NFile::NName::IsDevicePath(path)) - return; - #endif - if (fi.Find(path)) - name = fs2us(fi.Name); -} - -static void ConvertToLongNames(const UString &prefix, CObjectVector &items) -{ - FOR_VECTOR (i, items) - { - NWildcard::CItem &item = items[i]; - if (item.Recursive || item.PathParts.Size() != 1) - continue; - if (prefix.IsEmpty() && item.IsDriveItem()) - continue; - ConvertToLongName(prefix, item.PathParts.Front()); - } -} - -static void ConvertToLongNames(const UString &prefix, NWildcard::CCensorNode &node) -{ - ConvertToLongNames(prefix, node.IncludeItems); - ConvertToLongNames(prefix, node.ExcludeItems); - unsigned i; - for (i = 0; i < node.SubNodes.Size(); i++) - { - UString &name = node.SubNodes[i].Name; - if (prefix.IsEmpty() && NWildcard::IsDriveColonName(name)) - continue; - ConvertToLongName(prefix, name); - } - // mix folders with same name - for (i = 0; i < node.SubNodes.Size(); i++) - { - NWildcard::CCensorNode &nextNode1 = node.SubNodes[i]; - for (unsigned j = i + 1; j < node.SubNodes.Size();) - { - const NWildcard::CCensorNode &nextNode2 = node.SubNodes[j]; - if (nextNode1.Name.IsEqualTo_NoCase(nextNode2.Name)) - { - nextNode1.IncludeItems += nextNode2.IncludeItems; - nextNode1.ExcludeItems += nextNode2.ExcludeItems; - node.SubNodes.Delete(j); - } - else - j++; - } - } - for (i = 0; i < node.SubNodes.Size(); i++) - { - NWildcard::CCensorNode &nextNode = node.SubNodes[i]; - ConvertToLongNames(prefix + nextNode.Name + WCHAR_PATH_SEPARATOR, nextNode); - } -} - -void ConvertToLongNames(NWildcard::CCensor &censor) -{ - FOR_VECTOR (i, censor.Pairs) - { - NWildcard::CPair &pair = censor.Pairs[i]; - ConvertToLongNames(pair.Prefix, pair.Head); - } -} - -#endif - /* static NUpdateArchive::NPairAction::EEnum GetUpdatePairActionType(int i) { @@ -922,9 +858,45 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, options.CaseSensitive = g_CaseSensitive; } - options.LargePages = false; + + #if defined(_WIN32) && !defined(UNDER_CE) + NSecurity::EnablePrivilege_SymLink(); + #endif + + // options.LargePages = false; + if (parser[NKey::kLargePages].ThereIs) - options.LargePages = !parser[NKey::kLargePages].WithMinus; + { + unsigned slp = 0; + const UString &s = parser[NKey::kLargePages].PostStrings[0]; + if (s.IsEmpty()) + slp = 1; + else if (s != L"-") + { + if (!StringToUInt32(s, slp)) + throw CArcCmdLineException("Unsupported switch postfix for -slp", s); + } + + #ifdef _7ZIP_LARGE_PAGES + if (slp > + #ifndef UNDER_CE + (unsigned)NSecurity::Get_LargePages_RiskLevel() + #else + 0 + #endif + ) + { + SetLargePageSize(); + // note: this process also can inherit that Privilege from parent process + g_LargePagesMode = + #if defined(_WIN32) && !defined(UNDER_CE) + NSecurity::EnablePrivilege_LockMemory(); + #else + true; + #endif + } + #endif + } #ifndef UNDER_CE @@ -996,64 +968,6 @@ static Int32 FindCharset(const NCommandLineParser::CParser &parser, unsigned key } } -HRESULT EnumerateDirItemsAndSort( - NWildcard::CCensor &censor, - NWildcard::ECensorPathMode censorPathMode, - const UString &addPathPrefix, - UStringVector &sortedPaths, - UStringVector &sortedFullPaths, - CDirItemsStat &st, - IDirItemsCallback *callback) -{ - FStringVector paths; - - { - CDirItems dirItems; - dirItems.Callback = callback; - { - HRESULT res = EnumerateItems(censor, censorPathMode, addPathPrefix, dirItems); - st = dirItems.Stat; - RINOK(res); - } - - FOR_VECTOR (i, dirItems.Items) - { - const CDirItem &dirItem = dirItems.Items[i]; - if (!dirItem.IsDir()) - paths.Add(dirItems.GetPhyPath(i)); - } - } - - if (paths.Size() == 0) - throw CArcCmdLineException(kCannotFindArchive); - - UStringVector fullPaths; - - unsigned i; - - for (i = 0; i < paths.Size(); i++) - { - FString fullPath; - NFile::NDir::MyGetFullPathName(paths[i], fullPath); - fullPaths.Add(fs2us(fullPath)); - } - - CUIntVector indices; - SortFileNames(fullPaths, indices); - sortedPaths.ClearAndReserve(indices.Size()); - sortedFullPaths.ClearAndReserve(indices.Size()); - - for (i = 0; i < indices.Size(); i++) - { - unsigned index = indices[i]; - sortedPaths.AddInReserved(fs2us(paths[index])); - sortedFullPaths.AddInReserved(fullPaths[index]); - if (i > 0 && CompareFileNames(sortedFullPaths[i], sortedFullPaths[i - 1]) == 0) - throw CArcCmdLineException("Duplicate archive path:", sortedFullPaths[i]); - } - - return S_OK; -} static void SetBoolPair(NCommandLineParser::CParser &parser, unsigned switchID, CBoolPair &bp) { diff --git a/7zip/CPP/7zip/UI/Common/ArchiveCommandLine.h b/7zip/CPP/7zip/UI/Common/ArchiveCommandLine.h index 1c96601a1..9ed0825fa 100644 --- a/7zip/CPP/7zip/UI/Common/ArchiveCommandLine.h +++ b/7zip/CPP/7zip/UI/Common/ArchiveCommandLine.h @@ -6,14 +6,13 @@ #include "../../../Common/CommandLineParser.h" #include "../../../Common/Wildcard.h" +#include "EnumDirItems.h" + #include "Extract.h" #include "HashCalc.h" #include "Update.h" -struct CArcCmdLineException: public UString -{ - CArcCmdLineException(const char *a, const wchar_t *u = NULL); -}; +typedef CMessagePathException CArcCmdLineException; namespace NCommandType { enum EEnum { @@ -51,7 +50,7 @@ struct CArcCmdLineOptions { bool HelpMode; - bool LargePages; + // bool LargePages; bool CaseSensitiveChange; bool CaseSensitive; @@ -110,7 +109,7 @@ struct CArcCmdLineOptions UInt32 NumIterations; CArcCmdLineOptions(): - LargePages(false), + // LargePages(false), CaseSensitiveChange(false), CaseSensitive(false), @@ -134,13 +133,4 @@ class CArcCmdLineParser void Parse2(CArcCmdLineOptions &options); }; -HRESULT EnumerateDirItemsAndSort( - NWildcard::CCensor &censor, - NWildcard::ECensorPathMode pathMode, - const UString &addPathPrefix, - UStringVector &sortedPaths, - UStringVector &sortedFullPaths, - CDirItemsStat &st, - IDirItemsCallback *callback); - #endif diff --git a/7zip/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp b/7zip/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp index 083e7c849..de2f2e007 100644 --- a/7zip/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp +++ b/7zip/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp @@ -1182,7 +1182,9 @@ if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode) bool needDelete = true; if (needDelete) { + if (NFind::DoesFileExist(fullProcessedPath)) if (!DeleteFileAlways(fullProcessedPath)) + if (GetLastError() != ERROR_FILE_NOT_FOUND) { RINOK(SendMessageError_with_LastError(kCantDeleteOutputFile, fullProcessedPath)); return S_OK; @@ -1368,13 +1370,35 @@ if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode) // UInt64 ticks = GetCpuTicks(); bool res = _outFileStreamSpec->File.SetLength(_curSize); _fileLengthWasSet = res; - _outFileStreamSpec->File.SeekToBegin(); + // ticks = GetCpuTicks() - ticks; // printf("\nticks = %10d\n", (unsigned)ticks); if (!res) { RINOK(SendMessageError_with_LastError(kCantSetFileLen, fullProcessedPath)); } + + /* + _outFileStreamSpec->File.Close(); + ticks = GetCpuTicks() - ticks; + printf("\nticks = %10d\n", (unsigned)ticks); + return S_FALSE; + */ + + /* + File.SetLength() on FAT (xp64): is fast, but then File.Close() can be slow, + if we don't write any data. + File.SetLength() for remote share file (exFAT) can be slow in some cases, + and the Windows can return "network error" after 1 minute, + while remote file still can grow. + We need some way to detect such bad cases and disable PreAllocateOutFile mode. + */ + + res = _outFileStreamSpec->File.SeekToBegin(); + if (!res) + { + RINOK(SendMessageError_with_LastError("Can not seek to begin of file", fullProcessedPath)); + } } #ifdef SUPPORT_ALT_STREAMS diff --git a/7zip/CPP/7zip/UI/Common/ArchiveName.cpp b/7zip/CPP/7zip/UI/Common/ArchiveName.cpp index a3a44ce2e..9048edcef 100644 --- a/7zip/CPP/7zip/UI/Common/ArchiveName.cpp +++ b/7zip/CPP/7zip/UI/Common/ArchiveName.cpp @@ -2,6 +2,8 @@ #include "StdAfx.h" +#include "../../../Common/Wildcard.h" + #include "../../../Windows/FileDir.h" #include "../../../Windows/FileName.h" @@ -11,7 +13,7 @@ using namespace NWindows; using namespace NFile; -UString CreateArchiveName(const NFind::CFileInfo &fi, bool keepName) +static UString CreateArchiveName(const NFind::CFileInfo &fi, bool keepName) { FString resultName = fi.Name; if (!fi.IsDir() && !keepName) @@ -72,7 +74,82 @@ static FString CreateArchiveName2(const FString &path, bool fromPrev, bool keepN return resultName; } -UString CreateArchiveName(const UString &path, bool fromPrev, bool keepName) + +UString CreateArchiveName(const UStringVector &paths, const NFind::CFileInfo *fi) { - return Get_Correct_FsFile_Name(fs2us(CreateArchiveName2(us2fs(path), fromPrev, keepName))); + bool keepName = false; + /* + if (paths.Size() == 1) + { + const UString &name = paths[0]; + if (name.Len() > 4) + if (CompareFileNames(name.RightPtr(4), L".tar") == 0) + keepName = true; + } + */ + + UString name; + if (fi) + name = CreateArchiveName(*fi, keepName); + else + { + if (paths.IsEmpty()) + return L"archive"; + bool fromPrev = (paths.Size() > 1); + name = Get_Correct_FsFile_Name(fs2us(CreateArchiveName2(us2fs(paths.Front()), fromPrev, keepName))); + } + + UStringVector names; + + { + FOR_VECTOR (i, paths) + { + NFind::CFileInfo fi2; + const NFind::CFileInfo *fp; + if (fi && paths.Size() == 1) + fp = fi; + else + { + if (!fi2.Find(us2fs(paths[i]))) + continue; + fp = &fi2; + } + names.Add(fs2us(fp->Name)); + } + } + + UString postfix; + UInt32 index = 1; + + for (;;) + { + // we don't want cases when we include archive to itself. + // so we find first available name for archive + const UString name2 = name + postfix; + const UString name2_zip = name2 + L".zip"; + const UString name2_7z = name2 + L".7z"; + const UString name2_tar = name2 + L".tar"; + const UString name2_wim = name2 + L".wim"; + + unsigned i = 0; + + for (i = 0; i < names.Size(); i++) + { + const UString &fname = names[i]; + if ( 0 == CompareFileNames(fname, name2_zip) + || 0 == CompareFileNames(fname, name2_7z) + || 0 == CompareFileNames(fname, name2_tar) + || 0 == CompareFileNames(fname, name2_wim)) + break; + } + + if (i == names.Size()) + break; + index++; + postfix = "_"; + postfix.Add_UInt32(index); + } + + name += postfix; + return name; } diff --git a/7zip/CPP/7zip/UI/Common/ArchiveName.h b/7zip/CPP/7zip/UI/Common/ArchiveName.h index ec2f1e8d1..0d32645f4 100644 --- a/7zip/CPP/7zip/UI/Common/ArchiveName.h +++ b/7zip/CPP/7zip/UI/Common/ArchiveName.h @@ -3,11 +3,8 @@ #ifndef __ARCHIVE_NAME_H #define __ARCHIVE_NAME_H -#include "../../../Common/MyString.h" - #include "../../../Windows/FileFind.h" -UString CreateArchiveName(const UString &path, bool fromPrev, bool keepName); -UString CreateArchiveName(const NWindows::NFile::NFind::CFileInfo &fileInfo, bool keepName); +UString CreateArchiveName(const UStringVector &paths, const NWindows::NFile::NFind::CFileInfo *fi = NULL); #endif diff --git a/7zip/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp b/7zip/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp index fd2807ac1..4d9d9e148 100644 --- a/7zip/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp +++ b/7zip/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp @@ -102,7 +102,14 @@ STDMETHODIMP COpenCallbackImp::GetStream(const wchar_t *name, IInStream **inStre // if (!allowAbsVolPaths) if (!IsSafePath(name2)) return S_FALSE; - + + // #ifdef _WIN32 + // we don't want to support wildcards in names here here + if (name2.Find(L'?') >= 0 || + name2.Find(L'*') >= 0) + return S_FALSE; + // #endif + #endif diff --git a/7zip/CPP/7zip/UI/Common/Bench.cpp b/7zip/CPP/7zip/UI/Common/Bench.cpp index d0fead760..f849a8770 100644 --- a/7zip/CPP/7zip/UI/Common/Bench.cpp +++ b/7zip/CPP/7zip/UI/Common/Bench.cpp @@ -29,7 +29,6 @@ #endif #include "../../../../C/7zCrc.h" -#include "../../../../C/Alloc.h" #include "../../../../C/CpuArch.h" #ifndef _7ZIP_ST @@ -47,6 +46,7 @@ #include "../../../Common/IntToString.h" +#include "../../../Common/MyBuffer2.h" #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" @@ -94,80 +94,33 @@ static const UInt32 kAdditionalSize = (1 << 16); static const UInt32 kCompressedAdditionalSize = (1 << 10); static const UInt32 kMaxLzmaPropSize = 5; + + +#define ALLOC_WITH_HRESULT(_buffer_, _size_) \ + (_buffer_)->Alloc(_size_); \ + if (!(_buffer_)->IsAllocated()) return E_OUTOFMEMORY; + + class CBaseRandomGenerator { UInt32 A1; UInt32 A2; + UInt32 Salt; public: - CBaseRandomGenerator() { Init(); } + CBaseRandomGenerator(UInt32 salt = 0): Salt(salt) { Init(); } void Init() { A1 = 362436069; A2 = 521288629;} UInt32 GetRnd() { - return + return Salt ^ + ( ((A1 = 36969 * (A1 & 0xffff) + (A1 >> 16)) << 16) + - ((A2 = 18000 * (A2 & 0xffff) + (A2 >> 16)) ); - } -}; - - -static const unsigned kBufferAlignment = 1 << 4; - -struct CBenchBuffer -{ - size_t BufferSize; - - #ifdef _WIN32 - - Byte *Buffer; - - CBenchBuffer(): BufferSize(0), Buffer(NULL) {} - ~CBenchBuffer() { ::MidFree(Buffer); } - - void AllocAlignedMask(size_t size, size_t) - { - ::MidFree(Buffer); - BufferSize = 0; - Buffer = (Byte *)::MidAlloc(size); - if (Buffer) - BufferSize = size; - } - - #else - - Byte *Buffer; - Byte *_bufBase; - - CBenchBuffer(): BufferSize(0), Buffer(NULL), _bufBase(NULL){} - ~CBenchBuffer() { ::MidFree(_bufBase); } - - void AllocAlignedMask(size_t size, size_t alignMask) - { - ::MidFree(_bufBase); - Buffer = NULL; - BufferSize = 0; - _bufBase = (Byte *)::MidAlloc(size + alignMask); - - if (_bufBase) - { - // Buffer = (Byte *)(((uintptr_t)_bufBase + alignMask) & ~(uintptr_t)alignMask); - Buffer = (Byte *)(((ptrdiff_t)_bufBase + alignMask) & ~(ptrdiff_t)alignMask); - BufferSize = size; - } - } - - #endif - - bool Alloc(size_t size) - { - if (Buffer && BufferSize == size) - return true; - AllocAlignedMask(size, kBufferAlignment - 1); - return (Buffer != NULL || size == 0); + ((A2 = 18000 * (A2 & 0xffff) + (A2 >> 16)) ) + ); } }; -class CBenchRandomGenerator: public CBenchBuffer +class CBenchRandomGenerator: public CAlignedBuffer { static UInt32 GetVal(UInt32 &res, unsigned numBits) { @@ -184,23 +137,22 @@ class CBenchRandomGenerator: public CBenchBuffer public: - void GenerateSimpleRandom(CBaseRandomGenerator *_RG_) + void GenerateSimpleRandom(UInt32 salt) { - CBaseRandomGenerator rg = *_RG_; - const size_t bufSize = BufferSize; - Byte *buf = Buffer; + CBaseRandomGenerator rg(salt); + const size_t bufSize = Size(); + Byte *buf = (Byte *)*this; for (size_t i = 0; i < bufSize; i++) buf[i] = (Byte)rg.GetRnd(); - *_RG_ = rg; } - void GenerateLz(unsigned dictBits, CBaseRandomGenerator *_RG_) + void GenerateLz(unsigned dictBits, UInt32 salt) { - CBaseRandomGenerator rg = *_RG_; + CBaseRandomGenerator rg(salt); UInt32 pos = 0; UInt32 rep0 = 1; - const size_t bufSize = BufferSize; - Byte *buf = Buffer; + const size_t bufSize = Size(); + Byte *buf = (Byte *)*this; unsigned posBits = 1; while (pos < bufSize) @@ -255,8 +207,6 @@ class CBenchRandomGenerator: public CBenchBuffer *dest++ = *src++; } } - - *_RG_ = rg; } }; @@ -297,7 +247,7 @@ STDMETHODIMP CBenchmarkInStream::Read(void *data, UInt32 size, UInt32 *processed class CBenchmarkOutStream: public ISequentialOutStream, - public CBenchBuffer, + public CAlignedBuffer, public CMyUnknownImp { // bool _overflow; @@ -325,13 +275,13 @@ class CBenchmarkOutStream: STDMETHODIMP CBenchmarkOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize) { - size_t curSize = BufferSize - Pos; + size_t curSize = Size() - Pos; if (curSize > size) curSize = size; if (curSize != 0) { if (RealCopy) - memcpy(Buffer + Pos, data, curSize); + memcpy(((Byte *)*this) + Pos, data, curSize); if (CalcCrc) Crc = CrcUpdate(Crc, data, curSize); Pos += curSize; @@ -522,10 +472,9 @@ class CBenchProgressInfo: { public: CBenchProgressStatus *Status; - HRESULT Res; IBenchCallback *Callback; - CBenchProgressInfo(): Callback(0) {} + CBenchProgressInfo(): Callback(NULL) {} MY_UNKNOWN_IMP STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize); }; @@ -687,20 +636,39 @@ UInt64 GetDecompressRating(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt return props.GetDecompressRating(elapsedTime, freq, outSize, inSize, numIterations); } + + + +#ifndef _7ZIP_ST +struct CBenchSyncCommon +{ + bool ExitMode; + NSynchronization::CManualResetEvent StartEvent; + + CBenchSyncCommon(): ExitMode(false) {} +}; +#endif + + struct CEncoderInfo; struct CEncoderInfo { #ifndef _7ZIP_ST NWindows::CThread thread[2]; + NSynchronization::CManualResetEvent ReadyEvent; UInt32 NumDecoderSubThreads; + CBenchSyncCommon *Common; #endif + CMyComPtr _encoder; CMyComPtr _encoderFilter; CBenchProgressInfo *progressInfoSpec[2]; CMyComPtr progressInfo[2]; UInt64 NumIterations; + UInt32 Salt; + #ifdef USE_ALLOCA size_t AllocaSize; #endif @@ -739,26 +707,29 @@ struct CEncoderInfo const Byte *fileData; CBenchRandomGenerator rg; - CBenchBuffer rgCopy; // it must be 16-byte aligned !!! + CAlignedBuffer rgCopy; // it must be 16-byte aligned !!! CBenchmarkOutStream *propStreamSpec; CMyComPtr propStream; - // for decode + unsigned generateDictBits; COneMethodInfo _method; + + // for decode size_t _uncompressedDataSize; - HRESULT Init( - const COneMethodInfo &method, - unsigned generateDictBits, - CBaseRandomGenerator *rg); + HRESULT Generate(); HRESULT Encode(); HRESULT Decode(UInt32 decoderIndex); CEncoderInfo(): + #ifndef _7ZIP_ST + Common(NULL), + #endif + Salt(0), fileData(NULL), CheckCrc_Enc(true), CheckCrc_Dec(true), - outStreamSpec(0), callback(0), printCallback(0), propStreamSpec(0) {} + outStreamSpec(NULL), callback(NULL), printCallback(NULL), propStreamSpec(NULL) {} #ifndef _7ZIP_ST @@ -773,14 +744,15 @@ struct CEncoderInfo #endif res = encoder->Encode(); - encoder->Results[0] = res; } catch(...) { res = E_FAIL; } + encoder->Results[0] = res; if (res != S_OK) encoder->progressInfoSpec[0]->Status->SetResult(res); + encoder->ReadyEvent.Set(); return 0; } @@ -799,7 +771,12 @@ struct CEncoderInfo HRESULT CreateEncoderThread() { - return thread[0].Create(EncodeThreadFunction, this); + WRes res = 0; + if (!ReadyEvent.IsCreated()) + res = ReadyEvent.Create(); + if (res == 0) + res = thread[0].Create(EncodeThreadFunction, this); + return HRESULT_FROM_WIN32(res); } HRESULT CreateDecoderThread(unsigned index, bool callbackMode @@ -824,11 +801,10 @@ struct CEncoderInfo }; -HRESULT CEncoderInfo::Init( - const COneMethodInfo &method, - unsigned generateDictBits, - CBaseRandomGenerator *rgLoc) +HRESULT CEncoderInfo::Generate() { + const COneMethodInfo &method = _method; + // we need extra space, if input data is already compressed const size_t kCompressedBufferSize = kCompressedAdditionalSize + @@ -842,40 +818,39 @@ HRESULT CEncoderInfo::Init( if (!fileData) { - if (!rg.Alloc(kBufferSize)) - return E_OUTOFMEMORY; + ALLOC_WITH_HRESULT(&rg, kBufferSize); // DWORD ttt = GetTickCount(); if (generateDictBits == 0) - rg.GenerateSimpleRandom(rgLoc); + rg.GenerateSimpleRandom(Salt); else - rg.GenerateLz(generateDictBits, rgLoc); + rg.GenerateLz(generateDictBits, Salt); // printf("\n%d\n ", GetTickCount() - ttt); - crc = CrcCalc(rg.Buffer, rg.BufferSize); - uncompressedDataPtr = rg.Buffer; + crc = CrcCalc((const Byte *)rg, rg.Size()); + uncompressedDataPtr = (const Byte *)rg; } if (_encoderFilter) { - if (!rgCopy.Alloc(kBufferSize)) - return E_OUTOFMEMORY; + ALLOC_WITH_HRESULT(&rgCopy, kBufferSize); } - outStreamSpec = new CBenchmarkOutStream; - outStream = outStreamSpec; - if (!outStreamSpec->Alloc(kCompressedBufferSize)) - return E_OUTOFMEMORY; + if (!outStream) + { + outStreamSpec = new CBenchmarkOutStream; + outStream = outStreamSpec; + } + + ALLOC_WITH_HRESULT(outStreamSpec, kCompressedBufferSize) - propStreamSpec = 0; if (!propStream) { propStreamSpec = new CBenchmarkOutStream; propStream = propStreamSpec; } - if (!propStreamSpec->Alloc(kMaxLzmaPropSize)) - return E_OUTOFMEMORY; + ALLOC_WITH_HRESULT(propStreamSpec, kMaxLzmaPropSize); propStreamSpec->Init(true, false); @@ -962,6 +937,28 @@ static void My_FilterBench(ICompressFilter *filter, Byte *data, size_t size) HRESULT CEncoderInfo::Encode() { + RINOK(Generate()); + + #ifndef _7ZIP_ST + if (Common) + { + Results[0] = S_OK; + WRes wres = ReadyEvent.Set(); + if (wres == 0) + wres = Common->StartEvent.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + if (Common->ExitMode) + return S_OK; + } + else + #endif + { + CBenchProgressInfo *bpi = progressInfoSpec[0]; + bpi->SetStartTime(); + } + + CBenchInfo &bi = progressInfoSpec[0]->BenchInfo; bi.UnpackSize = 0; bi.PackSize = 0; @@ -998,10 +995,10 @@ HRESULT CEncoderInfo::Encode() if (_encoderFilter) { - memcpy(rgCopy.Buffer, uncompressedDataPtr, kBufferSize); + memcpy((Byte *)rgCopy, uncompressedDataPtr, kBufferSize); _encoderFilter->Init(); - My_FilterBench(_encoderFilter, rgCopy.Buffer, kBufferSize); - RINOK(WriteStream(outStream, rgCopy.Buffer, kBufferSize)); + My_FilterBench(_encoderFilter, (Byte *)rgCopy, kBufferSize); + RINOK(WriteStream(outStream, (const Byte *)rgCopy, kBufferSize)); } else { @@ -1079,7 +1076,7 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) if (setDecProps) { - RINOK(setDecProps->SetDecoderProperties2(propStreamSpec->Buffer, (UInt32)propStreamSpec->Pos)); + RINOK(setDecProps->SetDecoderProperties2((const Byte *)*propStreamSpec, (UInt32)propStreamSpec->Pos)); } { @@ -1107,7 +1104,7 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) prev = pi->BenchInfo.UnpackSize; } - inStreamSpec->Init(outStreamSpec->Buffer, compressedSize); + inStreamSpec->Init((const Byte *)*outStreamSpec, compressedSize); crcOutStreamSpec->Init(); UInt64 outSize = kBufferSize; @@ -1115,12 +1112,12 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) if (_decoderFilter) { - if (compressedSize > rgCopy.BufferSize) + if (compressedSize > rgCopy.Size()) return E_FAIL; - memcpy(rgCopy.Buffer, outStreamSpec->Buffer, compressedSize); + memcpy((Byte *)rgCopy, (const Byte *)*outStreamSpec, compressedSize); _decoderFilter->Init(); - My_FilterBench(_decoderFilter, rgCopy.Buffer, compressedSize); - RINOK(WriteStream(crcOutStream, rgCopy.Buffer, compressedSize)); + My_FilterBench(_decoderFilter, (Byte *)rgCopy, compressedSize); + RINOK(WriteStream(crcOutStream, (const Byte *)rgCopy, compressedSize)); } else { @@ -1144,7 +1141,7 @@ static const UInt32 kNumThreadsMax = (1 << 12); struct CBenchEncoders { CEncoderInfo *encoders; - CBenchEncoders(UInt32 num): encoders(0) { encoders = new CEncoderInfo[num]; } + CBenchEncoders(UInt32 num): encoders(NULL) { encoders = new CEncoderInfo[num]; } ~CBenchEncoders() { delete []encoders; } }; @@ -1158,6 +1155,57 @@ static UInt64 GetNumIterations(UInt64 numCommands, UInt64 complexInCommands) } + +#ifndef _7ZIP_ST + +// ---------- CBenchThreadsFlusher ---------- + +struct CBenchThreadsFlusher +{ + CBenchEncoders *EncodersSpec; + CBenchSyncCommon Common; + unsigned NumThreads; + bool NeedClose; + + CBenchThreadsFlusher(): NumThreads(0), NeedClose(false) {} + + ~CBenchThreadsFlusher() + { + StartAndWait(true); + } + + WRes StartAndWait(bool exitMode = false); +}; + + +WRes CBenchThreadsFlusher::StartAndWait(bool exitMode) +{ + if (!NeedClose) + return 0; + + Common.ExitMode = exitMode; + WRes res = Common.StartEvent.Set(); + + for (unsigned i = 0; i < NumThreads; i++) + { + NWindows::CThread &t = EncodersSpec->encoders[i].thread[0]; + if (t.IsCreated()) + { + WRes res2 = t.Wait(); + if (res2 == 0) + res2 = t.Close(); + if (res == S_OK) + res = res2; + } + } + NeedClose = false; + return res; +} + +#endif + + + static HRESULT MethodBench( DECL_EXTERNAL_CODECS_LOC_VARS UInt64 complexInCommands, @@ -1183,9 +1231,11 @@ static HRESULT MethodBench( COneMethodInfo method = method2; UInt64 methodId; UInt32 numStreams; - if (!FindMethod( + int codecIndex = FindMethod_Index( EXTERNAL_CODECS_LOC_VARS - method.MethodName, methodId, numStreams)) + method.MethodName, true, + methodId, numStreams); + if (codecIndex < 0) return E_NOTIMPL; if (numStreams != 1) return E_INVALIDARG; @@ -1207,6 +1257,8 @@ static HRESULT MethodBench( numSubDecoderThreads = 2; } } + + bool mtEncMode = (numEncoderThreads > 1); #endif CBenchEncoders encodersSpec(numEncoderThreads); @@ -1222,7 +1274,7 @@ static HRESULT MethodBench( { CCreatedCoder cod; - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS methodId, true, encoder._encoderFilter, cod)); + RINOK(CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS codecIndex, true, encoder._encoderFilter, cod)); encoder._encoder = cod.Coder; if (!encoder._encoder && !encoder._encoderFilter) return E_NOTIMPL; @@ -1239,16 +1291,13 @@ static HRESULT MethodBench( { CCreatedCoder cod; CMyComPtr &decoder = encoder._decoders[j]; - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS methodId, false, encoder._decoderFilter, cod)); + RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS methodId, false, encoder._decoderFilter, cod)); decoder = cod.Coder; if (!encoder._decoderFilter && !decoder) return E_NOTIMPL; } } - CBaseRandomGenerator rg; - rg.Init(); - UInt32 crc = 0; if (fileData) crc = CrcCalc(fileData, uncompressedDataSize); @@ -1257,22 +1306,38 @@ static HRESULT MethodBench( { CEncoderInfo &encoder = encoders[i]; encoder._method = method; + encoder.generateDictBits = generateDictBits; encoder._uncompressedDataSize = uncompressedDataSize; encoder.kBufferSize = uncompressedDataSize; encoder.fileData = fileData; encoder.crc = crc; - - RINOK(encoders[i].Init(method, generateDictBits, &rg)); } CBenchProgressStatus status; status.Res = S_OK; status.EncodeMode = true; + #ifndef _7ZIP_ST + CBenchThreadsFlusher encoderFlusher; + if (mtEncMode) + { + WRes wres = encoderFlusher.Common.StartEvent.Create(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + encoderFlusher.NumThreads = numEncoderThreads; + encoderFlusher.EncodersSpec = &encodersSpec; + encoderFlusher.NeedClose = true; + } + #endif + for (i = 0; i < numEncoderThreads; i++) { CEncoderInfo &encoder = encoders[i]; encoder.NumIterations = GetNumIterations(benchProps->GeComprCommands(uncompressedDataSize), complexInCommands); + encoder.Salt = g_CrcTable[i & 0xFF]; + encoder.Salt ^= (g_CrcTable[(i >> 8) & 0xFF] << 3); + // (g_CrcTable[0] == 0), and (encoder.Salt == 0) for first thread + // printf(" %8x", encoder.Salt); for (int j = 0; j < 2; j++) { @@ -1287,30 +1352,50 @@ static HRESULT MethodBench( CBenchProgressInfo *bpi = encoder.progressInfoSpec[0]; bpi->Callback = callback; bpi->BenchInfo.NumIterations = numEncoderThreads; - bpi->SetStartTime(); } #ifndef _7ZIP_ST - if (numEncoderThreads > 1) + if (mtEncMode) { #ifdef USE_ALLOCA encoder.AllocaSize = (i * 16 * 21) & 0x7FF; #endif + encoder.Common = &encoderFlusher.Common; RINOK(encoder.CreateEncoderThread()) } - else #endif - { - RINOK(encoder.Encode()); - } } - + + if (printCallback) + { + RINOK(printCallback->CheckBreak()); + } + #ifndef _7ZIP_ST - if (numEncoderThreads > 1) + if (mtEncMode) + { for (i = 0; i < numEncoderThreads; i++) - encoders[i].thread[0].Wait(); + { + CEncoderInfo &encoder = encoders[i]; + WRes wres = encoder.ReadyEvent.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + RINOK(encoder.Results[0]); + } + + CBenchProgressInfo *bpi = encoders[0].progressInfoSpec[0]; + bpi->SetStartTime(); + + WRes wres = encoderFlusher.StartAndWait(); + if (status.Res == 0 && wres != 0) + return HRESULT_FROM_WIN32(wres); + } + else #endif + { + RINOK(encoders[0].Encode()); + } RINOK(status.Res); @@ -1326,11 +1411,16 @@ static HRESULT MethodBench( CEncoderInfo &encoder = encoders[i]; info.UnpackSize += encoder.kBufferSize; info.PackSize += encoder.compressedSize; + // printf("\n%7d\n", encoder.compressedSize); } RINOK(callback->SetEncodeResult(info, true)); + + + // ---------- Decode ---------- + status.Res = S_OK; status.EncodeMode = false; @@ -1543,7 +1633,7 @@ struct CFreqThreads CFreqInfo *Items; UInt32 NumThreads; - CFreqThreads(): Items(0), NumThreads(0) {} + CFreqThreads(): Items(NULL), NumThreads(0) {} void WaitAll() { for (UInt32 i = 0; i < NumThreads; i++) @@ -1601,7 +1691,7 @@ struct CCrcThreads CCrcInfo *Items; UInt32 NumThreads; - CCrcThreads(): Items(0), NumThreads(0) {} + CCrcThreads(): Items(NULL), NumThreads(0) {} void WaitAll() { for (UInt32 i = 0; i < NumThreads; i++) @@ -1617,21 +1707,21 @@ struct CCrcThreads #endif -static UInt32 CrcCalc1(const Byte *buf, UInt32 size) +static UInt32 CrcCalc1(const Byte *buf, size_t size) { UInt32 crc = CRC_INIT_VAL;; - for (UInt32 i = 0; i < size; i++) + for (size_t i = 0; i < size; i++) crc = CRC_UPDATE_BYTE(crc, buf[i]); return CRC_GET_DIGEST(crc); } -static void RandGen(Byte *buf, UInt32 size, CBaseRandomGenerator &RG) +static void RandGen(Byte *buf, size_t size, CBaseRandomGenerator &RG) { - for (UInt32 i = 0; i < size; i++) + for (size_t i = 0; i < size; i++) buf[i] = (Byte)RG.GetRnd(); } -static UInt32 RandGenCrc(Byte *buf, UInt32 size, CBaseRandomGenerator &RG) +static UInt32 RandGenCrc(Byte *buf, size_t size, CBaseRandomGenerator &RG) { RandGen(buf, size, RG); return CrcCalc1(buf, size); @@ -1639,14 +1729,15 @@ static UInt32 RandGenCrc(Byte *buf, UInt32 size, CBaseRandomGenerator &RG) bool CrcInternalTest() { - CBenchBuffer buffer; - const UInt32 kBufferSize0 = (1 << 8); - const UInt32 kBufferSize1 = (1 << 10); - const UInt32 kCheckSize = (1 << 5); - if (!buffer.Alloc(kBufferSize0 + kBufferSize1)) + CAlignedBuffer buffer; + const size_t kBufferSize0 = (1 << 8); + const size_t kBufferSize1 = (1 << 10); + const unsigned kCheckSize = (1 << 5); + buffer.Alloc(kBufferSize0 + kBufferSize1); + if (!buffer.IsAllocated()) return false; - Byte *buf = buffer.Buffer; - UInt32 i; + Byte *buf = (Byte *)buffer; + size_t i; for (i = 0; i < kBufferSize0; i++) buf[i] = (Byte)i; UInt32 crc1 = CrcCalc1(buf, kBufferSize0); @@ -1655,7 +1746,7 @@ bool CrcInternalTest() CBaseRandomGenerator RG; RandGen(buf + kBufferSize0, kBufferSize1, RG); for (i = 0; i < kBufferSize0 + kBufferSize1 - kCheckSize; i++) - for (UInt32 j = 0; j < kCheckSize; j++) + for (unsigned j = 0; j < kCheckSize; j++) if (CrcCalc1(buf + i, j) != CrcCalc(buf + i, j)) return false; return true; @@ -1883,8 +1974,55 @@ AString GetProcessThreadsInfo(const NSystem::CProcessAffinity &ti) } +static void PrintSize(AString &s, UInt64 v) +{ + char c = 0; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'K'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'M'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'G'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'T'; + }}}} + else + { + PrintHex(s, v); + return; + } + char temp[32]; + ConvertUInt64ToString(v, temp); + s += temp; + if (c) + s += c; +} + + +#ifdef _7ZIP_LARGE_PAGES + extern bool g_LargePagesMode; +extern "C" +{ + extern SIZE_T g_LargePageSize; +} + +void Add_LargePages_String(AString &s) +{ + if (g_LargePagesMode || g_LargePageSize != 0) + { + s += " (LP-"; + PrintSize(s, g_LargePageSize); + #ifdef MY_CPU_X86_OR_AMD64 + if (CPU_IsSupported_PageGB()) + s += "-1G"; + #endif + if (!g_LargePagesMode) + s += "-NA"; + s += ")"; + } +} + +#endif + + static void PrintRequirements(IBenchPrintCallback &f, const char *sizeString, bool size_Defined, UInt64 size, const char *threadsString, UInt32 numThreads) @@ -1896,8 +2034,15 @@ static void PrintRequirements(IBenchPrintCallback &f, const char *sizeString, else f.Print(" ?"); f.Print(" MB"); - if (g_LargePagesMode) - f.Print(" LP"); + + #ifdef _7ZIP_LARGE_PAGES + { + AString s; + Add_LargePages_String(s); + f.Print(s); + } + #endif + f.Print(", # "); f.Print(threadsString); PrintNumber(f, numThreads, 3); @@ -2187,14 +2332,13 @@ static HRESULT CrcBench( methodName, hashID)) return E_NOTIMPL; - CBenchBuffer buffer; + CAlignedBuffer buffer; size_t totalSize = (size_t)bufferSize * numThreads; if (totalSize / numThreads != bufferSize) return E_OUTOFMEMORY; - if (!buffer.Alloc(totalSize)) - return E_OUTOFMEMORY; + ALLOC_WITH_HRESULT(&buffer, totalSize) - Byte *buf = buffer.Buffer; + Byte *buf = (Byte *)buffer; CBaseRandomGenerator RG; UInt32 bsize = (bufferSize == 0 ? 1 : bufferSize); UInt64 numIterations = complexInCommands * 256 / complexity / bsize; @@ -2537,26 +2681,7 @@ static const char * const k_PF[] = #endif -static void PrintSize(AString &s, UInt64 v) -{ - char c = 0; - if ((v & 0x3FF) == 0) { v >>= 10; c = 'K'; - if ((v & 0x3FF) == 0) { v >>= 10; c = 'M'; - if ((v & 0x3FF) == 0) { v >>= 10; c = 'G'; - if ((v & 0x3FF) == 0) { v >>= 10; c = 'T'; - }}}} - else - { - PrintHex(s, v); - return; - } - char temp[32]; - ConvertUInt64ToString(v, temp); - s += temp; - if (c) - s += c; -} - + static void PrintPage(AString &s, UInt32 v) { @@ -2600,7 +2725,8 @@ static void SysInfo_To_String(AString &s, const SYSTEM_INFO &si) PrintHex(s, si.wProcessorLevel); s += "."; PrintHex(s, si.wProcessorRevision); - if (si.dwActiveProcessorMask + 1 != ((UInt64)1 << si.dwNumberOfProcessors)) + if ((UInt64)si.dwActiveProcessorMask + 1 != ((UInt64)1 << si.dwNumberOfProcessors)) + if ((UInt64)si.dwActiveProcessorMask + 1 != 0 || si.dwNumberOfProcessors != sizeof(UInt64) * 8) { s += " act:"; PrintHex(s, si.dwActiveProcessorMask); @@ -2684,13 +2810,15 @@ void GetCpuName(AString &s) AString s2; x86cpuid_to_String(cpuid, s2); s += s2; - return; } + else + { #ifdef MY_CPU_AMD64 s += "x64"; #else s += "x86"; #endif + } } #else @@ -2701,6 +2829,10 @@ void GetCpuName(AString &s) #endif #endif + + #ifdef _7ZIP_LARGE_PAGES + Add_LargePages_String(s); + #endif } @@ -2723,6 +2855,27 @@ void GetCpuFeatures(AString &s) } +#ifdef _WIN32 +#ifndef UNDER_CE + +typedef void (WINAPI * Func_RtlGetVersion) (OSVERSIONINFOEXW *); + +static BOOL My_RtlGetVersion(OSVERSIONINFOEXW *vi) +{ + HMODULE ntdll = ::GetModuleHandleW(L"ntdll.dll"); + if (!ntdll) + return FALSE; + Func_RtlGetVersion func = (Func_RtlGetVersion)GetProcAddress(ntdll, "RtlGetVersion"); + if (!func) + return FALSE; + func(vi); + return TRUE; +} + +#endif +#endif + + HRESULT Bench( DECL_EXTERNAL_CODECS_LOC_VARS IBenchPrintCallback *printCallback, @@ -2733,7 +2886,7 @@ HRESULT Bench( bool multiDict) { if (!CrcInternalTest()) - return S_FALSE; + return E_FAIL; UInt32 numCPUs = 1; UInt64 ramSize = (UInt64)(sizeof(size_t)) << 29; @@ -2762,7 +2915,7 @@ HRESULT Bench( COneMethodInfo method; - CBenchBuffer fileDataBuffer; + CAlignedBuffer fileDataBuffer; { unsigned i; @@ -2787,10 +2940,9 @@ HRESULT Bench( return E_FAIL; if (len >= ((UInt32)1 << 31) || len == 0) return E_INVALIDARG; - if (!fileDataBuffer.Alloc((size_t)len)) - return E_OUTOFMEMORY; + ALLOC_WITH_HRESULT(&fileDataBuffer, (size_t)len); UInt32 processedSize; - file.Read(fileDataBuffer.Buffer, (UInt32)len, processedSize); + file.Read((Byte *)fileDataBuffer, (UInt32)len, processedSize); if (processedSize != len) return E_FAIL; if (printCallback) @@ -2857,6 +3009,30 @@ HRESULT Bench( if (printCallback) { + #ifdef _WIN32 + #ifndef UNDER_CE + { + AString s; + // OSVERSIONINFO vi; + OSVERSIONINFOEXW vi; + vi.dwOSVersionInfoSize = sizeof(vi); + // if (::GetVersionEx(&vi)) + if (My_RtlGetVersion(&vi)) + { + s += "Windows"; + if (vi.dwPlatformId != VER_PLATFORM_WIN32_NT) + s.Add_UInt32(vi.dwPlatformId); + s += " "; s.Add_UInt32(vi.dwMajorVersion); + s += "."; s.Add_UInt32(vi.dwMinorVersion); + s += " "; s.Add_UInt32(vi.dwBuildNumber); + // s += " "; s += GetAnsiString(vi.szCSDVersion); + } + printCallback->Print(s); + printCallback->NewLine(); + } + #endif + #endif + { AString s1, s2; GetSysInfo(s1, s2); @@ -2915,6 +3091,9 @@ HRESULT Bench( UInt64 start = ::GetTimeCount(); UInt32 sum = (UInt32)start; sum = CountCpuFreq(sum, (UInt32)(numMilCommands * 1000000 / kNumFreqCommands), g_BenchCpuFreqTemp); + if (sum == 0xF1541213) + if (printCallback) + printCallback->Print(""); const UInt64 realDelta = ::GetTimeCount() - start; start = realDelta; if (start == 0) @@ -2931,7 +3110,7 @@ HRESULT Bench( else { // PrintNumber(*printCallback, start, 0); - PrintNumber(*printCallback, mipsVal, 5 + ((sum == 0xF1541213) ? 1 : 0)); + PrintNumber(*printCallback, mipsVal, 5); } } /* @@ -2979,7 +3158,7 @@ HRESULT Bench( complexInCommands, true, numThreadsSpecified, method, - uncompressedDataSize, fileDataBuffer.Buffer, + uncompressedDataSize, (const Byte *)fileDataBuffer, kOldLzmaDictBits, printCallback, benchCallback, &benchProps); } @@ -3291,9 +3470,9 @@ HRESULT Bench( { res = TotalBench(EXTERNAL_CODECS_LOC_VARS complexInCommands, numThreads, - dictIsDefined || fileDataBuffer.Buffer, // forceUnpackSize - fileDataBuffer.Buffer ? fileDataBuffer.BufferSize : dict, - fileDataBuffer.Buffer, + dictIsDefined || fileDataBuffer.IsAllocated(), // forceUnpackSize + fileDataBuffer.IsAllocated() ? fileDataBuffer.Size() : dict, + (const Byte *)fileDataBuffer, printCallback, &callback); RINOK(res); } @@ -3383,9 +3562,9 @@ HRESULT Bench( } size_t uncompressedDataSize; - if (fileDataBuffer.Buffer) + if (fileDataBuffer.IsAllocated()) { - uncompressedDataSize = fileDataBuffer.BufferSize; + uncompressedDataSize = fileDataBuffer.Size(); } else { @@ -3399,7 +3578,7 @@ HRESULT Bench( complexInCommands, true, numThreads, method2, - uncompressedDataSize, fileDataBuffer.Buffer, + uncompressedDataSize, (const Byte *)fileDataBuffer, kOldLzmaDictBits, printCallback, &callback, &callback.BenchProps); f.NewLine(); RINOK(res); diff --git a/7zip/CPP/7zip/UI/Common/Bench.h b/7zip/CPP/7zip/UI/Common/Bench.h index ec51faee2..18a40a844 100644 --- a/7zip/CPP/7zip/UI/Common/Bench.h +++ b/7zip/CPP/7zip/UI/Common/Bench.h @@ -68,5 +68,10 @@ void GetSysInfo(AString &s1, AString &s2); void GetCpuName(AString &s); void GetCpuFeatures(AString &s); +#ifdef _7ZIP_LARGE_PAGES +void Add_LargePages_String(AString &s); +#else +// #define Add_LargePages_String +#endif #endif diff --git a/7zip/CPP/7zip/UI/Common/CompressCall.cpp b/7zip/CPP/7zip/UI/Common/CompressCall.cpp index c9aa5fdfe..42cb0d236 100644 --- a/7zip/CPP/7zip/UI/Common/CompressCall.cpp +++ b/7zip/CPP/7zip/UI/Common/CompressCall.cpp @@ -13,6 +13,7 @@ #include "../../../Windows/ErrorMsg.h" #include "../../../Windows/FileDir.h" #include "../../../Windows/FileMapping.h" +#include "../../../Windows/MemoryLock.h" #include "../../../Windows/ProcessUtils.h" #include "../../../Windows/Synchronization.h" @@ -94,6 +95,9 @@ static HRESULT Call7zGui(const UString ¶ms, static void AddLagePagesSwitch(UString ¶ms) { if (ReadLockMemoryEnable()) + #ifndef UNDER_CE + if (NSecurity::Get_LargePages_RiskLevel() == 0) + #endif params += " -slp"; } diff --git a/7zip/CPP/7zip/UI/Common/CompressCall2.cpp b/7zip/CPP/7zip/UI/Common/CompressCall2.cpp index fb67ed124..5bad774bc 100644 --- a/7zip/CPP/7zip/UI/Common/CompressCall2.cpp +++ b/7zip/CPP/7zip/UI/Common/CompressCall2.cpp @@ -1,10 +1,10 @@ -// CompressCall.cpp +// CompressCall2.cpp #include "StdAfx.h" #include "../../../Common/MyException.h" -#include "../../UI/common/ArchiveCommandLine.h" +#include "../../UI/Common/EnumDirItems.h" #include "../../UI/GUI/BenchmarkDialog.h" #include "../../UI/GUI/ExtractGUI.h" @@ -20,6 +20,7 @@ extern HWND g_HWND; #define MY_TRY_BEGIN HRESULT result; try { #define MY_TRY_FINISH } \ catch(CSystemException &e) { result = e.ErrorCode; } \ + catch(UString &s) { ErrorMessage(s); result = E_FAIL; } \ catch(...) { result = E_FAIL; } \ if (result != S_OK && result != E_ABORT) \ ErrorMessageHRESULT(result); diff --git a/7zip/CPP/7zip/UI/Common/EnumDirItems.cpp b/7zip/CPP/7zip/UI/Common/EnumDirItems.cpp index 0eee74430..088f07772 100644 --- a/7zip/CPP/7zip/UI/Common/EnumDirItems.cpp +++ b/7zip/CPP/7zip/UI/Common/EnumDirItems.cpp @@ -16,6 +16,7 @@ #endif #include "EnumDirItems.h" +#include "SortUtils.h" using namespace NWindows; using namespace NFile; @@ -925,3 +926,171 @@ void CDirItems::FillFixedReparse() } #endif + + + +static const char * const kCannotFindArchive = "Cannot find archive"; + +HRESULT EnumerateDirItemsAndSort( + NWildcard::CCensor &censor, + NWildcard::ECensorPathMode censorPathMode, + const UString &addPathPrefix, + UStringVector &sortedPaths, + UStringVector &sortedFullPaths, + CDirItemsStat &st, + IDirItemsCallback *callback) +{ + FStringVector paths; + + { + CDirItems dirItems; + dirItems.Callback = callback; + { + HRESULT res = EnumerateItems(censor, censorPathMode, addPathPrefix, dirItems); + st = dirItems.Stat; + RINOK(res); + } + + FOR_VECTOR (i, dirItems.Items) + { + const CDirItem &dirItem = dirItems.Items[i]; + if (!dirItem.IsDir()) + paths.Add(dirItems.GetPhyPath(i)); + } + } + + if (paths.Size() == 0) + { + // return S_OK; + throw CMessagePathException(kCannotFindArchive); + } + + UStringVector fullPaths; + + unsigned i; + + for (i = 0; i < paths.Size(); i++) + { + FString fullPath; + NFile::NDir::MyGetFullPathName(paths[i], fullPath); + fullPaths.Add(fs2us(fullPath)); + } + + CUIntVector indices; + SortFileNames(fullPaths, indices); + sortedPaths.ClearAndReserve(indices.Size()); + sortedFullPaths.ClearAndReserve(indices.Size()); + + for (i = 0; i < indices.Size(); i++) + { + unsigned index = indices[i]; + sortedPaths.AddInReserved(fs2us(paths[index])); + sortedFullPaths.AddInReserved(fullPaths[index]); + if (i > 0 && CompareFileNames(sortedFullPaths[i], sortedFullPaths[i - 1]) == 0) + throw CMessagePathException("Duplicate archive path:", sortedFullPaths[i]); + } + + return S_OK; +} + + + + +#ifdef _WIN32 + +// This code converts all short file names to long file names. + +static void ConvertToLongName(const UString &prefix, UString &name) +{ + if (name.IsEmpty() || DoesNameContainWildcard(name)) + return; + NFind::CFileInfo fi; + const FString path (us2fs(prefix + name)); + #ifndef UNDER_CE + if (NFile::NName::IsDevicePath(path)) + return; + #endif + if (fi.Find(path)) + name = fs2us(fi.Name); +} + +static void ConvertToLongNames(const UString &prefix, CObjectVector &items) +{ + FOR_VECTOR (i, items) + { + NWildcard::CItem &item = items[i]; + if (item.Recursive || item.PathParts.Size() != 1) + continue; + if (prefix.IsEmpty() && item.IsDriveItem()) + continue; + ConvertToLongName(prefix, item.PathParts.Front()); + } +} + +static void ConvertToLongNames(const UString &prefix, NWildcard::CCensorNode &node) +{ + ConvertToLongNames(prefix, node.IncludeItems); + ConvertToLongNames(prefix, node.ExcludeItems); + unsigned i; + for (i = 0; i < node.SubNodes.Size(); i++) + { + UString &name = node.SubNodes[i].Name; + if (prefix.IsEmpty() && NWildcard::IsDriveColonName(name)) + continue; + ConvertToLongName(prefix, name); + } + // mix folders with same name + for (i = 0; i < node.SubNodes.Size(); i++) + { + NWildcard::CCensorNode &nextNode1 = node.SubNodes[i]; + for (unsigned j = i + 1; j < node.SubNodes.Size();) + { + const NWildcard::CCensorNode &nextNode2 = node.SubNodes[j]; + if (nextNode1.Name.IsEqualTo_NoCase(nextNode2.Name)) + { + nextNode1.IncludeItems += nextNode2.IncludeItems; + nextNode1.ExcludeItems += nextNode2.ExcludeItems; + node.SubNodes.Delete(j); + } + else + j++; + } + } + for (i = 0; i < node.SubNodes.Size(); i++) + { + NWildcard::CCensorNode &nextNode = node.SubNodes[i]; + ConvertToLongNames(prefix + nextNode.Name + WCHAR_PATH_SEPARATOR, nextNode); + } +} + +void ConvertToLongNames(NWildcard::CCensor &censor) +{ + FOR_VECTOR (i, censor.Pairs) + { + NWildcard::CPair &pair = censor.Pairs[i]; + ConvertToLongNames(pair.Prefix, pair.Head); + } +} + +#endif + + +CMessagePathException::CMessagePathException(const char *a, const wchar_t *u) +{ + (*this) += a; + if (u) + { + Add_LF(); + (*this) += u; + } +} + +CMessagePathException::CMessagePathException(const wchar_t *a, const wchar_t *u) +{ + (*this) += a; + if (u) + { + Add_LF(); + (*this) += u; + } +} diff --git a/7zip/CPP/7zip/UI/Common/EnumDirItems.h b/7zip/CPP/7zip/UI/Common/EnumDirItems.h index 7afb800a0..6490bd509 100644 --- a/7zip/CPP/7zip/UI/Common/EnumDirItems.h +++ b/7zip/CPP/7zip/UI/Common/EnumDirItems.h @@ -18,4 +18,25 @@ HRESULT EnumerateItems( const UString &addPathPrefix, CDirItems &dirItems); + +struct CMessagePathException: public UString +{ + CMessagePathException(const char *a, const wchar_t *u = NULL); + CMessagePathException(const wchar_t *a, const wchar_t *u = NULL); +}; + + +HRESULT EnumerateDirItemsAndSort( + NWildcard::CCensor &censor, + NWildcard::ECensorPathMode pathMode, + const UString &addPathPrefix, + UStringVector &sortedPaths, + UStringVector &sortedFullPaths, + CDirItemsStat &st, + IDirItemsCallback *callback); + +#ifdef _WIN32 +void ConvertToLongNames(NWildcard::CCensor &censor); +#endif + #endif diff --git a/7zip/CPP/7zip/UI/Common/HashCalc.cpp b/7zip/CPP/7zip/UI/Common/HashCalc.cpp index 46a69de1e..c340ac7f0 100644 --- a/7zip/CPP/7zip/UI/Common/HashCalc.cpp +++ b/7zip/CPP/7zip/UI/Common/HashCalc.cpp @@ -230,7 +230,7 @@ HRESULT HashCalc( unsigned i; CHashBundle hb; RINOK(hb.SetMethods(EXTERNAL_CODECS_LOC_VARS options.Methods)); - hb.Init(); + // hb.Init(); hb.NumErrors = dirItems.Stat.NumErrors; diff --git a/7zip/CPP/7zip/UI/Common/HashCalc.h b/7zip/CPP/7zip/UI/Common/HashCalc.h index 38908e2e8..db5b39aa7 100644 --- a/7zip/CPP/7zip/UI/Common/HashCalc.h +++ b/7zip/CPP/7zip/UI/Common/HashCalc.h @@ -9,7 +9,6 @@ #include "../../Common/MethodProps.h" #include "DirItem.h" -#include "Property.h" const unsigned k_HashCalc_DigestSize_Max = 64; @@ -52,9 +51,13 @@ struct CHashBundle: public IHashCalc UInt64 CurSize; + UString MainName; + UString FirstFileName; + HRESULT SetMethods(DECL_EXTERNAL_CODECS_LOC_VARS const UStringVector &methods); - void Init() + // void Init() {} + CHashBundle() { NumDirs = NumFiles = NumAltStreams = FilesSize = AltStreamsSize = NumErrors = 0; } @@ -77,7 +80,7 @@ struct CHashBundle: public IHashCalc virtual HRESULT GetStream(const wchar_t *name, bool isFolder) x; \ virtual HRESULT OpenFileError(const FString &path, DWORD systemError) x; \ virtual HRESULT SetOperationResult(UInt64 fileSize, const CHashBundle &hb, bool showHash) x; \ - virtual HRESULT AfterLastFile(const CHashBundle &hb) x; \ + virtual HRESULT AfterLastFile(CHashBundle &hb) x; \ struct IHashCallbackUI: public IDirItemsCallback { diff --git a/7zip/CPP/7zip/UI/Common/OpenArchive.cpp b/7zip/CPP/7zip/UI/Common/OpenArchive.cpp index b9b4abd92..419c29ee7 100644 --- a/7zip/CPP/7zip/UI/Common/OpenArchive.cpp +++ b/7zip/CPP/7zip/UI/Common/OpenArchive.cpp @@ -563,6 +563,8 @@ HRESULT CArc::GetItemPathToParent(UInt32 index, UInt32 parent, UStringVector &pa UInt32 parentType = 0; RINOK(GetRawProps->GetParent(curIndex, &curParent, &parentType)); + // 18.06: fixed : we don't want to split name to parts + /* if (parentType != NParentType::kAltStream) { for (;;) @@ -576,6 +578,7 @@ HRESULT CArc::GetItemPathToParent(UInt32 index, UInt32 parent, UStringVector &pa s.DeleteFrom(pos); } } + */ parts.Insert(0, s); @@ -992,7 +995,7 @@ static void MakeCheckOrder(CCodecs *codecs, int index = orderIndices[i]; if (index < 0) continue; - const CArcInfoEx &ai = codecs->Formats[index]; + const CArcInfoEx &ai = codecs->Formats[(unsigned)index]; if (ai.SignatureOffset != 0) { orderIndices2.Add(index); @@ -2013,7 +2016,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } else { - const CArcInfoEx &ai = op.codecs->Formats[formatIndex]; + const CArcInfoEx &ai = op.codecs->Formats[(unsigned)formatIndex]; if (ai.FindExtension(extension) >= 0) { if (ai.Flags_FindSignature() && searchMarkerInHandler) @@ -2295,7 +2298,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) int index = orderIndices[i]; if (index < 0) continue; - const CArcInfoEx &ai = op.codecs->Formats[index]; + const CArcInfoEx &ai = op.codecs->Formats[(unsigned)index]; bool isDifficult = false; // if (ai.Version < 0x91F) // we don't use parser with old DLL (before 9.31) if (!ai.NewInterface) @@ -2327,7 +2330,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) if (isDifficult) { difficultFormats.Add(index); - difficultBools[index] = true; + difficultBools[(unsigned)index] = true; } } diff --git a/7zip/CPP/7zip/UI/Common/PropIDUtils.cpp b/7zip/CPP/7zip/UI/Common/PropIDUtils.cpp index 8a1948403..7702e222d 100644 --- a/7zip/CPP/7zip/UI/Common/PropIDUtils.cpp +++ b/7zip/CPP/7zip/UI/Common/PropIDUtils.cpp @@ -369,7 +369,7 @@ static void ParseSid(AString &s, const Byte *p, UInt32 lim, UInt32 &sidSize) int index = FindPairIndex(sid_32_Names, ARRAY_SIZE(sid_32_Names), v1); if (index >= 0) { - s += sid_32_Names[index].sz; + s += sid_32_Names[(unsigned)index].sz; return; } } @@ -379,7 +379,7 @@ static void ParseSid(AString &s, const Byte *p, UInt32 lim, UInt32 &sidSize) int index = FindPairIndex(sid_21_Names, ARRAY_SIZE(sid_21_Names), v4); if (index >= 0) { - s += sid_21_Names[index].sz; + s += sid_21_Names[(unsigned)index].sz; return; } } @@ -630,7 +630,7 @@ bool ConvertNtReparseToString(const Byte *data, UInt32 size, UString &s) { int index = FindPairIndex(k_ReparseTags, ARRAY_SIZE(k_ReparseTags), tag); if (index >= 0) - s += k_ReparseTags[index].sz; + s += k_ReparseTags[(unsigned)index].sz; else { s += "REPARSE:"; diff --git a/7zip/CPP/7zip/UI/Common/Update.cpp b/7zip/CPP/7zip/UI/Common/Update.cpp index 5d3c44849..8c7ae45f6 100644 --- a/7zip/CPP/7zip/UI/Common/Update.cpp +++ b/7zip/CPP/7zip/UI/Common/Update.cpp @@ -288,29 +288,27 @@ void CArchivePath::ParseFromPath(const UString &path, EArcNameMode mode) if (mode == k_ArcNameMode_Add) return; - if (mode == k_ArcNameMode_Exact) - { - BaseExtension.Empty(); - return; - } - int dotPos = Name.ReverseFind_Dot(); - if (dotPos < 0) - return; - if ((unsigned)dotPos == Name.Len() - 1) - { - Name.DeleteBack(); - BaseExtension.Empty(); - return; - } - const UString ext = Name.Ptr(dotPos + 1); - if (BaseExtension.IsEqualTo_NoCase(ext)) + if (mode != k_ArcNameMode_Exact) { - BaseExtension = ext; - Name.DeleteFrom(dotPos); + int dotPos = Name.ReverseFind_Dot(); + if (dotPos < 0) + return; + if ((unsigned)dotPos == Name.Len() - 1) + Name.DeleteBack(); + else + { + const UString ext = Name.Ptr(dotPos + 1); + if (BaseExtension.IsEqualTo_NoCase(ext)) + { + BaseExtension = ext; + Name.DeleteFrom(dotPos); + return; + } + } } - else - BaseExtension.Empty(); + + BaseExtension.Empty(); } UString CArchivePath::GetFinalPath() const @@ -327,6 +325,7 @@ UString CArchivePath::GetFinalPath() const UString CArchivePath::GetFinalVolPath() const { UString path = GetPathWithoutExt(); + // if BaseExtension is empty, we must ignore VolExtension also. if (!BaseExtension.IsEmpty()) { path += '.'; @@ -1046,10 +1045,6 @@ static HRESULT EnumerateInArchiveItems( #endif -#ifdef _WIN32 -void ConvertToLongNames(NWildcard::CCensor &censor); -#endif - HRESULT UpdateArchive( CCodecs *codecs, const CObjectVector &types, @@ -1170,7 +1165,7 @@ HRESULT UpdateArchive( { errorInfo.SystemError = ERROR_ACCESS_DENIED; errorInfo.Message = "The file is read-only"; - errorInfo.FileNames.Add(arcPath); + errorInfo.FileNames.Add(us2fs(arcPath)); return errorInfo.Get_HRESULT_Error(); } @@ -1381,6 +1376,31 @@ HRESULT UpdateArchive( unsigned ci; + + // self including protection + if (options.DeleteAfterCompressing) + { + for (ci = 0; ci < options.Commands.Size(); ci++) + { + CArchivePath &ap = options.Commands[ci].ArchivePath; + const FString path = us2fs(ap.GetFinalPath()); + // maybe we must compare absolute paths path here + FOR_VECTOR (i, dirItems.Items) + { + const FString phyPath = dirItems.GetPhyPath(i); + if (phyPath == path) + { + UString s; + s = "It is not allowed to include archive to itself"; + s.Add_LF(); + s += path; + throw s; + } + } + } + } + + for (ci = 0; ci < options.Commands.Size(); ci++) { CArchivePath &ap = options.Commands[ci].ArchivePath; @@ -1566,26 +1586,39 @@ HRESULT UpdateArchive( } CCurrentDirRestorer curDirRestorer; + + AStringVector paths; + AStringVector names; for (i = 0; i < fullPaths.Size(); i++) { const UString arcPath2 = fs2us(fullPaths[i]); const UString fileName = ExtractFileNameFromPath(arcPath2); - const AString path (GetAnsiString(arcPath2)); - const AString name (GetAnsiString(fileName)); + paths.Add(GetAnsiString(arcPath2)); + names.Add(GetAnsiString(fileName)); + // const AString path (GetAnsiString(arcPath2)); + // const AString name (GetAnsiString(fileName)); // Warning!!! MAPISendDocuments function changes Current directory // fnSend(0, ";", (LPSTR)(LPCSTR)path, (LPSTR)(LPCSTR)name, 0); + } - MapiFileDesc f; + CRecordVector files; + files.ClearAndSetSize(paths.Size()); + + for (i = 0; i < paths.Size(); i++) + { + MapiFileDesc &f = files[i]; memset(&f, 0, sizeof(f)); f.nPosition = 0xFFFFFFFF; - f.lpszPathName = (char *)(const char *)path; - f.lpszFileName = (char *)(const char *)name; - + f.lpszPathName = (char *)(const char *)paths[i]; + f.lpszFileName = (char *)(const char *)names[i]; + } + + { MapiMessage m; memset(&m, 0, sizeof(m)); - m.nFileCount = 1; - m.lpFiles = &f; + m.nFileCount = files.Size(); + m.lpFiles = &files.Front(); const AString addr (GetAnsiString(options.EMailAddress)); MapiRecipDesc rec; diff --git a/7zip/CPP/7zip/UI/Console/Console.dsp b/7zip/CPP/7zip/UI/Console/Console.dsp index 4145d726c..34918e38e 100644 --- a/7zip/CPP/7zip/UI/Console/Console.dsp +++ b/7zip/CPP/7zip/UI/Console/Console.dsp @@ -417,6 +417,10 @@ SOURCE=..\..\..\Common\MyBuffer.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\MyBuffer2.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\MyCom.h # End Source File # Begin Source File @@ -843,6 +847,15 @@ SOURCE=..\..\..\..\C\CpuArch.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\DllSecur.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sort.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/7zip/CPP/7zip/UI/Console/Console.mak b/7zip/CPP/7zip/UI/Console/Console.mak index 05a07e5ad..bd4c1da4f 100644 --- a/7zip/CPP/7zip/UI/Console/Console.mak +++ b/7zip/CPP/7zip/UI/Console/Console.mak @@ -1,3 +1,9 @@ +MY_CONSOLE = 1 + +!IFNDEF UNDER_CE +CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -D_7ZIP_LARGE_PAGES -DSUPPORT_DEVICE_FILE +!ENDIF + CONSOLE_OBJS = \ $O\BenchCon.obj \ $O\ConsoleClose.obj \ @@ -33,4 +39,5 @@ UI_COMMON_OBJS = \ $O\UpdatePair.obj \ $O\UpdateProduce.obj \ -# +C_OBJS = $(C_OBJS) \ + $O\DllSecur.obj \ diff --git a/7zip/CPP/7zip/UI/Console/Console.manifest b/7zip/CPP/7zip/UI/Console/Console.manifest new file mode 100644 index 000000000..58b68ced8 --- /dev/null +++ b/7zip/CPP/7zip/UI/Console/Console.manifest @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/7zip/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp b/7zip/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp index 897a2757f..21c2f0712 100644 --- a/7zip/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp +++ b/7zip/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp @@ -60,8 +60,9 @@ HRESULT CExtractScanConsole::ScanError(const FString &path, DWORD systemError) if (_se) { - *_se << endl << kError << NError::MyFormatMessage(systemError) << endl << - fs2us(path) << endl << endl; + *_se << endl << kError << NError::MyFormatMessage(systemError) << endl; + _se->NormalizePrint_UString(fs2us(path)); + *_se << endl << endl; _se->Flush(); } return HRESULT_FROM_WIN32(systemError); @@ -251,7 +252,9 @@ static const char * const kTab = " "; static void PrintFileInfo(CStdOutStream *_so, const wchar_t *path, const FILETIME *ft, const UInt64 *size) { - *_so << kTab << "Path: " << path << endl; + *_so << kTab << "Path: "; + _so->NormalizePrint_wstr(path); + *_so << endl; if (size && *size != (UInt64)(Int64)-1) { AString s; @@ -340,7 +343,10 @@ STDMETHODIMP CExtractCallbackConsole::PrepareOperation(const wchar_t *name, Int3 _tempU.Empty(); if (name) + { _tempU = name; + _so->Normalize_UString(_tempU); + } _so->PrintUString(_tempU, _tempA); if (position) *_so << " <" << *position << ">"; @@ -461,7 +467,10 @@ STDMETHODIMP CExtractCallbackConsole::SetOperationResult(Int32 opRes, Int32 encr *_se << s; if (!_currentName.IsEmpty()) - *_se << " : " << _currentName; + { + *_se << " : "; + _se->NormalizePrint_UString(_currentName); + } *_se << endl; _se->Flush(); } @@ -513,7 +522,11 @@ HRESULT CExtractCallbackConsole::BeforeOpen(const wchar_t *name, bool testMode) ClosePercents_for_so(); if (_so) - *_so << endl << (testMode ? kTesting : kExtracting) << name << endl; + { + *_so << endl << (testMode ? kTesting : kExtracting); + _so->NormalizePrint_wstr(name); + *_so << endl; + } if (NeedPercents()) _percent.Command = "Open"; @@ -573,8 +586,9 @@ void Print_ErrorFormatIndex_Warning(CStdOutStream *_so, const CCodecs *codecs, c { const CArcErrorInfo &er = arc.ErrorInfo; - UString s ("WARNING:\n"); - s += arc.Path; + *_so << "WARNING:\n"; + _so->NormalizePrint_UString(arc.Path); + UString s; if (arc.FormatIndex == er.ErrorFormatIndex) { s.Add_LF(); @@ -619,7 +633,10 @@ HRESULT CExtractCallbackConsole::OpenResult( { *_se << endl; if (level != 0) - *_se << arc.Path << endl; + { + _se->NormalizePrint_UString(arc.Path); + *_se << endl; + } } if (errorFlags != 0) @@ -653,7 +670,10 @@ HRESULT CExtractCallbackConsole::OpenResult( { *_so << endl; if (level != 0) - *_so << arc.Path << endl; + { + _so->NormalizePrint_UString(arc.Path); + *_so << endl; + } } if (warningFlags != 0) @@ -708,7 +728,9 @@ HRESULT CExtractCallbackConsole::OpenResult( _so->Flush(); if (_se) { - *_se << kError << name << endl; + *_se << kError; + _se->NormalizePrint_wstr(name); + *_se << endl; HRESULT res = Print_OpenArchive_Error(*_se, codecs, arcLink); RINOK(res); if (result == S_FALSE) diff --git a/7zip/CPP/7zip/UI/Console/HashCon.cpp b/7zip/CPP/7zip/UI/Console/HashCon.cpp index 8ff907537..762b21bb5 100644 --- a/7zip/CPP/7zip/UI/Console/HashCon.cpp +++ b/7zip/CPP/7zip/UI/Console/HashCon.cpp @@ -269,7 +269,7 @@ HRESULT CHashCallbackConsole::SetOperationResult(UInt64 fileSize, const CHashBun if (_fileName.IsEmpty()) *_so << kEmptyFileAlias; else - *_so << _fileName; + _so->NormalizePrint_UString(_fileName); } *_so << endl; } @@ -332,7 +332,7 @@ void CHashCallbackConsole::PrintProperty(const char *name, UInt64 value) *_so << name << s << endl; } -HRESULT CHashCallbackConsole::AfterLastFile(const CHashBundle &hb) +HRESULT CHashCallbackConsole::AfterLastFile(CHashBundle &hb) { ClosePercents2(); diff --git a/7zip/CPP/7zip/UI/Console/List.cpp b/7zip/CPP/7zip/UI/Console/List.cpp index 57d1d1bc1..416ef2c92 100644 --- a/7zip/CPP/7zip/UI/Console/List.cpp +++ b/7zip/CPP/7zip/UI/Console/List.cpp @@ -560,7 +560,7 @@ HRESULT CFieldPrinter::PrintItemInfo(UInt32 index, const CListStat &st) { if (!techMode) g_StdOut << temp; - g_StdOut.PrintUString(FilePath, TempAString); + g_StdOut.NormalizePrint_UString(FilePath, TempWString, TempAString); if (techMode) g_StdOut << MY_ENDL; continue; @@ -671,9 +671,10 @@ HRESULT CFieldPrinter::PrintItemInfo(UInt32 index, const CListStat &st) else if (prop.vt == VT_BSTR) { TempWString.SetFromBstr(prop.bstrVal); + // do we need multi-line support here ? + g_StdOut.Normalize_UString(TempWString); if (techMode) { - // replace CR/LF here. g_StdOut.PrintUString(TempWString, TempAString); } else @@ -815,9 +816,63 @@ static void PrintPropNameAndNumber_Signed(CStdOutStream &so, PROPID propID, Int6 so << val << endl; } -static void PrintPropPair(CStdOutStream &so, const char *name, const wchar_t *val) + +static void UString_Replace_CRLF_to_LF(UString &s) { - so << name << " = " << val << endl; + // s.Replace(L"\r\n", L"\n"); + wchar_t *src = s.GetBuf(); + wchar_t *dest = src; + for (;;) + { + wchar_t c = *src++; + if (c == 0) + break; + if (c == '\r' && *src == '\n') + { + src++; + c = '\n'; + } + *dest++ = c; + } + s.ReleaseBuf_SetEnd((unsigned)(dest - s.GetBuf())); +} + + +static void PrintPropVal_MultiLine(CStdOutStream &so, const wchar_t *val) +{ + UString s = val; + if (s.Find(L'\n') >= 0) + { + so << endl; + so << "{"; + so << endl; + UString_Replace_CRLF_to_LF(s); + so.Normalize_UString__LF_Allowed(s); + so << s; + so << endl; + so << "}"; + } + else + { + so.Normalize_UString(s); + so << s; + } + so << endl; +} + + +static void PrintPropPair(CStdOutStream &so, const char *name, const wchar_t *val, bool multiLine) +{ + so << name << " = "; + if (multiLine) + { + PrintPropVal_MultiLine(so, val); + return; + } + UString s = val; + so.Normalize_UString(s); + so << s; + so << endl; } @@ -831,9 +886,11 @@ static void PrintPropertyPair2(CStdOutStream &so, PROPID propID, const wchar_t * UString nameU; GetPropName(propID, name, nameA, nameU); if (!nameA.IsEmpty()) - PrintPropPair(so, nameA, s); + so << nameA; else - so << nameU << " = " << s << endl; + so << nameU; + so << " = "; + PrintPropVal_MultiLine(so, s); } } @@ -862,11 +919,11 @@ static void ErrorInfo_Print(CStdOutStream &so, const CArcErrorInfo &er) { PrintErrorFlags(so, "ERRORS:", er.GetErrorFlags()); if (!er.ErrorMessage.IsEmpty()) - PrintPropPair(so, "ERROR", er.ErrorMessage); + PrintPropPair(so, "ERROR", er.ErrorMessage, true); PrintErrorFlags(so, "WARNINGS:", er.GetWarningFlags()); if (!er.WarningMessage.IsEmpty()) - PrintPropPair(so, "WARNING", er.WarningMessage); + PrintPropPair(so, "WARNING", er.WarningMessage, true); } HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const CArchiveLink &arcLink) @@ -877,7 +934,7 @@ HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const const CArcErrorInfo &er = arc.ErrorInfo; so << "--\n"; - PrintPropPair(so, "Path", arc.Path); + PrintPropPair(so, "Path", arc.Path, false); if (er.ErrorFormatIndex >= 0) { if (er.ErrorFormatIndex == arc.FormatIndex) @@ -885,7 +942,7 @@ HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const else PrintArcTypeError(so, codecs->GetFormatNamePtr(er.ErrorFormatIndex), true); } - PrintPropPair(so, "Type", codecs->GetFormatNamePtr(arc.FormatIndex)); + PrintPropPair(so, "Type", codecs->GetFormatNamePtr(arc.FormatIndex), false); ErrorInfo_Print(so, er); @@ -943,7 +1000,8 @@ HRESULT Print_OpenArchive_Error(CStdOutStream &so, const CCodecs *codecs, const { if (arcLink.NonOpen_ErrorInfo.ErrorFormatIndex >= 0) { - so << arcLink.NonOpen_ArcPath << endl; + so.NormalizePrint_UString(arcLink.NonOpen_ArcPath); + so << endl; PrintArcTypeError(so, codecs->Formats[arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); } else @@ -1014,15 +1072,24 @@ HRESULT ListArchives(CCodecs *codecs, errorCode = ERROR_FILE_NOT_FOUND; lastError = HRESULT_FROM_WIN32(lastError);; g_StdOut.Flush(); - *g_ErrStream << endl << kError << NError::MyFormatMessage(errorCode) << - endl << arcPath << endl << endl; + if (g_ErrStream) + { + *g_ErrStream << endl << kError << NError::MyFormatMessage(errorCode) << endl; + g_ErrStream->NormalizePrint_UString(arcPath); + *g_ErrStream << endl << endl; + } numErrors++; continue; } if (fi.IsDir()) { g_StdOut.Flush(); - *g_ErrStream << endl << kError << arcPath << " is not a file" << endl << endl; + if (g_ErrStream) + { + *g_ErrStream << endl << kError; + g_ErrStream->NormalizePrint_UString(arcPath); + *g_ErrStream << " is not a file" << endl << endl; + } numErrors++; continue; } @@ -1061,7 +1128,9 @@ HRESULT ListArchives(CCodecs *codecs, if (enableHeaders) { - g_StdOut << endl << kListing << arcPath << endl << endl; + g_StdOut << endl << kListing; + g_StdOut.NormalizePrint_UString(arcPath); + g_StdOut << endl << endl; } HRESULT result = arcLink.Open_Strict(options, &openCallback); @@ -1070,22 +1139,28 @@ HRESULT ListArchives(CCodecs *codecs, { if (result == E_ABORT) return result; + if (result != S_FALSE) + lastError = result; g_StdOut.Flush(); - *g_ErrStream << endl << kError << arcPath << " : "; - if (result == S_FALSE) + if (g_ErrStream) { - Print_OpenArchive_Error(*g_ErrStream, codecs, arcLink); - } - else - { - lastError = result; - *g_ErrStream << "opening : "; - if (result == E_OUTOFMEMORY) - *g_ErrStream << "Can't allocate required memory"; + *g_ErrStream << endl << kError; + g_ErrStream->NormalizePrint_UString(arcPath); + *g_ErrStream << " : "; + if (result == S_FALSE) + { + Print_OpenArchive_Error(*g_ErrStream, codecs, arcLink); + } else - *g_ErrStream << NError::MyFormatMessage(result); + { + *g_ErrStream << "opening : "; + if (result == E_OUTOFMEMORY) + *g_ErrStream << "Can't allocate required memory"; + else + *g_ErrStream << NError::MyFormatMessage(result); + } + *g_ErrStream << endl; } - *g_ErrStream << endl; numErrors++; continue; } @@ -1255,7 +1330,7 @@ HRESULT ListArchives(CCodecs *codecs, if (arcLink.NonOpen_ErrorInfo.ErrorFormatIndex >= 0) { g_StdOut << "----------\n"; - PrintPropPair(g_StdOut, "Path", arcLink.NonOpen_ArcPath); + PrintPropPair(g_StdOut, "Path", arcLink.NonOpen_ArcPath, false); PrintArcTypeError(g_StdOut, codecs->Formats[arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); } } diff --git a/7zip/CPP/7zip/UI/Console/Main.cpp b/7zip/CPP/7zip/UI/Console/Main.cpp index 436a6cd19..333223bd0 100644 --- a/7zip/CPP/7zip/UI/Console/Main.cpp +++ b/7zip/CPP/7zip/UI/Console/Main.cpp @@ -10,10 +10,6 @@ #include "../../../../C/CpuArch.h" -#if defined( _7ZIP_LARGE_PAGES) -#include "../../../../C/Alloc.h" -#endif - #include "../../../Common/MyInitGuid.h" #include "../../../Common/CommandLineParser.h" @@ -25,13 +21,10 @@ #include "../../../Windows/ErrorMsg.h" -#ifdef _WIN32 -#include "../../../Windows/MemoryLock.h" -#endif - #include "../../../Windows/TimeUtils.h" #include "../Common/ArchiveCommandLine.h" +#include "../Common/Bench.h" #include "../Common/ExitCode.h" #include "../Common/Extract.h" @@ -64,8 +57,6 @@ using namespace NCommandLineParser; HINSTANCE g_hInstance = 0; #endif -bool g_LargePagesMode = false; - extern CStdOutStream *g_StdStream; extern CStdOutStream *g_ErrStream; @@ -96,7 +87,7 @@ static const char * const kHelpString = "a" #endif #endif - " [...] [...]\n" + " [...] [...] [@listfile]\n" "\n" "\n" " a : Add files to archive\n" @@ -112,8 +103,7 @@ static const char * const kHelpString = " x : eXtract files with full paths\n" "\n" "\n" - " -- : Stop switches parsing\n" - " @listfile : set path to listfile that contains file names\n" + " -- : Stop switches and @listfile parsing\n" " -ai[r[-|0]]{@listfile|!wildcard} : Include archives\n" " -ax[r[-|0]]{@listfile|!wildcard} : eXclude archives\n" " -ao{a|s|t|u} : set Overwrite mode\n" @@ -244,7 +234,8 @@ static void PrintWarningsPaths(const CErrorPathCodes &pc, CStdOutStream &so) { FOR_VECTOR(i, pc.Paths) { - so << pc.Paths[i] << " : "; + so.NormalizePrint_UString(fs2us(pc.Paths[i])); + so << " : "; so << NError::MyFormatMessage(pc.Codes[i]) << endl; } so << "----------------" << endl; @@ -383,6 +374,13 @@ static void PrintMemUsage(const char *s, UInt64 val) *g_StdStream << " " << s << " Memory ="; PrintNum(SHIFT_SIZE_VALUE(val, 20), 7); *g_StdStream << " MB"; + + #ifdef _7ZIP_LARGE_PAGES + AString lp; + Add_LargePages_String(lp); + if (!lp.IsEmpty()) + *g_StdStream << lp; + #endif } EXTERN_C_BEGIN @@ -524,6 +522,8 @@ int Main2( parser.Parse1(commandStrings, options); + g_StdOut.IsTerminalMode = options.IsStdOutTerminal; + g_StdErr.IsTerminalMode = options.IsStdErrTerminal; if (options.Number_for_Out != k_OutStream_stdout) g_StdStream = (options.Number_for_Out == k_OutStream_stderr ? &g_StdErr : NULL); @@ -541,24 +541,6 @@ int Main2( return 0; } - #if defined(_WIN32) && !defined(UNDER_CE) - NSecurity::EnablePrivilege_SymLink(); - #endif - - #ifdef _7ZIP_LARGE_PAGES - if (options.LargePages) - { - SetLargePageSize(); - // note: this process also can inherit that Privilege from parent process - g_LargePagesMode = - #if defined(_WIN32) && !defined(UNDER_CE) - NSecurity::EnablePrivilege_LockMemory(); - #else - true; - #endif - } - #endif - if (options.EnableHeaders) ShowCopyrightAndHelp(g_StdStream, false); @@ -932,7 +914,7 @@ int Main2( { hashCalc = &hb; ThrowException_if_Error(hb.SetMethods(EXTERNAL_CODECS_VARS_L options.HashMethods)); - hb.Init(); + // hb.Init(); } hresultMain = Extract( diff --git a/7zip/CPP/7zip/UI/Console/MainAr.cpp b/7zip/CPP/7zip/UI/Console/MainAr.cpp index 68e5fe97e..87bf57e85 100644 --- a/7zip/CPP/7zip/UI/Console/MainAr.cpp +++ b/7zip/CPP/7zip/UI/Console/MainAr.cpp @@ -2,6 +2,10 @@ #include "StdAfx.h" +#ifdef _WIN32 +#include "../../../../C/DllSecur.h" +#endif + #include "../../../Common/MyException.h" #include "../../../Common/StdOutStream.h" @@ -63,6 +67,10 @@ int MY_CDECL main try { + #ifdef _WIN32 + My_SetDefaultDllDirectories(); + #endif + res = Main2( #ifndef _WIN32 numArgs, args @@ -79,7 +87,7 @@ int MY_CDECL main PrintError(kUserBreakMessage); return (NExitCode::kUserBreak); } - catch(const CArcCmdLineException &e) + catch(const CMessagePathException &e) { PrintError(kException_CmdLine_Error_Message); if (g_ErrStream) diff --git a/7zip/CPP/7zip/UI/Console/PercentPrinter.cpp b/7zip/CPP/7zip/UI/Console/PercentPrinter.cpp index e3e8d5998..b2426878e 100644 --- a/7zip/CPP/7zip/UI/Console/PercentPrinter.cpp +++ b/7zip/CPP/7zip/UI/Console/PercentPrinter.cpp @@ -141,8 +141,9 @@ void CPercentPrinter::Print() { _s += ' '; - StdOut_Convert_UString_to_AString(FileName, _temp); - _temp.Replace('\n', ' '); + _tempU = FileName; + _so->Normalize_UString(_tempU); + StdOut_Convert_UString_to_AString(_tempU, _temp); if (_s.Len() + _temp.Len() > MaxLen) { unsigned len = FileName.Len(); @@ -153,8 +154,9 @@ void CPercentPrinter::Print() delta = 1; len -= delta; _tempU = FileName; - _tempU.Delete(len / 2, FileName.Len() - len); + _tempU.Delete(len / 2, _tempU.Len() - len); _tempU.Insert(len / 2, L" . "); + _so->Normalize_UString(_tempU); StdOut_Convert_UString_to_AString(_tempU, _temp); if (_s.Len() + _temp.Len() <= MaxLen) break; diff --git a/7zip/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp b/7zip/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp index e58e22d7e..cd232fff7 100644 --- a/7zip/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp +++ b/7zip/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp @@ -143,7 +143,9 @@ HRESULT CUpdateCallbackConsole::OpenResult( _so->Flush(); if (_se) { - *_se << kError << name << endl; + *_se << kError; + _se->NormalizePrint_wstr(name); + *_se << endl; HRESULT res = Print_OpenArchive_Error(*_se, codecs, arcLink); RINOK(res); _se->Flush(); @@ -185,7 +187,9 @@ void CCallbackConsoleBase::CommonError(const FString &path, DWORD systemError, b *_se << endl << (isWarning ? kWarning : kError) << NError::MyFormatMessage(systemError) - << endl << fs2us(path) << endl << endl; + << endl; + _se->NormalizePrint_UString(fs2us(path)); + *_se << endl << endl; _se->Flush(); } } @@ -281,8 +285,8 @@ HRESULT CUpdateCallbackConsole::StartArchive(const wchar_t *name, bool updating) if (_so) { *_so << (updating ? kUpdatingArchiveMessage : kCreatingArchiveMessage); - if (name != 0) - *_so << name; + if (name) + _so->NormalizePrint_wstr(name); else *_so << k_StdOut_ArcName; *_so << endl << endl; @@ -343,6 +347,7 @@ HRESULT CUpdateCallbackConsole::DeletingAfterArchiving(const FString &path, bool _tempA.Add_Space(); *_so << _tempA; _tempU = fs2us(path); + _so->Normalize_UString(_tempU); _so->PrintUString(_tempU, _tempA); *_so << endl; if (NeedFlush) @@ -475,7 +480,10 @@ HRESULT CCallbackConsoleBase::PrintProgress(const wchar_t *name, const char *com _tempU.Empty(); if (name) + { _tempU = name; + _so->Normalize_UString(_tempU); + } _so->PrintUString(_tempU, _tempA); *_so << endl; if (NeedFlush) @@ -568,7 +576,9 @@ HRESULT CUpdateCallbackConsole::ReportExtractResult(Int32 opRes, Int32 isEncrypt AString s; SetExtractErrorMessage(opRes, isEncrypted, s); - *_se << s << " : " << endl << name << endl << endl; + *_se << s << " : " << endl; + _se->NormalizePrint_wstr(name); + *_se << endl << endl; _se->Flush(); } return S_OK; diff --git a/7zip/CPP/7zip/UI/Console/makefile b/7zip/CPP/7zip/UI/Console/makefile index 98ad91c24..541b7681a 100644 --- a/7zip/CPP/7zip/UI/Console/makefile +++ b/7zip/CPP/7zip/UI/Console/makefile @@ -1,12 +1,7 @@ PROG = 7z.exe -MY_CONSOLE = 1 CFLAGS = $(CFLAGS) \ -DEXTERNAL_CODECS \ -!IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -D_7ZIP_LARGE_PAGES -DSUPPORT_DEVICE_FILE -!ENDIF - COMMON_OBJS = \ $O\CommandLineParser.obj \ $O\CRC.obj \ @@ -57,7 +52,7 @@ AR_COMMON_OBJS = \ COMPRESS_OBJS = \ $O\CopyCoder.obj \ -C_OBJS = \ +C_OBJS = $(C_OBJS) \ $O\Alloc.obj \ $O\CpuArch.obj \ $O\Sort.obj \ diff --git a/7zip/CPP/7zip/UI/Console/resource.rc b/7zip/CPP/7zip/UI/Console/resource.rc index 2f7ef0738..414427fb2 100644 --- a/7zip/CPP/7zip/UI/Console/resource.rc +++ b/7zip/CPP/7zip/UI/Console/resource.rc @@ -1,3 +1,7 @@ #include "../../MyVersionInfo.rc" MY_VERSION_INFO_APP("7-Zip Console" , "7z") + +#ifndef UNDER_CE +1 24 MOVEABLE PURE "Console.manifest" +#endif diff --git a/7zip/CPP/7zip/UI/Explorer/ContextMenu.cpp b/7zip/CPP/7zip/UI/Explorer/ContextMenu.cpp index 21de12a52..a99ca81d8 100644 --- a/7zip/CPP/7zip/UI/Explorer/ContextMenu.cpp +++ b/7zip/CPP/7zip/UI/Explorer/ContextMenu.cpp @@ -120,7 +120,11 @@ HRESULT CZipContextMenu::InitContextMenu(const wchar_t * /* folder */, const wch _isMenuForFM = true; _fileNames.Clear(); for (UInt32 i = 0; i < numFiles; i++) + { + // MessageBoxW(0, names[i], NULL, 0); + // OutputDebugStringW(names[i]); _fileNames.Add(names[i]); + } _dropMode = false; return S_OK; } @@ -423,6 +427,13 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, { // OutputDebugStringA("QueryContextMenu"); + /* + for (UInt32 i = 0; i < _fileNames.Size(); i++) + { + OutputDebugStringW(_fileNames[i]); + } + */ + LoadLangOneTime(); if (_fileNames.Size() == 0) return E_FAIL; @@ -582,7 +593,7 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, } } - const UString &fileName = _fileNames.Front(); + // const UString &fileName = _fileNames.Front(); if (needExtract) { @@ -639,12 +650,8 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, } } - UString arcName; - if (_fileNames.Size() == 1) - arcName = CreateArchiveName(fi0, false); - else - arcName = CreateArchiveName(fileName, _fileNames.Size() > 1, false); - + const UString arcName = CreateArchiveName(_fileNames, _fileNames.Size() == 1 ? &fi0 : NULL); + UString arcName7z = arcName; arcName7z += ".7z"; UString arcNameZip = arcName; diff --git a/7zip/CPP/7zip/UI/Explorer/Explorer.dsp b/7zip/CPP/7zip/UI/Explorer/Explorer.dsp index 2a6166ee1..bd91e5466 100644 --- a/7zip/CPP/7zip/UI/Explorer/Explorer.dsp +++ b/7zip/CPP/7zip/UI/Explorer/Explorer.dsp @@ -286,6 +286,15 @@ SOURCE=..\FileManager\RegistryUtils.h # PROP Default_Filter "" # Begin Source File +SOURCE=..\..\..\..\C\CpuArch.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\CpuArch.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -479,6 +488,14 @@ SOURCE=..\..\..\Windows\MemoryGlobal.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\MemoryLock.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\MemoryLock.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Menu.cpp # End Source File # Begin Source File diff --git a/7zip/CPP/7zip/UI/Explorer/makefile b/7zip/CPP/7zip/UI/Explorer/makefile index 1c8639bee..541122abe 100644 --- a/7zip/CPP/7zip/UI/Explorer/makefile +++ b/7zip/CPP/7zip/UI/Explorer/makefile @@ -34,6 +34,7 @@ WIN_OBJS = \ $O\FileFind.obj \ $O\FileIO.obj \ $O\FileName.obj \ + $O\MemoryLock.obj \ $O\Menu.obj \ $O\ProcessUtils.obj \ $O\Registry.obj \ @@ -67,6 +68,7 @@ FM_OBJS = \ $O\RegistryUtils.obj \ C_OBJS = \ + $O\CpuArch.obj \ $O\Threads.obj \ !include "../../7zip.mak" diff --git a/7zip/CPP/7zip/UI/Far/ExtractEngine.cpp b/7zip/CPP/7zip/UI/Far/ExtractEngine.cpp index 04bad7240..3567c0481 100644 --- a/7zip/CPP/7zip/UI/Far/ExtractEngine.cpp +++ b/7zip/CPP/7zip/UI/Far/ExtractEngine.cpp @@ -209,6 +209,8 @@ void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, AString &s) s = "Is not archive"; else if (opRes == NArchive::NExtract::NOperationResult::kHeadersError) s = "kHeaders Error"; + else if (opRes == NArchive::NExtract::NOperationResult::kWrongPassword) + s = "Wrong Password"; else { s = "Error #"; diff --git a/7zip/CPP/7zip/UI/Far/Far.cpp b/7zip/CPP/7zip/UI/Far/Far.cpp index cec4af52f..5d92bc406 100644 --- a/7zip/CPP/7zip/UI/Far/Far.cpp +++ b/7zip/CPP/7zip/UI/Far/Far.cpp @@ -327,7 +327,7 @@ HRESULT OpenArchive(const CSysString &fileName, } */ -static HANDLE MyOpenFilePluginW(const wchar_t *name) +static HANDLE MyOpenFilePluginW(const wchar_t *name, bool isAbortCodeSupported) { FString normalizedName = us2fs(name); normalizedName.Trim(); @@ -373,7 +373,12 @@ static HANDLE MyOpenFilePluginW(const wchar_t *name) archiverInfoResult, defaultName, openArchiveCallback); */ if (result == E_ABORT) - return (HANDLE)-2; + { + // fixed 18.06: + // OpenFilePlugin() is allowed to return (HANDLE)-2 as abort code + // OpenPlugin() is not allowed to return (HANDLE)-2. + return isAbortCodeSupported ? (HANDLE)-2 : INVALID_HANDLE_VALUE; + } UString errorMessage = agent->GetErrorMessage(); if (!errorMessage.IsEmpty()) @@ -403,7 +408,7 @@ static HANDLE MyOpenFilePluginW(const wchar_t *name) return (HANDLE)(plugin); } -static HANDLE MyOpenFilePlugin(const char *name) +static HANDLE MyOpenFilePlugin(const char *name, bool isAbortCodeSupported) { UINT codePage = #ifdef UNDER_CE @@ -411,7 +416,7 @@ static HANDLE MyOpenFilePlugin(const char *name) #else ::AreFileApisANSI() ? CP_ACP : CP_OEMCP; #endif - return MyOpenFilePluginW(GetUnicodeString(name, codePage)); + return MyOpenFilePluginW(GetUnicodeString(name, codePage), isAbortCodeSupported); } EXTERN_C HANDLE WINAPI OpenFilePlugin(char *name, const unsigned char * /* data */, int /* dataSize */) @@ -423,7 +428,7 @@ EXTERN_C HANDLE WINAPI OpenFilePlugin(char *name, const unsigned char * /* data // if (!Opt.ProcessShiftF1) return(INVALID_HANDLE_VALUE); } - return MyOpenFilePlugin(name); + return MyOpenFilePlugin(name, true); // isAbortCodeSupported MY_TRY_END2("OpenFilePlugin", INVALID_HANDLE_VALUE); } @@ -458,7 +463,7 @@ EXTERN_C HANDLE WINAPI OpenPlugin(int openFrom, INT_PTR item) fileName.DeleteBack(); fileName.DeleteFrontal(1); } - return MyOpenFilePlugin(fileName); + return MyOpenFilePlugin(fileName, false); // isAbortCodeSupported } if (openFrom == OPEN_PLUGINSMENU) @@ -470,7 +475,7 @@ EXTERN_C HANDLE WINAPI OpenPlugin(int openFrom, INT_PTR item) PluginPanelItem pluginPanelItem; if (!g_StartupInfo.ControlGetActivePanelCurrentItemInfo(pluginPanelItem)) throw 142134; - return MyOpenFilePlugin(pluginPanelItem.FindData.cFileName); + return MyOpenFilePlugin(pluginPanelItem.FindData.cFileName, false); // isAbortCodeSupported } case 1: diff --git a/7zip/CPP/7zip/UI/FileManager/7zFM.exe.manifest b/7zip/CPP/7zip/UI/FileManager/7zFM.exe.manifest index e82374515..6a13c9231 100644 --- a/7zip/CPP/7zip/UI/FileManager/7zFM.exe.manifest +++ b/7zip/CPP/7zip/UI/FileManager/7zFM.exe.manifest @@ -5,6 +5,13 @@ + + + + + + + true diff --git a/7zip/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp b/7zip/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp index b7fbb2359..7069e1aa8 100644 --- a/7zip/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp +++ b/7zip/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp @@ -468,7 +468,7 @@ static HRESULT UpdateFile(NFsFolder::CCopyStateIO &state, CFSTR inPath, CFSTR ou { if (NFind::DoesFileOrDirExist(outPath)) { - RINOK(SendMessageError(callback, NError::MyFormatMessage(ERROR_ALREADY_EXISTS), outPath)); + RINOK(SendMessageError(callback, NError::MyFormatMessage(ERROR_ALREADY_EXISTS), FString(outPath))); CFileInfo fi; if (fi.Find(inPath)) { diff --git a/7zip/CPP/7zip/UI/FileManager/EditDialog.cpp b/7zip/CPP/7zip/UI/FileManager/EditDialog.cpp new file mode 100644 index 000000000..7f596722c --- /dev/null +++ b/7zip/CPP/7zip/UI/FileManager/EditDialog.cpp @@ -0,0 +1,57 @@ +// EditDialog.cpp + +#include "StdAfx.h" + +#include "EditDialog.h" + +#ifdef LANG +#include "LangUtils.h" +#endif + +bool CEditDialog::OnInit() +{ + #ifdef LANG + LangSetDlgItems(*this, NULL, 0); + #endif + _edit.Attach(GetItem(IDE_EDIT)); + + SetText(Title); + _edit.SetText(Text); + + NormalizeSize(); + return CModalDialog::OnInit(); +} + +// #define MY_CLOSE_BUTTON__ID IDCANCEL +#define MY_CLOSE_BUTTON__ID IDCLOSE + +bool CEditDialog::OnSize(WPARAM /* wParam */, int xSize, int ySize) +{ + int mx, my; + GetMargins(8, mx, my); + int bx1, by; + GetItemSizes(MY_CLOSE_BUTTON__ID, bx1, by); + + // int bx2; + // GetItemSizes(IDOK, bx2, by); + + int y = ySize - my - by; + int x = xSize - mx - bx1; + + /* + RECT rect; + GetClientRect(&rect); + rect.top = y - my; + InvalidateRect(&rect); + */ + InvalidateRect(NULL); + + MoveItem(MY_CLOSE_BUTTON__ID, x, y, bx1, by); + // MoveItem(IDOK, x - mx - bx2, y, bx2, by); + /* + if (wParam == SIZE_MAXSHOW || wParam == SIZE_MAXIMIZED || wParam == SIZE_MAXHIDE) + mx = 0; + */ + _edit.Move(mx, my, xSize - mx * 2, y - my * 2); + return false; +} diff --git a/7zip/CPP/7zip/UI/FileManager/EditDialog.h b/7zip/CPP/7zip/UI/FileManager/EditDialog.h new file mode 100644 index 000000000..d820516a5 --- /dev/null +++ b/7zip/CPP/7zip/UI/FileManager/EditDialog.h @@ -0,0 +1,25 @@ +// EditDialog.h + +#ifndef __EDIT_DIALOG_H +#define __EDIT_DIALOG_H + +#include "../../../Windows/Control/Dialog.h" +#include "../../../Windows/Control/Edit.h" + +#include "EditDialogRes.h" + +class CEditDialog: public NWindows::NControl::CModalDialog +{ + NWindows::NControl::CEdit _edit; + virtual bool OnInit(); + virtual bool OnSize(WPARAM wParam, int xSize, int ySize); +public: + UString Title; + UString Text; + + INT_PTR Create(HWND wndParent = 0) { return CModalDialog::Create(IDD_EDIT_DLG, wndParent); } + + CEditDialog() {} +}; + +#endif diff --git a/7zip/CPP/7zip/UI/FileManager/EditDialog.rc b/7zip/CPP/7zip/UI/FileManager/EditDialog.rc new file mode 100644 index 000000000..cdb0b4450 --- /dev/null +++ b/7zip/CPP/7zip/UI/FileManager/EditDialog.rc @@ -0,0 +1,15 @@ +#include "EditDialogRes.h" +#include "../../GuiCommon.rc" + +#define xc 320 +#define yc 240 + +IDD_EDIT_DLG DIALOG 0, 0, xs, ys MY_MODAL_RESIZE_DIALOG_STYLE MY_FONT +CAPTION "Edit" +{ + // OK_CANCEL + MY_BUTTON__CLOSE + + EDITTEXT IDE_EDIT, m, m, xc, yc - bys - m, + ES_MULTILINE | ES_READONLY | WS_VSCROLL | WS_HSCROLL | ES_WANTRETURN +} diff --git a/7zip/CPP/7zip/UI/FileManager/EditDialogRes.h b/7zip/CPP/7zip/UI/FileManager/EditDialogRes.h new file mode 100644 index 000000000..58c5ca911 --- /dev/null +++ b/7zip/CPP/7zip/UI/FileManager/EditDialogRes.h @@ -0,0 +1,2 @@ +#define IDD_EDIT_DLG 94 +#define IDE_EDIT 100 diff --git a/7zip/CPP/7zip/UI/FileManager/ExtractCallback.cpp b/7zip/CPP/7zip/UI/FileManager/ExtractCallback.cpp index a5317e26a..1bc96ae37 100644 --- a/7zip/CPP/7zip/UI/FileManager/ExtractCallback.cpp +++ b/7zip/CPP/7zip/UI/FileManager/ExtractCallback.cpp @@ -759,11 +759,15 @@ STDMETHODIMP CExtractCallbackImp::AskWrite( destPathResultTemp = fs2us(destPathSys); } else + { + if (NFind::DoesFileExist(destPathSys)) if (!NDir::DeleteFileAlways(destPathSys)) + if (GetLastError() != ERROR_FILE_NOT_FOUND) { RINOK(MessageError("can not delete output file", destPathSys)); return E_ABORT; } + } } *writeAnswer = BoolToInt(true); return StringToBstr(destPathResultTemp, destPathResult); diff --git a/7zip/CPP/7zip/UI/FileManager/FM.cpp b/7zip/CPP/7zip/UI/FileManager/FM.cpp index ddde804b6..3e7f104d7 100644 --- a/7zip/CPP/7zip/UI/FileManager/FM.cpp +++ b/7zip/CPP/7zip/UI/FileManager/FM.cpp @@ -7,6 +7,9 @@ #include #include "../../../../C/Alloc.h" +#ifdef _WIN32 +#include "../../../../C/DllSecur.h" +#endif #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" @@ -383,6 +386,7 @@ static void SetMemoryLock() NSecurity::AddLockMemoryPrivilege(); if (ReadLockMemoryEnable()) + if (NSecurity::Get_LargePages_RiskLevel() == 0) { // note: child processes can inherit that Privilege g_LargePagesMode = NSecurity::EnablePrivilege_LockMemory(); @@ -659,6 +663,9 @@ int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, { try { + #ifdef _WIN32 + My_SetDefaultDllDirectories(); + #endif return WinMain2(nCmdShow); } catch (...) diff --git a/7zip/CPP/7zip/UI/FileManager/FM.dsp b/7zip/CPP/7zip/UI/FileManager/FM.dsp index 3b2fb2e34..69d285cb6 100644 --- a/7zip/CPP/7zip/UI/FileManager/FM.dsp +++ b/7zip/CPP/7zip/UI/FileManager/FM.dsp @@ -491,6 +491,14 @@ SOURCE=.\DialogSize.h # End Source File # Begin Source File +SOURCE=.\EditDialog.cpp +# End Source File +# Begin Source File + +SOURCE=.\EditDialog.h +# End Source File +# Begin Source File + SOURCE=.\LinkDialog.cpp # End Source File # Begin Source File @@ -704,10 +712,24 @@ SOURCE=..\..\..\..\C\Alloc.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\CpuArch.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\CpuArch.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\DllSecur.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sort.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/7zip/CPP/7zip/UI/FileManager/FM.mak b/7zip/CPP/7zip/UI/FileManager/FM.mak index 6a85ea2f9..654da627c 100644 --- a/7zip/CPP/7zip/UI/FileManager/FM.mak +++ b/7zip/CPP/7zip/UI/FileManager/FM.mak @@ -1,3 +1,16 @@ +CFLAGS = $(CFLAGS) \ + -DLANG \ + -DNEW_FOLDER_INTERFACE \ + +!IFDEF UNDER_CE +LIBS = $(LIBS) ceshell.lib Commctrl.lib +!ELSE +LIBS = $(LIBS) comctl32.lib htmlhelp.lib comdlg32.lib Mpr.lib Gdi32.lib +CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -DSUPPORT_DEVICE_FILE +LFLAGS = $(LFLAGS) /DELAYLOAD:mpr.dll +LIBS = $(LIBS) delayimp.lib +!ENDIF + FM_OBJS = \ $O\App.obj \ $O\BrowseDialog.obj \ @@ -46,6 +59,7 @@ FM_OBJS = \ $O\AboutDialog.obj \ $O\ComboDialog.obj \ $O\CopyDialog.obj \ + $O\EditDialog.obj \ $O\EditPage.obj \ $O\LangPage.obj \ $O\ListViewDialog.obj \ @@ -72,6 +86,9 @@ WIN_OBJS = $(WIN_OBJS) \ !ENDIF +C_OBJS = $(C_OBJS) \ + $O\DllSecur.obj \ + AGENT_OBJS = \ $O\Agent.obj \ $O\AgentOut.obj \ diff --git a/7zip/CPP/7zip/UI/FileManager/FSFolder.cpp b/7zip/CPP/7zip/UI/FileManager/FSFolder.cpp index 94cd04cbb..9e7202148 100644 --- a/7zip/CPP/7zip/UI/FileManager/FSFolder.cpp +++ b/7zip/CPP/7zip/UI/FileManager/FSFolder.cpp @@ -532,9 +532,8 @@ STDMETHODIMP CFSFolder::GetNumRawProps(UInt32 *numProps) return S_OK; } -STDMETHODIMP CFSFolder::GetRawPropInfo(UInt32 index, BSTR *name, PROPID *propID) +STDMETHODIMP CFSFolder::GetRawPropInfo(UInt32 /* index */, BSTR *name, PROPID *propID) { - index = index; *name = NULL; *propID = kpidNtReparse; return S_OK; diff --git a/7zip/CPP/7zip/UI/FileManager/FSFolderCopy.cpp b/7zip/CPP/7zip/UI/FileManager/FSFolderCopy.cpp index 546f3d274..7f95b1d20 100644 --- a/7zip/CPP/7zip/UI/FileManager/FSFolderCopy.cpp +++ b/7zip/CPP/7zip/UI/FileManager/FSFolderCopy.cpp @@ -161,8 +161,6 @@ static DWORD CALLBACK CopyProgressRoutine( LPVOID lpData // from CopyFileEx ) { - TotalFileSize = TotalFileSize; - // TotalBytesTransferred = TotalBytesTransferred; // StreamSize = StreamSize; // StreamBytesTransferred = StreamBytesTransferred; // dwStreamNumber = dwStreamNumber; diff --git a/7zip/CPP/7zip/UI/FileManager/LangUtils.cpp b/7zip/CPP/7zip/UI/FileManager/LangUtils.cpp index 94e9ecba3..bcaa5f1eb 100644 --- a/7zip/CPP/7zip/UI/FileManager/LangUtils.cpp +++ b/7zip/CPP/7zip/UI/FileManager/LangUtils.cpp @@ -58,6 +58,7 @@ static const CIDLangPair kLangPairs[] = { IDCANCEL, 402 }, { IDYES, 406 }, { IDNO, 407 }, + { IDCLOSE, 408 }, { IDHELP, 409 } }; diff --git a/7zip/CPP/7zip/UI/FileManager/ListViewDialog.cpp b/7zip/CPP/7zip/UI/FileManager/ListViewDialog.cpp index 80cdd7f62..a42e790b7 100644 --- a/7zip/CPP/7zip/UI/FileManager/ListViewDialog.cpp +++ b/7zip/CPP/7zip/UI/FileManager/ListViewDialog.cpp @@ -2,6 +2,9 @@ #include "StdAfx.h" +#include "../../../Windows/Clipboard.h" + +#include "EditDialog.h" #include "ListViewDialog.h" #include "RegistryUtils.h" @@ -11,6 +14,23 @@ using namespace NWindows; +static const unsigned kOneStringMaxSize = 1024; + + +static void ListView_GetSelected(NControl::CListView &listView, CUIntVector &vector) +{ + vector.Clear(); + int index = -1; + for (;;) + { + index = listView.GetNextSelectedItem(index); + if (index < 0) + break; + vector.Add(index); + } +} + + bool CListViewDialog::OnInit() { #ifdef LANG @@ -18,28 +38,92 @@ bool CListViewDialog::OnInit() #endif _listView.Attach(GetItem(IDL_LISTVIEW)); + if (NumColumns > 1) + { + LONG_PTR style = _listView.GetStyle(); + style &= ~(LONG_PTR)LVS_NOCOLUMNHEADER; + _listView.SetStyle(style); + } + CFmSettings st; st.Load(); + + DWORD exStyle = 0; + if (st.SingleClick) - _listView.SetExtendedListViewStyle(LVS_EX_ONECLICKACTIVATE | LVS_EX_TRACKSELECT); + exStyle |= LVS_EX_ONECLICKACTIVATE | LVS_EX_TRACKSELECT; + + exStyle |= LVS_EX_FULLROWSELECT; + if (exStyle != 0) + _listView.SetExtendedListViewStyle(exStyle); + SetText(Title); + const int kWidth = 400; + LVCOLUMN columnInfo; columnInfo.mask = LVCF_FMT | LVCF_WIDTH | LVCF_SUBITEM; columnInfo.fmt = LVCFMT_LEFT; columnInfo.iSubItem = 0; - columnInfo.cx = 200; + columnInfo.cx = kWidth; + columnInfo.pszText = NULL; // (TCHAR *)(const TCHAR *)""; // "Property" + + if (NumColumns > 1) + { + columnInfo.cx = 100; + /* + // Windows always uses LVCFMT_LEFT for first column. + // if we need LVCFMT_RIGHT, we can create dummy column and then remove it - _listView.InsertColumn(0, &columnInfo); + // columnInfo.mask |= LVCF_TEXT; + _listView.InsertColumn(0, &columnInfo); + + columnInfo.iSubItem = 1; + columnInfo.fmt = LVCFMT_RIGHT; + _listView.InsertColumn(1, &columnInfo); + _listView.DeleteColumn(0); + */ + } + // else + _listView.InsertColumn(0, &columnInfo); + if (NumColumns > 1) + { + // columnInfo.fmt = LVCFMT_LEFT; + columnInfo.cx = kWidth - columnInfo.cx; + columnInfo.iSubItem = 1; + // columnInfo.pszText = NULL; // (TCHAR *)(const TCHAR *)""; // "Value" + _listView.InsertColumn(1, &columnInfo); + } + + + UString s; + FOR_VECTOR (i, Strings) + { _listView.InsertItem(i, Strings[i]); - if (Strings.Size() > 0) + if (NumColumns > 1 && i < Values.Size()) + { + s = Values[i]; + if (s.Len() > kOneStringMaxSize) + { + s.DeleteFrom(kOneStringMaxSize); + s += " ..."; + } + s.Replace(L"\r\n", L" "); + s.Replace(L"\n", L" "); + _listView.SetSubItem(i, 1, s); + } + } + + if (SelectFirst && Strings.Size() > 0) _listView.SetItemState_FocusedSelected(0); _listView.SetColumnWidthAuto(0); + if (NumColumns > 1) + _listView.SetColumnWidthAuto(1); StringsWereChanged = false; NormalizeSize(); @@ -74,8 +158,103 @@ bool CListViewDialog::OnSize(WPARAM /* wParam */, int xSize, int ySize) return false; } + extern bool g_LVN_ITEMACTIVATE_Support; +void CListViewDialog::CopyToClipboard() +{ + CUIntVector indexes; + ListView_GetSelected(_listView, indexes); + UString s; + + FOR_VECTOR (i, indexes) + { + unsigned index = indexes[i]; + s += Strings[index]; + if (NumColumns > 1 && index < Values.Size()) + { + const UString &v = Values[index]; + // if (!v.IsEmpty()) + { + s += ": "; + s += v; + } + } + // if (indexes.Size() > 1) + { + s += + #ifdef _WIN32 + "\r\n" + #else + "\n" + #endif + ; + } + } + + ClipboardSetText(*this, s); +} + + +void CListViewDialog::ShowItemInfo() +{ + CUIntVector indexes; + ListView_GetSelected(_listView, indexes); + if (indexes.Size() != 1) + return; + unsigned index = indexes[0]; + + CEditDialog dlg; + if (NumColumns == 1) + dlg.Text = Strings[index]; + else + { + dlg.Title = Strings[index]; + if (index < Values.Size()) + dlg.Text = Values[index]; + } + + #ifdef _WIN32 + if (dlg.Text.Find(L'\r') < 0) + dlg.Text.Replace(L"\n", L"\r\n"); + #endif + + dlg.Create(*this); +} + + +void CListViewDialog::DeleteItems() +{ + for (;;) + { + int index = _listView.GetNextSelectedItem(-1); + if (index < 0) + break; + StringsWereChanged = true; + _listView.DeleteItem(index); + if ((unsigned)index < Strings.Size()) + Strings.Delete(index); + if ((unsigned)index < Values.Size()) + Values.Delete(index); + } + int focusedIndex = _listView.GetFocusedItem(); + if (focusedIndex >= 0) + _listView.SetItemState_FocusedSelected(focusedIndex); + _listView.SetColumnWidthAuto(0); +} + + +void CListViewDialog::OnEnter() +{ + if (IsKeyDown(VK_MENU) + || NumColumns > 1) + { + ShowItemInfo(); + return; + } + OnOK(); +} + bool CListViewDialog::OnNotify(UINT /* controlID */, LPNMHDR header) { if (header->hwndFrom != _listView) @@ -85,7 +264,7 @@ bool CListViewDialog::OnNotify(UINT /* controlID */, LPNMHDR header) case LVN_ITEMACTIVATE: if (g_LVN_ITEMACTIVATE_Support) { - OnOK(); + OnEnter(); return true; } break; @@ -93,7 +272,7 @@ bool CListViewDialog::OnNotify(UINT /* controlID */, LPNMHDR header) case NM_RETURN: // probabably it's unused if (!g_LVN_ITEMACTIVATE_Support) { - OnOK(); + OnEnter(); return true; } break; @@ -107,19 +286,7 @@ bool CListViewDialog::OnNotify(UINT /* controlID */, LPNMHDR header) { if (!DeleteIsAllowed) return false; - for (;;) - { - int index = _listView.GetNextSelectedItem(-1); - if (index < 0) - break; - StringsWereChanged = true; - _listView.DeleteItem(index); - Strings.Delete(index); - } - int focusedIndex = _listView.GetFocusedItem(); - if (focusedIndex >= 0) - _listView.SetItemState_FocusedSelected(focusedIndex); - _listView.SetColumnWidthAuto(0); + DeleteItems(); return true; } case 'A': @@ -129,6 +296,17 @@ bool CListViewDialog::OnNotify(UINT /* controlID */, LPNMHDR header) _listView.SelectAll(); return true; } + break; + } + case VK_INSERT: + case 'C': + { + if (IsKeyDown(VK_CONTROL)) + { + CopyToClipboard(); + return true; + } + break; } } } diff --git a/7zip/CPP/7zip/UI/FileManager/ListViewDialog.h b/7zip/CPP/7zip/UI/FileManager/ListViewDialog.h index b7ba38309..00206afd2 100644 --- a/7zip/CPP/7zip/UI/FileManager/ListViewDialog.h +++ b/7zip/CPP/7zip/UI/FileManager/ListViewDialog.h @@ -15,16 +15,32 @@ class CListViewDialog: public NWindows::NControl::CModalDialog virtual bool OnInit(); virtual bool OnSize(WPARAM wParam, int xSize, int ySize); virtual bool OnNotify(UINT controlID, LPNMHDR header); + void CopyToClipboard(); + void DeleteItems(); + void ShowItemInfo(); + void OnEnter(); public: UString Title; + + bool SelectFirst; bool DeleteIsAllowed; bool StringsWereChanged; + UStringVector Strings; + UStringVector Values; + int FocusedItemIndex; + unsigned NumColumns; INT_PTR Create(HWND wndParent = 0) { return CModalDialog::Create(IDD_LISTVIEW, wndParent); } - CListViewDialog(): DeleteIsAllowed(false) {} + CListViewDialog(): + SelectFirst(false), + DeleteIsAllowed(false), + StringsWereChanged(false), + FocusedItemIndex(-1), + NumColumns(1) + {} }; #endif diff --git a/7zip/CPP/7zip/UI/FileManager/ListViewDialog.rc b/7zip/CPP/7zip/UI/FileManager/ListViewDialog.rc index 14d3b2e6c..961d224a9 100644 --- a/7zip/CPP/7zip/UI/FileManager/ListViewDialog.rc +++ b/7zip/CPP/7zip/UI/FileManager/ListViewDialog.rc @@ -1,8 +1,8 @@ #include "ListViewDialogRes.h" #include "../../GuiCommon.rc" -#define xc 320 -#define yc 240 +#define xc 440 +#define yc 320 IDD_LISTVIEW DIALOG 0, 0, xs, ys MY_MODAL_RESIZE_DIALOG_STYLE MY_FONT CAPTION "ListView" diff --git a/7zip/CPP/7zip/UI/FileManager/Panel.cpp b/7zip/CPP/7zip/UI/FileManager/Panel.cpp index 973e64ddd..afb5649a1 100644 --- a/7zip/CPP/7zip/UI/FileManager/Panel.cpp +++ b/7zip/CPP/7zip/UI/FileManager/Panel.cpp @@ -879,8 +879,9 @@ void CPanel::AddToArchive() FOR_VECTOR (i, indices) names.Add(curPrefix + GetItemRelPath2(indices[i])); - bool fromPrev = (names.Size() > 1); - const UString arcName = CreateArchiveName(names.Front(), fromPrev, false); + + const UString arcName = CreateArchiveName(names); + HRESULT res = CompressFiles(destCurDirPrefix, arcName, L"", true, // addExtension names, false, true, false); diff --git a/7zip/CPP/7zip/UI/FileManager/PanelCopy.cpp b/7zip/CPP/7zip/UI/FileManager/PanelCopy.cpp index d07def1d5..c3416cc7d 100644 --- a/7zip/CPP/7zip/UI/FileManager/PanelCopy.cpp +++ b/7zip/CPP/7zip/UI/FileManager/PanelCopy.cpp @@ -16,7 +16,11 @@ using namespace NWindows; class CPanelCopyThread: public CProgressThreadVirt { + bool ResultsWereShown; + bool NeedShowRes; + HRESULT ProcessVirt(); + virtual void ProcessWasFinished_GuiVirt(); public: const CCopyToOptions *options; CMyComPtr FolderOperations; @@ -25,14 +29,34 @@ class CPanelCopyThread: public CProgressThreadVirt CMyComPtr ExtractCallback; CHashBundle Hash; - UString FirstFilePath; + // UString FirstFilePath; HRESULT Result; - - CPanelCopyThread(): Result(E_FAIL) {} + void ShowFinalResults(HWND hwnd); + + CPanelCopyThread(): + Result(E_FAIL), + ResultsWereShown(false), + NeedShowRes(false) + {} }; + +void CPanelCopyThread::ShowFinalResults(HWND hwnd) +{ + if (NeedShowRes) + if (!ResultsWereShown) + { + ResultsWereShown = true; + ShowHashResults(Hash, hwnd); + } +} +void CPanelCopyThread::ProcessWasFinished_GuiVirt() +{ + ShowFinalResults(*this); +} + HRESULT CPanelCopyThread::ProcessVirt() { /* @@ -69,11 +93,15 @@ HRESULT CPanelCopyThread::ProcessVirt() BoolToInt(options->replaceAltStreamChars), options->folder, ExtractCallback); - if (Result == S_OK && !ExtractCallbackSpec->ThereAreMessageErrors && - (!options->hashMethods.IsEmpty() || options->testMode)) + if (Result == S_OK && !ExtractCallbackSpec->ThereAreMessageErrors) { - CProgressMessageBoxPair &pair = GetMessagePair(false); // GetMessagePair(ExtractCallbackSpec->Hash.NumErrors != 0); - AddHashBundleRes(pair.Message, Hash, FirstFilePath); + if (!options->hashMethods.IsEmpty()) + NeedShowRes = true; + else if (options->testMode) + { + CProgressMessageBoxPair &pair = GetMessagePair(false); // GetMessagePair(ExtractCallbackSpec->Hash.NumErrors != 0); + AddHashBundleRes(pair.Message, Hash); + } } return Result; @@ -92,7 +120,6 @@ static void ThrowException_if_Error(HRESULT res) #endif */ - HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &indices, UStringVector *messages, bool &usePassword, UString &password) @@ -102,7 +129,7 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind UString errorMessage = LangString(IDS_OPERATION_IS_NOT_SUPPORTED); if (options.showErrorMessages) MessageBox_Error(errorMessage); - else if (messages != 0) + else if (messages) messages->Add(errorMessage); return E_FAIL; } @@ -124,14 +151,17 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind extracter.ExtractCallback = extracter.ExtractCallbackSpec; extracter.options = &options; - extracter.ExtractCallbackSpec->ProgressDialog = &extracter.ProgressDialog; - extracter.ProgressDialog.CompressingMode = false; + extracter.ExtractCallbackSpec->ProgressDialog = &extracter; + extracter.CompressingMode = false; extracter.ExtractCallbackSpec->StreamMode = options.streamMode; if (indices.Size() == 1) - extracter.FirstFilePath = GetItemRelPath(indices[0]); + { + extracter.Hash.FirstFileName = GetItemRelPath(indices[0]); + extracter.Hash.MainName = extracter.Hash.FirstFileName; + } if (options.VirtFileSystem) { @@ -159,7 +189,7 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind extracter.ExtractCallbackSpec->SetHashCalc(&extracter.Hash); } - extracter.Hash.Init(); + // extracter.Hash.Init(); UString title; { @@ -185,9 +215,9 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind UString progressWindowTitle ("7-Zip"); // LangString(IDS_APP_TITLE); - extracter.ProgressDialog.MainWindow = GetParent(); - extracter.ProgressDialog.MainTitle = progressWindowTitle; - extracter.ProgressDialog.MainAddTitle = title + L' '; + extracter.MainWindow = GetParent(); + extracter.MainTitle = progressWindowTitle; + extracter.MainAddTitle = title + L' '; extracter.ExtractCallbackSpec->OverwriteMode = NExtract::NOverwriteMode::kAsk; extracter.ExtractCallbackSpec->Init(); @@ -199,8 +229,9 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind RINOK(extracter.Create(title, GetParent())); - if (messages != 0) - *messages = extracter.ProgressDialog.Sync.Messages; + + if (messages) + *messages = extracter.Sync.Messages; res = extracter.Result; if (res == S_OK && extracter.ExtractCallbackSpec->IsOK()) @@ -208,6 +239,9 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind usePassword = extracter.ExtractCallbackSpec->PasswordIsDefined; password = extracter.ExtractCallbackSpec->Password; } + + extracter.ShowFinalResults(_window); + } RefreshTitleAlways(); @@ -296,8 +330,8 @@ HRESULT CPanel::CopyFrom(bool moveMode, const UString &folderPrefix, const UStri NWindows::CThread thread; RINOK(thread.Create(CThreadUpdate::MyThreadFunction, &updater)); updater.ProgressDialog.Create(title, thread, GetParent()); - - if (messages != 0) + + if (messages) *messages = updater.ProgressDialog.Sync.Messages; res = updater.Result; @@ -308,7 +342,7 @@ HRESULT CPanel::CopyFrom(bool moveMode, const UString &folderPrefix, const UStri UString errorMessage = LangString(IDS_OPERATION_IS_NOT_SUPPORTED); if (showErrorMessages) MessageBox_Error(errorMessage); - else if (messages != 0) + else if (messages) messages->Add(errorMessage); return E_ABORT; } diff --git a/7zip/CPP/7zip/UI/FileManager/PanelCrc.cpp b/7zip/CPP/7zip/UI/FileManager/PanelCrc.cpp index 227c45330..1cfbfe5d5 100644 --- a/7zip/CPP/7zip/UI/FileManager/PanelCrc.cpp +++ b/7zip/CPP/7zip/UI/FileManager/PanelCrc.cpp @@ -139,18 +139,44 @@ DWORD CDirEnumerator::GetNextFile(NFind::CFileInfo &fi, bool &filled, FString &r class CThreadCrc: public CProgressThreadVirt { + bool ResultsWereShown; + bool WasFinished; + HRESULT ProcessVirt(); + virtual void ProcessWasFinished_GuiVirt(); public: CDirEnumerator Enumerator; CHashBundle Hash; + // FString FirstFilePath; void SetStatus(const UString &s); void AddErrorMessage(DWORD systemError, const FChar *name); + void ShowFinalResults(HWND hwnd); + + CThreadCrc(): + ResultsWereShown(false), + WasFinished(false) + {} }; +void CThreadCrc::ShowFinalResults(HWND hwnd) +{ + if (WasFinished) + if (!ResultsWereShown) + { + ResultsWereShown = true; + ShowHashResults(Hash, hwnd); + } +} + +void CThreadCrc::ProcessWasFinished_GuiVirt() +{ + ShowFinalResults(*this); +} + void CThreadCrc::AddErrorMessage(DWORD systemError, const FChar *name) { - ProgressDialog.Sync.AddError_Code_Name(systemError, fs2us(Enumerator.BasePrefix + name)); + Sync.AddError_Code_Name(systemError, fs2us(Enumerator.BasePrefix + name)); Hash.NumErrors++; } @@ -162,18 +188,18 @@ void CThreadCrc::SetStatus(const UString &s2) s.Add_Space_if_NotEmpty(); s += fs2us(Enumerator.BasePrefix); } - ProgressDialog.Sync.Set_Status(s); + Sync.Set_Status(s); } HRESULT CThreadCrc::ProcessVirt() { - Hash.Init(); + // Hash.Init(); CMyBuffer buf; if (!buf.Allocate(kBufSize)) return E_OUTOFMEMORY; - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; SetStatus(LangString(IDS_SCANNING)); @@ -233,7 +259,6 @@ HRESULT CThreadCrc::ProcessVirt() Enumerator.Init(); FString tempPath; - FString firstFilePath; bool isFirstFile = true; UInt64 errorsFilesSize = 0; @@ -264,7 +289,7 @@ HRESULT CThreadCrc::ProcessVirt() } if (isFirstFile) { - firstFilePath = path; + Hash.FirstFileName = path; isFirstFile = false; } sync.Set_FilePath(fs2us(path)); @@ -303,12 +328,13 @@ HRESULT CThreadCrc::ProcessVirt() SetStatus(L""); CProgressMessageBoxPair &pair = GetMessagePair(Hash.NumErrors != 0); - AddHashBundleRes(pair.Message, Hash, fs2us(firstFilePath)); + WasFinished = true; LangString(IDS_CHECKSUM_INFORMATION, pair.Title); return S_OK; } + HRESULT CApp::CalculateCrc2(const UString &methodName) { unsigned srcPanelIndex = GetFocusedPanelIndex(); @@ -338,14 +364,19 @@ HRESULT CApp::CalculateCrc2(const UString &methodName) { CThreadCrc t; + { UStringVector methods; methods.Add(methodName); RINOK(t.Hash.SetMethods(EXTERNAL_CODECS_VARS_G methods)); } + FOR_VECTOR (i, indices) t.Enumerator.FilePaths.Add(us2fs(srcPanel.GetItemRelPath(indices[i]))); + if (t.Enumerator.FilePaths.Size() == 1) + t.Hash.MainName = t.Enumerator.FilePaths[0]; + UString basePrefix = srcPanel.GetFsPath(); UString basePrefix2 = basePrefix; if (basePrefix2.Back() == ':') @@ -360,17 +391,20 @@ HRESULT CApp::CalculateCrc2(const UString &methodName) t.Enumerator.EnterToDirs = !GetFlatMode(); - t.ProgressDialog.ShowCompressionInfo = false; + t.ShowCompressionInfo = false; UString title = LangString(IDS_CHECKSUM_CALCULATING); - t.ProgressDialog.MainWindow = _window; - t.ProgressDialog.MainTitle = "7-Zip"; // LangString(IDS_APP_TITLE); - t.ProgressDialog.MainAddTitle = title; - t.ProgressDialog.MainAddTitle.Add_Space(); + t.MainWindow = _window; + t.MainTitle = "7-Zip"; // LangString(IDS_APP_TITLE); + t.MainAddTitle = title; + t.MainAddTitle.Add_Space(); RINOK(t.Create(title, _window)); + + t.ShowFinalResults(_window); } + RefreshTitleAlways(); return S_OK; } diff --git a/7zip/CPP/7zip/UI/FileManager/PanelDrag.cpp b/7zip/CPP/7zip/UI/FileManager/PanelDrag.cpp index b6fca5949..6a781e225 100644 --- a/7zip/CPP/7zip/UI/FileManager/PanelDrag.cpp +++ b/7zip/CPP/7zip/UI/FileManager/PanelDrag.cpp @@ -81,15 +81,17 @@ CDataObject::CDataObject() STDMETHODIMP CDataObject::SetData(LPFORMATETC etc, STGMEDIUM *medium, BOOL /* release */) { - if (etc->cfFormat == m_SetFolderFormat && etc->tymed == TYMED_HGLOBAL && - etc->dwAspect == DVASPECT_CONTENT && medium->tymed == TYMED_HGLOBAL) + if (etc->cfFormat == m_SetFolderFormat + && etc->tymed == TYMED_HGLOBAL + && etc->dwAspect == DVASPECT_CONTENT + && medium->tymed == TYMED_HGLOBAL) { Path.Empty(); - if (medium->hGlobal == 0) + if (!medium->hGlobal) return S_OK; size_t size = GlobalSize(medium->hGlobal) / sizeof(wchar_t); const wchar_t *src = (const wchar_t *)GlobalLock(medium->hGlobal); - if (src != 0) + if (src) { for (size_t i = 0; i < size; i++) { @@ -109,13 +111,13 @@ static HGLOBAL DuplicateGlobalMem(HGLOBAL srcGlobal) { SIZE_T size = GlobalSize(srcGlobal); const void *src = GlobalLock(srcGlobal); - if (src == 0) + if (!src) return 0; HGLOBAL destGlobal = GlobalAlloc(GHND | GMEM_SHARE, size); - if (destGlobal != 0) + if (destGlobal) { void *dest = GlobalLock(destGlobal); - if (dest == 0) + if (!dest) { GlobalFree(destGlobal); destGlobal = 0; @@ -136,7 +138,7 @@ STDMETHODIMP CDataObject::GetData(LPFORMATETC etc, LPSTGMEDIUM medium) medium->tymed = m_Etc.tymed; medium->pUnkForRelease = 0; medium->hGlobal = DuplicateGlobalMem(hGlobal); - if (medium->hGlobal == 0) + if (!medium->hGlobal) return E_OUTOFMEMORY; return S_OK; } @@ -261,7 +263,7 @@ static bool CopyNamesToHGlobal(NMemory::CGlobal &hgDrop, const UStringVector &na NMemory::CGlobalLock dropLock(hgDrop); DROPFILES* dropFiles = (DROPFILES*)dropLock.GetPointer(); - if (dropFiles == 0) + if (!dropFiles) return false; dropFiles->fNC = FALSE; dropFiles->pt.x = 0; @@ -291,7 +293,7 @@ static bool CopyNamesToHGlobal(NMemory::CGlobal &hgDrop, const UStringVector &na NMemory::CGlobalLock dropLock(hgDrop); DROPFILES* dropFiles = (DROPFILES*)dropLock.GetPointer(); - if (dropFiles == 0) + if (!dropFiles) return false; dropFiles->fNC = FALSE; dropFiles->pt.x = 0; @@ -336,7 +338,11 @@ void CPanel::OnDrag(LPNMLISTVIEW /* nmListView */) dirPrefix = us2fs(GetFsPath()); else { - tempDirectory.Create(kTempDirPrefix); + if (!tempDirectory.Create(kTempDirPrefix)) + { + MessageBox_Error(L"Can't create temp folder"); + return; + } dirPrefix = tempDirectory.GetPath(); // dirPrefix2 = dirPrefix; NFile::NName::NormalizeDirPathPrefix(dirPrefix); @@ -394,11 +400,67 @@ void CPanel::OnDrag(LPNMLISTVIEW /* nmListView */) dropSourceSpec->DataObjectSpec = dataObjectSpec; dropSourceSpec->DataObject = dataObjectSpec; - bool moveIsAllowed = isFSFolder; + + /* + CTime - file creation timestamp. + There are two operations in Windows with Drag and Drop: + COPY_OPERATION - icon with Plus sign - CTime will be set as current_time. + MOVE_OPERATION - icon without Plus sign - CTime will be preserved + + Note: if we call DoDragDrop() with (effectsOK = DROPEFFECT_MOVE), then + it will use MOVE_OPERATION and CTime will be preserved. + But MoveFile() function doesn't preserve CTime, if different volumes are used. + Why it's so? + Does DoDragDrop() use some another function (not MoveFile())? + + if (effectsOK == DROPEFFECT_COPY) it works as COPY_OPERATION + + if (effectsOK == DROPEFFECT_MOVE) drag works as MOVE_OPERATION + if (effectsOK == (DROPEFFECT_COPY | DROPEFFECT_MOVE)) + { + if we drag file to same volume, then Windows suggests: + CTRL - COPY_OPERATION + [default] - MOVE_OPERATION + + if we drag file to another volume, then Windows suggests + [default] - COPY_OPERATION + SHIFT - MOVE_OPERATION + } + + We want to use MOVE_OPERATION for extracting from archive (open in 7-Zip) to Explorer: + It has the following advantages: + 1) it uses fast MOVE_OPERATION instead of slow COPY_OPERATION and DELETE, if same volume. + 2) it preserved CTime + + Some another programs support only COPY_OPERATION. + So we can use (DROPEFFECT_COPY | DROPEFFECT_MOVE) + + Also another program can return from DoDragDrop() before + files using. But we delete temp folder after DoDragDrop(), + and another program can't open input files in that case. + + We create objects: + IDropSource *dropSource + IDataObject *dataObject + if DropTarget is 7-Zip window, then 7-Zip's + IDropTarget::DragOver() sets Path in IDataObject. + and + IDropSource::QueryContinueDrag() sets NeedPostCopy, if Path is not epmty. + So we can detect destination path after DoDragDrop(). + Now we don't know any good way to detect destination path for D&D to Explorer. + */ + + bool moveIsAllowed = isFSFolder; + /* DWORD effectsOK = DROPEFFECT_COPY; if (moveIsAllowed) effectsOK |= DROPEFFECT_MOVE; + */ + + // 18.04: was changed + DWORD effectsOK = DROPEFFECT_MOVE | DROPEFFECT_COPY; + DWORD effect; _panelCallback->DragBegin(); @@ -418,7 +480,8 @@ void CPanel::OnDrag(LPNMLISTVIEW /* nmListView */) NFile::NName::NormalizeDirPathPrefix(dataObjectSpec->Path); CCopyToOptions options; options.folder = dataObjectSpec->Path; - options.moveMode = (effect == DROPEFFECT_MOVE); + // if MOVE is not allowed, we just use COPY operation + options.moveMode = (effect == DROPEFFECT_MOVE && moveIsAllowed); res = CopyTo(options, indices, &dropSourceSpec->Messages); } /* @@ -475,7 +538,7 @@ static void MySetDropHighlighted(HWND hWnd, int index, bool enable) void CDropTarget::RemoveSelection() { - if (m_SelectionIndex >= 0 && m_Panel != 0) + if (m_SelectionIndex >= 0 && m_Panel) MySetDropHighlighted(m_Panel->_listView, m_SelectionIndex, false); m_SelectionIndex = -1; } @@ -555,7 +618,7 @@ void CDropTarget::PositionCursor(POINTL ptl) bool CDropTarget::IsFsFolderPath() const { - if (!m_IsAppTarget && m_Panel != 0) + if (!m_IsAppTarget && m_Panel) return (m_Panel->IsFSFolder() || (m_Panel->IsFSDrivesFolder() && m_SelectionIndex >= 0)); return false; } @@ -614,7 +677,7 @@ static void GetNamesFromDataObject(IDataObject *dataObject, UStringVector &names size_t blockSize = GlobalSize(medium.hGlobal); NMemory::CGlobalLock dropLock(medium.hGlobal); const DROPFILES* dropFiles = (DROPFILES*)dropLock.GetPointer(); - if (dropFiles == 0) + if (!dropFiles) return; if (blockSize < dropFiles->pFiles) return; @@ -629,7 +692,7 @@ static void GetNamesFromDataObject(IDataObject *dataObject, UStringVector &names bool CDropTarget::IsItSameDrive() const { - if (m_Panel == 0) + if (!m_Panel) return false; if (!IsFsFolderPath()) return false; @@ -662,6 +725,21 @@ bool CDropTarget::IsItSameDrive() const return true; } + +/* + There are 2 different actions, when we drag to 7-Zip: + 1) Drag from any external program except of Explorer to "7-Zip" FS folder. + We want to create new archive for that operation. + 2) all another operation work as usual file COPY/MOVE + - Drag from "7-Zip" FS to "7-Zip" FS. + COPY/MOVE are supported. + - Drag to open archive in 7-Zip. + We want to update archive. + We replace COPY to MOVE. + - Drag from "7-Zip" archive to "7-Zip" FS. + We replace COPY to MOVE. +*/ + DWORD CDropTarget::GetEffect(DWORD keyState, POINTL /* pt */, DWORD allowedEffect) { if (!m_DropIsAllowed || !m_PanelDropIsAllowed) @@ -671,10 +749,12 @@ DWORD CDropTarget::GetEffect(DWORD keyState, POINTL /* pt */, DWORD allowedEffec allowedEffect &= ~DROPEFFECT_MOVE; DWORD effect = 0; + if (keyState & MK_CONTROL) effect = allowedEffect & DROPEFFECT_COPY; else if (keyState & MK_SHIFT) effect = allowedEffect & DROPEFFECT_MOVE; + if (effect == 0) { if (allowedEffect & DROPEFFECT_COPY) @@ -716,10 +796,10 @@ bool CDropTarget::SetPath(bool enablePath) const path = GetTargetPath(); size_t size = path.Len() + 1; medium.hGlobal = GlobalAlloc(GHND | GMEM_SHARE, size * sizeof(wchar_t)); - if (medium.hGlobal == 0) + if (!medium.hGlobal) return false; wchar_t *dest = (wchar_t *)GlobalLock(medium.hGlobal); - if (dest == 0) + if (!dest) { GlobalUnlock(medium.hGlobal); return false; @@ -860,8 +940,10 @@ void CPanel::CompressDropFiles(const UStringVector &fileNames, const UString &fo if (IsFolderInTemp(folderPath2F)) folderPath2 = ROOT_FS_FOLDER; } - const UString archiveName = CreateArchiveName(fileNames.Front(), (fileNames.Size() > 1), false); - CompressFiles(folderPath2, archiveName, L"", + + const UString arcName = CreateArchiveName(fileNames); + + CompressFiles(folderPath2, arcName, L"", true, // addExtension fileNames, false, // email diff --git a/7zip/CPP/7zip/UI/FileManager/PanelFolderChange.cpp b/7zip/CPP/7zip/UI/FileManager/PanelFolderChange.cpp index 1f93f8624..38b83124d 100644 --- a/7zip/CPP/7zip/UI/FileManager/PanelFolderChange.cpp +++ b/7zip/CPP/7zip/UI/FileManager/PanelFolderChange.cpp @@ -625,6 +625,7 @@ void CPanel::FoldersHistory() { CListViewDialog listViewDialog; listViewDialog.DeleteIsAllowed = true; + listViewDialog.SelectFirst = true; LangString(IDS_FOLDERS_HISTORY, listViewDialog.Title); _appState->FolderHistory.GetList(listViewDialog.Strings); if (listViewDialog.Create(GetParent()) != IDOK) diff --git a/7zip/CPP/7zip/UI/FileManager/PanelItemOpen.cpp b/7zip/CPP/7zip/UI/FileManager/PanelItemOpen.cpp index f7c05a39e..89f49bf69 100644 --- a/7zip/CPP/7zip/UI/FileManager/PanelItemOpen.cpp +++ b/7zip/CPP/7zip/UI/FileManager/PanelItemOpen.cpp @@ -674,7 +674,7 @@ static const char * const kStartExtensions = #endif " exe bat ps1 com" " chm" - " msi doc xls ppt pps wps wpt wks xlr wdb vsd pub" + " msi doc dot xls ppt pps wps wpt wks xlr wdb vsd pub" " docx docm dotx dotm xlsx xlsm xltx xltm xlsb xps" " xlam pptx pptm potx potm ppam ppsx ppsm xsn" @@ -1151,7 +1151,7 @@ HRESULT CPanel::OnOpenItemChanged(UInt32 index, const wchar_t *fullFilePath, CThreadCopyFrom t; t.UpdateCallbackSpec = new CUpdateCallback100Imp; t.UpdateCallback = t.UpdateCallbackSpec; - t.UpdateCallbackSpec->ProgressDialog = &t.ProgressDialog; + t.UpdateCallbackSpec->ProgressDialog = &t; t.ItemIndex = index; t.FullPath = fullFilePath; t.FolderOperations = _folderOperations; diff --git a/7zip/CPP/7zip/UI/FileManager/PanelMenu.cpp b/7zip/CPP/7zip/UI/FileManager/PanelMenu.cpp index 3db65be4a..11c6d7c1a 100644 --- a/7zip/CPP/7zip/UI/FileManager/PanelMenu.cpp +++ b/7zip/CPP/7zip/UI/FileManager/PanelMenu.cpp @@ -16,6 +16,7 @@ #include "App.h" #include "FormatUtils.h" #include "LangUtils.h" +#include "ListViewDialog.h" #include "MyLoadMenu.h" #include "PropertyName.h" @@ -50,17 +51,40 @@ void CPanel::InvokeSystemCommand(const char *command) contextMenu->InvokeCommand(&ci); } -static const char * const kSeparator = "----------------------------\n"; -static const char * const kSeparatorSmall = "----\n"; -static const char * const kPropValueSeparator = ": "; +static const char * const kSeparator = "------------------------"; +static const char * const kSeparatorSmall = "----------------"; extern UString ConvertSizeToString(UInt64 value) throw(); bool IsSizeProp(UINT propID) throw(); UString GetOpenArcErrorMessage(UInt32 errorFlags); + +static void AddListAscii(CListViewDialog &dialog, const char *s) +{ + dialog.Strings.Add((UString)s); + dialog.Values.AddNew(); +} + +static void AddSeparator(CListViewDialog &dialog) +{ + AddListAscii(dialog, kSeparator); +} + +static void AddSeparatorSmall(CListViewDialog &dialog) +{ + AddListAscii(dialog, kSeparatorSmall); +} + +static void AddPropertyPair(const UString &name, const UString &val, CListViewDialog &dialog) +{ + dialog.Strings.Add(name); + dialog.Values.Add(val); +} + + static void AddPropertyString(PROPID propID, const wchar_t *nameBSTR, - const NCOM::CPropVariant &prop, UString &s) + const NCOM::CPropVariant &prop, CListViewDialog &dialog) { if (prop.vt != VT_EMPTY) { @@ -87,23 +111,16 @@ static void AddPropertyString(PROPID propID, const wchar_t *nameBSTR, if (!val.IsEmpty()) { - s += GetNameOfProperty(propID, nameBSTR); - s += kPropValueSeparator; - /* - if (propID == kpidComment) - s.Add_LF(); - */ - s += val; - s.Add_LF(); + AddPropertyPair(GetNameOfProperty(propID, nameBSTR), val, dialog); } } } -static void AddPropertyString(PROPID propID, UInt64 val, UString &s) +static void AddPropertyString(PROPID propID, UInt64 val, CListViewDialog &dialog) { NCOM::CPropVariant prop = val; - AddPropertyString(propID, NULL, prop, s); + AddPropertyString(propID, NULL, prop, dialog); } @@ -137,7 +154,9 @@ void CPanel::Properties() } { - UString message; + CListViewDialog message; + // message.DeleteIsAllowed = false; + // message.SelectFirst = false; CRecordVector operatedIndices; GetOperatedItemIndices(operatedIndices); @@ -205,15 +224,12 @@ void CPanel::Properties() } } } - message += GetNameOfProperty(propID, name); - message += kPropValueSeparator; - message += s.Ptr(); - message.Add_LF(); + AddPropertyPair(GetNameOfProperty(propID, name), (UString)s.Ptr(), message); } } } - message += kSeparator; + AddSeparator(message); } else if (operatedIndices.Size() >= 1) { @@ -239,8 +255,7 @@ void CPanel::Properties() { wchar_t temp[32]; ConvertUInt32ToString(operatedIndices.Size(), temp); - message += MyFormatNew(g_App.LangString_N_SELECTED_ITEMS, temp); - message.Add_LF(); + AddPropertyPair(L"", MyFormatNew(g_App.LangString_N_SELECTED_ITEMS, temp), message); } if (numDirs != 0) @@ -250,7 +265,7 @@ void CPanel::Properties() AddPropertyString(kpidSize, unpackSize, message); AddPropertyString(kpidPackSize, packSize, message); - message += kSeparator; + AddSeparator(message); } @@ -309,7 +324,7 @@ void CPanel::Properties() { const int kNumSpecProps = ARRAY_SIZE(kSpecProps); - message += kSeparator; + AddSeparator(message); for (Int32 i = -(int)kNumSpecProps; i < (Int32)numProps; i++) { @@ -334,7 +349,7 @@ void CPanel::Properties() UInt32 numProps; if (getProps->GetArcNumProps2(level, &numProps) == S_OK) { - message += kSeparatorSmall; + AddSeparatorSmall(message); for (Int32 i = 0; i < (Int32)numProps; i++) { CMyComBSTR name; @@ -352,10 +367,15 @@ void CPanel::Properties() } } } - ::MessageBoxW(*(this), message, LangString(IDS_PROPERTIES), MB_OK); + + message.Title = LangString(IDS_PROPERTIES); + message.NumColumns = 2; + message.Create(GetParent()); } } + + void CPanel::EditCut() { // InvokeSystemCommand("cut"); diff --git a/7zip/CPP/7zip/UI/FileManager/PanelOperations.cpp b/7zip/CPP/7zip/UI/FileManager/PanelOperations.cpp index d049a3ce6..00bda1a98 100644 --- a/7zip/CPP/7zip/UI/FileManager/PanelOperations.cpp +++ b/7zip/CPP/7zip/UI/FileManager/PanelOperations.cpp @@ -79,10 +79,10 @@ HRESULT CThreadFolderOperations::DoOperation(CPanel &panel, const UString &progr { UpdateCallbackSpec = new CUpdateCallback100Imp; UpdateCallback = UpdateCallbackSpec; - UpdateCallbackSpec->ProgressDialog = &ProgressDialog; + UpdateCallbackSpec->ProgressDialog = this; - ProgressDialog.WaitMode = true; - ProgressDialog.Sync.FinalMessage.ErrorMessage.Title = titleError; + WaitMode = true; + Sync.FinalMessage.ErrorMessage.Title = titleError; Result = S_OK; UpdateCallbackSpec->Init(); @@ -95,11 +95,11 @@ HRESULT CThreadFolderOperations::DoOperation(CPanel &panel, const UString &progr } - ProgressDialog.MainWindow = panel._mainWindow; // panel.GetParent() - ProgressDialog.MainTitle = "7-Zip"; // LangString(IDS_APP_TITLE); - ProgressDialog.MainAddTitle = progressTitle + L' '; + MainWindow = panel._mainWindow; // panel.GetParent() + MainTitle = "7-Zip"; // LangString(IDS_APP_TITLE); + MainAddTitle = progressTitle + L' '; - RINOK(Create(progressTitle, ProgressDialog.MainWindow)); + RINOK(Create(progressTitle, MainWindow)); return Result; } diff --git a/7zip/CPP/7zip/UI/FileManager/PanelSplitFile.cpp b/7zip/CPP/7zip/UI/FileManager/PanelSplitFile.cpp index 7cab2cf93..7c633323e 100644 --- a/7zip/CPP/7zip/UI/FileManager/PanelSplitFile.cpp +++ b/7zip/CPP/7zip/UI/FileManager/PanelSplitFile.cpp @@ -157,7 +157,7 @@ HRESULT CThreadSplit::ProcessVirt() if (!inFile.GetLength(length)) return GetLastError(); - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; sync.Set_NumBytesTotal(length); UInt64 pos = 0; @@ -306,7 +306,7 @@ void CApp::Split() CThreadSplit spliter; spliter.NumVolumes = numVolumes; - CProgressDialog &progressDialog = spliter.ProgressDialog; + CProgressDialog &progressDialog = spliter; UString progressWindowTitle ("7-Zip"); // LangString(IDS_APP_TITLE, 0x03000000); UString title = LangString(IDS_SPLITTING); @@ -362,7 +362,7 @@ HRESULT CThreadCombine::ProcessVirt() return res; } - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; sync.Set_NumBytesTotal(TotalSize); CMyBuffer bufferObject; @@ -534,7 +534,7 @@ void CApp::Combine() return; } - CProgressDialog &progressDialog = combiner.ProgressDialog; + CProgressDialog &progressDialog = combiner; progressDialog.ShowCompressionInfo = false; UString progressWindowTitle ("7-Zip"); // LangString(IDS_APP_TITLE, 0x03000000); diff --git a/7zip/CPP/7zip/UI/FileManager/ProgressDialog2.cpp b/7zip/CPP/7zip/UI/FileManager/ProgressDialog2.cpp index 762f2492e..28e3eae73 100644 --- a/7zip/CPP/7zip/UI/FileManager/ProgressDialog2.cpp +++ b/7zip/CPP/7zip/UI/FileManager/ProgressDialog2.cpp @@ -236,7 +236,10 @@ void CProgressSync::AddError_Code_Name(DWORD systemError, const wchar_t *name) AddError_Message_Name(s, name); } -CProgressDialog::CProgressDialog(): _timer(0), CompressingMode(true), MainWindow(0) +CProgressDialog::CProgressDialog(): + _timer(0), + CompressingMode(true), + MainWindow(0) { _isDir = false; @@ -974,7 +977,9 @@ bool CProgressDialog::OnExternalCloseMessage() ::SendMessage(GetItem(IDCANCEL), BM_SETSTYLE, BS_DEFPUSHBUTTON, MAKELPARAM(TRUE, 0)); HideItem(IDB_PROGRESS_BACKGROUND); HideItem(IDB_PAUSE); - + + ProcessWasFinished_GuiVirt(); + bool thereAreMessages; CProgressFinalMessage fm; { @@ -982,6 +987,7 @@ bool CProgressDialog::OnExternalCloseMessage() thereAreMessages = !Sync.Messages.IsEmpty(); fm = Sync.FinalMessage; } + if (!fm.ErrorMessage.Message.IsEmpty()) { MessagesDisplayed = true; @@ -992,6 +998,7 @@ bool CProgressDialog::OnExternalCloseMessage() else if (!thereAreMessages) { MessagesDisplayed = true; + if (!fm.OkMessage.Message.IsEmpty()) { if (fm.OkMessage.Title.IsEmpty()) @@ -1245,11 +1252,24 @@ void CProgressDialog::ProcessWasFinished() } +static THREAD_FUNC_DECL MyThreadFunction(void *param) +{ + CProgressThreadVirt *p = (CProgressThreadVirt *)param; + try + { + p->Process(); + p->ThreadFinishedOK = true; + } + catch (...) { p->Result = E_FAIL; } + return 0; +} + + HRESULT CProgressThreadVirt::Create(const UString &title, HWND parentWindow) { NWindows::CThread thread; RINOK(thread.Create(MyThreadFunction, this)); - ProgressDialog.Create(title, thread, parentWindow); + CProgressDialog::Create(title, thread, parentWindow); return S_OK; } @@ -1265,7 +1285,7 @@ static void AddMessageToString(UString &dest, const UString &src) void CProgressThreadVirt::Process() { - CProgressCloser closer(ProgressDialog); + CProgressCloser closer(*this); UString m; try { Result = ProcessVirt(); } catch(const wchar_t *s) { m = s; } @@ -1293,7 +1313,7 @@ void CProgressThreadVirt::Process() } } - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; NSynchronization::CCriticalSectionLock lock(sync._cs); if (m.IsEmpty()) { diff --git a/7zip/CPP/7zip/UI/FileManager/ProgressDialog2.h b/7zip/CPP/7zip/UI/FileManager/ProgressDialog2.h index 03f836da2..6c4213ac9 100644 --- a/7zip/CPP/7zip/UI/FileManager/ProgressDialog2.h +++ b/7zip/CPP/7zip/UI/FileManager/ProgressDialog2.h @@ -261,7 +261,18 @@ class CProgressDialog: public NWindows::NControl::CModalDialog INT_PTR Create(const UString &title, NWindows::CThread &thread, HWND wndParent = 0); + + /* how it works: + 1) the working thread calls ProcessWasFinished() + that sends kCloseMessage message to CProgressDialog (GUI) thread + 2) CProgressDialog (GUI) thread receives kCloseMessage message and + calls ProcessWasFinished_GuiVirt(); + So we can implement ProcessWasFinished_GuiVirt() and show special + results window in GUI thread with CProgressDialog as parent window + */ + void ProcessWasFinished(); + virtual void ProcessWasFinished_GuiVirt() {} }; @@ -273,7 +284,8 @@ class CProgressCloser ~CProgressCloser() { _p->ProcessWasFinished(); } }; -class CProgressThreadVirt + +class CProgressThreadVirt: public CProgressDialog { protected: FStringVector ErrorPaths; @@ -281,33 +293,59 @@ class CProgressThreadVirt // error if any of HRESULT, ErrorMessage, ErrorPath virtual HRESULT ProcessVirt() = 0; - void Process(); public: HRESULT Result; bool ThreadFinishedOK; // if there is no fatal exception - CProgressDialog ProgressDialog; - - static THREAD_FUNC_DECL MyThreadFunction(void *param) - { - CProgressThreadVirt *p = (CProgressThreadVirt *)param; - try - { - p->Process(); - p->ThreadFinishedOK = true; - } - catch (...) { p->Result = E_FAIL; } - return 0; - } + void Process(); void AddErrorPath(const FString &path) { ErrorPaths.Add(path); } HRESULT Create(const UString &title, HWND parentWindow = 0); CProgressThreadVirt(): Result(E_FAIL), ThreadFinishedOK(false) {} CProgressMessageBoxPair &GetMessagePair(bool isError) { return isError ? FinalMessage.ErrorMessage : FinalMessage.OkMessage; } - }; UString HResultToMessage(HRESULT errorCode); +/* +how it works: + +client code inherits CProgressThreadVirt and calls +CProgressThreadVirt::Create() +{ + it creates new thread that calls CProgressThreadVirt::Process(); + it creates modal progress dialog window with ProgressDialog.Create() +} + +CProgressThreadVirt::Process() +{ + { + ProcessVirt(); // virtual function that must implement real work + } + if (exceptions) or FinalMessage.ErrorMessage.Message + { + set message to ProgressDialog.Sync.FinalMessage.ErrorMessage.Message + } + else if (FinalMessage.OkMessage.Message) + { + set message to ProgressDialog.Sync.FinalMessage.OkMessage + } + + PostMsg(kCloseMessage); +} + + +CProgressDialog::OnExternalCloseMessage() +{ + if (ProgressDialog.Sync.FinalMessage) + { + WorkWasFinishedVirt(); + Show (ProgressDialog.Sync.FinalMessage) + MessagesDisplayed = true; + } +} + +*/ + #endif diff --git a/7zip/CPP/7zip/UI/FileManager/makefile b/7zip/CPP/7zip/UI/FileManager/makefile index d9e81780c..4525d8462 100644 --- a/7zip/CPP/7zip/UI/FileManager/makefile +++ b/7zip/CPP/7zip/UI/FileManager/makefile @@ -1,16 +1,7 @@ PROG = 7zFM.exe CFLAGS = $(CFLAGS) \ - -DLANG \ - -DNEW_FOLDER_INTERFACE \ -DEXTERNAL_CODECS \ -!IFDEF UNDER_CE -LIBS = $(LIBS) ceshell.lib Commctrl.lib -!ELSE -LIBS = $(LIBS) comctl32.lib htmlhelp.lib comdlg32.lib Mpr.lib Gdi32.lib -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -DSUPPORT_DEVICE_FILE -!ENDIF - !include "FM.mak" COMMON_OBJS = \ @@ -102,8 +93,9 @@ GUI_OBJS = \ COMPRESS_OBJS = \ $O\CopyCoder.obj \ -C_OBJS = \ +C_OBJS = $(C_OBJS) \ $O\Alloc.obj \ + $O\CpuArch.obj \ $O\Sort.obj \ $O\Threads.obj \ diff --git a/7zip/CPP/7zip/UI/FileManager/resource.rc b/7zip/CPP/7zip/UI/FileManager/resource.rc index a166129b5..6d259eb02 100644 --- a/7zip/CPP/7zip/UI/FileManager/resource.rc +++ b/7zip/CPP/7zip/UI/FileManager/resource.rc @@ -247,6 +247,7 @@ END #include "BrowseDialog.rc" #include "ComboDialog.rc" #include "CopyDialog.rc" +#include "EditDialog.rc" #include "EditPage.rc" #include "FoldersPage.rc" #include "LangPage.rc" diff --git a/7zip/CPP/7zip/UI/GUI/7zG.exe.manifest b/7zip/CPP/7zip/UI/GUI/7zG.exe.manifest index 68c6047e9..39f516cd4 100644 --- a/7zip/CPP/7zip/UI/GUI/7zG.exe.manifest +++ b/7zip/CPP/7zip/UI/GUI/7zG.exe.manifest @@ -5,6 +5,13 @@ + + + + + + + true diff --git a/7zip/CPP/7zip/UI/GUI/BenchmarkDialog.cpp b/7zip/CPP/7zip/UI/GUI/BenchmarkDialog.cpp index 59dc78b3b..e6dc57ba6 100644 --- a/7zip/CPP/7zip/UI/GUI/BenchmarkDialog.cpp +++ b/7zip/CPP/7zip/UI/GUI/BenchmarkDialog.cpp @@ -324,7 +324,6 @@ static void PrintSize_MB(UString &s, UInt64 size) s += kMB; } -extern bool g_LargePagesMode; UInt32 CBenchmarkDialog::OnChangeDictionary() { @@ -338,8 +337,14 @@ UInt32 CBenchmarkDialog::OnChangeDictionary() s += " / "; PrintSize_MB(s, ramSize); } - if (g_LargePagesMode) - s += " LP"; + + #ifdef _7ZIP_LARGE_PAGES + { + AString s2; + Add_LargePages_String(s2); + s += s2; + } + #endif SetItemText(IDT_BENCH_MEMORY_VAL, s); diff --git a/7zip/CPP/7zip/UI/GUI/ExtractGUI.cpp b/7zip/CPP/7zip/UI/GUI/ExtractGUI.cpp index 5071d1f0d..1e37efb83 100644 --- a/7zip/CPP/7zip/UI/GUI/ExtractGUI.cpp +++ b/7zip/CPP/7zip/UI/GUI/ExtractGUI.cpp @@ -52,22 +52,9 @@ static void AddValuePair(UString &s, UINT resourceID, UInt64 value, bool addColo static void AddSizePair(UString &s, UINT resourceID, UInt64 value) { - { - wchar_t sz[32]; - AddLangString(s, resourceID); - s += ": "; - ConvertUInt64ToString(value, sz); - s += MyFormatNew(IDS_FILE_SIZE, sz); - } - // s += sz; - if (value >= (1 << 20)) - { - char sz[32]; - ConvertUInt64ToString(value >> 20, sz); - s += " ("; - s += sz; - s += " MB)"; - } + AddLangString(s, resourceID); + s += ": "; + AddSizeValue(s, value); s.Add_LF(); } @@ -86,19 +73,36 @@ class CThreadExtracting: public CProgressThreadVirt UStringVector *ArchivePathsFull; const NWildcard::CCensorNode *WildcardCensor; const CExtractOptions *Options; + #ifndef _SFX CHashBundle *HashBundle; + virtual void ProcessWasFinished_GuiVirt(); #endif + CMyComPtr ExtractCallback; UString Title; + + CPropNameValPairs Pairs; }; + +#ifndef _SFX +void CThreadExtracting::ProcessWasFinished_GuiVirt() +{ + if (HashBundle && !Pairs.IsEmpty()) + ShowHashResults(Pairs, *this); +} +#endif + HRESULT CThreadExtracting::ProcessVirt() { CDecompressStat Stat; + #ifndef _SFX + /* if (HashBundle) HashBundle->Init(); + */ #endif HRESULT res = Extract(codecs, @@ -109,16 +113,23 @@ HRESULT CThreadExtracting::ProcessVirt() HashBundle, #endif FinalMessage.ErrorMessage.Message, Stat); + #ifndef _SFX - if (res == S_OK && Options->TestMode && ExtractCallbackSpec->IsOK()) + if (res == S_OK && ExtractCallbackSpec->IsOK()) { - UString s; + if (HashBundle) + { + AddValuePair(Pairs, IDS_ARCHIVES_COLON, Stat.NumArchives); + AddSizeValuePair(Pairs, IDS_PROP_PACKED_SIZE, Stat.PackSize); + AddHashBundleRes(Pairs, *HashBundle); + } + else if (Options->TestMode) + { + UString s; - AddValuePair(s, IDS_ARCHIVES_COLON, Stat.NumArchives, false); - AddSizePair(s, IDS_PROP_PACKED_SIZE, Stat.PackSize); + AddValuePair(s, IDS_ARCHIVES_COLON, Stat.NumArchives, false); + AddSizePair(s, IDS_PROP_PACKED_SIZE, Stat.PackSize); - if (!HashBundle) - { if (Stat.NumFolders != 0) AddValuePair(s, IDS_PROP_FOLDERS, Stat.NumFolders); AddValuePair(s, IDS_PROP_FILES, Stat.NumFiles); @@ -129,24 +140,19 @@ HRESULT CThreadExtracting::ProcessVirt() AddValuePair(s, IDS_PROP_NUM_ALT_STREAMS, Stat.NumAltStreams); AddSizePair(s, IDS_PROP_ALT_STREAMS_SIZE, Stat.AltStreams_UnpackSize); } - } - - if (HashBundle) - { s.Add_LF(); - AddHashBundleRes(s, *HashBundle, UString()); + AddLangString(s, IDS_MESSAGE_NO_ERRORS); + FinalMessage.OkMessage.Title = Title; + FinalMessage.OkMessage.Message = s; } - - s.Add_LF(); - AddLangString(s, IDS_MESSAGE_NO_ERRORS); - - FinalMessage.OkMessage.Title = Title; - FinalMessage.OkMessage.Message = s; } #endif + return res; } + + HRESULT ExtractGUI( CCodecs *codecs, const CObjectVector &formatIndices, @@ -252,11 +258,11 @@ HRESULT ExtractGUI( extracter.Title = title; extracter.ExtractCallbackSpec = extractCallback; - extracter.ExtractCallbackSpec->ProgressDialog = &extracter.ProgressDialog; + extracter.ExtractCallbackSpec->ProgressDialog = &extracter; extracter.ExtractCallback = extractCallback; extracter.ExtractCallbackSpec->Init(); - extracter.ProgressDialog.CompressingMode = false; + extracter.CompressingMode = false; extracter.ArchivePaths = &archivePaths; extracter.ArchivePathsFull = &archivePathsFull; @@ -266,10 +272,9 @@ HRESULT ExtractGUI( extracter.HashBundle = hb; #endif - extracter.ProgressDialog.IconID = IDI_ICON; + extracter.IconID = IDI_ICON; RINOK(extracter.Create(title, hwndParent)); - messageWasDisplayed = extracter.ThreadFinishedOK & - extracter.ProgressDialog.MessagesDisplayed; + messageWasDisplayed = extracter.ThreadFinishedOK && extracter.MessagesDisplayed; return extracter.Result; } diff --git a/7zip/CPP/7zip/UI/GUI/GUI.cpp b/7zip/CPP/7zip/UI/GUI/GUI.cpp index 6bbfb7405..2674ec98e 100644 --- a/7zip/CPP/7zip/UI/GUI/GUI.cpp +++ b/7zip/CPP/7zip/UI/GUI/GUI.cpp @@ -2,12 +2,14 @@ #include "StdAfx.h" +#ifdef _WIN32 +#include "../../../../C/DllSecur.h" +#endif + #include "../../../Common/MyWindows.h" #include -#include "../../../../C/Alloc.h" - #include "../../../Common/MyInitGuid.h" #include "../../../Common/CommandLineParser.h" @@ -16,9 +18,6 @@ #include "../../../Windows/FileDir.h" #include "../../../Windows/NtCheck.h" -#ifdef _WIN32 -#include "../../../Windows/MemoryLock.h" -#endif #include "../Common/ArchiveCommandLine.h" #include "../Common/ExitCode.h" @@ -37,8 +36,6 @@ using namespace NWindows; HINSTANCE g_hInstance; -bool g_LargePagesMode = false; - #ifndef UNDER_CE DWORD g_ComCtl32Version; @@ -126,24 +123,6 @@ static int Main2() parser.Parse1(commandStrings, options); parser.Parse2(options); - #if defined(_WIN32) && !defined(UNDER_CE) - NSecurity::EnablePrivilege_SymLink(); - #endif - - #ifdef _7ZIP_LARGE_PAGES - if (options.LargePages) - { - SetLargePageSize(); - // note: this process also can inherit that Privilege from parent process - g_LargePagesMode = - #if defined(_WIN32) && !defined(UNDER_CE) - NSecurity::EnablePrivilege_LockMemory(); - #else - true; - #endif - } - #endif - CREATE_CODECS_OBJECT codecs->CaseSensitiveChange = options.CaseSensitiveChange; @@ -397,13 +376,17 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, // setlocale(LC_COLLATE, ".ACP"); try { + #ifdef _WIN32 + My_SetDefaultDllDirectories(); + #endif + return Main2(); } catch(const CNewException &) { return ShowMemErrorMessage(); } - catch(const CArcCmdLineException &e) + catch(const CMessagePathException &e) { ErrorMessage(e); return NExitCode::kUserError; diff --git a/7zip/CPP/7zip/UI/GUI/GUI.dsp b/7zip/CPP/7zip/UI/GUI/GUI.dsp index c022088e3..b70e28480 100644 --- a/7zip/CPP/7zip/UI/GUI/GUI.dsp +++ b/7zip/CPP/7zip/UI/GUI/GUI.dsp @@ -425,6 +425,14 @@ SOURCE=.\CompressDialog.h # End Source File # Begin Source File +SOURCE=..\FileManager\EditDialog.cpp +# End Source File +# Begin Source File + +SOURCE=..\FileManager\EditDialog.h +# End Source File +# Begin Source File + SOURCE=.\ExtractDialog.cpp # End Source File # Begin Source File @@ -433,6 +441,14 @@ SOURCE=.\ExtractDialog.h # End Source File # Begin Source File +SOURCE=..\FileManager\ListViewDialog.cpp +# End Source File +# Begin Source File + +SOURCE=..\FileManager\ListViewDialog.h +# End Source File +# Begin Source File + SOURCE=..\FileManager\OverwriteDialog.cpp # End Source File # Begin Source File @@ -794,6 +810,15 @@ SOURCE=..\..\..\..\C\CpuArch.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\DllSecur.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sort.c !IF "$(CFG)" == "GUI - Win32 Release" @@ -975,6 +1000,14 @@ SOURCE=..\..\..\Windows\Control\Static.h # End Group # Begin Source File +SOURCE=..\..\..\Windows\Clipboard.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\Clipboard.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\COM.h # End Source File # Begin Source File @@ -1051,6 +1084,14 @@ SOURCE=..\..\..\Windows\FileSystem.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\MemoryGlobal.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\MemoryGlobal.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\MemoryLock.cpp # End Source File # Begin Source File diff --git a/7zip/CPP/7zip/UI/GUI/HashGUI.cpp b/7zip/CPP/7zip/UI/GUI/HashGUI.cpp index cc93f07e6..37cd65e91 100644 --- a/7zip/CPP/7zip/UI/GUI/HashGUI.cpp +++ b/7zip/CPP/7zip/UI/GUI/HashGUI.cpp @@ -9,6 +9,7 @@ #include "../FileManager/FormatUtils.h" #include "../FileManager/LangUtils.h" +#include "../FileManager/ListViewDialog.h" #include "../FileManager/OverwriteDialogRes.h" #include "../FileManager/ProgressDialog2.h" #include "../FileManager/ProgressDialog2Res.h" @@ -19,13 +20,19 @@ using namespace NWindows; + + class CHashCallbackGUI: public CProgressThreadVirt, public IHashCallbackUI { UInt64 NumFiles; bool _curIsFolder; UString FirstFileName; + // UString MainPath; + + CPropNameValPairs PropNameValPairs; HRESULT ProcessVirt(); + virtual void ProcessWasFinished_GuiVirt(); public: const NWildcard::CCensor *censor; @@ -40,49 +47,62 @@ class CHashCallbackGUI: public CProgressThreadVirt, public IHashCallbackUI void AddErrorMessage(DWORD systemError, const wchar_t *name) { - ProgressDialog.Sync.AddError_Code_Name(systemError, name); + Sync.AddError_Code_Name(systemError, name); } }; -static void AddValuePair(UString &s, UINT resourceID, UInt64 value) + +void AddValuePair(CPropNameValPairs &pairs, UINT resourceID, UInt64 value) { - AddLangString(s, resourceID); - s += ": "; + CProperty &pair = pairs.AddNew(); + AddLangString(pair.Name, resourceID); char sz[32]; ConvertUInt64ToString(value, sz); - s += sz; - s.Add_LF(); + pair.Value = sz; } -static void AddSizeValuePair(UString &s, UINT resourceID, UInt64 value) + +void AddSizeValue(UString &s, UInt64 value) { - AddLangString(s, resourceID); - s += ": "; { wchar_t sz[32]; ConvertUInt64ToString(value, sz); s += MyFormatNew(IDS_FILE_SIZE, sz); } + if (value >= (1 << 10)) { + char c; + if (value >= ((UInt64)10 << 30)) { value >>= 30; c = 'G'; } + else if (value >= (10 << 20)) { value >>= 20; c = 'M'; } + else { value >>= 10; c = 'K'; } char sz[32]; - ConvertUInt64ToString(value >> 20, sz); + ConvertUInt64ToString(value, sz); s += " ("; s += sz; - s += " MB)"; - s.Add_LF(); + s += " "; + s += (wchar_t)c; + s += "iB)"; } } +void AddSizeValuePair(CPropNameValPairs &pairs, UINT resourceID, UInt64 value) +{ + CProperty &pair = pairs.AddNew(); + LangString(resourceID, pair.Name); + AddSizeValue(pair.Value, value); +} + + HRESULT CHashCallbackGUI::StartScanning() { - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; sync.Set_Status(LangString(IDS_SCANNING)); return CheckBreak(); } HRESULT CHashCallbackGUI::ScanProgress(const CDirItemsStat &st, const FString &path, bool isDir) { - return ProgressDialog.Sync.ScanProgress(st.NumFiles, st.GetTotalBytes(), path, isDir); + return Sync.ScanProgress(st.NumFiles, st.GetTotalBytes(), path, isDir); } HRESULT CHashCallbackGUI::ScanError(const FString &path, DWORD systemError) @@ -98,26 +118,26 @@ HRESULT CHashCallbackGUI::FinishScanning(const CDirItemsStat &st) HRESULT CHashCallbackGUI::CheckBreak() { - return ProgressDialog.Sync.CheckStop(); + return Sync.CheckStop(); } HRESULT CHashCallbackGUI::SetNumFiles(UInt64 numFiles) { - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; sync.Set_NumFilesTotal(numFiles); return CheckBreak(); } HRESULT CHashCallbackGUI::SetTotal(UInt64 size) { - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; sync.Set_NumBytesTotal(size); return CheckBreak(); } HRESULT CHashCallbackGUI::SetCompleted(const UInt64 *completed) { - return ProgressDialog.Sync.Set_NumBytesCur(completed); + return Sync.Set_NumBytesCur(completed); } HRESULT CHashCallbackGUI::BeforeFirstFile(const CHashBundle & /* hb */) @@ -130,7 +150,7 @@ HRESULT CHashCallbackGUI::GetStream(const wchar_t *name, bool isFolder) if (NumFiles == 0) FirstFileName = name; _curIsFolder = isFolder; - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; sync.Set_FilePath(name, isFolder); return CheckBreak(); } @@ -147,49 +167,53 @@ HRESULT CHashCallbackGUI::OpenFileError(const FString &path, DWORD systemError) HRESULT CHashCallbackGUI::SetOperationResult(UInt64 /* fileSize */, const CHashBundle & /* hb */, bool /* showHash */) { - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; if (!_curIsFolder) NumFiles++; sync.Set_NumFilesCur(NumFiles); return CheckBreak(); } -static void AddHashString(UString &s, const CHasherState &h, unsigned digestIndex, const wchar_t *title) +static void AddHashString(CProperty &s, const CHasherState &h, unsigned digestIndex) { - s += title; - s.Add_Space(); char temp[k_HashCalc_DigestSize_Max * 2 + 4]; AddHashHexToString(temp, h.Digests[digestIndex], h.DigestSize); - s += temp; - s.Add_LF(); + s.Value = temp; } -static void AddHashResString(UString &s, const CHasherState &h, unsigned digestIndex, UInt32 resID) +static void AddHashResString(CPropNameValPairs &s, const CHasherState &h, unsigned digestIndex, UInt32 resID) { - UString s2 = LangString(resID); + CProperty &pair = s.AddNew(); + UString &s2 = pair.Name; + LangString(resID, s2); UString name (h.Name); s2.Replace(L"CRC", name); - AddHashString(s, h, digestIndex, s2); + s2.Replace(L":", L""); + AddHashString(pair, h, digestIndex); } -void AddHashBundleRes(UString &s, const CHashBundle &hb, const UString &firstFileName) + +void AddHashBundleRes(CPropNameValPairs &s, const CHashBundle &hb) { if (hb.NumErrors != 0) - { AddValuePair(s, IDS_PROP_NUM_ERRORS, hb.NumErrors); - s.Add_LF(); - } - - if (hb.NumFiles == 1 && hb.NumDirs == 0 && !firstFileName.IsEmpty()) + + if (hb.NumFiles == 1 && hb.NumDirs == 0 && !hb.FirstFileName.IsEmpty()) { - AddLangString(s, IDS_PROP_NAME); - s += ": "; - s += firstFileName; - s.Add_LF(); + CProperty &pair = s.AddNew(); + LangString(IDS_PROP_NAME, pair.Name); + pair.Value = hb.FirstFileName; } else { - AddValuePair(s, IDS_PROP_FOLDERS, hb.NumDirs); + if (!hb.MainName.IsEmpty()) + { + CProperty &pair = s.AddNew(); + LangString(IDS_PROP_NAME, pair.Name); + pair.Value = hb.MainName; + } + if (hb.NumDirs != 0) + AddValuePair(s, IDS_PROP_FOLDERS, hb.NumDirs); AddValuePair(s, IDS_PROP_FILES, hb.NumFiles); } @@ -197,25 +221,18 @@ void AddHashBundleRes(UString &s, const CHashBundle &hb, const UString &firstFil if (hb.NumAltStreams != 0) { - s.Add_LF(); AddValuePair(s, IDS_PROP_NUM_ALT_STREAMS, hb.NumAltStreams); AddSizeValuePair(s, IDS_PROP_ALT_STREAMS_SIZE, hb.AltStreamsSize); } - if (hb.NumErrors == 0 && hb.Hashers.IsEmpty()) - { - s.Add_LF(); - AddLangString(s, IDS_MESSAGE_NO_ERRORS); - } - FOR_VECTOR (i, hb.Hashers) { - s.Add_LF(); const CHasherState &h = hb.Hashers[i]; if (hb.NumFiles == 1 && hb.NumDirs == 0) { - s += h.Name; - AddHashString(s, h, k_HashCalc_Index_DataSum, L":"); + CProperty &pair = s.AddNew(); + pair.Name += h.Name; + AddHashString(pair, h, k_HashCalc_Index_DataSum); } else { @@ -229,32 +246,57 @@ void AddHashBundleRes(UString &s, const CHashBundle &hb, const UString &firstFil } } -HRESULT CHashCallbackGUI::AfterLastFile(const CHashBundle &hb) + +void AddHashBundleRes(UString &s, const CHashBundle &hb) { - UString s; - AddHashBundleRes(s, hb, FirstFileName); + CPropNameValPairs pairs; + AddHashBundleRes(pairs, hb); - CProgressSync &sync = ProgressDialog.Sync; + FOR_VECTOR (i, pairs) + { + const CProperty &pair = pairs[i]; + s += pair.Name; + s += ": "; + s += pair.Value; + s.Add_LF(); + } + + if (hb.NumErrors == 0 && hb.Hashers.IsEmpty()) + { + s.Add_LF(); + AddLangString(s, IDS_MESSAGE_NO_ERRORS); + s.Add_LF(); + } +} + + +HRESULT CHashCallbackGUI::AfterLastFile(CHashBundle &hb) +{ + hb.FirstFileName = FirstFileName; + // MainPath + AddHashBundleRes(PropNameValPairs, hb); + + CProgressSync &sync = Sync; sync.Set_NumFilesCur(hb.NumFiles); - CProgressMessageBoxPair &pair = GetMessagePair(hb.NumErrors != 0); - pair.Message = s; - LangString(IDS_CHECKSUM_INFORMATION, pair.Title); + // CProgressMessageBoxPair &pair = GetMessagePair(hb.NumErrors != 0); + // pair.Message = s; + // LangString(IDS_CHECKSUM_INFORMATION, pair.Title); return S_OK; } + HRESULT CHashCallbackGUI::ProcessVirt() { NumFiles = 0; - AString errorInfo; HRESULT res = HashCalc(EXTERNAL_CODECS_LOC_VARS *censor, *options, errorInfo, this); - return res; } + HRESULT HashCalcGUI( DECL_EXTERNAL_CODECS_LOC_VARS const NWildcard::CCensor &censor, @@ -268,15 +310,49 @@ HRESULT HashCalcGUI( t.censor = &censor; t.options = &options; - t.ProgressDialog.ShowCompressionInfo = false; + t.ShowCompressionInfo = false; const UString title = LangString(IDS_CHECKSUM_CALCULATING); - t.ProgressDialog.MainTitle = "7-Zip"; // LangString(IDS_APP_TITLE); - t.ProgressDialog.MainAddTitle = title; - t.ProgressDialog.MainAddTitle.Add_Space(); + t.MainTitle = "7-Zip"; // LangString(IDS_APP_TITLE); + t.MainAddTitle = title; + t.MainAddTitle.Add_Space(); RINOK(t.Create(title)); - messageWasDisplayed = t.ThreadFinishedOK && t.ProgressDialog.MessagesDisplayed; + messageWasDisplayed = t.ThreadFinishedOK && t.MessagesDisplayed; return S_OK; } + + +void ShowHashResults(const CPropNameValPairs &propPairs, HWND hwnd) +{ + CListViewDialog lv; + + FOR_VECTOR (i, propPairs) + { + const CProperty &pair = propPairs[i]; + lv.Strings.Add(pair.Name); + lv.Values.Add(pair.Value); + } + + lv.Title = LangString(IDS_CHECKSUM_INFORMATION); + lv.DeleteIsAllowed = true; + lv.SelectFirst = false; + lv.NumColumns = 2; + + lv.Create(hwnd); +} + + +void ShowHashResults(const CHashBundle &hb, HWND hwnd) +{ + CPropNameValPairs propPairs; + AddHashBundleRes(propPairs, hb); + ShowHashResults(propPairs, hwnd); +} + + +void CHashCallbackGUI::ProcessWasFinished_GuiVirt() +{ + ShowHashResults(PropNameValPairs, *this); +} diff --git a/7zip/CPP/7zip/UI/GUI/HashGUI.h b/7zip/CPP/7zip/UI/GUI/HashGUI.h index 40340365a..826445352 100644 --- a/7zip/CPP/7zip/UI/GUI/HashGUI.h +++ b/7zip/CPP/7zip/UI/GUI/HashGUI.h @@ -4,6 +4,7 @@ #define __HASH_GUI_H #include "../Common/HashCalc.h" +#include "../Common/Property.h" HRESULT HashCalcGUI( DECL_EXTERNAL_CODECS_LOC_VARS @@ -11,6 +12,16 @@ HRESULT HashCalcGUI( const CHashOptions &options, bool &messageWasDisplayed); -void AddHashBundleRes(UString &s, const CHashBundle &hb, const UString &firstFileName); +typedef CObjectVector CPropNameValPairs; + +void AddValuePair(CPropNameValPairs &pairs, UINT resourceID, UInt64 value); +void AddSizeValue(UString &s, UInt64 value); +void AddSizeValuePair(CPropNameValPairs &pairs, UINT resourceID, UInt64 value); + +void AddHashBundleRes(CPropNameValPairs &s, const CHashBundle &hb); +void AddHashBundleRes(UString &s, const CHashBundle &hb); + +void ShowHashResults(const CPropNameValPairs &propPairs, HWND hwnd); +void ShowHashResults(const CHashBundle &hb, HWND hwnd); #endif diff --git a/7zip/CPP/7zip/UI/GUI/UpdateGUI.cpp b/7zip/CPP/7zip/UI/GUI/UpdateGUI.cpp index 261b1e2b8..2e738d3a9 100644 --- a/7zip/CPP/7zip/UI/GUI/UpdateGUI.cpp +++ b/7zip/CPP/7zip/UI/GUI/UpdateGUI.cpp @@ -460,7 +460,7 @@ HRESULT UpdateGUI( tu.cmdArcPath = &cmdArcPath; tu.UpdateCallbackGUI = callback; - tu.UpdateCallbackGUI->ProgressDialog = &tu.ProgressDialog; + tu.UpdateCallbackGUI->ProgressDialog = &tu; tu.UpdateCallbackGUI->Init(); UString title = LangString(IDS_PROGRESS_COMPRESSING); @@ -476,10 +476,10 @@ HRESULT UpdateGUI( tu.WildcardCensor = &censor; tu.Options = &options; - tu.ProgressDialog.IconID = IDI_ICON; + tu.IconID = IDI_ICON; RINOK(tu.Create(title, hwndParent)); - messageWasDisplayed = tu.ThreadFinishedOK && tu.ProgressDialog.MessagesDisplayed; + messageWasDisplayed = tu.ThreadFinishedOK && tu.MessagesDisplayed; return tu.Result; } diff --git a/7zip/CPP/7zip/UI/GUI/makefile b/7zip/CPP/7zip/UI/GUI/makefile index bcafa0e57..f8416022c 100644 --- a/7zip/CPP/7zip/UI/GUI/makefile +++ b/7zip/CPP/7zip/UI/GUI/makefile @@ -36,6 +36,7 @@ COMMON_OBJS = \ $O\Wildcard.obj \ WIN_OBJS = \ + $O\Clipboard.obj \ $O\CommonDialog.obj \ $O\DLL.obj \ $O\ErrorMsg.obj \ @@ -45,6 +46,7 @@ WIN_OBJS = \ $O\FileLink.obj \ $O\FileName.obj \ $O\FileSystem.obj \ + $O\MemoryGlobal.obj \ $O\MemoryLock.obj \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ @@ -102,10 +104,12 @@ AR_COMMON_OBJS = \ $O\OutStreamWithCRC.obj \ FM_OBJS = \ + $O\EditDialog.obj \ $O\ExtractCallback.obj \ $O\FormatUtils.obj \ $O\HelpUtils.obj \ $O\LangUtils.obj \ + $O\ListViewDialog.obj \ $O\OpenCallback.obj \ $O\ProgramLocation.obj \ $O\PropertyName.obj \ @@ -130,6 +134,7 @@ COMPRESS_OBJS = \ C_OBJS = \ $O\Alloc.obj \ $O\CpuArch.obj \ + $O\DllSecur.obj \ $O\Sort.obj \ $O\Threads.obj \ diff --git a/7zip/CPP/7zip/UI/GUI/resource.rc b/7zip/CPP/7zip/UI/GUI/resource.rc index 13a09c646..04af8157c 100644 --- a/7zip/CPP/7zip/UI/GUI/resource.rc +++ b/7zip/CPP/7zip/UI/GUI/resource.rc @@ -21,3 +21,5 @@ IDI_ICON ICON "FM.ico" #include "Extract.rc" #include "../FileManager/BrowseDialog.rc" #include "../FileManager/ComboDialog.rc" +#include "../FileManager/EditDialog.rc" +#include "../FileManager/ListViewDialog.rc" diff --git a/7zip/CPP/Build.mak b/7zip/CPP/Build.mak index 5bcd1b8b5..f53286b18 100644 --- a/7zip/CPP/Build.mak +++ b/7zip/CPP/Build.mak @@ -4,19 +4,20 @@ LIBS = $(LIBS) oleaut32.lib ole32.lib CFLAGS = $(CFLAGS) -DUNICODE -D_UNICODE !ENDIF -# CFLAGS = $(CFLAGS) -FAsc -Fa$O/Asm/ - !IFNDEF O -!IFDEF CPU -O=$(CPU) +!IFDEF PLATFORM +O=$(PLATFORM) !ELSE -O=O +O=o !ENDIF !ENDIF -!IF "$(CPU)" == "AMD64" +# CFLAGS = $(CFLAGS) -FAsc -Fa$O/asm/ + + +!IF "$(PLATFORM)" == "x64" MY_ML = ml64 -Dx64 -WX -!ELSEIF "$(CPU)" == "ARM" +!ELSEIF "$(PLATFORM)" == "arm" MY_ML = armasm -WX !ELSE MY_ML = ml -WX @@ -29,16 +30,16 @@ RFLAGS = $(RFLAGS) -dUNDER_CE LFLAGS = $(LFLAGS) /ENTRY:mainACRTStartup !ENDIF !ELSE -!IFNDEF NEW_COMPILER +!IFDEF OLD_COMPILER LFLAGS = $(LFLAGS) -OPT:NOWIN98 !ENDIF -!IF "$(CPU)" != "ARM" && "$(CPU)" != "ARM64" +!IF "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" CFLAGS = $(CFLAGS) -Gr !ENDIF LIBS = $(LIBS) user32.lib advapi32.lib shell32.lib !ENDIF -!IF "$(CPU)" == "ARM" +!IF "$(PLATFORM)" == "arm" COMPL_ASM = $(MY_ML) $** $O/$(*B).obj !ELSE COMPL_ASM = $(MY_ML) -c -Fo$O/ $** @@ -46,19 +47,19 @@ COMPL_ASM = $(MY_ML) -c -Fo$O/ $** CFLAGS = $(CFLAGS) -nologo -c -Fo$O/ -W4 -WX -EHsc -Gy -GR- -GF -!IFDEF MY_STATIC_LINK +!IFDEF MY_DYNAMIC_LINK +CFLAGS = $(CFLAGS) -MD +!ELSE !IFNDEF MY_SINGLE_THREAD CFLAGS = $(CFLAGS) -MT !ENDIF -!ELSE -CFLAGS = $(CFLAGS) -MD !ENDIF -!IFDEF NEW_COMPILER +!IFNDEF OLD_COMPILER CFLAGS = $(CFLAGS) -GS- -Zc:forScope -Zc:wchar_t !IFNDEF UNDER_CE CFLAGS = $(CFLAGS) -MP2 -!IFNDEF CPU +!IFNDEF PLATFORM # CFLAGS = $(CFLAGS) -arch:IA32 !ENDIF !ENDIF @@ -66,7 +67,17 @@ CFLAGS = $(CFLAGS) -MP2 CFLAGS = $(CFLAGS) !ENDIF -!IF "$(CPU)" == "AMD64" +!IFDEF MY_CONSOLE +CFLAGS = $(CFLAGS) -D_CONSOLE +!ENDIF + +!IFNDEF UNDER_CE +!IF "$(PLATFORM)" == "arm" +CFLAGS = $(CFLAGS) -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE +!ENDIF +!ENDIF + +!IF "$(PLATFORM)" == "x64" CFLAGS_O1 = $(CFLAGS) -O1 !ELSE CFLAGS_O1 = $(CFLAGS) -O1 @@ -82,14 +93,16 @@ LFLAGS = $(LFLAGS) /LARGEADDRESSAWARE !IFDEF DEF_FILE LFLAGS = $(LFLAGS) -DLL -DEF:$(DEF_FILE) !ELSE -!IF "$(CPU)" != "ARM" && "$(CPU)" != "ARM64" +!IF defined(MY_FIXED) && "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" LFLAGS = $(LFLAGS) /FIXED +!ELSE +LFLAGS = $(LFLAGS) /FIXED:NO !ENDIF # /BASE:0x400000 !ENDIF -# !IF "$(CPU)" == "AMD64" +# !IF "$(PLATFORM)" == "x64" !IFDEF SUB_SYS_VER @@ -115,10 +128,11 @@ COMPLB = $(CC) $(CFLAGS_O1) -Yu"StdAfx.h" -Fp$O/a.pch $< # COMPLB_O2 = $(CC) $(CFLAGS_O2) -Yu"StdAfx.h" -Fp$O/a.pch $< COMPLB_O2 = $(CC) $(CFLAGS_O2) $< -CCOMPL_PCH = $(CC) $(CFLAGS_O2) -Yc"Precomp.h" -Fp$O/a.pch $** -CCOMPL_USE = $(CC) $(CFLAGS_O2) -Yu"Precomp.h" -Fp$O/a.pch $** -CCOMPL = $(CC) $(CFLAGS_O2) $** -CCOMPLB = $(CC) $(CFLAGS_O2) $< +CFLAGS_C_ALL = $(CFLAGS_O2) $(CFLAGS_C_SPEC) +CCOMPL_PCH = $(CC) $(CFLAGS_C_ALL) -Yc"Precomp.h" -Fp$O/a.pch $** +CCOMPL_USE = $(CC) $(CFLAGS_C_ALL) -Yu"Precomp.h" -Fp$O/a.pch $** +CCOMPL = $(CC) $(CFLAGS_C_ALL) $** +CCOMPLB = $(CC) $(CFLAGS_C_ALL) $< all: $(PROGPATH) @@ -128,10 +142,10 @@ clean: $O: if not exist "$O" mkdir "$O" -$O/Asm: - if not exist "$O/Asm" mkdir "$O/Asm" +$O/asm: + if not exist "$O/asm" mkdir "$O/asm" -$(PROGPATH): $O $O/Asm $(OBJS) $(DEF_FILE) +$(PROGPATH): $O $O/asm $(OBJS) $(DEF_FILE) link $(LFLAGS) -out:$(PROGPATH) $(OBJS) $(LIBS) !IFNDEF NO_DEFAULT_RES diff --git a/7zip/CPP/Common/ListFileUtils.cpp b/7zip/CPP/Common/ListFileUtils.cpp index 9112003b1..edd37eb13 100644 --- a/7zip/CPP/Common/ListFileUtils.cpp +++ b/7zip/CPP/Common/ListFileUtils.cpp @@ -25,14 +25,21 @@ static void AddName(UStringVector &strings, UString &s) strings.Add(s); } -bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage) +bool ReadNamesFromListFile2(CFSTR fileName, UStringVector &strings, UINT codePage, DWORD &lastError) { + lastError = 0; NWindows::NFile::NIO::CInFile file; if (!file.Open(fileName)) + { + lastError = ::GetLastError(); return false; + } UInt64 fileSize; if (!file.GetLength(fileSize)) + { + lastError = ::GetLastError(); return false; + } if (fileSize >= ((UInt32)1 << 31) - 32) return false; UString u; @@ -43,7 +50,10 @@ bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage CByteArr buf((size_t)fileSize); UInt32 processed; if (!file.Read(buf, (UInt32)fileSize, processed)) + { + lastError = ::GetLastError(); return false; + } if (processed != fileSize) return false; file.Close(); @@ -74,7 +84,10 @@ bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage char *p = s.GetBuf((unsigned)fileSize); UInt32 processed; if (!file.Read(p, (UInt32)fileSize, processed)) + { + lastError = ::GetLastError(); return false; + } if (processed != fileSize) return false; file.Close(); @@ -94,7 +107,7 @@ bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage } const wchar_t kGoodBOM = 0xFEFF; - const wchar_t kBadBOM = 0xFFFE; + // const wchar_t kBadBOM = 0xFFFE; UString s; unsigned i = 0; @@ -102,8 +115,10 @@ bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage for (; i < u.Len(); i++) { wchar_t c = u[i]; + /* if (c == kGoodBOM || c == kBadBOM) return false; + */ if (c == '\n' || c == 0xD) { AddName(strings, s); diff --git a/7zip/CPP/Common/ListFileUtils.h b/7zip/CPP/Common/ListFileUtils.h index e8d833fdb..a91e4b112 100644 --- a/7zip/CPP/Common/ListFileUtils.h +++ b/7zip/CPP/Common/ListFileUtils.h @@ -9,6 +9,10 @@ #define MY__CP_UTF16 1200 #define MY__CP_UTF16BE 1201 -bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage = CP_OEMCP); +// bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage = CP_OEMCP); + + // = CP_OEMCP +bool ReadNamesFromListFile2(CFSTR fileName, UStringVector &strings, UINT codePage, + DWORD &lastError); #endif diff --git a/7zip/CPP/Common/MyBuffer2.h b/7zip/CPP/Common/MyBuffer2.h index ba98d137d..d61a72ef1 100644 --- a/7zip/CPP/Common/MyBuffer2.h +++ b/7zip/CPP/Common/MyBuffer2.h @@ -5,7 +5,7 @@ #include "../../C/Alloc.h" -#include "Defs.h" +#include "MyTypes.h" class CMidBuffer { @@ -15,7 +15,7 @@ class CMidBuffer CLASS_NO_COPY(CMidBuffer) public: - CMidBuffer(): _data(NULL), _size(0) {}; + CMidBuffer(): _data(NULL), _size(0) {} ~CMidBuffer() { ::MidFree(_data); } void Free() { ::MidFree(_data); _data = NULL; _size = 0; } @@ -29,12 +29,12 @@ class CMidBuffer { if (!_data || size > _size) { + ::MidFree(_data); const size_t kMinSize = (size_t)1 << 16; if (size < kMinSize) size = kMinSize; - ::MidFree(_data); _size = 0; - _data = 0; + _data = NULL; _data = (Byte *)::MidAlloc(size); if (_data) _size = size; @@ -42,4 +42,59 @@ class CMidBuffer } }; + +class CAlignedBuffer +{ + Byte *_data; + size_t _size; + + CLASS_NO_COPY(CAlignedBuffer) + +public: + CAlignedBuffer(): _data(NULL), _size(0) {} + ~CAlignedBuffer() + { + ISzAlloc_Free(&g_AlignedAlloc, _data); + } + + void Free() + { + ISzAlloc_Free(&g_AlignedAlloc, _data); + _data = NULL; + _size = 0; + } + + bool IsAllocated() const { return _data != NULL; } + operator Byte *() { return _data; } + operator const Byte *() const { return _data; } + size_t Size() const { return _size; } + + void Alloc(size_t size) + { + if (!_data || size != _size) + { + ISzAlloc_Free(&g_AlignedAlloc, _data); + _size = 0; + _data = NULL; + _data = (Byte *)ISzAlloc_Alloc(&g_AlignedAlloc, size); + if (_data) + _size = size; + } + } + + void AllocAtLeast(size_t size) + { + if (!_data || size > _size) + { + ISzAlloc_Free(&g_AlignedAlloc, _data); + _size = 0; + _data = NULL; + _data = (Byte *)ISzAlloc_Alloc(&g_AlignedAlloc, size); + if (_data) + _size = size; + } + } +}; + + #endif diff --git a/7zip/CPP/Common/MyString.h b/7zip/CPP/Common/MyString.h index 299115ddd..6682d8f18 100644 --- a/7zip/CPP/Common/MyString.h +++ b/7zip/CPP/Common/MyString.h @@ -310,6 +310,7 @@ class AString void ReplaceOneCharAtPos(unsigned pos, char c) { _chars[pos] = c; } + char *GetBuf() { return _chars; } /* GetBuf(minLen): provides the buffer that can store at least (minLen) characters and additional null terminator. 9.35: GetBuf doesn't preserve old characters and terminator */ @@ -548,6 +549,8 @@ class UString void ReplaceOneCharAtPos(unsigned pos, wchar_t c) { _chars[pos] = c; } + wchar_t *GetBuf() { return _chars; } + wchar_t *GetBuf(unsigned minLen) { if (minLen > _limit) @@ -574,7 +577,7 @@ class UString } UString &operator=(wchar_t c); - UString &operator=(char c) { return (*this)=((wchar_t)c); } + UString &operator=(char c) { return (*this)=((wchar_t)(unsigned char)c); } UString &operator=(const wchar_t *s); UString &operator=(const UString &s); void SetFrom(const wchar_t *s, unsigned len); // no check diff --git a/7zip/CPP/Common/MyWindows.h b/7zip/CPP/Common/MyWindows.h index a8db9e656..db3f35d82 100644 --- a/7zip/CPP/Common/MyWindows.h +++ b/7zip/CPP/Common/MyWindows.h @@ -16,6 +16,7 @@ #include // for wchar_t #include +// #include // for uintptr_t #include "MyGuidDef.h" @@ -53,6 +54,8 @@ typedef long BOOL; // typedef size_t ULONG_PTR; typedef size_t DWORD_PTR; +// typedef uintptr_t UINT_PTR; +// typedef ptrdiff_t UINT_PTR; typedef Int64 LONGLONG; typedef UInt64 ULONGLONG; @@ -81,6 +84,8 @@ typedef struct _FILETIME typedef ULONG PROPID; typedef LONG SCODE; +#define ERROR_NEGATIVE_SEEK 131L + #define S_OK ((HRESULT)0x00000000L) #define S_FALSE ((HRESULT)0x00000001L) #define E_NOTIMPL ((HRESULT)0x80004001L) diff --git a/7zip/CPP/Common/StdOutStream.cpp b/7zip/CPP/Common/StdOutStream.cpp index 19e0329f2..8236072d3 100644 --- a/7zip/CPP/Common/StdOutStream.cpp +++ b/7zip/CPP/Common/StdOutStream.cpp @@ -75,6 +75,65 @@ void CStdOutStream::PrintUString(const UString &s, AString &temp) *this << (const char *)temp; } + +static const wchar_t kReplaceChar = '_'; + +void CStdOutStream::Normalize_UString__LF_Allowed(UString &s) +{ + unsigned len = s.Len(); + wchar_t *d = s.GetBuf(); + + if (IsTerminalMode) + for (unsigned i = 0; i < len; i++) + { + wchar_t c = d[i]; + if (c <= 13 && c >= 7 && c != '\n') + d[i] = kReplaceChar; + } +} + +void CStdOutStream::Normalize_UString(UString &s) +{ + unsigned len = s.Len(); + wchar_t *d = s.GetBuf(); + + if (IsTerminalMode) + for (unsigned i = 0; i < len; i++) + { + wchar_t c = d[i]; + if (c <= 13 && c >= 7) + d[i] = kReplaceChar; + } + else + for (unsigned i = 0; i < len; i++) + { + wchar_t c = d[i]; + if (c == '\n') + d[i] = kReplaceChar; + } +} + +void CStdOutStream::NormalizePrint_UString(const UString &s, UString &tempU, AString &tempA) +{ + tempU = s; + Normalize_UString(tempU); + PrintUString(tempU, tempA); +} + +void CStdOutStream::NormalizePrint_UString(const UString &s) +{ + NormalizePrint_wstr(s); +} + +void CStdOutStream::NormalizePrint_wstr(const wchar_t *s) +{ + UString tempU = s; + Normalize_UString(tempU); + AString tempA; + PrintUString(tempU, tempA); +} + + CStdOutStream & CStdOutStream::operator<<(Int32 number) throw() { char s[32]; diff --git a/7zip/CPP/Common/StdOutStream.h b/7zip/CPP/Common/StdOutStream.h index 1837bfb0a..2e637e8fe 100644 --- a/7zip/CPP/Common/StdOutStream.h +++ b/7zip/CPP/Common/StdOutStream.h @@ -13,7 +13,9 @@ class CStdOutStream FILE *_stream; bool _streamIsOpen; public: - CStdOutStream(): _stream(0), _streamIsOpen(false) {}; + bool IsTerminalMode; + + CStdOutStream(): _stream(0), _streamIsOpen(false), IsTerminalMode(false) {}; CStdOutStream(FILE *stream): _stream(stream), _streamIsOpen(false) {}; ~CStdOutStream() { Close(); } @@ -50,6 +52,13 @@ class CStdOutStream CStdOutStream & operator<<(const wchar_t *s); void PrintUString(const UString &s, AString &temp); + + void Normalize_UString__LF_Allowed(UString &s); + void Normalize_UString(UString &s); + + void NormalizePrint_UString(const UString &s, UString &tempU, AString &tempA); + void NormalizePrint_UString(const UString &s); + void NormalizePrint_wstr(const wchar_t *s); }; CStdOutStream & endl(CStdOutStream & outStream) throw(); diff --git a/7zip/CPP/Windows/Control/Dialog.cpp b/7zip/CPP/Windows/Control/Dialog.cpp index 9df3ef5ef..020694a6a 100644 --- a/7zip/CPP/Windows/Control/Dialog.cpp +++ b/7zip/CPP/Windows/Control/Dialog.cpp @@ -72,6 +72,7 @@ bool CDialog::OnButtonClicked(int buttonID, HWND /* buttonHWND */) { case IDOK: OnOK(); break; case IDCANCEL: OnCancel(); break; + case IDCLOSE: OnClose(); break; case IDHELP: OnHelp(); break; default: return false; } diff --git a/7zip/CPP/Windows/Control/Dialog.h b/7zip/CPP/Windows/Control/Dialog.h index 59b9f4193..33caa5b27 100644 --- a/7zip/CPP/Windows/Control/Dialog.h +++ b/7zip/CPP/Windows/Control/Dialog.h @@ -105,6 +105,7 @@ class CDialog: public CWindow virtual bool OnButtonClicked(int buttonID, HWND buttonHWND); virtual void OnOK() {}; virtual void OnCancel() {}; + virtual void OnClose() {} virtual bool OnNotify(UINT /* controlID */, LPNMHDR /* lParam */) { return false; } virtual bool OnTimer(WPARAM /* timerID */, LPARAM /* callback */) { return false; } @@ -133,6 +134,7 @@ class CModelessDialog: public CDialog #endif virtual void OnOK() { Destroy(); } virtual void OnCancel() { Destroy(); } + virtual void OnClose() { Destroy(); } }; class CModalDialog: public CDialog @@ -147,6 +149,7 @@ class CModalDialog: public CDialog bool End(INT_PTR result) { return BOOLToBool(::EndDialog(_window, result)); } virtual void OnOK() { End(IDOK); } virtual void OnCancel() { End(IDCANCEL); } + virtual void OnClose() { End(IDCLOSE); } }; class CDialogChildControl: public NWindows::CWindow diff --git a/7zip/CPP/Windows/MemoryLock.cpp b/7zip/CPP/Windows/MemoryLock.cpp index b628a03e8..f9d08a6e6 100644 --- a/7zip/CPP/Windows/MemoryLock.cpp +++ b/7zip/CPP/Windows/MemoryLock.cpp @@ -2,6 +2,8 @@ #include "StdAfx.h" +#include "../../C/CpuArch.h" + #include "MemoryLock.h" namespace NWindows { @@ -67,6 +69,44 @@ bool EnablePrivilege(LPCTSTR privilegeName, bool enable) return res; } + + +typedef void (WINAPI * Func_RtlGetVersion) (OSVERSIONINFOEXW *); + +/* + We suppose that Window 10 works incorrectly with "Large Pages" at: + - Windows 10 1703 (15063) + - Windows 10 1709 (16299) + + - Windows 10 1809 (17763) on some CPUs that have no 1 GB page support. + We need more information about that new BUG in Windows. +*/ + +unsigned Get_LargePages_RiskLevel() +{ + OSVERSIONINFOEXW vi; + HMODULE ntdll = ::GetModuleHandleW(L"ntdll.dll"); + if (!ntdll) + return 0; + Func_RtlGetVersion func = (Func_RtlGetVersion)GetProcAddress(ntdll, "RtlGetVersion"); + if (!func) + return 0; + func(&vi); + if (vi.dwPlatformId != VER_PLATFORM_WIN32_NT) + return 0; + if (vi.dwMajorVersion + vi.dwMinorVersion != 10) + return 0; + if (vi.dwBuildNumber <= 16299) + return 1; + + #ifdef MY_CPU_X86_OR_AMD64 + if (!CPU_IsSupported_PageGB()) + return 1; + #endif + + return 0; +} + #endif }} diff --git a/7zip/CPP/Windows/MemoryLock.h b/7zip/CPP/Windows/MemoryLock.h index 4a5dafcf0..dcaf182e9 100644 --- a/7zip/CPP/Windows/MemoryLock.h +++ b/7zip/CPP/Windows/MemoryLock.h @@ -31,6 +31,8 @@ inline void EnablePrivilege_SymLink() // Do we need to set SE_BACKUP_NAME ? } +unsigned Get_LargePages_RiskLevel(); + #endif }} diff --git a/7zip/CPP/Windows/Registry.cpp b/7zip/CPP/Windows/Registry.cpp index c000e6289..a2893131a 100644 --- a/7zip/CPP/Windows/Registry.cpp +++ b/7zip/CPP/Windows/Registry.cpp @@ -152,7 +152,7 @@ LONG CKey::SetValue(LPCWSTR name, LPCWSTR value) MYASSERT(value != NULL); MYASSERT(_object != NULL); if (g_IsNT) - return RegSetValueExW(_object, name, NULL, REG_SZ, + return RegSetValueExW(_object, name, 0, REG_SZ, (const BYTE * )value, (DWORD)((wcslen(value) + 1) * sizeof(wchar_t))); return SetValue(name == 0 ? 0 : (LPCSTR)GetSystemString(name), value == 0 ? 0 : (LPCSTR)GetSystemString(value)); diff --git a/7zip/CPP/Windows/Shell.cpp b/7zip/CPP/Windows/Shell.cpp index 382d774fd..b424e67ca 100644 --- a/7zip/CPP/Windows/Shell.cpp +++ b/7zip/CPP/Windows/Shell.cpp @@ -2,6 +2,11 @@ #include "StdAfx.h" +/* +#include +#include +*/ + #include "../Common/MyCom.h" #ifndef _UNICODE #include "../Common/StringConvert.h" @@ -114,9 +119,22 @@ UString CDrop::QueryFileName(UINT fileIndex) void CDrop::QueryFileNames(UStringVector &fileNames) { UINT numFiles = QueryCountOfFiles(); + /* + char s[100]; + sprintf(s, "QueryFileNames: %d files", numFiles); + OutputDebugStringA(s); + */ fileNames.ClearAndReserve(numFiles); for (UINT i = 0; i < numFiles; i++) - fileNames.AddInReserved(QueryFileName(i)); + { + const UString s2 = QueryFileName(i); + if (!s2.IsEmpty()) + fileNames.AddInReserved(s2); + /* + OutputDebugStringW(L"file ---"); + OutputDebugStringW(s2); + */ + } } diff --git a/7zip/DOC/7zFormat.txt b/7zip/DOC/7zFormat.txt index 55b52b1b9..74cdfa418 100644 --- a/7zip/DOC/7zFormat.txt +++ b/7zip/DOC/7zFormat.txt @@ -1,4 +1,4 @@ -7z Format description (4.59) +7z Format description (18.06) ---------------------------- This file contains description of 7z archive format. @@ -175,7 +175,7 @@ SignatureHeader ArchiveVersion { BYTE Major; // now = 0 - BYTE Minor; // now = 2 + BYTE Minor; // now = 4 }; UINT32 StartHeaderCRC; @@ -399,7 +399,7 @@ FilesInfo UINT64 DataIndex [] for(Definded Items) - UINT64 Time + REAL_UINT64 Time [] kNames: (0x11) diff --git a/7zip/DOC/7zip.inf b/7zip/DOC/7zip.inf index 9bd74d527..b4e18e839 100644 --- a/7zip/DOC/7zip.inf +++ b/7zip/DOC/7zip.inf @@ -10,8 +10,8 @@ AppName = "7-Zip" InstallDir = %CE1%\%AppName% [Strings] -AppVer = "18.01" -AppDate = "2018-01-28" +AppVer = "19.00" +AppDate = "2019-01-21" [CEDevice] ; ProcessorType = 2577 ; ARM diff --git a/7zip/DOC/7zip.nsi b/7zip/DOC/7zip.nsi index 5483e5fba..5a806ae77 100644 --- a/7zip/DOC/7zip.nsi +++ b/7zip/DOC/7zip.nsi @@ -1,8 +1,8 @@ ;-------------------------------- ;Defines -!define VERSION_MAJOR 18 -!define VERSION_MINOR 01 +!define VERSION_MAJOR 19 +!define VERSION_MINOR 00 !define VERSION_POSTFIX_FULL "" !ifdef WIN64 !ifdef IA64 @@ -220,6 +220,7 @@ Section File ja.txt File ka.txt File kaa.txt + File kab.txt File kk.txt File ko.txt File ku.txt @@ -421,6 +422,7 @@ Section Uninstall Delete $INSTDIR\Lang\ja.txt Delete $INSTDIR\Lang\ka.txt Delete $INSTDIR\Lang\kaa.txt + Delete $INSTDIR\Lang\kab.txt Delete $INSTDIR\Lang\kk.txt Delete $INSTDIR\Lang\ko.txt Delete $INSTDIR\Lang\ku.txt diff --git a/7zip/DOC/7zip.wxs b/7zip/DOC/7zip.wxs index 96969d931..338d58290 100644 --- a/7zip/DOC/7zip.wxs +++ b/7zip/DOC/7zip.wxs @@ -1,7 +1,7 @@ - - + + @@ -104,6 +104,8 @@ + + @@ -294,6 +296,7 @@ + diff --git a/7zip/DOC/License.txt b/7zip/DOC/License.txt index cf3741b67..d6b672bb0 100644 --- a/7zip/DOC/License.txt +++ b/7zip/DOC/License.txt @@ -3,7 +3,7 @@ License for use and distribution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 7-Zip Copyright (C) 1999-2018 Igor Pavlov. + 7-Zip Copyright (C) 1999-2019 Igor Pavlov. The licenses for files are: diff --git a/7zip/DOC/Methods.txt b/7zip/DOC/Methods.txt index fe38c78c2..d4a1b1dd4 100644 --- a/7zip/DOC/Methods.txt +++ b/7zip/DOC/Methods.txt @@ -1,8 +1,8 @@ 7-Zip method IDs for 7z and xz archives --------------------------------------- -Version: 17.01 -Date: 2017-05-27 +Version: 18.06 +Date: 2018-06-30 Each compression or crypto method in 7z is associated with unique binary value (ID). The length of ID in bytes is arbitrary but it can not exceed 63 bits (8 bytes). @@ -132,6 +132,12 @@ List of defined IDs 04 - LZ4 05 - LZ5 06 - LIZARD + + 12 xx - reserverd (Denis Anisimov) + + 01 - WavPack2 + FE - eSplitter + FF - RawSplitter 06.. - Crypto diff --git a/7zip/DOC/lzma.txt b/7zip/DOC/lzma.txt index 87d727b55..a65988fe0 100644 --- a/7zip/DOC/lzma.txt +++ b/7zip/DOC/lzma.txt @@ -33,7 +33,7 @@ If you want to use old interfaces you can download previous version of LZMA SDK from sourceforge.net site. To use ANSI-C LZMA Decoder you need the following files: -1) LzmaDec.h + LzmaDec.c + Types.h +1) LzmaDec.h + LzmaDec.c + 7zTypes.h + Precomp.h + Compiler.h Look example code: C/Util/Lzma/LzmaUtil.c @@ -70,7 +70,7 @@ You can use p = p; operator to disable compiler warnings. Single-call Decompressing ------------------------- When to use: RAM->RAM decompressing -Compile files: LzmaDec.h + LzmaDec.c + Types.h +Compile files: LzmaDec.h + LzmaDec.c + 7zTypes.h Compile defines: no defines Memory Requirements: - Input buffer: compressed size @@ -124,7 +124,7 @@ Multi-call State Decompressing (zlib-like interface) ---------------------------------------------------- When to use: file->file decompressing -Compile files: LzmaDec.h + LzmaDec.c + Types.h +Compile files: LzmaDec.h + LzmaDec.c + 7zTypes.h Memory Requirements: - Buffer for input stream: any size (for example, 16 KB) @@ -167,7 +167,7 @@ How To compress data -------------------- Compile files: - Types.h + 7zTypes.h Threads.h LzmaEnc.h LzmaEnc.c diff --git a/7zip/DOC/readme.txt b/7zip/DOC/readme.txt index 9907ff83b..0d1aebf7b 100644 --- a/7zip/DOC/readme.txt +++ b/7zip/DOC/readme.txt @@ -1,9 +1,9 @@ -7-Zip 18.01 Sources +7-Zip 19.00 Sources ------------------- 7-Zip is a file archiver for Windows. -7-Zip Copyright (C) 1999-2018 Igor Pavlov. +7-Zip Copyright (C) 1999-2019 Igor Pavlov. License Info @@ -44,28 +44,50 @@ LZMA SDK is written and placed in the public domain by Igor Pavlov. How to compile -------------- -To compile sources you need Visual C++ 6.0. -For compiling some files you also need -new Platform SDK from Microsoft' Site: -http://www.microsoft.com/msdownload/platformsdk/sdkupdate/psdk-full.htm -or -http://www.microsoft.com/msdownload/platformsdk/sdkupdate/XPSP2FULLInstall.htm -or -http://www.microsoft.com/msdownload/platformsdk/sdkupdate/ - -If you use MSVC6, specify SDK directories at top of directories lists: + +To compile the sources to Windows binaries you need Visual Studio compiler and/or Windows SDK. +You can use latest Windows Studio 2017 to compile binaries for x86, x64 and arm64 platforms. +Also you can use old compilers for some platforms: + x86 : Visual C++ 6.0 with Platform SDK + x64 : Windows Server 2003 R2 Platform SDK + arm64 : Windows Studio 2017 + arm : Windows Studio 2017 + ia64 (itanium) : Windows Server 2003 R2 Platform SDK + arm for Windows CE : Standard SDK for Windows CE 5.0 + +If you use MSVC6, specify also Platform SDK directories at top of directories lists: Tools / Options / Directories - Include files - Library files - -To compile 7-Zip for AMD64 and IA64 you need: - Windows Server 2003 SP1 Platform SDK from microsoft.com - Also you need Microsoft Macro Assembler: - ml.exe for x86 - - ml64.exe for AMD64 -You can use ml.exe from Windows SDK for Windows Vista or some other version. + - ml64.exe for x64 +You can use ml.exe from Windows SDK for Windows Vista or some later versions. + +There are two ways to compile 7-Zip binaries: +1) via makefile in command line. +2) via dsp file in Visual Studio. + +The dsp file compiling can be used for development and debug purposes. +The final 7-Zip binaries are compiled via makefiles, that provide best +optimization options. + +How to compile with makefile +---------------------------- + +Some macronames can be defined for compiling with makefile: + +PLATFORM + with possible values: x64, x86, arm64, arm, ia64 + +OLD_COMPILER + for old VC compiler, like MSCV 6.0. + +MY_DYNAMIC_LINK + for dynamic linking to the run-time library (msvcrt.dll). + The default makefile option is static linking to the run-time library. + Compiling under Unix/Linux diff --git a/7zip/DOC/src-history.txt b/7zip/DOC/src-history.txt index 6f637d016..51921cd0b 100644 --- a/7zip/DOC/src-history.txt +++ b/7zip/DOC/src-history.txt @@ -1,6 +1,50 @@ HISTORY of the 7-Zip source code -------------------------------- +19.00 2019-02-21 +------------------------- +- Encryption strength for 7z archives was increased: + the size of random initialization vector was increased from 64-bit to 128-bit, + and the pseudo-random number generator was improved. +- Some bugs were fixed. + + +18.06 2018-12-30 +------------------------- +- The speed for LZMA/LZMA2 compressing was increased by 3-10%, + and there are minor changes in compression ratio. +- Some bugs were fixed. +- The bug in 7-Zip 18.02-18.05 was fixed: + There was memory leak in multithreading xz decoder - XzDecMt_Decode(), + if xz stream contains only one block. +- 7-Zip 18.02-18.05 used only one CPU thread for bz2 archive creation. +- The changes for MSVS compiler makefiles: + - the makefiles now use "PLATFORM" macroname with values (x64, x86, arm64) + instead of "CPU" macroname with values (AMD64, ARM64). + - the makefiles by default now use static version of the run-time library. + + +18.05 2018-04-30 +------------------------- +- The speed for LZMA/LZMA2 compressing was increased + by 8% for fastest/fast compression levels and + by 3% for normal/maximum compression levels. +- Previous versions of 7-Zip could work incorrectly in "Large memory pages" mode in + Windows 10 because of some BUG with "Large Pages" in Windows 10. + Now 7-Zip doesn't use "Large Pages" on Windows 10 up to revision 1709 (16299). + + +18.03 beta 2018-03-04 +------------------------- +- Asm\x86\LzmaDecOpt.asm: new optimized LZMA decoder written in asm + for x64 with about 30% higher speed than main version of LZMA decoder written in C. +- The speed for single-thread LZMA/LZMA2 decoder written in C was increased by 3%. +- 7-Zip now can use multi-threading for 7z/LZMA2 decoding, + if there are multiple independent data chunks in LZMA2 stream. +- 7-Zip now can use multi-threading for xz decoding, + if there are multiple blocks in xz stream. + + 17.00 beta 2017-04-29 ------------------------- - NewHandler.h / NewHandler.cpp: