diff options
| author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2021-07-15 02:04:24 +0000 |
|---|---|---|
| committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2021-07-15 02:04:24 +0000 |
| commit | 10816c2185ea9a744225a78cd23b79100c56e1ed (patch) | |
| tree | 3768027f18b5ab3fd307da9f0e225d6df6792786 | |
| parent | 11ccf82c7d2e456657d10d7ce14c70b9e168d6fb (diff) | |
| parent | 69030f7920e318dc77f3f6f617fcc947c86638f9 (diff) | |
| download | zstd-android12-mainline-captiveportallogin-release.tar.gz | |
Snap for 7550930 from 69030f7920e318dc77f3f6f617fcc947c86638f9 to mainline-captiveportallogin-releaseandroid-mainline-12.0.0_r6android-mainline-12.0.0_r23android12-mainline-captiveportallogin-release
Change-Id: Ie66ea23674966b2c79ea37298fbc6273f4b2f330
206 files changed, 12425 insertions, 36676 deletions
diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml new file mode 100644 index 00000000..bb88de57 --- /dev/null +++ b/.github/workflows/generic-dev.yml @@ -0,0 +1,209 @@ +name: generic-dev + +on: + pull_request: + branches: [ dev, master, actionsTest ] + +jobs: + +# Dev PR jobs that still have to be migrated from travis +# +# icc (need self-hosted) +# versionTag +# valgrindTest (keeps failing for some reason. need investigation) +# staticAnalyze (need trusty so need self-hosted) +# pcc-fuzz: (need trusty so need self-hosted) +# min-decomp-macros (flakey) +# +# setting up self-hosted is pretty straightforward, but +# I need admins permissions to the repo for that it looks like +# So I'm tabling that for now +# +# The master branch exclusive jobs will be in a separate +# workflow file (the osx tests and meson build that is) + + benchmarking: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: make benchmarking + run: make benchmarking + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: make test + run: make test + + gcc-6-7-libzstd: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: gcc-6 + gcc-7 + libzstdmt compilation + run: | + make gcc6install gcc7install + CC=gcc-6 CFLAGS=-Werror make -j all + make clean + CC=gcc-7 CFLAGS=-Werror make -j all + make clean + LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt + make -C tests zbufftest-dll + + gcc-8-asan-ubsan-testzstd: + runs-on: ubuntu-16.04 # fails on 18.04 + steps: + - uses: actions/checkout@v2 + - name: gcc-8 + ASan + UBSan + Test Zstd + run: | + make gcc8install + CC=gcc-8 CFLAGS="-Werror" make -j all + make clean + CC=gcc-8 make -j uasan-test-zstd </dev/null + + gcc-6-asan-ubsan-testzstd-32bit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: gcc-6 + ASan + UBSan + Test Zstd, 32bit mode + run: | + make gcc6install libc6install + CC=gcc-6 CFLAGS="-Werror -m32" make -j all32 + make clean + CC=gcc-6 make -j uasan-test-zstd32 + + clang-38-msan-testzstd: + runs-on: ubuntu-16.04 # fails on 18.04 + steps: + - uses: actions/checkout@v2 + - name: clang-3.8 + MSan + Test Zstd + run: | + # make clang38install (doesn't work) + sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main" + sudo apt-get update + sudo apt-get install clang-3.8 + CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 + + cmake-build-and-test-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: cmake build and test check + run: | + FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild + cp -r ./ "../zstd source" + cd "../zstd source" + FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild + + gcc-8-asan-ubsan-fuzz: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: gcc-8 + ASan + UBSan + Fuzz Test + run: | + make gcc8install + CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest + + gcc-6-asan-ubsan-fuzz32: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: gcc-6 + ASan + UBSan + Fuzz Test 32bit + run: | + make gcc6install libc6install + CC=gcc-6 CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest + + clang-38-msan-fuzz: + runs-on: ubuntu-16.04 # fails on 18.04 + steps: + - uses: actions/checkout@v2 + - name: clang-3.8 + MSan + Fuzz Test + run: | + # make clang38install (doesn't work) + sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main" + sudo apt-get update + sudo apt-get install clang-3.8 + CC=clang-3.8 make clean msan-fuzztest + + asan-ubsan-msan-regression: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: ASan + UBSan + MSan + Regression Test + run: | + make -j uasanregressiontest + make clean + make -j msanregressiontest + + cpp-gnu90-c99-compatibility: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: C++, gnu90 and c99 compatibility + run: | + make cxxtest + make clean + make gnu90build + make clean + make c99build + make clean + make travis-install + + mingw-cross-compilation: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: mingw cross-compilation + run: | + # sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work) + sudo apt-get install gcc-mingw-w64 + CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd + +# TODO: Broken test - fix and uncomment +# armbuild: +# runs-on: ubuntu-16.04 # doesn't work on latest +# steps: +# - uses: actions/checkout@v2 +# - name: ARM Build Test +# run: | +# make arminstall +# make armbuild + +# TODO: Broken test - fix and uncomment +# armfuzz: +# runs-on: ubuntu-16.04 # doesn't work on latest +# steps: +# - uses: actions/checkout@v2 +# - name: Qemu ARM emulation + Fuzz Test +# run: | +# make arminstall +# make armfuzz + + bourne-shell: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Bourne shell compatibility (shellcheck) + run: | + wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz + tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz + shellcheck-v0.7.1/shellcheck --shell=sh --severity=warning --exclude=SC2010 tests/playTests.sh + + icc: + name: icc-check + runs-on: ubuntu-latest + steps: + - name: install icc + run: | + export DEBIAN_FRONTEND=noninteractive + sudo apt-get -qqq update + sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg + sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB + sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB + sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" + sudo apt-get update + sudo apt-get install -y intel-basekit intel-hpckit + - uses: actions/checkout@v2 + - name: make check + run: | + make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check diff --git a/.github/workflows/generic-release.yml b/.github/workflows/generic-release.yml new file mode 100644 index 00000000..de4a1cb9 --- /dev/null +++ b/.github/workflows/generic-release.yml @@ -0,0 +1,69 @@ +name: generic-release + +on: + pull_request: + # This will eventually only be for pushes to master + # but for dogfooding purposes, I'm running it even + # on dev pushes + branches: [ dev, master, actionsTest ] + +jobs: + # missing jobs + # + # ppc64le + fuzz test + # Qemu PPC64 + Fuzz test + # Qemu aarch64 + Fuzz Test (on Xenial) + # versions comp + # meson test + + osx: + runs-on: macos-10.15 + steps: + - uses: actions/checkout@v2 + - name: OS-X + run: | + make test + # make -c lib all (need to fix. not working right now) + + zbuff: + runs-on: ubuntu-16.04 + steps: + - uses: actions/checkout@v2 + - name: zbuff test + run: | + make -C tests test-zbuff + + tsan: + runs-on: ubuntu-16.04 + steps: + - uses: actions/checkout@v2 + - name: thread sanitizer + run: | + sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main" + sudo apt-get update + sudo apt-get install clang-3.8 + CC=clang-3.8 make tsan-test-zstream + CC=clang-3.8 make tsan-fuzztest + zlib-wrapper: + runs-on: ubuntu-16.04 + steps: + - uses: actions/checkout@v2 + - name: zlib wrapper test + run: | + make gpp6install valgrindinstall + make -C zlibWrapper test + make -C zlibWrapper valgrindTest + + lz4-threadpool-partial-libs: + runs-on: ubuntu-16.04 + steps: + - uses: actions/checkout@v2 + - name: LZ4, thread pool, and partial libs testslib wrapper test + run: | + make lz4install + make -C tests test-lz4 + make check < /dev/null | tee # mess with lz4 console detection + make clean + make -C tests test-pool + make clean + bash tests/libzstd_partial_builds.sh diff --git a/.github/workflows/linux-kernel.yml b/.github/workflows/linux-kernel.yml new file mode 100644 index 00000000..35871ff0 --- /dev/null +++ b/.github/workflows/linux-kernel.yml @@ -0,0 +1,13 @@ +name: linux-kernel + +on: + pull_request: + branches: [ dev, master, actionsTest ] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Create linux kernel library + build + test + run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9309ba92..0e14345d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,21 +3,28 @@ on: [pull_request] jobs: Fuzzing: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + sanitizer: [address, undefined, memory] steps: - - name: Build Fuzzers + - name: Build Fuzzers (${{ matrix.sanitizer }}) + id: build uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master with: oss-fuzz-project-name: 'zstd' dry-run: false - - name: Run Fuzzers + sanitizer: ${{ matrix.sanitizer }} + - name: Run Fuzzers (${{ matrix.sanitizer }}) uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master with: oss-fuzz-project-name: 'zstd' fuzz-seconds: 600 dry-run: false + sanitizer: ${{ matrix.sanitizer }} - name: Upload Crash uses: actions/upload-artifact@v1 - if: failure() + if: failure() && steps.build.outcome == 'success' with: - name: artifacts + name: ${{ matrix.sanitizer }}-artifacts path: ./out/artifacts @@ -22,6 +22,8 @@ zstdmt # Test artefacts tmp* +*.zst +*.zstd dictionary. dictionary NUL @@ -32,6 +34,7 @@ bin/ .buckd/ buck-out/ build-* +*.gcda # Other files .directory diff --git a/.travis.yml b/.travis.yml index a4046287..226d4c01 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,22 +25,6 @@ env: matrix: fast_finish: true include: - - name: icc - compiler: icc - env: - - C_COMPILER=icc - - CXX_COMPILER=icpc - install: - - source /opt/intel/inteloneapi/compiler/latest/env/vars.sh - addons: - apt: - sources: - - sourceline: 'deb https://apt.repos.intel.com/oneapi all main' - key_url: 'https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB' - packages: - - intel-oneapi-icc - script: make check - - name: arm64 # ~2.5 mn os: linux arch: arm64 @@ -92,14 +76,14 @@ matrix: - name: Minimal Decompressor Macros # ~5mn script: - - make clean - - make -j all check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror" - - make clean - - make -j all check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT" - - make clean - - make -j all check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG" - - make clean - - make -j all check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" + - make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror" + - make clean && make check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror" + - make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT" + - make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT" + - make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG" + - make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG" + - make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" + - make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" - name: cmake build and test check # ~6mn script: @@ -232,11 +216,12 @@ matrix: - make -C zlibWrapper test - make -C zlibWrapper valgrindTest - - name: LZ4, thread pool, and partial libs tests # ~2mn + - name: LZ4, thread pool, and partial libs tests # ~4mn if: branch = master script: - make lz4install - make -C tests test-lz4 + - make check < /dev/null | tee # mess with lz4 console detection - make clean - make -C tests test-pool - make clean @@ -1,3 +1,46 @@ +package { + default_applicable_licenses: ["external_zstd_license"], +} + +// Added automatically by a large-scale-change that took the approach of +// 'apply every license found to every target'. While this makes sure we respect +// every license restriction, it may not be entirely correct. +// +// e.g. GPL in an MIT project might only apply to the contrib/ directory. +// +// Please consider splitting the single license below into multiple licenses, +// taking care not to lose any license_kind information, and overriding the +// default license using the 'licenses: [...]' property on targets as needed. +// +// For unused files, consider creating a 'fileGroup' with "//visibility:private" +// to attach the license to, and including a comment whether the files may be +// used in the current project. +// +// large-scale-change included anything that looked like it might be a license +// text as a license_text. e.g. LICENSE, NOTICE, COPYING etc. +// +// Please consider removing redundant or irrelevant files from 'license_text:'. +// See: http://go/android-license-faq +license { + name: "external_zstd_license", + visibility: [":__subpackages__"], + license_kinds: [ + "SPDX-license-identifier-BSD", + "SPDX-license-identifier-CC0-1.0", + "SPDX-license-identifier-GPL", + "SPDX-license-identifier-GPL-2.0", + "SPDX-license-identifier-GPL-3.0", + "SPDX-license-identifier-LGPL", + "SPDX-license-identifier-MIT", + "SPDX-license-identifier-Zlib", + "legacy_unencumbered", + ], + license_text: [ + "COPYING", + "LICENSE", + ], +} + cc_library { name: "libzstd", apex_available: [ @@ -1,4 +1,62 @@ -v1.4.5 +v1.4.7 +perf: stronger --long mode at high compression levels, by @senhuang42 +perf: stronger --patch-from at high compression levels, thanks to --long improvements +perf: faster dictionary compression at medium compression levels, by @felixhandte +perf: small speed & memory usage improvements for ZSTD_compress2(), by @terrelln +perf: improved fast compression speeds with Visual Studio, by @animalize +cli : Set nb of threads with environment variable ZSTD_NBTHREADS, by @senhuang42 +cli : accept decompressing files with *.zstd suffix +cli : provide a condensed summary by default when processing multiple files +cli : fix : stdin input no longer confused as user prompt +cli : improve accuracy of several error messages +api : new sequence ingestion API, by @senhuang42 +api : shared thread pool: control total nb of threads used by multiple compression jobs, by @marxin +api : new ZSTD_getDictID_fromCDict(), by @LuAPi +api : zlibWrapper only uses public API, and is compatible with dynamic library, by @terrelln +api : fix : multithreaded compression has predictable output even in special cases (see #2327) (issue not accessible from cli) +api : fix : dictionary compression correctly respects dictionary compression level (see #2303) (issue not accessible from cli) +build: fix cmake script when using path with spaces, by @terrelln +build: improved compile-time detection of aarch64/neon platforms, by @bsdimp +build: Fix building on AIX 5.1, by @likema +build: compile paramgrill with cmake on Windows, requested by @mirh +doc : clarify repcode updates in format specification, by @felixhandte + +v1.4.6 +fix : Always return dstSize_tooSmall when that is the case +fix : Fix ZSTD_initCStream_advanced() with static allocation and no dictionary +perf: Improve small block decompression speed by 20%+, by @terrelln +perf: Reduce compression stack usage by 1 KB, by @terrelln +perf: Improve decompression speed by improving ZSTD_wildcopy, by @helloguo (#2252, #2256) +perf: Improve histogram construction, by @cyan4973 (#2253) +cli : Add --output-dir-mirror option, by @xxie24 (#2219) +cli : Warn when (de)compressing multiple files into a single output, by @senhuang42 (#2279) +cli : Improved progress bar and status summary when (de)compressing multiple files, by @senhuang42 (#2283) +cli : Call stat less often, by @felixhandte (#2262) +cli : Allow --patch-from XXX and --filelist XXX in addition to --patch-from=XXX and --filelist=XXX, by @cyan4973 (#2250) +cli : Allow --patch-from to compress stdin with --stream-size, by @bimbashrestha (#2206) +api : Do not install zbuff.h, since it has long been deprecated, by @cyan4973 (#2166). +api : Fix ZSTD_CCtx_setParameter() with ZSTD_c_compressionLevel to make 0 mean default level, by @i-do-cpp (#2291) +api : Rename ZSTDMT_NBTHREADS_MAX to ZSTDMT_NBWORKERS_MAX, by @marxin (#2228). +build: Install pkg-config file with CMake and MinGW, by @tonytheodore (#2183) +build: Install DLL with CMake on Windows, by @BioDataAnalysis (#2221) +build: Fix DLL install location with CMake, by @xantares and @bimbashrestha (#2186) +build: Add ZSTD_NO_UNUSED_FUNCTIONS macro to hide unused functions +build: Add ZSTD_NO_INTRINSICS macro to avoid explicit intrinsics +build: Add STATIC_BMI2 macro for compile time detection of BMI2 on MSVC, by @Niadb (#2258) +build: Fix -Wcomma warnings, by @cwoffenden +build: Remove distutils requirement for meson build, by @neheb (#2197) +build: Fix cli compilation with uclibc +build: Fix cli compilation without st_mtime, by @ffontaine (#2246) +build: Fix shadowing warnings in library +build: Fix single file library compilation with Enscripten, by @yoshihitoh (#2227) +misc: Improve single file library and include dictBuilder, by @cwoffenden +misc: Allow compression dictionaries with missing symbols +misc: Add freestanding translation script in contrib/freestanding_lib +misc: Collect all of zstd's libc dependencies into zstd_deps.h +doc : Add ZSTD_versionString() to manual, by @animalize +doc : Fix documentation for ZSTD_CCtxParams_setParameter(), by @felixhandte (#2270) + +v1.4.5 (May 22, 2020) fix : Compression ratio regression on huge files (> 3 GB) using high levels (--ultra) and multithreading, by @terrelln perf: Improved decompression speed: x64 : +10% (clang) / +5% (gcc); ARM : from +15% to +50%, depending on SoC, by @terrelln perf: Automatically downsizes ZSTD_DCtx when too large for too long (#2069, by @bimbashreshta) @@ -24,7 +82,7 @@ misc: Edit-distance match finder in contrib/ doc : Improved beginner CONTRIBUTING.md docs doc : New issue templates for zstd -v1.4.4 +v1.4.4 (Nov 6, 2019) perf: Improved decompression speed, by > 10%, by @terrelln perf: Better compression speed when re-using a context, by @felixhandte perf: Fix compression ratio when compressing large files with small dictionary, by @senhuang42 @@ -51,18 +109,18 @@ pack: modified pkgconfig, for better integration into openwrt, requested by @neh misc: Improved documentation : ZSTD_CLEVEL, DYNAMIC_BMI2, ZSTD_CDict, function deprecation, zstd format misc: fixed educational decoder : accept larger literals section, and removed UNALIGNED() macro -v1.4.3 +v1.4.3 (Aug 20, 2019) bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709) bug: Fix Buffer Overflow in legacy v0.3 decompression by @felixhandte (#1722) build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705) -v1.4.2 +v1.4.2 (Jul 26, 2019) bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696) bug: Fix seekable decompression in-memory API by @iburinoc (#1695) misc: Validate blocks are smaller than size limit by @vivekmg (#1685) misc: Restructure source files by @ephiepark (#1679) -v1.4.1 +v1.4.1 (Jul 20, 2019) bug: Fix data corruption in niche use cases by @terrelln (#1659) bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595) bug: Fix out of bounds read by @terrelln (#1590) @@ -92,7 +150,7 @@ build: Visual Studio: fix linking by @absotively (#1639) build: Fix MinGW-W64 build by @myzhang1029 (#1600) misc: Expand decodecorpus coverage by @ephiepark (#1664) -v1.4.0 +v1.4.0 (Apr 17, 2019) perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln api: Move the advanced API, including all functions in the staging section, to the stable section api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress @@ -129,7 +187,7 @@ misc: Optimize dictionary memory usage in corner cases misc: Improve the dictionary builder on small or homogeneous data misc: Fix spelling across the repo by @jsoref -v1.3.8 +v1.3.8 (Dec 28, 2018) perf: better decompression speed on large files (+7%) and cold dictionaries (+15%) perf: slightly better compression ratio at high compression modes api : finalized advanced API, last stage before "stable" status @@ -151,14 +209,14 @@ doc : clarified zstd_compression_format.md, by @ulikunitz misc: fixed zstdgrep, returns 1 on failure, by @lzutao misc: NEWS renamed as CHANGELOG, in accordance with fboss -v1.3.7 +v1.3.7 (Oct 20, 2018) perf: slightly better decompression speed on clang (depending on hardware target) fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10 build: no longer build backtrace by default in release mode; restrict further automatic mode build: control backtrace support through build macro BACKTRACE misc: added man pages for zstdless and zstdgrep, by @samrussell -v1.3.6 +v1.3.6 (Oct 6, 2018) perf: much faster dictionary builder, by @jenniferliu perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte perf: faster dictionary decompression when using a very large number of dictionaries simultaneously @@ -172,7 +230,7 @@ build: Read Legacy format is limited to v0.5+ by default. Can be changed at comp doc : zstd_compression_format.md updated to match wording in IETF RFC 8478 misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97 -v1.3.5 +v1.3.5 (Jun 29, 2018) perf: much faster dictionary compression, by @felixhandte perf: small quality improvement for dictionary generation, by @terrelln perf: slightly improved high compression levels (notably level 19) @@ -187,7 +245,7 @@ build: make and make all are compatible with -j doc : clarify zstd_compression_format.md, updated for IETF RFC process misc: pzstd compatible with reproducible compilation, by @lamby -v1.3.4 +v1.3.4 (Mar 27, 2018) perf: faster speed (especially decoding speed) on recent cpus (haswell+) perf: much better performance associating --long with multi-threading, by @terrelln perf: better compression at levels 13-15 @@ -205,7 +263,7 @@ build: VS2017 scripts, by @HaydnTrigg misc: all /contrib projects fixed misc: added /contrib/docker script by @gyscos -v1.3.3 +v1.3.3 (Dec 21, 2017) perf: faster zstd_opt strategy (levels 16-19) fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen cli : fix : content size written in header by default @@ -217,7 +275,7 @@ api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` mac build: fix : compilation under rhel6 and centos6, reported by @pixelb build: added `check` target -v1.3.2 +v1.3.2 (Oct 10, 2017) new : long range mode, using --long command, by Stella Lau (@stellamplau) new : ability to generate and decode magicless frames (#591) changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode @@ -240,7 +298,7 @@ example : added streaming_memory_usage license : changed /examples license to BSD + GPLv2 license : fix a few header files to reflect new license (#825) -v1.3.1 +v1.3.1 (Aug 21, 2017) New license : BSD + GPLv2 perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk) perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760) @@ -255,7 +313,7 @@ new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cru new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau) updated : contrib/linux-kernel, by Nick Terrell (@terrelln) -v1.3.0 +v1.3.0 (Jul 6, 2017) cli : new : `--list` command, by Paul Cruz cli : changed : xz/lzma support enabled by default cli : changed : `-t *` continue processing list after a decompression error @@ -270,7 +328,7 @@ tools : decodecorpus can generate random dictionary-compressed samples, by Paul new : contrib/seekable_format, demo and API, by Sean Purcell changed : contrib/linux-kernel, updated version and license, by Nick Terrell -v1.2.0 +v1.2.0 (May 5, 2017) cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable) cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell cli : new : zstdmt symlink hardwired to `zstd -T0` @@ -292,7 +350,7 @@ build: enabled Multi-threading support for *BSD, by Baptiste Daroussin tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target. new : contrib/linux-kernel version, by Nick Terrell -v1.1.4 +v1.1.4 (Mar 18, 2017) cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski cli : new : advanced benchmark command --priority=rt cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 @@ -308,7 +366,7 @@ build : improved cmake script, by @Majlen build : added -Wformat-security flag, as recommended by Padraig Brady doc : new : educational decoder, by Sean Purcell -v1.1.3 +v1.1.3 (Feb 7, 2017) cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`) cli : new : experimental target `make zstdmt`, with multi-threading support cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano. @@ -324,7 +382,7 @@ API : fix : all symbols properly exposed in libzstd, by Nick Terrell build : support for Solaris target, by Przemyslaw Skibinski doc : clarified specification, by Sean Purcell -v1.1.2 +v1.1.2 (Dec 15, 2016) API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize() API : zbuff : changed : prototypes now generate deprecation warnings @@ -341,7 +399,7 @@ zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski install : better compatibility with FreeBSD, by Dimitry Andric source tree : changed : zbuff source files moved to lib/deprecated -v1.1.1 +v1.1.1 (Nov 2, 2016) New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption New : doc/zstd_manual.html, by Przemyslaw Skibinski Improved : slightly better compression ratio at --ultra levels (>= 20) @@ -352,7 +410,7 @@ Changed : zstd_errors.h is now installed within /include (and replaces errors_pu Updated man page Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets -v1.1.0 +v1.1.0 (Sep 28, 2016) New : contrib/pzstd, parallel version of zstd, by Nick Terrell added : NetBSD install target (#338) Improved : speed for batches of small files @@ -366,7 +424,7 @@ Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado ( Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365) Fixed : zstd-pgo, reported by octoploid (#329) -v1.0.0 +v1.0.0 (Sep 1, 2016) Change Licensing, all project is now BSD, Copyright Facebook Small decompression speed improvement API : Streaming API supports legacy format @@ -375,7 +433,7 @@ CLI supports legacy formats v0.4+ Fixed : compression fails on certain huge files, reported by Jesse McGrew Enhanced documentation, by Przemyslaw Skibinski -v0.8.1 +v0.8.1 (Aug 18, 2016) New streaming API Changed : --ultra now enables levels beyond 19 Changed : -i# now selects benchmark time in second @@ -384,7 +442,7 @@ Fixed : speed regression on specific patterns (#272) Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291) Fixed : ICC compilation, by Przemyslaw Skibinski -v0.8.0 +v0.8.0 (Aug 2, 2016) Improved : better speed on clang and gcc -O2, thanks to Eric Biggers New : Build on FreeBSD and DragonFly, thanks to JrMarino Changed : modified API : ZSTD_compressEnd() @@ -397,17 +455,17 @@ Modified : minor compression level adaptations Updated : compression format specification to v0.2.0 changed : zstd.h moved to /lib directory -v0.7.5 +v0.7.5 (Aug 1, 2016) Transition version, supporting decoding of v0.8.x -v0.7.4 +v0.7.4 (Jul 17, 2016) Added : homebrew for Mac, by Daniel Cade Added : more examples Fixed : segfault when using small dictionaries, reported by Felix Handte Modified : default compression level for CLI is now 3 Updated : specification, to v0.1.1 -v0.7.3 +v0.7.3 (Jul 9, 2016) New : compression format specification New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. New : `ZSTD_getDecompressedSize()` @@ -419,18 +477,18 @@ fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section) modified : legacy functions no longer need magic number -v0.7.2 +v0.7.2 (Jul 4, 2016) fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski. fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner. fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner. -v0.7.1 +v0.7.1 (Jun 23, 2016) fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв fixed : corruption issue, reported by cj modified : checksum enabled by default in command line mode -v0.7.0 +v0.7.0 (Jun 17, 2016) New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski New : Command `--rm`, to remove source file after successful de/compression New : Visual build scripts, by Christophe Chevalier @@ -443,7 +501,7 @@ API : support for custom malloc/free functions New : controllable Dictionary ID New : Support for skippable frames -v0.6.1 +v0.6.1 (May 13, 2016) New : zlib wrapper API, thanks to Przemyslaw Skibinski New : Ability to compile compressor / decompressor separately Changed : new lib directory structure @@ -453,103 +511,103 @@ Fixed : null-string roundtrip (#176) New : benchmark mode can select directory as input Experimental : midipix support, VMS support -v0.6.0 +v0.6.0 (Apr 13, 2016) Stronger high compression modes, thanks to Przemyslaw Skibinski API : ZSTD_getFrameParams() provides size of decompressed content New : highest compression modes require `--ultra` command to fully unleash their capacity Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner -v0.5.1 +v0.5.1 (Feb 18, 2016) New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski Changed : Dictionary builder integrated into libzstd and zstd cli Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`. Fix : high compression modes for big-endian platforms New : zstd cli : `-t` | `--test` command -v0.5.0 +v0.5.0 (Feb 5, 2016) New : dictionary builder utility Changed : streaming & dictionary API Improved : better compression of small data -v0.4.7 +v0.4.7 (Jan 22, 2016) Improved : small compression speed improvement in HC mode Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default fix : bt search bug -v0.4.6 +v0.4.6 (Jan 13, 2016) fix : fast compression mode on Windows New : cmake configuration file, thanks to Artyom Dymchenko Improved : high compression mode on repetitive data New : block-level API New : ZSTD_duplicateCCtx() -v0.4.5 +v0.4.5 (Dec 18, 2015) new : -m/--multiple : compress/decompress multiple files -v0.4.4 +v0.4.4 (Dec 14, 2015) Fixed : high compression modes for Windows 32 bits new : external dictionary API extended to buffered mode and accessible through command line new : windows DLL project, thanks to Christophe Chevalier -v0.4.3 : +v0.4.3 (Dec 7, 2015) new : external dictionary API new : zstd-frugal -v0.4.2 : +v0.4.2 (Dec 2, 2015) Generic minor improvements for small blocks Fixed : big-endian compatibility, by Peter Harris (#85) -v0.4.1 +v0.4.1 (Dec 1, 2015) Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben) removed `zstd.c` -v0.4.0 +v0.4.0 (Nov 29, 2015) Command line utility compatible with high compression levels Removed zstdhc => merged into zstd Added : ZBUFF API (see zstd_buffered.h) Rolling buffer support -v0.3.6 +v0.3.6 (Nov 10, 2015) small blocks params -v0.3.5 +v0.3.5 (Nov 9, 2015) minor generic compression improvements -v0.3.4 +v0.3.4 (Nov 6, 2015) Faster fast cLevels -v0.3.3 +v0.3.3 (Nov 5, 2015) Small compression ratio improvement -v0.3.2 +v0.3.2 (Nov 2, 2015) Fixed Visual Studio -v0.3.1 : +v0.3.1 (Nov 2, 2015) Small compression ratio improvement -v0.3 +v0.3 (Oct 30, 2015) HC mode : compression levels 2-26 -v0.2.2 +v0.2.2 (Oct 28, 2015) Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier -v0.2.1 +v0.2.1 (Oct 24, 2015) Fix : Read errors, advanced fuzzer tests, by Hanno Böck -v0.2.0 +v0.2.0 (Oct 22, 2015) **Breaking format change** Faster decompression speed Can still decode v0.1 format -v0.1.3 +v0.1.3 (Oct 15, 2015) fix uninitialization warning, reported by Evan Nemerson -v0.1.2 +v0.1.2 (Sep 11, 2015) frame concatenation support -v0.1.1 +v0.1.1 (Aug 27, 2015) fix compression bug detects write-flush errors -v0.1.0 +v0.1.0 (Aug 25, 2015) first release diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 637e3718..bb85d587 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -126,6 +126,20 @@ just `contrib/largeNbDicts` and nothing else, you can run: scan-build make -C contrib/largeNbDicts largeNbDicts ``` +### Pitfalls of static analysis +`scan-build` is part of our regular CI suite. Other static analyzers are not. + +It can be useful to look at additional static analyzers once in a while (and we do), but it's not a good idea to multiply the nb of analyzers run continuously at each commit and PR. The reasons are : + +- Static analyzers are full of false positive. The signal to noise ratio is actually pretty low. +- A good CI policy is "zero-warning tolerance". That means that all issues must be solved, including false positives. This quickly becomes a tedious workload. +- Multiple static analyzers will feature multiple kind of false positives, sometimes applying to the same code but in different ways leading to : + + torteous code, trying to please multiple constraints, hurting readability and therefore maintenance. Sometimes, such complexity introduce other more subtle bugs, that are just out of scope of the analyzers. + + sometimes, these constraints are mutually exclusive : if one try to solve one, the other static analyzer will complain, they can't be both happy at the same time. +- As if that was not enough, the list of false positives change with each version. It's hard enough to follow one static analyzer, but multiple ones with their own update agenda, this quickly becomes a massive velocity reducer. + +This is different from running a static analyzer once in a while, looking at the output, and __cherry picking__ a few warnings that seem helpful, either because they detected a genuine risk of bug, or because it helps expressing the code in a way which is more readable or more difficult to misuse. These kind of reports can be useful, and are accepted. + ## Performance Performance is extremely important for zstd and we only merge pull requests whose performance landscape and corresponding trade-offs have been adequately analyzed, reproduced, and presented. @@ -9,11 +9,11 @@ third_party { type: GIT value: "https://github.com/facebook/zstd" } - version: "v1.4.5" + version: "v1.4.7" license_type: RESTRICTED last_upgrade_date { year: 2020 - month: 6 - day: 3 + month: 12 + day: 17 } } @@ -8,6 +8,9 @@ # You may select, at your option, one of the above-listed licenses. # ################################################################ +# verbose mode (print commands) on V=1 or VERBOSE=1 +Q = $(if $(filter 1,$(V) $(VERBOSE)),,@) + PRGDIR = programs ZSTDDIR = lib BUILDIR = build @@ -28,9 +31,9 @@ VOID = /dev/null TARGET_SYSTEM ?= $(OS) ifneq (,$(filter Windows%,$(TARGET_SYSTEM))) -EXT =.exe + EXT =.exe else -EXT = + EXT = endif ## default: Build lib-release and zstd-release @@ -46,8 +49,8 @@ allmost: allzstd zlibwrapper # skip zwrapper, can't build that on alternate architectures without the proper zlib installed .PHONY: allzstd allzstd: lib-all - $(MAKE) -C $(PRGDIR) all - $(MAKE) -C $(TESTDIR) all + $(Q)$(MAKE) -C $(PRGDIR) all + $(Q)$(MAKE) -C $(TESTDIR) all .PHONY: all32 all32: @@ -55,18 +58,19 @@ all32: $(MAKE) -C $(TESTDIR) all32 .PHONY: lib lib-release libzstd.a +lib-all : lib lib lib-release lib-all : - @$(MAKE) -C $(ZSTDDIR) $@ + $(Q)$(MAKE) -C $(ZSTDDIR) $@ .PHONY: zstd zstd-release zstd zstd-release: - @$(MAKE) -C $(PRGDIR) $@ - cp $(PRGDIR)/zstd$(EXT) . + $(Q)$(MAKE) -C $(PRGDIR) $@ + $(Q)ln -sf $(PRGDIR)/zstd$(EXT) zstd$(EXT) .PHONY: zstdmt zstdmt: - @$(MAKE) -C $(PRGDIR) $@ - cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT) + $(Q)$(MAKE) -C $(PRGDIR) $@ + $(Q)cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT) .PHONY: zlibwrapper zlibwrapper: lib @@ -75,16 +79,16 @@ zlibwrapper: lib ## test: run long-duration tests .PHONY: test DEBUGLEVEL ?= 1 -test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror +test: MOREFLAGS += -g -Werror test: - MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants + DEBUGLEVEL=$(DEBUGLEVEL) MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ - ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder test + ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@ ## shortest: same as `make check` .PHONY: shortest shortest: - $(MAKE) -C $(TESTDIR) $@ + $(Q)$(MAKE) -C $(TESTDIR) $@ ## check: run basic tests for `zstd` cli .PHONY: check @@ -97,10 +101,10 @@ automated_benchmarking: .PHONY: benchmarking benchmarking: automated_benchmarking -## examples: build all examples in `/examples` directory +## examples: build all examples in `examples/` directory .PHONY: examples examples: lib - CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all + $(MAKE) -C examples all ## manual: generate API documentation in html format .PHONY: manual @@ -117,6 +121,7 @@ man: contrib: lib $(MAKE) -C contrib/pzstd all $(MAKE) -C contrib/seekable_format/examples all + $(MAKE) -C contrib/seekable_format/tests test $(MAKE) -C contrib/largeNbDicts all cd contrib/single_file_libs/ ; ./build_decoder_test.sh cd contrib/single_file_libs/ ; ./build_library_test.sh @@ -127,17 +132,18 @@ cleanTabs: .PHONY: clean clean: - @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) - @$(MAKE) -C $(PRGDIR) $@ > $(VOID) - @$(MAKE) -C $(TESTDIR) $@ > $(VOID) - @$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) - @$(MAKE) -C examples/ $@ > $(VOID) - @$(MAKE) -C contrib/gen_html $@ > $(VOID) - @$(MAKE) -C contrib/pzstd $@ > $(VOID) - @$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID) - @$(MAKE) -C contrib/largeNbDicts $@ > $(VOID) - @$(RM) zstd$(EXT) zstdmt$(EXT) tmp* - @$(RM) -r lz4 + $(Q)$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) + $(Q)$(MAKE) -C $(PRGDIR) $@ > $(VOID) + $(Q)$(MAKE) -C $(TESTDIR) $@ > $(VOID) + $(Q)$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) + $(Q)$(MAKE) -C examples/ $@ > $(VOID) + $(Q)$(MAKE) -C contrib/gen_html $@ > $(VOID) + $(Q)$(MAKE) -C contrib/pzstd $@ > $(VOID) + $(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID) + $(Q)$(MAKE) -C contrib/seekable_format/tests $@ > $(VOID) + $(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID) + $(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp* + $(Q)$(RM) -r lz4 @echo Cleaning completed #------------------------------------------------------------------------------ @@ -161,7 +167,7 @@ EGREP = egrep $(EGREP_OPTIONS) ## list: Print all targets and their descriptions (if provided) .PHONY: list list: - @TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \ + $(Q)TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \ | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \ | $(EGREP) -v -e '^[^[:alnum:]]' | sort); \ { \ @@ -176,13 +182,13 @@ list: .PHONY: install armtest usan asan uasan install: - @$(MAKE) -C $(ZSTDDIR) $@ - @$(MAKE) -C $(PRGDIR) $@ + $(Q)$(MAKE) -C $(ZSTDDIR) $@ + $(Q)$(MAKE) -C $(PRGDIR) $@ .PHONY: uninstall uninstall: - @$(MAKE) -C $(ZSTDDIR) $@ - @$(MAKE) -C $(PRGDIR) $@ + $(Q)$(MAKE) -C $(ZSTDDIR) $@ + $(Q)$(MAKE) -C $(PRGDIR) $@ .PHONY: travis-install travis-install: @@ -176,6 +176,12 @@ Going into `build` directory, you will find additional possibilities: You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo. The output binary will be in `buck-out/gen/programs/`. +## Testing + +You can run quick local smoke tests by executing the `playTest.sh` script from the `src/tests` directory. +Two env variables `$ZSTD_BIN` and `$DATAGEN_BIN` are needed for the test script to locate the zstd and datagen binary. +For information on CI testing, please refer to TESTING.md + ## Status Zstandard is currently deployed within Facebook. It is used continuously to compress large amounts of data in multiple formats and use cases. diff --git a/appveyor.yml b/appveyor.yml index 5d77b310..169c66bd 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -162,6 +162,8 @@ - if [%TEST%]==[cmake] ( mkdir build\cmake\build && cd build\cmake\build && + SET FUZZERTEST=-T2mn && + SET ZSTREAM_TESTTIME=-T2mn && cmake -G "Visual Studio 14 2015 Win64" .. && cd ..\..\.. && make clean @@ -194,7 +196,7 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2" + SCRIPT: "CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2" - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x86" @@ -285,5 +287,6 @@ - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% - if [%HOST%]==[mingw] ( set "CC=%COMPILER%" && + make clean && make check ) diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index 9b5d7efb..a0505778 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -7,7 +7,7 @@ # in the COPYING file in the root directory of this source tree). # ################################################################ -cmake_minimum_required(VERSION 2.8.9 FATAL_ERROR) +cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR) # As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies. # Set and use the newest cmake policies that are validated to work diff --git a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake index 18681988..62389717 100644 --- a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake +++ b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake @@ -60,7 +60,6 @@ macro(ADD_ZSTD_COMPILATION_FLAGS) CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO) if( ${flag_var} ) separate_arguments(${flag_var}) - list(REMOVE_DUPLICATES ${flag_var}) string(REPLACE ";" " " ${flag_var} "${${flag_var}}") endif() endforeach () diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 666da60c..088c8760 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -134,16 +134,37 @@ if (ZSTD_BUILD_STATIC) OUTPUT_NAME ${STATIC_LIBRARY_BASE_NAME}) endif () -if (UNIX) +if (UNIX OR MINGW) # pkg-config set(PREFIX "${CMAKE_INSTALL_PREFIX}") - set(LIBDIR "${CMAKE_INSTALL_LIBDIR}") - set(INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}") + set(EXEC_PREFIX "\\$$\{prefix}") + set(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}") + set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}") set(VERSION "${zstd_VERSION}") + + string(LENGTH "${PREFIX}" PREFIX_LENGTH) + string(SUBSTRING "${LIBDIR}" 0 ${PREFIX_LENGTH} LIBDIR_PREFIX) + string(SUBSTRING "${LIBDIR}" ${PREFIX_LENGTH} -1 LIBDIR_SUFFIX) + string(SUBSTRING "${INCLUDEDIR}" 0 ${PREFIX_LENGTH} INCLUDEDIR_PREFIX) + string(SUBSTRING "${INCLUDEDIR}" ${PREFIX_LENGTH} -1 INCLUDEDIR_SUFFIX) + + if ("${INCLUDEDIR_PREFIX}" STREQUAL "${PREFIX}") + set(INCLUDEDIR_PREFIX "\\$$\{prefix}") + endif() + if ("${LIBDIR_PREFIX}" STREQUAL "${PREFIX}") + set(LIBDIR_PREFIX "\\$$\{exec_prefix}") + endif() + add_custom_target(libzstd.pc ALL - ${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc" - -DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}" - -P "${CMAKE_CURRENT_SOURCE_DIR}/pkgconfig.cmake" + ${CMAKE_COMMAND} + -DIN=${LIBRARY_DIR}/libzstd.pc.in + -DOUT="libzstd.pc" + -DPREFIX="${PREFIX}" + -DEXEC_PREFIX="${EXEC_PREFIX}" + -DINCLUDEDIR="${INCLUDEDIR_PREFIX}${INCLUDEDIR_SUFFIX}" + -DLIBDIR="${LIBDIR_PREFIX}${LIBDIR_SUFFIX}" + -DVERSION="${VERSION}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/pkgconfig.cmake COMMENT "Creating pkg-config file") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") @@ -152,9 +173,7 @@ endif () # install target install(FILES "${LIBRARY_DIR}/zstd.h" - "${LIBRARY_DIR}/deprecated/zbuff.h" "${LIBRARY_DIR}/dictBuilder/zdict.h" - "${LIBRARY_DIR}/dictBuilder/cover.h" "${LIBRARY_DIR}/common/zstd_errors.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") @@ -163,6 +182,7 @@ install(TARGETS ${library_targets} INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" + RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" ) # uninstall target diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt index 95d60f5f..34eca910 100644 --- a/build/cmake/tests/CMakeLists.txt +++ b/build/cmake/tests/CMakeLists.txt @@ -84,9 +84,9 @@ add_test(NAME zstreamtest COMMAND zstreamtest ${ZSTD_ZSTREAM_FLAGS}) # # playTests.sh # -AddTestFlagsOption(ZSTD_PLAYTESTS_FLAGS "--test-large-data" +AddTestFlagsOption(ZSTD_PLAYTESTS_FLAGS "$ENV{PLAYTESTS_FLAGS}" "Semicolon-separated list of flags to pass to the playTests.sh test") -add_test(NAME playTests COMMAND sh -c "${TESTS_DIR}/playTests.sh" ${ZSTD_PLAYTESTS_FLAGS}) +add_test(NAME playTests COMMAND sh -c "\"${TESTS_DIR}/playTests.sh\" ${ZSTD_PLAYTESTS_FLAGS}") if (ZSTD_BUILD_PROGRAMS) set_property(TEST playTests APPEND PROPERTY ENVIRONMENT "ZSTD_BIN=$<TARGET_FILE:zstd>" @@ -100,7 +100,9 @@ endif() # Label the "Medium" set of tests (see TESTING.md) set_property(TEST fuzzer zstreamtest playTests APPEND PROPERTY LABELS Medium) +add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c) if (UNIX) - add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c) target_link_libraries(paramgrill libzstd_static m) #m is math library +else() + target_link_libraries(paramgrill libzstd_static) endif () diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index 6fb3df8e..17806c8d 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -127,6 +127,5 @@ pkgconfig.generate(libzstd, url: 'http://www.zstd.net/') install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'), - join_paths(zstd_rootdir, 'lib/deprecated/zbuff.h'), join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.h'), join_paths(zstd_rootdir, 'lib/common/zstd_errors.h')) diff --git a/build/meson/meson.build b/build/meson/meson.build index 970cf3c9..2a425b2f 100644 --- a/build/meson/meson.build +++ b/build/meson/meson.build @@ -22,7 +22,6 @@ project('zstd', cc = meson.get_compiler('c') cxx = meson.get_compiler('cpp') pkgconfig = import('pkgconfig') -python3 = import('python').find_installation() windows_mod = import('windows') host_machine_os = host_machine.system() @@ -40,8 +39,8 @@ compiler_msvc = 'msvc' zstd_version = meson.project_version() zstd_h_file = join_paths(meson.current_source_dir(), '../../lib/zstd.h') -GetZstdLibraryVersion_py = files('GetZstdLibraryVersion.py') -r = run_command(python3, GetZstdLibraryVersion_py, zstd_h_file) +GetZstdLibraryVersion_py = find_program('GetZstdLibraryVersion.py', native : true) +r = run_command(GetZstdLibraryVersion_py, zstd_h_file) if r.returncode() == 0 zstd_version = r.stdout().strip() message('Project version is now: @0@'.format(zstd_version)) diff --git a/contrib/freestanding_lib/freestanding.py b/contrib/freestanding_lib/freestanding.py new file mode 100755 index 00000000..9e91a489 --- /dev/null +++ b/contrib/freestanding_lib/freestanding.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +# ################################################################ +# Copyright (c) 2020-2020, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ########################################################################## + +import argparse +import contextlib +import os +import re +import shutil +import sys +from typing import Optional + + +INCLUDED_SUBDIRS = ["common", "compress", "decompress"] + +SKIPPED_FILES = [ + "common/mem.h", + "common/zstd_deps.h", + "common/pool.c", + "common/pool.h", + "common/threading.c", + "common/threading.h", + "compress/zstdmt_compress.h", + "compress/zstdmt_compress.c", +] + +XXHASH_FILES = [ + "common/xxhash.c", + "common/xxhash.h", +] + + +class FileLines(object): + def __init__(self, filename): + self.filename = filename + with open(self.filename, "r") as f: + self.lines = f.readlines() + + def write(self): + with open(self.filename, "w") as f: + f.write("".join(self.lines)) + + +class PartialPreprocessor(object): + """ + Looks for simple ifdefs and ifndefs and replaces them. + Handles && and ||. + Has fancy logic to handle translating elifs to ifs. + Only looks for macros in the first part of the expression with no + parens. + Does not handle multi-line macros (only looks in first line). + """ + def __init__(self, defs: [(str, Optional[str])], replaces: [(str, str)], undefs: [str]): + MACRO_GROUP = r"(?P<macro>[a-zA-Z_][a-zA-Z_0-9]*)" + ELIF_GROUP = r"(?P<elif>el)?" + OP_GROUP = r"(?P<op>&&|\|\|)?" + + self._defs = {macro:value for macro, value in defs} + self._replaces = {macro:value for macro, value in replaces} + self._defs.update(self._replaces) + self._undefs = set(undefs) + + self._define = re.compile(r"\s*#\s*define") + self._if = re.compile(r"\s*#\s*if") + self._elif = re.compile(r"\s*#\s*(?P<elif>el)if") + self._else = re.compile(r"\s*#\s*(?P<else>else)") + self._endif = re.compile(r"\s*#\s*endif") + + self._ifdef = re.compile(fr"\s*#\s*if(?P<not>n)?def {MACRO_GROUP}\s*") + self._if_defined = re.compile( + fr"\s*#\s*{ELIF_GROUP}if\s+(?P<not>!)?\s*defined\s*\(\s*{MACRO_GROUP}\s*\)\s*{OP_GROUP}" + ) + self._if_defined_value = re.compile( + fr"\s*#\s*{ELIF_GROUP}if\s+defined\s*\(\s*{MACRO_GROUP}\s*\)\s*" + fr"(?P<op>&&)\s*" + fr"(?P<openp>\()?\s*" + fr"(?P<macro2>[a-zA-Z_][a-zA-Z_0-9]*)\s*" + fr"(?P<cmp>[=><!]+)\s*" + fr"(?P<value>[0-9]*)\s*" + fr"(?P<closep>\))?\s*" + ) + self._if_true = re.compile( + fr"\s*#\s*{ELIF_GROUP}if\s+{MACRO_GROUP}\s*{OP_GROUP}" + ) + + self._c_comment = re.compile(r"/\*.*?\*/") + self._cpp_comment = re.compile(r"//") + + def _log(self, *args, **kwargs): + print(*args, **kwargs) + + def _strip_comments(self, line): + # First strip c-style comments (may include //) + while True: + m = self._c_comment.search(line) + if m is None: + break + line = line[:m.start()] + line[m.end():] + + # Then strip cpp-style comments + m = self._cpp_comment.search(line) + if m is not None: + line = line[:m.start()] + + return line + + def _fixup_indentation(self, macro, replace: [str]): + if len(replace) == 0: + return replace + if len(replace) == 1 and self._define.match(replace[0]) is None: + # If there is only one line, only replace defines + return replace + + + all_pound = True + for line in replace: + if not line.startswith('#'): + all_pound = False + if all_pound: + replace = [line[1:] for line in replace] + + min_spaces = len(replace[0]) + for line in replace: + spaces = 0 + for i, c in enumerate(line): + if c != ' ': + # Non-preprocessor line ==> skip the fixup + if not all_pound and c != '#': + return replace + spaces = i + break + min_spaces = min(min_spaces, spaces) + + replace = [line[min_spaces:] for line in replace] + + if all_pound: + replace = ["#" + line for line in replace] + + return replace + + def _handle_if_block(self, macro, idx, is_true, prepend): + """ + Remove the #if or #elif block starting on this line. + """ + REMOVE_ONE = 0 + KEEP_ONE = 1 + REMOVE_REST = 2 + + if is_true: + state = KEEP_ONE + else: + state = REMOVE_ONE + + line = self._inlines[idx] + is_if = self._if.match(line) is not None + assert is_if or self._elif.match(line) is not None + depth = 0 + + start_idx = idx + + idx += 1 + replace = prepend + finished = False + while idx < len(self._inlines): + line = self._inlines[idx] + # Nested if statement + if self._if.match(line): + depth += 1 + idx += 1 + continue + # We're inside a nested statement + if depth > 0: + if self._endif.match(line): + depth -= 1 + idx += 1 + continue + + # We're at the original depth + + # Looking only for an endif. + # We've found a true statement, but haven't + # completely elided the if block, so we just + # remove the remainder. + if state == REMOVE_REST: + if self._endif.match(line): + if is_if: + # Remove the endif because we took the first if + idx += 1 + finished = True + break + idx += 1 + continue + + if state == KEEP_ONE: + m = self._elif.match(line) + if self._endif.match(line): + replace += self._inlines[start_idx + 1:idx] + idx += 1 + finished = True + break + if self._elif.match(line) or self._else.match(line): + replace += self._inlines[start_idx + 1:idx] + state = REMOVE_REST + idx += 1 + continue + + if state == REMOVE_ONE: + m = self._elif.match(line) + if m is not None: + if is_if: + idx += 1 + b = m.start('elif') + e = m.end('elif') + assert e - b == 2 + replace.append(line[:b] + line[e:]) + finished = True + break + m = self._else.match(line) + if m is not None: + if is_if: + idx += 1 + while self._endif.match(self._inlines[idx]) is None: + replace.append(self._inlines[idx]) + idx += 1 + idx += 1 + finished = True + break + if self._endif.match(line): + if is_if: + # Remove the endif because no other elifs + idx += 1 + finished = True + break + idx += 1 + continue + if not finished: + raise RuntimeError("Unterminated if block!") + + replace = self._fixup_indentation(macro, replace) + + self._log(f"\tHardwiring {macro}") + if start_idx > 0: + self._log(f"\t\t {self._inlines[start_idx - 1][:-1]}") + for x in range(start_idx, idx): + self._log(f"\t\t- {self._inlines[x][:-1]}") + for line in replace: + self._log(f"\t\t+ {line[:-1]}") + if idx < len(self._inlines): + self._log(f"\t\t {self._inlines[idx][:-1]}") + + return idx, replace + + def _preprocess_once(self): + outlines = [] + idx = 0 + changed = False + while idx < len(self._inlines): + line = self._inlines[idx] + sline = self._strip_comments(line) + m = self._ifdef.fullmatch(sline) + if_true = False + if m is None: + m = self._if_defined_value.fullmatch(sline) + if m is None: + m = self._if_defined.match(sline) + if m is None: + m = self._if_true.match(sline) + if_true = (m is not None) + if m is None: + outlines.append(line) + idx += 1 + continue + + groups = m.groupdict() + macro = groups['macro'] + op = groups.get('op') + + if not (macro in self._defs or macro in self._undefs): + outlines.append(line) + idx += 1 + continue + + defined = macro in self._defs + + # Needed variables set: + # resolved: Is the statement fully resolved? + # is_true: If resolved, is the statement true? + ifdef = False + if if_true: + if not defined: + outlines.append(line) + idx += 1 + continue + + defined_value = self._defs[macro] + is_int = True + try: + defined_value = int(defined_value) + except TypeError: + is_int = False + except ValueError: + is_int = False + + resolved = is_int + is_true = (defined_value != 0) + + if resolved and op is not None: + if op == '&&': + resolved = not is_true + else: + assert op == '||' + resolved = is_true + + else: + ifdef = groups.get('not') is None + elseif = groups.get('elif') is not None + + macro2 = groups.get('macro2') + cmp = groups.get('cmp') + value = groups.get('value') + openp = groups.get('openp') + closep = groups.get('closep') + + is_true = (ifdef == defined) + resolved = True + if op is not None: + if op == '&&': + resolved = not is_true + else: + assert op == '||' + resolved = is_true + + if macro2 is not None and not resolved: + assert ifdef and defined and op == '&&' and cmp is not None + # If the statment is true, but we have a single value check, then + # check the value. + defined_value = self._defs[macro] + are_ints = True + try: + defined_value = int(defined_value) + value = int(value) + except TypeError: + are_ints = False + except ValueError: + are_ints = False + if ( + macro == macro2 and + ((openp is None) == (closep is None)) and + are_ints + ): + resolved = True + if cmp == '<': + is_true = defined_value < value + elif cmp == '<=': + is_true = defined_value <= value + elif cmp == '==': + is_true = defined_value == value + elif cmp == '!=': + is_true = defined_value != value + elif cmp == '>=': + is_true = defined_value >= value + elif cmp == '>': + is_true = defined_value > value + else: + resolved = False + + if op is not None and not resolved: + # Remove the first op in the line + spaces + if op == '&&': + opre = op + else: + assert op == '||' + opre = r'\|\|' + needle = re.compile(fr"(?P<if>\s*#\s*(el)?if\s+).*?(?P<op>{opre}\s*)") + match = needle.match(line) + assert match is not None + newline = line[:match.end('if')] + line[match.end('op'):] + + self._log(f"\tHardwiring partially resolved {macro}") + self._log(f"\t\t- {line[:-1]}") + self._log(f"\t\t+ {newline[:-1]}") + + outlines.append(newline) + idx += 1 + continue + + # Skip any statements we cannot fully compute + if not resolved: + outlines.append(line) + idx += 1 + continue + + prepend = [] + if macro in self._replaces: + assert not ifdef + assert op is None + value = self._replaces.pop(macro) + prepend = [f"#define {macro} {value}\n"] + + idx, replace = self._handle_if_block(macro, idx, is_true, prepend) + outlines += replace + changed = True + + return changed, outlines + + def preprocess(self, filename): + with open(filename, 'r') as f: + self._inlines = f.readlines() + changed = True + iters = 0 + while changed: + iters += 1 + changed, outlines = self._preprocess_once() + self._inlines = outlines + + with open(filename, 'w') as f: + f.write(''.join(self._inlines)) + + +class Freestanding(object): + def __init__( + self, zstd_deps: str, mem: str, source_lib: str, output_lib: str, + external_xxhash: bool, xxh64_state: Optional[str], + xxh64_prefix: Optional[str], rewritten_includes: [(str, str)], + defs: [(str, Optional[str])], replaces: [(str, str)], + undefs: [str], excludes: [str] + ): + self._zstd_deps = zstd_deps + self._mem = mem + self._src_lib = source_lib + self._dst_lib = output_lib + self._external_xxhash = external_xxhash + self._xxh64_state = xxh64_state + self._xxh64_prefix = xxh64_prefix + self._rewritten_includes = rewritten_includes + self._defs = defs + self._replaces = replaces + self._undefs = undefs + self._excludes = excludes + + def _dst_lib_file_paths(self): + """ + Yields all the file paths in the dst_lib. + """ + for root, dirname, filenames in os.walk(self._dst_lib): + for filename in filenames: + filepath = os.path.join(root, filename) + yield filepath + + def _log(self, *args, **kwargs): + print(*args, **kwargs) + + def _copy_file(self, lib_path): + if not (lib_path.endswith(".c") or lib_path.endswith(".h")): + return + if lib_path in SKIPPED_FILES: + self._log(f"\tSkipping file: {lib_path}") + return + if self._external_xxhash and lib_path in XXHASH_FILES: + self._log(f"\tSkipping xxhash file: {lib_path}") + return + + src_path = os.path.join(self._src_lib, lib_path) + dst_path = os.path.join(self._dst_lib, lib_path) + self._log(f"\tCopying: {src_path} -> {dst_path}") + shutil.copyfile(src_path, dst_path) + + def _copy_source_lib(self): + self._log("Copying source library into output library") + + assert os.path.exists(self._src_lib) + os.makedirs(self._dst_lib, exist_ok=True) + self._copy_file("zstd.h") + for subdir in INCLUDED_SUBDIRS: + src_dir = os.path.join(self._src_lib, subdir) + dst_dir = os.path.join(self._dst_lib, subdir) + + assert os.path.exists(src_dir) + os.makedirs(dst_dir, exist_ok=True) + + for filename in os.listdir(src_dir): + lib_path = os.path.join(subdir, filename) + self._copy_file(lib_path) + + def _copy_zstd_deps(self): + dst_zstd_deps = os.path.join(self._dst_lib, "common", "zstd_deps.h") + self._log(f"Copying zstd_deps: {self._zstd_deps} -> {dst_zstd_deps}") + shutil.copyfile(self._zstd_deps, dst_zstd_deps) + + def _copy_mem(self): + dst_mem = os.path.join(self._dst_lib, "common", "mem.h") + self._log(f"Copying mem: {self._mem} -> {dst_mem}") + shutil.copyfile(self._mem, dst_mem) + + def _hardwire_preprocessor(self, name: str, value: Optional[str] = None, undef=False): + """ + If value=None then hardwire that it is defined, but not what the value is. + If undef=True then value must be None. + If value='' then the macro is defined to '' exactly. + """ + assert not (undef and value is not None) + for filepath in self._dst_lib_file_paths(): + file = FileLines(filepath) + + def _hardwire_defines(self): + self._log("Hardwiring macros") + partial_preprocessor = PartialPreprocessor(self._defs, self._replaces, self._undefs) + for filepath in self._dst_lib_file_paths(): + partial_preprocessor.preprocess(filepath) + + def _remove_excludes(self): + self._log("Removing excluded sections") + for exclude in self._excludes: + self._log(f"\tRemoving excluded sections for: {exclude}") + begin_re = re.compile(f"BEGIN {exclude}") + end_re = re.compile(f"END {exclude}") + for filepath in self._dst_lib_file_paths(): + file = FileLines(filepath) + outlines = [] + skipped = [] + emit = True + for line in file.lines: + if emit and begin_re.search(line) is not None: + assert end_re.search(line) is None + emit = False + if emit: + outlines.append(line) + else: + skipped.append(line) + if end_re.search(line) is not None: + assert begin_re.search(line) is None + self._log(f"\t\tRemoving excluded section: {exclude}") + for s in skipped: + self._log(f"\t\t\t- {s}") + emit = True + skipped = [] + if not emit: + raise RuntimeError("Excluded section unfinished!") + file.lines = outlines + file.write() + + def _rewrite_include(self, original, rewritten): + self._log(f"\tRewriting include: {original} -> {rewritten}") + regex = re.compile(f"\\s*#\\s*include\\s*(?P<include>{original})") + for filepath in self._dst_lib_file_paths(): + file = FileLines(filepath) + for i, line in enumerate(file.lines): + match = regex.match(line) + if match is None: + continue + s = match.start('include') + e = match.end('include') + file.lines[i] = line[:s] + rewritten + line[e:] + file.write() + + def _rewrite_includes(self): + self._log("Rewriting includes") + for original, rewritten in self._rewritten_includes: + self._rewrite_include(original, rewritten) + + def _replace_xxh64_prefix(self): + if self._xxh64_prefix is None: + return + self._log(f"Replacing XXH64 prefix with {self._xxh64_prefix}") + replacements = [] + if self._xxh64_state is not None: + replacements.append( + (re.compile(r"([^\w]|^)(?P<orig>XXH64_state_t)([^\w]|$)"), self._xxh64_state) + ) + if self._xxh64_prefix is not None: + replacements.append( + (re.compile(r"([^\w]|^)(?P<orig>XXH64)_"), self._xxh64_prefix) + ) + for filepath in self._dst_lib_file_paths(): + file = FileLines(filepath) + for i, line in enumerate(file.lines): + modified = False + for regex, replacement in replacements: + match = regex.search(line) + while match is not None: + modified = True + b = match.start('orig') + e = match.end('orig') + line = line[:b] + replacement + line[e:] + match = regex.search(line) + if modified: + self._log(f"\t- {file.lines[i][:-1]}") + self._log(f"\t+ {line[:-1]}") + file.lines[i] = line + file.write() + + def go(self): + self._copy_source_lib() + self._copy_zstd_deps() + self._copy_mem() + self._hardwire_defines() + self._remove_excludes() + self._rewrite_includes() + self._replace_xxh64_prefix() + + +def parse_optional_pair(defines: [str]) -> [(str, Optional[str])]: + output = [] + for define in defines: + parsed = define.split('=') + if len(parsed) == 1: + output.append((parsed[0], None)) + elif len(parsed) == 2: + output.append((parsed[0], parsed[1])) + else: + raise RuntimeError(f"Bad define: {define}") + return output + + +def parse_pair(rewritten_includes: [str]) -> [(str, str)]: + output = [] + for rewritten_include in rewritten_includes: + parsed = rewritten_include.split('=') + if len(parsed) == 2: + output.append((parsed[0], parsed[1])) + else: + raise RuntimeError(f"Bad rewritten include: {rewritten_include}") + return output + + + +def main(name, args): + parser = argparse.ArgumentParser(prog=name) + parser.add_argument("--zstd-deps", default="zstd_deps.h", help="Zstd dependencies file") + parser.add_argument("--mem", default="mem.h", help="Memory module") + parser.add_argument("--source-lib", default="../../lib", help="Location of the zstd library") + parser.add_argument("--output-lib", default="./freestanding_lib", help="Where to output the freestanding zstd library") + parser.add_argument("--xxhash", default=None, help="Alternate external xxhash include e.g. --xxhash='<xxhash.h>'. If set xxhash is not included.") + parser.add_argument("--xxh64-state", default=None, help="Alternate XXH64 state type (excluding _) e.g. --xxh64-state='struct xxh64_state'") + parser.add_argument("--xxh64-prefix", default=None, help="Alternate XXH64 function prefix (excluding _) e.g. --xxh64-prefix=xxh64") + parser.add_argument("--rewrite-include", default=[], dest="rewritten_includes", action="append", help="Rewrite an include REGEX=NEW (e.g. '<stddef\\.h>=<linux/types.h>')") + parser.add_argument("-D", "--define", default=[], dest="defs", action="append", help="Pre-define this macro (can be passed multiple times)") + parser.add_argument("-U", "--undefine", default=[], dest="undefs", action="append", help="Pre-undefine this macro (can be passed mutliple times)") + parser.add_argument("-R", "--replace", default=[], dest="replaces", action="append", help="Pre-define this macro and replace the first ifndef block with its definition") + parser.add_argument("-E", "--exclude", default=[], dest="excludes", action="append", help="Exclude all lines between 'BEGIN <EXCLUDE>' and 'END <EXCLUDE>'") + args = parser.parse_args(args) + + # Always remove threading + if "ZSTD_MULTITHREAD" not in args.undefs: + args.undefs.append("ZSTD_MULTITHREAD") + + args.defs = parse_optional_pair(args.defs) + for name, _ in args.defs: + if name in args.undefs: + raise RuntimeError(f"{name} is both defined and undefined!") + + args.replaces = parse_pair(args.replaces) + for name, _ in args.replaces: + if name in args.undefs or name in args.defs: + raise RuntimeError(f"{name} is both replaced and (un)defined!") + + args.rewritten_includes = parse_pair(args.rewritten_includes) + + external_xxhash = False + if args.xxhash is not None: + external_xxhash = True + args.rewritten_includes.append(('"(\\.\\./common/)?xxhash.h"', args.xxhash)) + + if args.xxh64_prefix is not None: + if not external_xxhash: + raise RuntimeError("--xxh64-prefix may only be used with --xxhash provided") + + if args.xxh64_state is not None: + if not external_xxhash: + raise RuntimeError("--xxh64-state may only be used with --xxhash provided") + + Freestanding( + args.zstd_deps, + args.mem, + args.source_lib, + args.output_lib, + external_xxhash, + args.xxh64_state, + args.xxh64_prefix, + args.rewritten_includes, + args.defs, + args.replaces, + args.undefs, + args.excludes + ).go() + +if __name__ == "__main__": + main(sys.argv[0], sys.argv[1:]) diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index c84d48b2..ddbb3e08 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -156,6 +156,18 @@ createDictionaryBuffer(const char* dictionaryName, } } +static ZSTD_CDict* createCDictForDedicatedDictSearch(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); + ZSTD_CCtxParams_init(params, compressionLevel); + ZSTD_CCtxParams_setParameter(params, ZSTD_c_enableDedicatedDictSearch, 1); + ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, compressionLevel); + + ZSTD_CDict* cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, params, ZSTD_defaultCMem); + + ZSTD_freeCCtxParams(params); + return cdict; +} /*! BMK_loadFiles() : * Loads `buffer`, with content from files listed within `fileNamesTable`. @@ -449,12 +461,14 @@ static void freeCDictCollection(cdict_collection_t cdictc) } /* returns .buffers=NULL if operation fails */ -static cdict_collection_t createCDictCollection(const void* dictBuffer, size_t dictSize, size_t nbCDict, int cLevel) +static cdict_collection_t createCDictCollection(const void* dictBuffer, size_t dictSize, size_t nbCDict, int cLevel, int dedicatedDictSearch) { ZSTD_CDict** const cdicts = malloc(nbCDict * sizeof(ZSTD_CDict*)); if (cdicts==NULL) return kNullCDictCollection; for (size_t dictNb=0; dictNb < nbCDict; dictNb++) { - cdicts[dictNb] = ZSTD_createCDict(dictBuffer, dictSize, cLevel); + cdicts[dictNb] = dedicatedDictSearch ? + createCDictForDedicatedDictSearch(dictBuffer, dictSize, cLevel) : + ZSTD_createCDict(dictBuffer, dictSize, cLevel); CONTROL(cdicts[dictNb] != NULL); } cdict_collection_t cdictc; @@ -720,7 +734,8 @@ int bench(const char** fileNameTable, unsigned nbFiles, const char* dictionary, size_t blockSize, int clevel, unsigned nbDictMax, unsigned nbBlocks, - unsigned nbRounds, int benchCompression) + unsigned nbRounds, int benchCompression, + int dedicatedDictSearch) { int result = 0; @@ -775,7 +790,9 @@ int bench(const char** fileNameTable, unsigned nbFiles, DICTSIZE); CONTROL(dictBuffer.ptr != NULL); - ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, clevel); + ZSTD_CDict* const cdict = dedicatedDictSearch ? + createCDictForDedicatedDictSearch(dictBuffer.ptr, dictBuffer.size, clevel) : + ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, clevel); CONTROL(cdict != NULL); size_t const cTotalSizeNoDict = compressBlocks(NULL, dstSlices, srcSlices, NULL, clevel); @@ -798,7 +815,7 @@ int bench(const char** fileNameTable, unsigned nbFiles, unsigned const nbDicts = nbDictMax ? nbDictMax : nbBlocks; - cdict_collection_t const cdictionaries = createCDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts, clevel); + cdict_collection_t const cdictionaries = createCDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts, clevel, dedicatedDictSearch); CONTROL(cdictionaries.cdicts != NULL); ddict_collection_t const ddictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts); @@ -924,6 +941,7 @@ int main (int argc, const char** argv) { int recursiveMode = 0; int benchCompression = 1; + int dedicatedDictSearch = 0; unsigned nbRounds = BENCH_TIME_DEFAULT_S; const char* const exeName = argv[0]; @@ -953,6 +971,7 @@ int main (int argc, const char** argv) if (longCommandWArg(&argument, "--nbDicts=")) { nbDicts = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--nbBlocks=")) { nbBlocks = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--clevel=")) { cLevel = (int)readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--dedicated-dict-search")) { dedicatedDictSearch = 1; continue; } if (longCommandWArg(&argument, "-")) { cLevel = (int)readU32FromChar(&argument); continue; } /* anything that's not a command is a filename */ nameTable[nameIdx++] = argument; @@ -970,7 +989,7 @@ int main (int argc, const char** argv) nameTable = NULL; /* UTIL_createFileNamesTable() takes ownership of nameTable */ } - int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds, benchCompression); + int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds, benchCompression, dedicatedDictSearch); UTIL_freeFileNamesTable(filenameTable); free(nameTable); diff --git a/contrib/linux-kernel/0000-cover-letter.patch b/contrib/linux-kernel/0000-cover-letter.patch deleted file mode 100644 index d57ef27e..00000000 --- a/contrib/linux-kernel/0000-cover-letter.patch +++ /dev/null @@ -1,122 +0,0 @@ -From 308795a7713ca6fcd468b60fba9a2fca99cee6a0 Mon Sep 17 00:00:00 2001 -From: Nick Terrell <terrelln@fb.com> -Date: Tue, 8 Aug 2017 19:20:25 -0700 -Subject: [PATCH v5 0/5] Add xxhash and zstd modules - -Hi all, - -This patch set adds xxhash, zstd compression, and zstd decompression -modules. It also adds zstd support to BtrFS and SquashFS. - -Each patch has relevant summaries, benchmarks, and tests. - -Best, -Nick Terrell - -Changelog: - -v1 -> v2: -- Make pointer in lib/xxhash.c:394 non-const (1/5) -- Use div_u64() for division of u64s (2/5) -- Reduce stack usage of ZSTD_compressSequences(), ZSTD_buildSeqTable(), - ZSTD_decompressSequencesLong(), FSE_buildDTable(), FSE_decompress_wksp(), - HUF_writeCTable(), HUF_readStats(), HUF_readCTable(), - HUF_compressWeights(), HUF_readDTableX2(), and HUF_readDTableX4() (2/5) -- No zstd function uses more than 400 B of stack space (2/5) - -v2 -> v3: -- Work around gcc-7 bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388 - (2/5) -- Fix bug in dictionary compression from upstream commit cc1522351f (2/5) -- Port upstream BtrFS commits e1ddce71d6, 389a6cfc2a, and 6acafd1eff (3/5) -- Change default compression level for BtrFS to 3 (3/5) - -v3 -> v4: -- Fix compiler warnings (2/5) -- Add missing includes (3/5) -- Fix minor linter warnings (3/5, 4/5) -- Add crypto patch (5/5) - -v4 -> v5: -- Fix rare compression bug from upstream commit 308047eb5d (2/5) -- Fix bug introduced in v3 when working around the gcc-7 bug (2/5) -- Fix ZSTD_DStream initialization code in squashfs (4/5) -- Fix patch documentation for patches written by Sean Purcell (4/5) - -Nick Terrell (5): - lib: Add xxhash module - lib: Add zstd modules - btrfs: Add zstd support - squashfs: Add zstd support - crypto: Add zstd support - - crypto/Kconfig | 9 + - crypto/Makefile | 1 + - crypto/testmgr.c | 10 + - crypto/testmgr.h | 71 + - crypto/zstd.c | 265 ++++ - fs/btrfs/Kconfig | 2 + - fs/btrfs/Makefile | 2 +- - fs/btrfs/compression.c | 1 + - fs/btrfs/compression.h | 6 +- - fs/btrfs/ctree.h | 1 + - fs/btrfs/disk-io.c | 2 + - fs/btrfs/ioctl.c | 6 +- - fs/btrfs/props.c | 6 + - fs/btrfs/super.c | 12 +- - fs/btrfs/sysfs.c | 2 + - fs/btrfs/zstd.c | 432 ++++++ - fs/squashfs/Kconfig | 14 + - fs/squashfs/Makefile | 1 + - fs/squashfs/decompressor.c | 7 + - fs/squashfs/decompressor.h | 4 + - fs/squashfs/squashfs_fs.h | 1 + - fs/squashfs/zstd_wrapper.c | 151 ++ - include/linux/xxhash.h | 236 +++ - include/linux/zstd.h | 1157 +++++++++++++++ - include/uapi/linux/btrfs.h | 8 +- - lib/Kconfig | 11 + - lib/Makefile | 3 + - lib/xxhash.c | 500 +++++++ - lib/zstd/Makefile | 18 + - lib/zstd/bitstream.h | 374 +++++ - lib/zstd/compress.c | 3484 ++++++++++++++++++++++++++++++++++++++++++++ - lib/zstd/decompress.c | 2528 ++++++++++++++++++++++++++++++++ - lib/zstd/entropy_common.c | 243 +++ - lib/zstd/error_private.h | 53 + - lib/zstd/fse.h | 575 ++++++++ - lib/zstd/fse_compress.c | 795 ++++++++++ - lib/zstd/fse_decompress.c | 332 +++++ - lib/zstd/huf.h | 212 +++ - lib/zstd/huf_compress.c | 770 ++++++++++ - lib/zstd/huf_decompress.c | 960 ++++++++++++ - lib/zstd/mem.h | 151 ++ - lib/zstd/zstd_common.c | 75 + - lib/zstd/zstd_internal.h | 263 ++++ - lib/zstd/zstd_opt.h | 1014 +++++++++++++ - 44 files changed, 14756 insertions(+), 12 deletions(-) - create mode 100644 crypto/zstd.c - create mode 100644 fs/btrfs/zstd.c - create mode 100644 fs/squashfs/zstd_wrapper.c - create mode 100644 include/linux/xxhash.h - create mode 100644 include/linux/zstd.h - create mode 100644 lib/xxhash.c - create mode 100644 lib/zstd/Makefile - create mode 100644 lib/zstd/bitstream.h - create mode 100644 lib/zstd/compress.c - create mode 100644 lib/zstd/decompress.c - create mode 100644 lib/zstd/entropy_common.c - create mode 100644 lib/zstd/error_private.h - create mode 100644 lib/zstd/fse.h - create mode 100644 lib/zstd/fse_compress.c - create mode 100644 lib/zstd/fse_decompress.c - create mode 100644 lib/zstd/huf.h - create mode 100644 lib/zstd/huf_compress.c - create mode 100644 lib/zstd/huf_decompress.c - create mode 100644 lib/zstd/mem.h - create mode 100644 lib/zstd/zstd_common.c - create mode 100644 lib/zstd/zstd_internal.h - create mode 100644 lib/zstd/zstd_opt.h - --- -2.9.3 diff --git a/contrib/linux-kernel/0001-lib-Add-xxhash-module.patch b/contrib/linux-kernel/0001-lib-Add-xxhash-module.patch deleted file mode 100644 index 83f09924..00000000 --- a/contrib/linux-kernel/0001-lib-Add-xxhash-module.patch +++ /dev/null @@ -1,862 +0,0 @@ -From a4b1ffb6e89bbccd519f9afa0910635668436105 Mon Sep 17 00:00:00 2001 -From: Nick Terrell <terrelln@fb.com> -Date: Mon, 17 Jul 2017 17:07:18 -0700 -Subject: [PATCH v5 1/5] lib: Add xxhash module - -Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an -extremely fast non-cryptographic hash algorithm for checksumming. -The zstd compression and decompression modules added in the next patch -require xxhash. I extracted it out from zstd since it is useful on its -own. I copied the code from the upstream XXHash source repository and -translated it into kernel style. I ran benchmarks and tests in the kernel -and tests in userland. - -I benchmarked xxhash as a special character device. I ran in four modes, -no-op, xxh32, xxh64, and crc32. The no-op mode simply copies the data to -kernel space and ignores it. The xxh32, xxh64, and crc32 modes compute -hashes on the copied data. I also ran it with four different buffer sizes. -The benchmark file is located in the upstream zstd source repository under -`contrib/linux-kernel/xxhash_test.c` [1]. - -I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM. -The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor, -16 GB of RAM, and a SSD. I benchmarked using the file `filesystem.squashfs` -from `ubuntu-16.10-desktop-amd64.iso`, which is 1,536,217,088 B large. -Run the following commands for the benchmark: - - modprobe xxhash_test - mknod xxhash_test c 245 0 - time cp filesystem.squashfs xxhash_test - -The time is reported by the time of the userland `cp`. -The GB/s is computed with - - 1,536,217,008 B / time(buffer size, hash) - -which includes the time to copy from userland. -The Normalized GB/s is computed with - - 1,536,217,088 B / (time(buffer size, hash) - time(buffer size, none)). - - -| Buffer Size (B) | Hash | Time (s) | GB/s | Adjusted GB/s | -|-----------------|-------|----------|------|---------------| -| 1024 | none | 0.408 | 3.77 | - | -| 1024 | xxh32 | 0.649 | 2.37 | 6.37 | -| 1024 | xxh64 | 0.542 | 2.83 | 11.46 | -| 1024 | crc32 | 1.290 | 1.19 | 1.74 | -| 4096 | none | 0.380 | 4.04 | - | -| 4096 | xxh32 | 0.645 | 2.38 | 5.79 | -| 4096 | xxh64 | 0.500 | 3.07 | 12.80 | -| 4096 | crc32 | 1.168 | 1.32 | 1.95 | -| 8192 | none | 0.351 | 4.38 | - | -| 8192 | xxh32 | 0.614 | 2.50 | 5.84 | -| 8192 | xxh64 | 0.464 | 3.31 | 13.60 | -| 8192 | crc32 | 1.163 | 1.32 | 1.89 | -| 16384 | none | 0.346 | 4.43 | - | -| 16384 | xxh32 | 0.590 | 2.60 | 6.30 | -| 16384 | xxh64 | 0.466 | 3.30 | 12.80 | -| 16384 | crc32 | 1.183 | 1.30 | 1.84 | - -Tested in userland using the test-suite in the zstd repo under -`contrib/linux-kernel/test/XXHashUserlandTest.cpp` [2] by mocking the -kernel functions. A line in each branch of every function in `xxhash.c` -was commented out to ensure that the test-suite fails. Additionally -tested while testing zstd and with SMHasher [3]. - -[1] https://phabricator.intern.facebook.com/P57526246 -[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/XXHashUserlandTest.cpp -[3] https://github.com/aappleby/smhasher - -zstd source repository: https://github.com/facebook/zstd -XXHash source repository: https://github.com/cyan4973/xxhash - -Signed-off-by: Nick Terrell <terrelln@fb.com> ---- -v1 -> v2: -- Make pointer in lib/xxhash.c:394 non-const - - include/linux/xxhash.h | 236 +++++++++++++++++++++++ - lib/Kconfig | 3 + - lib/Makefile | 1 + - lib/xxhash.c | 500 +++++++++++++++++++++++++++++++++++++++++++++++++ - 4 files changed, 740 insertions(+) - create mode 100644 include/linux/xxhash.h - create mode 100644 lib/xxhash.c - -diff --git a/include/linux/xxhash.h b/include/linux/xxhash.h -new file mode 100644 -index 0000000..9e1f42c ---- /dev/null -+++ b/include/linux/xxhash.h -@@ -0,0 +1,236 @@ -+/* -+ * xxHash - Extremely Fast Hash algorithm -+ * Copyright (C) 2012-2016, Yann Collet. -+ * -+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above -+ * copyright notice, this list of conditions and the following disclaimer -+ * in the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ * -+ * You can contact the author at: -+ * - xxHash homepage: http://cyan4973.github.io/xxHash/ -+ * - xxHash source repository: https://github.com/Cyan4973/xxHash -+ */ -+ -+/* -+ * Notice extracted from xxHash homepage: -+ * -+ * xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -+ * It also successfully passes all tests from the SMHasher suite. -+ * -+ * Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 -+ * Duo @3GHz) -+ * -+ * Name Speed Q.Score Author -+ * xxHash 5.4 GB/s 10 -+ * CrapWow 3.2 GB/s 2 Andrew -+ * MumurHash 3a 2.7 GB/s 10 Austin Appleby -+ * SpookyHash 2.0 GB/s 10 Bob Jenkins -+ * SBox 1.4 GB/s 9 Bret Mulvey -+ * Lookup3 1.2 GB/s 9 Bob Jenkins -+ * SuperFastHash 1.2 GB/s 1 Paul Hsieh -+ * CityHash64 1.05 GB/s 10 Pike & Alakuijala -+ * FNV 0.55 GB/s 5 Fowler, Noll, Vo -+ * CRC32 0.43 GB/s 9 -+ * MD5-32 0.33 GB/s 10 Ronald L. Rivest -+ * SHA1-32 0.28 GB/s 10 -+ * -+ * Q.Score is a measure of quality of the hash function. -+ * It depends on successfully passing SMHasher test set. -+ * 10 is a perfect score. -+ * -+ * A 64-bits version, named xxh64 offers much better speed, -+ * but for 64-bits applications only. -+ * Name Speed on 64 bits Speed on 32 bits -+ * xxh64 13.8 GB/s 1.9 GB/s -+ * xxh32 6.8 GB/s 6.0 GB/s -+ */ -+ -+#ifndef XXHASH_H -+#define XXHASH_H -+ -+#include <linux/types.h> -+ -+/*-**************************** -+ * Simple Hash Functions -+ *****************************/ -+ -+/** -+ * xxh32() - calculate the 32-bit hash of the input with a given seed. -+ * -+ * @input: The data to hash. -+ * @length: The length of the data to hash. -+ * @seed: The seed can be used to alter the result predictably. -+ * -+ * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s -+ * -+ * Return: The 32-bit hash of the data. -+ */ -+uint32_t xxh32(const void *input, size_t length, uint32_t seed); -+ -+/** -+ * xxh64() - calculate the 64-bit hash of the input with a given seed. -+ * -+ * @input: The data to hash. -+ * @length: The length of the data to hash. -+ * @seed: The seed can be used to alter the result predictably. -+ * -+ * This function runs 2x faster on 64-bit systems, but slower on 32-bit systems. -+ * -+ * Return: The 64-bit hash of the data. -+ */ -+uint64_t xxh64(const void *input, size_t length, uint64_t seed); -+ -+/*-**************************** -+ * Streaming Hash Functions -+ *****************************/ -+ -+/* -+ * These definitions are only meant to allow allocation of XXH state -+ * statically, on stack, or in a struct for example. -+ * Do not use members directly. -+ */ -+ -+/** -+ * struct xxh32_state - private xxh32 state, do not use members directly -+ */ -+struct xxh32_state { -+ uint32_t total_len_32; -+ uint32_t large_len; -+ uint32_t v1; -+ uint32_t v2; -+ uint32_t v3; -+ uint32_t v4; -+ uint32_t mem32[4]; -+ uint32_t memsize; -+}; -+ -+/** -+ * struct xxh32_state - private xxh64 state, do not use members directly -+ */ -+struct xxh64_state { -+ uint64_t total_len; -+ uint64_t v1; -+ uint64_t v2; -+ uint64_t v3; -+ uint64_t v4; -+ uint64_t mem64[4]; -+ uint32_t memsize; -+}; -+ -+/** -+ * xxh32_reset() - reset the xxh32 state to start a new hashing operation -+ * -+ * @state: The xxh32 state to reset. -+ * @seed: Initialize the hash state with this seed. -+ * -+ * Call this function on any xxh32_state to prepare for a new hashing operation. -+ */ -+void xxh32_reset(struct xxh32_state *state, uint32_t seed); -+ -+/** -+ * xxh32_update() - hash the data given and update the xxh32 state -+ * -+ * @state: The xxh32 state to update. -+ * @input: The data to hash. -+ * @length: The length of the data to hash. -+ * -+ * After calling xxh32_reset() call xxh32_update() as many times as necessary. -+ * -+ * Return: Zero on success, otherwise an error code. -+ */ -+int xxh32_update(struct xxh32_state *state, const void *input, size_t length); -+ -+/** -+ * xxh32_digest() - produce the current xxh32 hash -+ * -+ * @state: Produce the current xxh32 hash of this state. -+ * -+ * A hash value can be produced at any time. It is still possible to continue -+ * inserting input into the hash state after a call to xxh32_digest(), and -+ * generate new hashes later on, by calling xxh32_digest() again. -+ * -+ * Return: The xxh32 hash stored in the state. -+ */ -+uint32_t xxh32_digest(const struct xxh32_state *state); -+ -+/** -+ * xxh64_reset() - reset the xxh64 state to start a new hashing operation -+ * -+ * @state: The xxh64 state to reset. -+ * @seed: Initialize the hash state with this seed. -+ */ -+void xxh64_reset(struct xxh64_state *state, uint64_t seed); -+ -+/** -+ * xxh64_update() - hash the data given and update the xxh64 state -+ * @state: The xxh64 state to update. -+ * @input: The data to hash. -+ * @length: The length of the data to hash. -+ * -+ * After calling xxh64_reset() call xxh64_update() as many times as necessary. -+ * -+ * Return: Zero on success, otherwise an error code. -+ */ -+int xxh64_update(struct xxh64_state *state, const void *input, size_t length); -+ -+/** -+ * xxh64_digest() - produce the current xxh64 hash -+ * -+ * @state: Produce the current xxh64 hash of this state. -+ * -+ * A hash value can be produced at any time. It is still possible to continue -+ * inserting input into the hash state after a call to xxh64_digest(), and -+ * generate new hashes later on, by calling xxh64_digest() again. -+ * -+ * Return: The xxh64 hash stored in the state. -+ */ -+uint64_t xxh64_digest(const struct xxh64_state *state); -+ -+/*-************************** -+ * Utils -+ ***************************/ -+ -+/** -+ * xxh32_copy_state() - copy the source state into the destination state -+ * -+ * @src: The source xxh32 state. -+ * @dst: The destination xxh32 state. -+ */ -+void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src); -+ -+/** -+ * xxh64_copy_state() - copy the source state into the destination state -+ * -+ * @src: The source xxh64 state. -+ * @dst: The destination xxh64 state. -+ */ -+void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src); -+ -+#endif /* XXHASH_H */ -diff --git a/lib/Kconfig b/lib/Kconfig -index 6762529..5e7541f 100644 ---- a/lib/Kconfig -+++ b/lib/Kconfig -@@ -192,6 +192,9 @@ config CRC8 - when they need to do cyclic redundancy check according CRC8 - algorithm. Module will be called crc8. - -+config XXHASH -+ tristate -+ - config AUDIT_GENERIC - bool - depends on AUDIT && !AUDIT_ARCH -diff --git a/lib/Makefile b/lib/Makefile -index 40c1837..d06b68a 100644 ---- a/lib/Makefile -+++ b/lib/Makefile -@@ -102,6 +102,7 @@ obj-$(CONFIG_CRC4) += crc4.o - obj-$(CONFIG_CRC7) += crc7.o - obj-$(CONFIG_LIBCRC32C) += libcrc32c.o - obj-$(CONFIG_CRC8) += crc8.o -+obj-$(CONFIG_XXHASH) += xxhash.o - obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o - - obj-$(CONFIG_842_COMPRESS) += 842/ -diff --git a/lib/xxhash.c b/lib/xxhash.c -new file mode 100644 -index 0000000..aa61e2a ---- /dev/null -+++ b/lib/xxhash.c -@@ -0,0 +1,500 @@ -+/* -+ * xxHash - Extremely Fast Hash algorithm -+ * Copyright (C) 2012-2016, Yann Collet. -+ * -+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above -+ * copyright notice, this list of conditions and the following disclaimer -+ * in the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ * -+ * You can contact the author at: -+ * - xxHash homepage: http://cyan4973.github.io/xxHash/ -+ * - xxHash source repository: https://github.com/Cyan4973/xxHash -+ */ -+ -+#include <asm/unaligned.h> -+#include <linux/errno.h> -+#include <linux/compiler.h> -+#include <linux/kernel.h> -+#include <linux/module.h> -+#include <linux/string.h> -+#include <linux/xxhash.h> -+ -+/*-************************************* -+ * Macros -+ **************************************/ -+#define xxh_rotl32(x, r) ((x << r) | (x >> (32 - r))) -+#define xxh_rotl64(x, r) ((x << r) | (x >> (64 - r))) -+ -+#ifdef __LITTLE_ENDIAN -+# define XXH_CPU_LITTLE_ENDIAN 1 -+#else -+# define XXH_CPU_LITTLE_ENDIAN 0 -+#endif -+ -+/*-************************************* -+ * Constants -+ **************************************/ -+static const uint32_t PRIME32_1 = 2654435761U; -+static const uint32_t PRIME32_2 = 2246822519U; -+static const uint32_t PRIME32_3 = 3266489917U; -+static const uint32_t PRIME32_4 = 668265263U; -+static const uint32_t PRIME32_5 = 374761393U; -+ -+static const uint64_t PRIME64_1 = 11400714785074694791ULL; -+static const uint64_t PRIME64_2 = 14029467366897019727ULL; -+static const uint64_t PRIME64_3 = 1609587929392839161ULL; -+static const uint64_t PRIME64_4 = 9650029242287828579ULL; -+static const uint64_t PRIME64_5 = 2870177450012600261ULL; -+ -+/*-************************** -+ * Utils -+ ***************************/ -+void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src) -+{ -+ memcpy(dst, src, sizeof(*dst)); -+} -+EXPORT_SYMBOL(xxh32_copy_state); -+ -+void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src) -+{ -+ memcpy(dst, src, sizeof(*dst)); -+} -+EXPORT_SYMBOL(xxh64_copy_state); -+ -+/*-*************************** -+ * Simple Hash Functions -+ ****************************/ -+static uint32_t xxh32_round(uint32_t seed, const uint32_t input) -+{ -+ seed += input * PRIME32_2; -+ seed = xxh_rotl32(seed, 13); -+ seed *= PRIME32_1; -+ return seed; -+} -+ -+uint32_t xxh32(const void *input, const size_t len, const uint32_t seed) -+{ -+ const uint8_t *p = (const uint8_t *)input; -+ const uint8_t *b_end = p + len; -+ uint32_t h32; -+ -+ if (len >= 16) { -+ const uint8_t *const limit = b_end - 16; -+ uint32_t v1 = seed + PRIME32_1 + PRIME32_2; -+ uint32_t v2 = seed + PRIME32_2; -+ uint32_t v3 = seed + 0; -+ uint32_t v4 = seed - PRIME32_1; -+ -+ do { -+ v1 = xxh32_round(v1, get_unaligned_le32(p)); -+ p += 4; -+ v2 = xxh32_round(v2, get_unaligned_le32(p)); -+ p += 4; -+ v3 = xxh32_round(v3, get_unaligned_le32(p)); -+ p += 4; -+ v4 = xxh32_round(v4, get_unaligned_le32(p)); -+ p += 4; -+ } while (p <= limit); -+ -+ h32 = xxh_rotl32(v1, 1) + xxh_rotl32(v2, 7) + -+ xxh_rotl32(v3, 12) + xxh_rotl32(v4, 18); -+ } else { -+ h32 = seed + PRIME32_5; -+ } -+ -+ h32 += (uint32_t)len; -+ -+ while (p + 4 <= b_end) { -+ h32 += get_unaligned_le32(p) * PRIME32_3; -+ h32 = xxh_rotl32(h32, 17) * PRIME32_4; -+ p += 4; -+ } -+ -+ while (p < b_end) { -+ h32 += (*p) * PRIME32_5; -+ h32 = xxh_rotl32(h32, 11) * PRIME32_1; -+ p++; -+ } -+ -+ h32 ^= h32 >> 15; -+ h32 *= PRIME32_2; -+ h32 ^= h32 >> 13; -+ h32 *= PRIME32_3; -+ h32 ^= h32 >> 16; -+ -+ return h32; -+} -+EXPORT_SYMBOL(xxh32); -+ -+static uint64_t xxh64_round(uint64_t acc, const uint64_t input) -+{ -+ acc += input * PRIME64_2; -+ acc = xxh_rotl64(acc, 31); -+ acc *= PRIME64_1; -+ return acc; -+} -+ -+static uint64_t xxh64_merge_round(uint64_t acc, uint64_t val) -+{ -+ val = xxh64_round(0, val); -+ acc ^= val; -+ acc = acc * PRIME64_1 + PRIME64_4; -+ return acc; -+} -+ -+uint64_t xxh64(const void *input, const size_t len, const uint64_t seed) -+{ -+ const uint8_t *p = (const uint8_t *)input; -+ const uint8_t *const b_end = p + len; -+ uint64_t h64; -+ -+ if (len >= 32) { -+ const uint8_t *const limit = b_end - 32; -+ uint64_t v1 = seed + PRIME64_1 + PRIME64_2; -+ uint64_t v2 = seed + PRIME64_2; -+ uint64_t v3 = seed + 0; -+ uint64_t v4 = seed - PRIME64_1; -+ -+ do { -+ v1 = xxh64_round(v1, get_unaligned_le64(p)); -+ p += 8; -+ v2 = xxh64_round(v2, get_unaligned_le64(p)); -+ p += 8; -+ v3 = xxh64_round(v3, get_unaligned_le64(p)); -+ p += 8; -+ v4 = xxh64_round(v4, get_unaligned_le64(p)); -+ p += 8; -+ } while (p <= limit); -+ -+ h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) + -+ xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18); -+ h64 = xxh64_merge_round(h64, v1); -+ h64 = xxh64_merge_round(h64, v2); -+ h64 = xxh64_merge_round(h64, v3); -+ h64 = xxh64_merge_round(h64, v4); -+ -+ } else { -+ h64 = seed + PRIME64_5; -+ } -+ -+ h64 += (uint64_t)len; -+ -+ while (p + 8 <= b_end) { -+ const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p)); -+ -+ h64 ^= k1; -+ h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4; -+ p += 8; -+ } -+ -+ if (p + 4 <= b_end) { -+ h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1; -+ h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; -+ p += 4; -+ } -+ -+ while (p < b_end) { -+ h64 ^= (*p) * PRIME64_5; -+ h64 = xxh_rotl64(h64, 11) * PRIME64_1; -+ p++; -+ } -+ -+ h64 ^= h64 >> 33; -+ h64 *= PRIME64_2; -+ h64 ^= h64 >> 29; -+ h64 *= PRIME64_3; -+ h64 ^= h64 >> 32; -+ -+ return h64; -+} -+EXPORT_SYMBOL(xxh64); -+ -+/*-************************************************** -+ * Advanced Hash Functions -+ ***************************************************/ -+void xxh32_reset(struct xxh32_state *statePtr, const uint32_t seed) -+{ -+ /* use a local state for memcpy() to avoid strict-aliasing warnings */ -+ struct xxh32_state state; -+ -+ memset(&state, 0, sizeof(state)); -+ state.v1 = seed + PRIME32_1 + PRIME32_2; -+ state.v2 = seed + PRIME32_2; -+ state.v3 = seed + 0; -+ state.v4 = seed - PRIME32_1; -+ memcpy(statePtr, &state, sizeof(state)); -+} -+EXPORT_SYMBOL(xxh32_reset); -+ -+void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed) -+{ -+ /* use a local state for memcpy() to avoid strict-aliasing warnings */ -+ struct xxh64_state state; -+ -+ memset(&state, 0, sizeof(state)); -+ state.v1 = seed + PRIME64_1 + PRIME64_2; -+ state.v2 = seed + PRIME64_2; -+ state.v3 = seed + 0; -+ state.v4 = seed - PRIME64_1; -+ memcpy(statePtr, &state, sizeof(state)); -+} -+EXPORT_SYMBOL(xxh64_reset); -+ -+int xxh32_update(struct xxh32_state *state, const void *input, const size_t len) -+{ -+ const uint8_t *p = (const uint8_t *)input; -+ const uint8_t *const b_end = p + len; -+ -+ if (input == NULL) -+ return -EINVAL; -+ -+ state->total_len_32 += (uint32_t)len; -+ state->large_len |= (len >= 16) | (state->total_len_32 >= 16); -+ -+ if (state->memsize + len < 16) { /* fill in tmp buffer */ -+ memcpy((uint8_t *)(state->mem32) + state->memsize, input, len); -+ state->memsize += (uint32_t)len; -+ return 0; -+ } -+ -+ if (state->memsize) { /* some data left from previous update */ -+ const uint32_t *p32 = state->mem32; -+ -+ memcpy((uint8_t *)(state->mem32) + state->memsize, input, -+ 16 - state->memsize); -+ -+ state->v1 = xxh32_round(state->v1, get_unaligned_le32(p32)); -+ p32++; -+ state->v2 = xxh32_round(state->v2, get_unaligned_le32(p32)); -+ p32++; -+ state->v3 = xxh32_round(state->v3, get_unaligned_le32(p32)); -+ p32++; -+ state->v4 = xxh32_round(state->v4, get_unaligned_le32(p32)); -+ p32++; -+ -+ p += 16-state->memsize; -+ state->memsize = 0; -+ } -+ -+ if (p <= b_end - 16) { -+ const uint8_t *const limit = b_end - 16; -+ uint32_t v1 = state->v1; -+ uint32_t v2 = state->v2; -+ uint32_t v3 = state->v3; -+ uint32_t v4 = state->v4; -+ -+ do { -+ v1 = xxh32_round(v1, get_unaligned_le32(p)); -+ p += 4; -+ v2 = xxh32_round(v2, get_unaligned_le32(p)); -+ p += 4; -+ v3 = xxh32_round(v3, get_unaligned_le32(p)); -+ p += 4; -+ v4 = xxh32_round(v4, get_unaligned_le32(p)); -+ p += 4; -+ } while (p <= limit); -+ -+ state->v1 = v1; -+ state->v2 = v2; -+ state->v3 = v3; -+ state->v4 = v4; -+ } -+ -+ if (p < b_end) { -+ memcpy(state->mem32, p, (size_t)(b_end-p)); -+ state->memsize = (uint32_t)(b_end-p); -+ } -+ -+ return 0; -+} -+EXPORT_SYMBOL(xxh32_update); -+ -+uint32_t xxh32_digest(const struct xxh32_state *state) -+{ -+ const uint8_t *p = (const uint8_t *)state->mem32; -+ const uint8_t *const b_end = (const uint8_t *)(state->mem32) + -+ state->memsize; -+ uint32_t h32; -+ -+ if (state->large_len) { -+ h32 = xxh_rotl32(state->v1, 1) + xxh_rotl32(state->v2, 7) + -+ xxh_rotl32(state->v3, 12) + xxh_rotl32(state->v4, 18); -+ } else { -+ h32 = state->v3 /* == seed */ + PRIME32_5; -+ } -+ -+ h32 += state->total_len_32; -+ -+ while (p + 4 <= b_end) { -+ h32 += get_unaligned_le32(p) * PRIME32_3; -+ h32 = xxh_rotl32(h32, 17) * PRIME32_4; -+ p += 4; -+ } -+ -+ while (p < b_end) { -+ h32 += (*p) * PRIME32_5; -+ h32 = xxh_rotl32(h32, 11) * PRIME32_1; -+ p++; -+ } -+ -+ h32 ^= h32 >> 15; -+ h32 *= PRIME32_2; -+ h32 ^= h32 >> 13; -+ h32 *= PRIME32_3; -+ h32 ^= h32 >> 16; -+ -+ return h32; -+} -+EXPORT_SYMBOL(xxh32_digest); -+ -+int xxh64_update(struct xxh64_state *state, const void *input, const size_t len) -+{ -+ const uint8_t *p = (const uint8_t *)input; -+ const uint8_t *const b_end = p + len; -+ -+ if (input == NULL) -+ return -EINVAL; -+ -+ state->total_len += len; -+ -+ if (state->memsize + len < 32) { /* fill in tmp buffer */ -+ memcpy(((uint8_t *)state->mem64) + state->memsize, input, len); -+ state->memsize += (uint32_t)len; -+ return 0; -+ } -+ -+ if (state->memsize) { /* tmp buffer is full */ -+ uint64_t *p64 = state->mem64; -+ -+ memcpy(((uint8_t *)p64) + state->memsize, input, -+ 32 - state->memsize); -+ -+ state->v1 = xxh64_round(state->v1, get_unaligned_le64(p64)); -+ p64++; -+ state->v2 = xxh64_round(state->v2, get_unaligned_le64(p64)); -+ p64++; -+ state->v3 = xxh64_round(state->v3, get_unaligned_le64(p64)); -+ p64++; -+ state->v4 = xxh64_round(state->v4, get_unaligned_le64(p64)); -+ -+ p += 32 - state->memsize; -+ state->memsize = 0; -+ } -+ -+ if (p + 32 <= b_end) { -+ const uint8_t *const limit = b_end - 32; -+ uint64_t v1 = state->v1; -+ uint64_t v2 = state->v2; -+ uint64_t v3 = state->v3; -+ uint64_t v4 = state->v4; -+ -+ do { -+ v1 = xxh64_round(v1, get_unaligned_le64(p)); -+ p += 8; -+ v2 = xxh64_round(v2, get_unaligned_le64(p)); -+ p += 8; -+ v3 = xxh64_round(v3, get_unaligned_le64(p)); -+ p += 8; -+ v4 = xxh64_round(v4, get_unaligned_le64(p)); -+ p += 8; -+ } while (p <= limit); -+ -+ state->v1 = v1; -+ state->v2 = v2; -+ state->v3 = v3; -+ state->v4 = v4; -+ } -+ -+ if (p < b_end) { -+ memcpy(state->mem64, p, (size_t)(b_end-p)); -+ state->memsize = (uint32_t)(b_end - p); -+ } -+ -+ return 0; -+} -+EXPORT_SYMBOL(xxh64_update); -+ -+uint64_t xxh64_digest(const struct xxh64_state *state) -+{ -+ const uint8_t *p = (const uint8_t *)state->mem64; -+ const uint8_t *const b_end = (const uint8_t *)state->mem64 + -+ state->memsize; -+ uint64_t h64; -+ -+ if (state->total_len >= 32) { -+ const uint64_t v1 = state->v1; -+ const uint64_t v2 = state->v2; -+ const uint64_t v3 = state->v3; -+ const uint64_t v4 = state->v4; -+ -+ h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) + -+ xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18); -+ h64 = xxh64_merge_round(h64, v1); -+ h64 = xxh64_merge_round(h64, v2); -+ h64 = xxh64_merge_round(h64, v3); -+ h64 = xxh64_merge_round(h64, v4); -+ } else { -+ h64 = state->v3 + PRIME64_5; -+ } -+ -+ h64 += (uint64_t)state->total_len; -+ -+ while (p + 8 <= b_end) { -+ const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p)); -+ -+ h64 ^= k1; -+ h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4; -+ p += 8; -+ } -+ -+ if (p + 4 <= b_end) { -+ h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1; -+ h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; -+ p += 4; -+ } -+ -+ while (p < b_end) { -+ h64 ^= (*p) * PRIME64_5; -+ h64 = xxh_rotl64(h64, 11) * PRIME64_1; -+ p++; -+ } -+ -+ h64 ^= h64 >> 33; -+ h64 *= PRIME64_2; -+ h64 ^= h64 >> 29; -+ h64 *= PRIME64_3; -+ h64 ^= h64 >> 32; -+ -+ return h64; -+} -+EXPORT_SYMBOL(xxh64_digest); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("xxHash"); --- -2.9.3 diff --git a/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch b/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch deleted file mode 100644 index 0232d2d4..00000000 --- a/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch +++ /dev/null @@ -1,13285 +0,0 @@ -From 2b29ec569f8438a0307debd29873859ca6d407fc Mon Sep 17 00:00:00 2001 -From: Nick Terrell <terrelln@fb.com> -Date: Mon, 17 Jul 2017 17:08:19 -0700 -Subject: [PATCH v5 2/5] lib: Add zstd modules - -Add zstd compression and decompression kernel modules. -zstd offers a wide variety of compression speed and quality trade-offs. -It can compress at speeds approaching lz4, and quality approaching lzma. -zstd decompressions at speeds more than twice as fast as zlib, and -decompression speed remains roughly the same across all compression levels. - -The code was ported from the upstream zstd source repository. The -`linux/zstd.h` header was modified to match linux kernel style. -The cross-platform and allocation code was stripped out. Instead zstd -requires the caller to pass a preallocated workspace. The source files -were clang-formatted [1] to match the Linux Kernel style as much as -possible. Otherwise, the code was unmodified. We would like to avoid -as much further manual modification to the source code as possible, so it -will be easier to keep the kernel zstd up to date. - -I benchmarked zstd compression as a special character device. I ran zstd -and zlib compression at several levels, as well as performing no -compression, which measure the time spent copying the data to kernel space. -Data is passed to the compressor 4096 B at a time. The benchmark file is -located in the upstream zstd source repository under -`contrib/linux-kernel/zstd_compress_test.c` [2]. - -I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM. -The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor, -16 GB of RAM, and a SSD. I benchmarked using `silesia.tar` [3], which is -211,988,480 B large. Run the following commands for the benchmark: - - sudo modprobe zstd_compress_test - sudo mknod zstd_compress_test c 245 0 - sudo cp silesia.tar zstd_compress_test - -The time is reported by the time of the userland `cp`. -The MB/s is computed with - - 1,536,217,008 B / time(buffer size, hash) - -which includes the time to copy from userland. -The Adjusted MB/s is computed with - - 1,536,217,088 B / (time(buffer size, hash) - time(buffer size, none)). - -The memory reported is the amount of memory the compressor requests. - -| Method | Size (B) | Time (s) | Ratio | MB/s | Adj MB/s | Mem (MB) | -|----------|----------|----------|-------|---------|----------|----------| -| none | 11988480 | 0.100 | 1 | 2119.88 | - | - | -| zstd -1 | 73645762 | 1.044 | 2.878 | 203.05 | 224.56 | 1.23 | -| zstd -3 | 66988878 | 1.761 | 3.165 | 120.38 | 127.63 | 2.47 | -| zstd -5 | 65001259 | 2.563 | 3.261 | 82.71 | 86.07 | 2.86 | -| zstd -10 | 60165346 | 13.242 | 3.523 | 16.01 | 16.13 | 13.22 | -| zstd -15 | 58009756 | 47.601 | 3.654 | 4.45 | 4.46 | 21.61 | -| zstd -19 | 54014593 | 102.835 | 3.925 | 2.06 | 2.06 | 60.15 | -| zlib -1 | 77260026 | 2.895 | 2.744 | 73.23 | 75.85 | 0.27 | -| zlib -3 | 72972206 | 4.116 | 2.905 | 51.50 | 52.79 | 0.27 | -| zlib -6 | 68190360 | 9.633 | 3.109 | 22.01 | 22.24 | 0.27 | -| zlib -9 | 67613382 | 22.554 | 3.135 | 9.40 | 9.44 | 0.27 | - -I benchmarked zstd decompression using the same method on the same machine. -The benchmark file is located in the upstream zstd repo under -`contrib/linux-kernel/zstd_decompress_test.c` [4]. The memory reported is -the amount of memory required to decompress data compressed with the given -compression level. If you know the maximum size of your input, you can -reduce the memory usage of decompression irrespective of the compression -level. - -| Method | Time (s) | MB/s | Adjusted MB/s | Memory (MB) | -|----------|----------|---------|---------------|-------------| -| none | 0.025 | 8479.54 | - | - | -| zstd -1 | 0.358 | 592.15 | 636.60 | 0.84 | -| zstd -3 | 0.396 | 535.32 | 571.40 | 1.46 | -| zstd -5 | 0.396 | 535.32 | 571.40 | 1.46 | -| zstd -10 | 0.374 | 566.81 | 607.42 | 2.51 | -| zstd -15 | 0.379 | 559.34 | 598.84 | 4.61 | -| zstd -19 | 0.412 | 514.54 | 547.77 | 8.80 | -| zlib -1 | 0.940 | 225.52 | 231.68 | 0.04 | -| zlib -3 | 0.883 | 240.08 | 247.07 | 0.04 | -| zlib -6 | 0.844 | 251.17 | 258.84 | 0.04 | -| zlib -9 | 0.837 | 253.27 | 287.64 | 0.04 | - -Tested in userland using the test-suite in the zstd repo under -`contrib/linux-kernel/test/UserlandTest.cpp` [5] by mocking the kernel -functions. Fuzz tested using libfuzzer [6] with the fuzz harnesses under -`contrib/linux-kernel/test/{RoundTripCrash.c,DecompressCrash.c}` [7] [8] -with ASAN, UBSAN, and MSAN. Additionally, it was tested while testing the -BtrFS and SquashFS patches coming next. - -[1] https://clang.llvm.org/docs/ClangFormat.html -[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/zstd_compress_test.c -[3] http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia -[4] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/zstd_decompress_test.c -[5] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/UserlandTest.cpp -[6] http://llvm.org/docs/LibFuzzer.html -[7] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/RoundTripCrash.c -[8] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/DecompressCrash.c - -zstd source repository: https://github.com/facebook/zstd - -Signed-off-by: Nick Terrell <terrelln@fb.com> ---- -v1 -> v2: -- Use div_u64() for division of u64s -- Reduce stack usage of ZSTD_compressSequences(), ZSTD_buildSeqTable(), - ZSTD_decompressSequencesLong(), FSE_buildDTable(), FSE_decompress_wksp(), - HUF_writeCTable(), HUF_readStats(), HUF_readCTable(), - HUF_compressWeights(), HUF_readDTableX2(), and HUF_readDTableX4() -- No function uses more than 400 B of stack space - -v2 -> v3: -- Work around gcc-7 bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388 -- Fix bug in dictionary compression from upstream commit cc1522351f - -v3 -> v4: -- Fix minor compiler warnings - -v4 -> v5: -- Fix rare compression bug from upstream commit 308047eb5d -- Fix bug introduced in v3 when working around the gcc-7 bug - - include/linux/zstd.h | 1155 +++++++++++++++ - lib/Kconfig | 8 + - lib/Makefile | 2 + - lib/zstd/Makefile | 18 + - lib/zstd/bitstream.h | 374 +++++ - lib/zstd/compress.c | 3482 +++++++++++++++++++++++++++++++++++++++++++++ - lib/zstd/decompress.c | 2526 ++++++++++++++++++++++++++++++++ - lib/zstd/entropy_common.c | 243 ++++ - lib/zstd/error_private.h | 51 + - lib/zstd/fse.h | 575 ++++++++ - lib/zstd/fse_compress.c | 795 +++++++++++ - lib/zstd/fse_decompress.c | 332 +++++ - lib/zstd/huf.h | 212 +++ - lib/zstd/huf_compress.c | 770 ++++++++++ - lib/zstd/huf_decompress.c | 960 +++++++++++++ - lib/zstd/mem.h | 149 ++ - lib/zstd/zstd_common.c | 73 + - lib/zstd/zstd_internal.h | 261 ++++ - lib/zstd/zstd_opt.h | 1012 +++++++++++++ - 19 files changed, 12998 insertions(+) - create mode 100644 include/linux/zstd.h - create mode 100644 lib/zstd/Makefile - create mode 100644 lib/zstd/bitstream.h - create mode 100644 lib/zstd/compress.c - create mode 100644 lib/zstd/decompress.c - create mode 100644 lib/zstd/entropy_common.c - create mode 100644 lib/zstd/error_private.h - create mode 100644 lib/zstd/fse.h - create mode 100644 lib/zstd/fse_compress.c - create mode 100644 lib/zstd/fse_decompress.c - create mode 100644 lib/zstd/huf.h - create mode 100644 lib/zstd/huf_compress.c - create mode 100644 lib/zstd/huf_decompress.c - create mode 100644 lib/zstd/mem.h - create mode 100644 lib/zstd/zstd_common.c - create mode 100644 lib/zstd/zstd_internal.h - create mode 100644 lib/zstd/zstd_opt.h - -diff --git a/include/linux/zstd.h b/include/linux/zstd.h -new file mode 100644 -index 0000000..305efd0 ---- /dev/null -+++ b/include/linux/zstd.h -@@ -0,0 +1,1155 @@ -+/* -+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. -+ * All rights reserved. -+ * -+ * This source code is licensed under the BSD-style license found in the -+ * LICENSE file in the root directory of https://github.com/facebook/zstd. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ */ -+ -+#ifndef ZSTD_H -+#define ZSTD_H -+ -+/* ====== Dependency ======*/ -+#include <linux/types.h> /* size_t */ -+ -+ -+/*-***************************************************************************** -+ * Introduction -+ * -+ * zstd, short for Zstandard, is a fast lossless compression algorithm, -+ * targeting real-time compression scenarios at zlib-level and better -+ * compression ratios. The zstd compression library provides in-memory -+ * compression and decompression functions. The library supports compression -+ * levels from 1 up to ZSTD_maxCLevel() which is 22. Levels >= 20, labeled -+ * ultra, should be used with caution, as they require more memory. -+ * Compression can be done in: -+ * - a single step, reusing a context (described as Explicit memory management) -+ * - unbounded multiple steps (described as Streaming compression) -+ * The compression ratio achievable on small data can be highly improved using -+ * compression with a dictionary in: -+ * - a single step (described as Simple dictionary API) -+ * - a single step, reusing a dictionary (described as Fast dictionary API) -+ ******************************************************************************/ -+ -+/*====== Helper functions ======*/ -+ -+/** -+ * enum ZSTD_ErrorCode - zstd error codes -+ * -+ * Functions that return size_t can be checked for errors using ZSTD_isError() -+ * and the ZSTD_ErrorCode can be extracted using ZSTD_getErrorCode(). -+ */ -+typedef enum { -+ ZSTD_error_no_error, -+ ZSTD_error_GENERIC, -+ ZSTD_error_prefix_unknown, -+ ZSTD_error_version_unsupported, -+ ZSTD_error_parameter_unknown, -+ ZSTD_error_frameParameter_unsupported, -+ ZSTD_error_frameParameter_unsupportedBy32bits, -+ ZSTD_error_frameParameter_windowTooLarge, -+ ZSTD_error_compressionParameter_unsupported, -+ ZSTD_error_init_missing, -+ ZSTD_error_memory_allocation, -+ ZSTD_error_stage_wrong, -+ ZSTD_error_dstSize_tooSmall, -+ ZSTD_error_srcSize_wrong, -+ ZSTD_error_corruption_detected, -+ ZSTD_error_checksum_wrong, -+ ZSTD_error_tableLog_tooLarge, -+ ZSTD_error_maxSymbolValue_tooLarge, -+ ZSTD_error_maxSymbolValue_tooSmall, -+ ZSTD_error_dictionary_corrupted, -+ ZSTD_error_dictionary_wrong, -+ ZSTD_error_dictionaryCreation_failed, -+ ZSTD_error_maxCode -+} ZSTD_ErrorCode; -+ -+/** -+ * ZSTD_maxCLevel() - maximum compression level available -+ * -+ * Return: Maximum compression level available. -+ */ -+int ZSTD_maxCLevel(void); -+/** -+ * ZSTD_compressBound() - maximum compressed size in worst case scenario -+ * @srcSize: The size of the data to compress. -+ * -+ * Return: The maximum compressed size in the worst case scenario. -+ */ -+size_t ZSTD_compressBound(size_t srcSize); -+/** -+ * ZSTD_isError() - tells if a size_t function result is an error code -+ * @code: The function result to check for error. -+ * -+ * Return: Non-zero iff the code is an error. -+ */ -+static __attribute__((unused)) unsigned int ZSTD_isError(size_t code) -+{ -+ return code > (size_t)-ZSTD_error_maxCode; -+} -+/** -+ * ZSTD_getErrorCode() - translates an error function result to a ZSTD_ErrorCode -+ * @functionResult: The result of a function for which ZSTD_isError() is true. -+ * -+ * Return: The ZSTD_ErrorCode corresponding to the functionResult or 0 -+ * if the functionResult isn't an error. -+ */ -+static __attribute__((unused)) ZSTD_ErrorCode ZSTD_getErrorCode( -+ size_t functionResult) -+{ -+ if (!ZSTD_isError(functionResult)) -+ return (ZSTD_ErrorCode)0; -+ return (ZSTD_ErrorCode)(0 - functionResult); -+} -+ -+/** -+ * enum ZSTD_strategy - zstd compression search strategy -+ * -+ * From faster to stronger. -+ */ -+typedef enum { -+ ZSTD_fast, -+ ZSTD_dfast, -+ ZSTD_greedy, -+ ZSTD_lazy, -+ ZSTD_lazy2, -+ ZSTD_btlazy2, -+ ZSTD_btopt, -+ ZSTD_btopt2 -+} ZSTD_strategy; -+ -+/** -+ * struct ZSTD_compressionParameters - zstd compression parameters -+ * @windowLog: Log of the largest match distance. Larger means more -+ * compression, and more memory needed during decompression. -+ * @chainLog: Fully searched segment. Larger means more compression, slower, -+ * and more memory (useless for fast). -+ * @hashLog: Dispatch table. Larger means more compression, -+ * slower, and more memory. -+ * @searchLog: Number of searches. Larger means more compression and slower. -+ * @searchLength: Match length searched. Larger means faster decompression, -+ * sometimes less compression. -+ * @targetLength: Acceptable match size for optimal parser (only). Larger means -+ * more compression, and slower. -+ * @strategy: The zstd compression strategy. -+ */ -+typedef struct { -+ unsigned int windowLog; -+ unsigned int chainLog; -+ unsigned int hashLog; -+ unsigned int searchLog; -+ unsigned int searchLength; -+ unsigned int targetLength; -+ ZSTD_strategy strategy; -+} ZSTD_compressionParameters; -+ -+/** -+ * struct ZSTD_frameParameters - zstd frame parameters -+ * @contentSizeFlag: Controls whether content size will be present in the frame -+ * header (when known). -+ * @checksumFlag: Controls whether a 32-bit checksum is generated at the end -+ * of the frame for error detection. -+ * @noDictIDFlag: Controls whether dictID will be saved into the frame header -+ * when using dictionary compression. -+ * -+ * The default value is all fields set to 0. -+ */ -+typedef struct { -+ unsigned int contentSizeFlag; -+ unsigned int checksumFlag; -+ unsigned int noDictIDFlag; -+} ZSTD_frameParameters; -+ -+/** -+ * struct ZSTD_parameters - zstd parameters -+ * @cParams: The compression parameters. -+ * @fParams: The frame parameters. -+ */ -+typedef struct { -+ ZSTD_compressionParameters cParams; -+ ZSTD_frameParameters fParams; -+} ZSTD_parameters; -+ -+/** -+ * ZSTD_getCParams() - returns ZSTD_compressionParameters for selected level -+ * @compressionLevel: The compression level from 1 to ZSTD_maxCLevel(). -+ * @estimatedSrcSize: The estimated source size to compress or 0 if unknown. -+ * @dictSize: The dictionary size or 0 if a dictionary isn't being used. -+ * -+ * Return: The selected ZSTD_compressionParameters. -+ */ -+ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, -+ unsigned long long estimatedSrcSize, size_t dictSize); -+ -+/** -+ * ZSTD_getParams() - returns ZSTD_parameters for selected level -+ * @compressionLevel: The compression level from 1 to ZSTD_maxCLevel(). -+ * @estimatedSrcSize: The estimated source size to compress or 0 if unknown. -+ * @dictSize: The dictionary size or 0 if a dictionary isn't being used. -+ * -+ * The same as ZSTD_getCParams() except also selects the default frame -+ * parameters (all zero). -+ * -+ * Return: The selected ZSTD_parameters. -+ */ -+ZSTD_parameters ZSTD_getParams(int compressionLevel, -+ unsigned long long estimatedSrcSize, size_t dictSize); -+ -+/*-************************************* -+ * Explicit memory management -+ **************************************/ -+ -+/** -+ * ZSTD_CCtxWorkspaceBound() - amount of memory needed to initialize a ZSTD_CCtx -+ * @cParams: The compression parameters to be used for compression. -+ * -+ * If multiple compression parameters might be used, the caller must call -+ * ZSTD_CCtxWorkspaceBound() for each set of parameters and use the maximum -+ * size. -+ * -+ * Return: A lower bound on the size of the workspace that is passed to -+ * ZSTD_initCCtx(). -+ */ -+size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams); -+ -+/** -+ * struct ZSTD_CCtx - the zstd compression context -+ * -+ * When compressing many times it is recommended to allocate a context just once -+ * and reuse it for each successive compression operation. -+ */ -+typedef struct ZSTD_CCtx_s ZSTD_CCtx; -+/** -+ * ZSTD_initCCtx() - initialize a zstd compression context -+ * @workspace: The workspace to emplace the context into. It must outlive -+ * the returned context. -+ * @workspaceSize: The size of workspace. Use ZSTD_CCtxWorkspaceBound() to -+ * determine how large the workspace must be. -+ * -+ * Return: A compression context emplaced into workspace. -+ */ -+ZSTD_CCtx *ZSTD_initCCtx(void *workspace, size_t workspaceSize); -+ -+/** -+ * ZSTD_compressCCtx() - compress src into dst -+ * @ctx: The context. Must have been initialized with a workspace at -+ * least as large as ZSTD_CCtxWorkspaceBound(params.cParams). -+ * @dst: The buffer to compress src into. -+ * @dstCapacity: The size of the destination buffer. May be any size, but -+ * ZSTD_compressBound(srcSize) is guaranteed to be large enough. -+ * @src: The data to compress. -+ * @srcSize: The size of the data to compress. -+ * @params: The parameters to use for compression. See ZSTD_getParams(). -+ * -+ * Return: The compressed size or an error, which can be checked using -+ * ZSTD_isError(). -+ */ -+size_t ZSTD_compressCCtx(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, -+ const void *src, size_t srcSize, ZSTD_parameters params); -+ -+/** -+ * ZSTD_DCtxWorkspaceBound() - amount of memory needed to initialize a ZSTD_DCtx -+ * -+ * Return: A lower bound on the size of the workspace that is passed to -+ * ZSTD_initDCtx(). -+ */ -+size_t ZSTD_DCtxWorkspaceBound(void); -+ -+/** -+ * struct ZSTD_DCtx - the zstd decompression context -+ * -+ * When decompressing many times it is recommended to allocate a context just -+ * once and reuse it for each successive decompression operation. -+ */ -+typedef struct ZSTD_DCtx_s ZSTD_DCtx; -+/** -+ * ZSTD_initDCtx() - initialize a zstd decompression context -+ * @workspace: The workspace to emplace the context into. It must outlive -+ * the returned context. -+ * @workspaceSize: The size of workspace. Use ZSTD_DCtxWorkspaceBound() to -+ * determine how large the workspace must be. -+ * -+ * Return: A decompression context emplaced into workspace. -+ */ -+ZSTD_DCtx *ZSTD_initDCtx(void *workspace, size_t workspaceSize); -+ -+/** -+ * ZSTD_decompressDCtx() - decompress zstd compressed src into dst -+ * @ctx: The decompression context. -+ * @dst: The buffer to decompress src into. -+ * @dstCapacity: The size of the destination buffer. Must be at least as large -+ * as the decompressed size. If the caller cannot upper bound the -+ * decompressed size, then it's better to use the streaming API. -+ * @src: The zstd compressed data to decompress. Multiple concatenated -+ * frames and skippable frames are allowed. -+ * @srcSize: The exact size of the data to decompress. -+ * -+ * Return: The decompressed size or an error, which can be checked using -+ * ZSTD_isError(). -+ */ -+size_t ZSTD_decompressDCtx(ZSTD_DCtx *ctx, void *dst, size_t dstCapacity, -+ const void *src, size_t srcSize); -+ -+/*-************************ -+ * Simple dictionary API -+ **************************/ -+ -+/** -+ * ZSTD_compress_usingDict() - compress src into dst using a dictionary -+ * @ctx: The context. Must have been initialized with a workspace at -+ * least as large as ZSTD_CCtxWorkspaceBound(params.cParams). -+ * @dst: The buffer to compress src into. -+ * @dstCapacity: The size of the destination buffer. May be any size, but -+ * ZSTD_compressBound(srcSize) is guaranteed to be large enough. -+ * @src: The data to compress. -+ * @srcSize: The size of the data to compress. -+ * @dict: The dictionary to use for compression. -+ * @dictSize: The size of the dictionary. -+ * @params: The parameters to use for compression. See ZSTD_getParams(). -+ * -+ * Compression using a predefined dictionary. The same dictionary must be used -+ * during decompression. -+ * -+ * Return: The compressed size or an error, which can be checked using -+ * ZSTD_isError(). -+ */ -+size_t ZSTD_compress_usingDict(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, -+ const void *src, size_t srcSize, const void *dict, size_t dictSize, -+ ZSTD_parameters params); -+ -+/** -+ * ZSTD_decompress_usingDict() - decompress src into dst using a dictionary -+ * @ctx: The decompression context. -+ * @dst: The buffer to decompress src into. -+ * @dstCapacity: The size of the destination buffer. Must be at least as large -+ * as the decompressed size. If the caller cannot upper bound the -+ * decompressed size, then it's better to use the streaming API. -+ * @src: The zstd compressed data to decompress. Multiple concatenated -+ * frames and skippable frames are allowed. -+ * @srcSize: The exact size of the data to decompress. -+ * @dict: The dictionary to use for decompression. The same dictionary -+ * must've been used to compress the data. -+ * @dictSize: The size of the dictionary. -+ * -+ * Return: The decompressed size or an error, which can be checked using -+ * ZSTD_isError(). -+ */ -+size_t ZSTD_decompress_usingDict(ZSTD_DCtx *ctx, void *dst, size_t dstCapacity, -+ const void *src, size_t srcSize, const void *dict, size_t dictSize); -+ -+/*-************************** -+ * Fast dictionary API -+ ***************************/ -+ -+/** -+ * ZSTD_CDictWorkspaceBound() - memory needed to initialize a ZSTD_CDict -+ * @cParams: The compression parameters to be used for compression. -+ * -+ * Return: A lower bound on the size of the workspace that is passed to -+ * ZSTD_initCDict(). -+ */ -+size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams); -+ -+/** -+ * struct ZSTD_CDict - a digested dictionary to be used for compression -+ */ -+typedef struct ZSTD_CDict_s ZSTD_CDict; -+ -+/** -+ * ZSTD_initCDict() - initialize a digested dictionary for compression -+ * @dictBuffer: The dictionary to digest. The buffer is referenced by the -+ * ZSTD_CDict so it must outlive the returned ZSTD_CDict. -+ * @dictSize: The size of the dictionary. -+ * @params: The parameters to use for compression. See ZSTD_getParams(). -+ * @workspace: The workspace. It must outlive the returned ZSTD_CDict. -+ * @workspaceSize: The workspace size. Must be at least -+ * ZSTD_CDictWorkspaceBound(params.cParams). -+ * -+ * When compressing multiple messages / blocks with the same dictionary it is -+ * recommended to load it just once. The ZSTD_CDict merely references the -+ * dictBuffer, so it must outlive the returned ZSTD_CDict. -+ * -+ * Return: The digested dictionary emplaced into workspace. -+ */ -+ZSTD_CDict *ZSTD_initCDict(const void *dictBuffer, size_t dictSize, -+ ZSTD_parameters params, void *workspace, size_t workspaceSize); -+ -+/** -+ * ZSTD_compress_usingCDict() - compress src into dst using a ZSTD_CDict -+ * @ctx: The context. Must have been initialized with a workspace at -+ * least as large as ZSTD_CCtxWorkspaceBound(cParams) where -+ * cParams are the compression parameters used to initialize the -+ * cdict. -+ * @dst: The buffer to compress src into. -+ * @dstCapacity: The size of the destination buffer. May be any size, but -+ * ZSTD_compressBound(srcSize) is guaranteed to be large enough. -+ * @src: The data to compress. -+ * @srcSize: The size of the data to compress. -+ * @cdict: The digested dictionary to use for compression. -+ * @params: The parameters to use for compression. See ZSTD_getParams(). -+ * -+ * Compression using a digested dictionary. The same dictionary must be used -+ * during decompression. -+ * -+ * Return: The compressed size or an error, which can be checked using -+ * ZSTD_isError(). -+ */ -+size_t ZSTD_compress_usingCDict(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, -+ const void *src, size_t srcSize, const ZSTD_CDict *cdict); -+ -+ -+/** -+ * ZSTD_DDictWorkspaceBound() - memory needed to initialize a ZSTD_DDict -+ * -+ * Return: A lower bound on the size of the workspace that is passed to -+ * ZSTD_initDDict(). -+ */ -+size_t ZSTD_DDictWorkspaceBound(void); -+ -+/** -+ * struct ZSTD_DDict - a digested dictionary to be used for decompression -+ */ -+typedef struct ZSTD_DDict_s ZSTD_DDict; -+ -+/** -+ * ZSTD_initDDict() - initialize a digested dictionary for decompression -+ * @dictBuffer: The dictionary to digest. The buffer is referenced by the -+ * ZSTD_DDict so it must outlive the returned ZSTD_DDict. -+ * @dictSize: The size of the dictionary. -+ * @workspace: The workspace. It must outlive the returned ZSTD_DDict. -+ * @workspaceSize: The workspace size. Must be at least -+ * ZSTD_DDictWorkspaceBound(). -+ * -+ * When decompressing multiple messages / blocks with the same dictionary it is -+ * recommended to load it just once. The ZSTD_DDict merely references the -+ * dictBuffer, so it must outlive the returned ZSTD_DDict. -+ * -+ * Return: The digested dictionary emplaced into workspace. -+ */ -+ZSTD_DDict *ZSTD_initDDict(const void *dictBuffer, size_t dictSize, -+ void *workspace, size_t workspaceSize); -+ -+/** -+ * ZSTD_decompress_usingDDict() - decompress src into dst using a ZSTD_DDict -+ * @ctx: The decompression context. -+ * @dst: The buffer to decompress src into. -+ * @dstCapacity: The size of the destination buffer. Must be at least as large -+ * as the decompressed size. If the caller cannot upper bound the -+ * decompressed size, then it's better to use the streaming API. -+ * @src: The zstd compressed data to decompress. Multiple concatenated -+ * frames and skippable frames are allowed. -+ * @srcSize: The exact size of the data to decompress. -+ * @ddict: The digested dictionary to use for decompression. The same -+ * dictionary must've been used to compress the data. -+ * -+ * Return: The decompressed size or an error, which can be checked using -+ * ZSTD_isError(). -+ */ -+size_t ZSTD_decompress_usingDDict(ZSTD_DCtx *dctx, void *dst, -+ size_t dstCapacity, const void *src, size_t srcSize, -+ const ZSTD_DDict *ddict); -+ -+ -+/*-************************** -+ * Streaming -+ ***************************/ -+ -+/** -+ * struct ZSTD_inBuffer - input buffer for streaming -+ * @src: Start of the input buffer. -+ * @size: Size of the input buffer. -+ * @pos: Position where reading stopped. Will be updated. -+ * Necessarily 0 <= pos <= size. -+ */ -+typedef struct ZSTD_inBuffer_s { -+ const void *src; -+ size_t size; -+ size_t pos; -+} ZSTD_inBuffer; -+ -+/** -+ * struct ZSTD_outBuffer - output buffer for streaming -+ * @dst: Start of the output buffer. -+ * @size: Size of the output buffer. -+ * @pos: Position where writing stopped. Will be updated. -+ * Necessarily 0 <= pos <= size. -+ */ -+typedef struct ZSTD_outBuffer_s { -+ void *dst; -+ size_t size; -+ size_t pos; -+} ZSTD_outBuffer; -+ -+ -+ -+/*-***************************************************************************** -+ * Streaming compression - HowTo -+ * -+ * A ZSTD_CStream object is required to track streaming operation. -+ * Use ZSTD_initCStream() to initialize a ZSTD_CStream object. -+ * ZSTD_CStream objects can be reused multiple times on consecutive compression -+ * operations. It is recommended to re-use ZSTD_CStream in situations where many -+ * streaming operations will be achieved consecutively. Use one separate -+ * ZSTD_CStream per thread for parallel execution. -+ * -+ * Use ZSTD_compressStream() repetitively to consume input stream. -+ * The function will automatically update both `pos` fields. -+ * Note that it may not consume the entire input, in which case `pos < size`, -+ * and it's up to the caller to present again remaining data. -+ * It returns a hint for the preferred number of bytes to use as an input for -+ * the next function call. -+ * -+ * At any moment, it's possible to flush whatever data remains within internal -+ * buffer, using ZSTD_flushStream(). `output->pos` will be updated. There might -+ * still be some content left within the internal buffer if `output->size` is -+ * too small. It returns the number of bytes left in the internal buffer and -+ * must be called until it returns 0. -+ * -+ * ZSTD_endStream() instructs to finish a frame. It will perform a flush and -+ * write frame epilogue. The epilogue is required for decoders to consider a -+ * frame completed. Similar to ZSTD_flushStream(), it may not be able to flush -+ * the full content if `output->size` is too small. In which case, call again -+ * ZSTD_endStream() to complete the flush. It returns the number of bytes left -+ * in the internal buffer and must be called until it returns 0. -+ ******************************************************************************/ -+ -+/** -+ * ZSTD_CStreamWorkspaceBound() - memory needed to initialize a ZSTD_CStream -+ * @cParams: The compression parameters to be used for compression. -+ * -+ * Return: A lower bound on the size of the workspace that is passed to -+ * ZSTD_initCStream() and ZSTD_initCStream_usingCDict(). -+ */ -+size_t ZSTD_CStreamWorkspaceBound(ZSTD_compressionParameters cParams); -+ -+/** -+ * struct ZSTD_CStream - the zstd streaming compression context -+ */ -+typedef struct ZSTD_CStream_s ZSTD_CStream; -+ -+/*===== ZSTD_CStream management functions =====*/ -+/** -+ * ZSTD_initCStream() - initialize a zstd streaming compression context -+ * @params: The zstd compression parameters. -+ * @pledgedSrcSize: If params.fParams.contentSizeFlag == 1 then the caller must -+ * pass the source size (zero means empty source). Otherwise, -+ * the caller may optionally pass the source size, or zero if -+ * unknown. -+ * @workspace: The workspace to emplace the context into. It must outlive -+ * the returned context. -+ * @workspaceSize: The size of workspace. -+ * Use ZSTD_CStreamWorkspaceBound(params.cParams) to determine -+ * how large the workspace must be. -+ * -+ * Return: The zstd streaming compression context. -+ */ -+ZSTD_CStream *ZSTD_initCStream(ZSTD_parameters params, -+ unsigned long long pledgedSrcSize, void *workspace, -+ size_t workspaceSize); -+ -+/** -+ * ZSTD_initCStream_usingCDict() - initialize a streaming compression context -+ * @cdict: The digested dictionary to use for compression. -+ * @pledgedSrcSize: Optionally the source size, or zero if unknown. -+ * @workspace: The workspace to emplace the context into. It must outlive -+ * the returned context. -+ * @workspaceSize: The size of workspace. Call ZSTD_CStreamWorkspaceBound() -+ * with the cParams used to initialize the cdict to determine -+ * how large the workspace must be. -+ * -+ * Return: The zstd streaming compression context. -+ */ -+ZSTD_CStream *ZSTD_initCStream_usingCDict(const ZSTD_CDict *cdict, -+ unsigned long long pledgedSrcSize, void *workspace, -+ size_t workspaceSize); -+ -+/*===== Streaming compression functions =====*/ -+/** -+ * ZSTD_resetCStream() - reset the context using parameters from creation -+ * @zcs: The zstd streaming compression context to reset. -+ * @pledgedSrcSize: Optionally the source size, or zero if unknown. -+ * -+ * Resets the context using the parameters from creation. Skips dictionary -+ * loading, since it can be reused. If `pledgedSrcSize` is non-zero the frame -+ * content size is always written into the frame header. -+ * -+ * Return: Zero or an error, which can be checked using ZSTD_isError(). -+ */ -+size_t ZSTD_resetCStream(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize); -+/** -+ * ZSTD_compressStream() - streaming compress some of input into output -+ * @zcs: The zstd streaming compression context. -+ * @output: Destination buffer. `output->pos` is updated to indicate how much -+ * compressed data was written. -+ * @input: Source buffer. `input->pos` is updated to indicate how much data was -+ * read. Note that it may not consume the entire input, in which case -+ * `input->pos < input->size`, and it's up to the caller to present -+ * remaining data again. -+ * -+ * The `input` and `output` buffers may be any size. Guaranteed to make some -+ * forward progress if `input` and `output` are not empty. -+ * -+ * Return: A hint for the number of bytes to use as the input for the next -+ * function call or an error, which can be checked using -+ * ZSTD_isError(). -+ */ -+size_t ZSTD_compressStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output, -+ ZSTD_inBuffer *input); -+/** -+ * ZSTD_flushStream() - flush internal buffers into output -+ * @zcs: The zstd streaming compression context. -+ * @output: Destination buffer. `output->pos` is updated to indicate how much -+ * compressed data was written. -+ * -+ * ZSTD_flushStream() must be called until it returns 0, meaning all the data -+ * has been flushed. Since ZSTD_flushStream() causes a block to be ended, -+ * calling it too often will degrade the compression ratio. -+ * -+ * Return: The number of bytes still present within internal buffers or an -+ * error, which can be checked using ZSTD_isError(). -+ */ -+size_t ZSTD_flushStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output); -+/** -+ * ZSTD_endStream() - flush internal buffers into output and end the frame -+ * @zcs: The zstd streaming compression context. -+ * @output: Destination buffer. `output->pos` is updated to indicate how much -+ * compressed data was written. -+ * -+ * ZSTD_endStream() must be called until it returns 0, meaning all the data has -+ * been flushed and the frame epilogue has been written. -+ * -+ * Return: The number of bytes still present within internal buffers or an -+ * error, which can be checked using ZSTD_isError(). -+ */ -+size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output); -+ -+/** -+ * ZSTD_CStreamInSize() - recommended size for the input buffer -+ * -+ * Return: The recommended size for the input buffer. -+ */ -+size_t ZSTD_CStreamInSize(void); -+/** -+ * ZSTD_CStreamOutSize() - recommended size for the output buffer -+ * -+ * When the output buffer is at least this large, it is guaranteed to be large -+ * enough to flush at least one complete compressed block. -+ * -+ * Return: The recommended size for the output buffer. -+ */ -+size_t ZSTD_CStreamOutSize(void); -+ -+ -+ -+/*-***************************************************************************** -+ * Streaming decompression - HowTo -+ * -+ * A ZSTD_DStream object is required to track streaming operations. -+ * Use ZSTD_initDStream() to initialize a ZSTD_DStream object. -+ * ZSTD_DStream objects can be re-used multiple times. -+ * -+ * Use ZSTD_decompressStream() repetitively to consume your input. -+ * The function will update both `pos` fields. -+ * If `input->pos < input->size`, some input has not been consumed. -+ * It's up to the caller to present again remaining data. -+ * If `output->pos < output->size`, decoder has flushed everything it could. -+ * Returns 0 iff a frame is completely decoded and fully flushed. -+ * Otherwise it returns a suggested next input size that will never load more -+ * than the current frame. -+ ******************************************************************************/ -+ -+/** -+ * ZSTD_DStreamWorkspaceBound() - memory needed to initialize a ZSTD_DStream -+ * @maxWindowSize: The maximum window size allowed for compressed frames. -+ * -+ * Return: A lower bound on the size of the workspace that is passed to -+ * ZSTD_initDStream() and ZSTD_initDStream_usingDDict(). -+ */ -+size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize); -+ -+/** -+ * struct ZSTD_DStream - the zstd streaming decompression context -+ */ -+typedef struct ZSTD_DStream_s ZSTD_DStream; -+/*===== ZSTD_DStream management functions =====*/ -+/** -+ * ZSTD_initDStream() - initialize a zstd streaming decompression context -+ * @maxWindowSize: The maximum window size allowed for compressed frames. -+ * @workspace: The workspace to emplace the context into. It must outlive -+ * the returned context. -+ * @workspaceSize: The size of workspace. -+ * Use ZSTD_DStreamWorkspaceBound(maxWindowSize) to determine -+ * how large the workspace must be. -+ * -+ * Return: The zstd streaming decompression context. -+ */ -+ZSTD_DStream *ZSTD_initDStream(size_t maxWindowSize, void *workspace, -+ size_t workspaceSize); -+/** -+ * ZSTD_initDStream_usingDDict() - initialize streaming decompression context -+ * @maxWindowSize: The maximum window size allowed for compressed frames. -+ * @ddict: The digested dictionary to use for decompression. -+ * @workspace: The workspace to emplace the context into. It must outlive -+ * the returned context. -+ * @workspaceSize: The size of workspace. -+ * Use ZSTD_DStreamWorkspaceBound(maxWindowSize) to determine -+ * how large the workspace must be. -+ * -+ * Return: The zstd streaming decompression context. -+ */ -+ZSTD_DStream *ZSTD_initDStream_usingDDict(size_t maxWindowSize, -+ const ZSTD_DDict *ddict, void *workspace, size_t workspaceSize); -+ -+/*===== Streaming decompression functions =====*/ -+/** -+ * ZSTD_resetDStream() - reset the context using parameters from creation -+ * @zds: The zstd streaming decompression context to reset. -+ * -+ * Resets the context using the parameters from creation. Skips dictionary -+ * loading, since it can be reused. -+ * -+ * Return: Zero or an error, which can be checked using ZSTD_isError(). -+ */ -+size_t ZSTD_resetDStream(ZSTD_DStream *zds); -+/** -+ * ZSTD_decompressStream() - streaming decompress some of input into output -+ * @zds: The zstd streaming decompression context. -+ * @output: Destination buffer. `output.pos` is updated to indicate how much -+ * decompressed data was written. -+ * @input: Source buffer. `input.pos` is updated to indicate how much data was -+ * read. Note that it may not consume the entire input, in which case -+ * `input.pos < input.size`, and it's up to the caller to present -+ * remaining data again. -+ * -+ * The `input` and `output` buffers may be any size. Guaranteed to make some -+ * forward progress if `input` and `output` are not empty. -+ * ZSTD_decompressStream() will not consume the last byte of the frame until -+ * the entire frame is flushed. -+ * -+ * Return: Returns 0 iff a frame is completely decoded and fully flushed. -+ * Otherwise returns a hint for the number of bytes to use as the input -+ * for the next function call or an error, which can be checked using -+ * ZSTD_isError(). The size hint will never load more than the frame. -+ */ -+size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, -+ ZSTD_inBuffer *input); -+ -+/** -+ * ZSTD_DStreamInSize() - recommended size for the input buffer -+ * -+ * Return: The recommended size for the input buffer. -+ */ -+size_t ZSTD_DStreamInSize(void); -+/** -+ * ZSTD_DStreamOutSize() - recommended size for the output buffer -+ * -+ * When the output buffer is at least this large, it is guaranteed to be large -+ * enough to flush at least one complete decompressed block. -+ * -+ * Return: The recommended size for the output buffer. -+ */ -+size_t ZSTD_DStreamOutSize(void); -+ -+ -+/* --- Constants ---*/ -+#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */ -+#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U -+ -+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) -+#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) -+ -+#define ZSTD_WINDOWLOG_MAX_32 27 -+#define ZSTD_WINDOWLOG_MAX_64 27 -+#define ZSTD_WINDOWLOG_MAX \ -+ ((unsigned int)(sizeof(size_t) == 4 \ -+ ? ZSTD_WINDOWLOG_MAX_32 \ -+ : ZSTD_WINDOWLOG_MAX_64)) -+#define ZSTD_WINDOWLOG_MIN 10 -+#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX -+#define ZSTD_HASHLOG_MIN 6 -+#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) -+#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN -+#define ZSTD_HASHLOG3_MAX 17 -+#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) -+#define ZSTD_SEARCHLOG_MIN 1 -+/* only for ZSTD_fast, other strategies are limited to 6 */ -+#define ZSTD_SEARCHLENGTH_MAX 7 -+/* only for ZSTD_btopt, other strategies are limited to 4 */ -+#define ZSTD_SEARCHLENGTH_MIN 3 -+#define ZSTD_TARGETLENGTH_MIN 4 -+#define ZSTD_TARGETLENGTH_MAX 999 -+ -+/* for static allocation */ -+#define ZSTD_FRAMEHEADERSIZE_MAX 18 -+#define ZSTD_FRAMEHEADERSIZE_MIN 6 -+static const size_t ZSTD_frameHeaderSize_prefix = 5; -+static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN; -+static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; -+/* magic number + skippable frame length */ -+static const size_t ZSTD_skippableHeaderSize = 8; -+ -+ -+/*-************************************* -+ * Compressed size functions -+ **************************************/ -+ -+/** -+ * ZSTD_findFrameCompressedSize() - returns the size of a compressed frame -+ * @src: Source buffer. It should point to the start of a zstd encoded frame -+ * or a skippable frame. -+ * @srcSize: The size of the source buffer. It must be at least as large as the -+ * size of the frame. -+ * -+ * Return: The compressed size of the frame pointed to by `src` or an error, -+ * which can be check with ZSTD_isError(). -+ * Suitable to pass to ZSTD_decompress() or similar functions. -+ */ -+size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize); -+ -+/*-************************************* -+ * Decompressed size functions -+ **************************************/ -+/** -+ * ZSTD_getFrameContentSize() - returns the content size in a zstd frame header -+ * @src: It should point to the start of a zstd encoded frame. -+ * @srcSize: The size of the source buffer. It must be at least as large as the -+ * frame header. `ZSTD_frameHeaderSize_max` is always large enough. -+ * -+ * Return: The frame content size stored in the frame header if known. -+ * `ZSTD_CONTENTSIZE_UNKNOWN` if the content size isn't stored in the -+ * frame header. `ZSTD_CONTENTSIZE_ERROR` on invalid input. -+ */ -+unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); -+ -+/** -+ * ZSTD_findDecompressedSize() - returns decompressed size of a series of frames -+ * @src: It should point to the start of a series of zstd encoded and/or -+ * skippable frames. -+ * @srcSize: The exact size of the series of frames. -+ * -+ * If any zstd encoded frame in the series doesn't have the frame content size -+ * set, `ZSTD_CONTENTSIZE_UNKNOWN` is returned. But frame content size is always -+ * set when using ZSTD_compress(). The decompressed size can be very large. -+ * If the source is untrusted, the decompressed size could be wrong or -+ * intentionally modified. Always ensure the result fits within the -+ * application's authorized limits. ZSTD_findDecompressedSize() handles multiple -+ * frames, and so it must traverse the input to read each frame header. This is -+ * efficient as most of the data is skipped, however it does mean that all frame -+ * data must be present and valid. -+ * -+ * Return: Decompressed size of all the data contained in the frames if known. -+ * `ZSTD_CONTENTSIZE_UNKNOWN` if the decompressed size is unknown. -+ * `ZSTD_CONTENTSIZE_ERROR` if an error occurred. -+ */ -+unsigned long long ZSTD_findDecompressedSize(const void *src, size_t srcSize); -+ -+/*-************************************* -+ * Advanced compression functions -+ **************************************/ -+/** -+ * ZSTD_checkCParams() - ensure parameter values remain within authorized range -+ * @cParams: The zstd compression parameters. -+ * -+ * Return: Zero or an error, which can be checked using ZSTD_isError(). -+ */ -+size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams); -+ -+/** -+ * ZSTD_adjustCParams() - optimize parameters for a given srcSize and dictSize -+ * @srcSize: Optionally the estimated source size, or zero if unknown. -+ * @dictSize: Optionally the estimated dictionary size, or zero if unknown. -+ * -+ * Return: The optimized parameters. -+ */ -+ZSTD_compressionParameters ZSTD_adjustCParams( -+ ZSTD_compressionParameters cParams, unsigned long long srcSize, -+ size_t dictSize); -+ -+/*--- Advanced decompression functions ---*/ -+ -+/** -+ * ZSTD_isFrame() - returns true iff the buffer starts with a valid frame -+ * @buffer: The source buffer to check. -+ * @size: The size of the source buffer, must be at least 4 bytes. -+ * -+ * Return: True iff the buffer starts with a zstd or skippable frame identifier. -+ */ -+unsigned int ZSTD_isFrame(const void *buffer, size_t size); -+ -+/** -+ * ZSTD_getDictID_fromDict() - returns the dictionary id stored in a dictionary -+ * @dict: The dictionary buffer. -+ * @dictSize: The size of the dictionary buffer. -+ * -+ * Return: The dictionary id stored within the dictionary or 0 if the -+ * dictionary is not a zstd dictionary. If it returns 0 the -+ * dictionary can still be loaded as a content-only dictionary. -+ */ -+unsigned int ZSTD_getDictID_fromDict(const void *dict, size_t dictSize); -+ -+/** -+ * ZSTD_getDictID_fromDDict() - returns the dictionary id stored in a ZSTD_DDict -+ * @ddict: The ddict to find the id of. -+ * -+ * Return: The dictionary id stored within `ddict` or 0 if the dictionary is not -+ * a zstd dictionary. If it returns 0 `ddict` will be loaded as a -+ * content-only dictionary. -+ */ -+unsigned int ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict); -+ -+/** -+ * ZSTD_getDictID_fromFrame() - returns the dictionary id stored in a zstd frame -+ * @src: Source buffer. It must be a zstd encoded frame. -+ * @srcSize: The size of the source buffer. It must be at least as large as the -+ * frame header. `ZSTD_frameHeaderSize_max` is always large enough. -+ * -+ * Return: The dictionary id required to decompress the frame stored within -+ * `src` or 0 if the dictionary id could not be decoded. It can return -+ * 0 if the frame does not require a dictionary, the dictionary id -+ * wasn't stored in the frame, `src` is not a zstd frame, or `srcSize` -+ * is too small. -+ */ -+unsigned int ZSTD_getDictID_fromFrame(const void *src, size_t srcSize); -+ -+/** -+ * struct ZSTD_frameParams - zstd frame parameters stored in the frame header -+ * @frameContentSize: The frame content size, or 0 if not present. -+ * @windowSize: The window size, or 0 if the frame is a skippable frame. -+ * @dictID: The dictionary id, or 0 if not present. -+ * @checksumFlag: Whether a checksum was used. -+ */ -+typedef struct { -+ unsigned long long frameContentSize; -+ unsigned int windowSize; -+ unsigned int dictID; -+ unsigned int checksumFlag; -+} ZSTD_frameParams; -+ -+/** -+ * ZSTD_getFrameParams() - extracts parameters from a zstd or skippable frame -+ * @fparamsPtr: On success the frame parameters are written here. -+ * @src: The source buffer. It must point to a zstd or skippable frame. -+ * @srcSize: The size of the source buffer. `ZSTD_frameHeaderSize_max` is -+ * always large enough to succeed. -+ * -+ * Return: 0 on success. If more data is required it returns how many bytes -+ * must be provided to make forward progress. Otherwise it returns -+ * an error, which can be checked using ZSTD_isError(). -+ */ -+size_t ZSTD_getFrameParams(ZSTD_frameParams *fparamsPtr, const void *src, -+ size_t srcSize); -+ -+/*-***************************************************************************** -+ * Buffer-less and synchronous inner streaming functions -+ * -+ * This is an advanced API, giving full control over buffer management, for -+ * users which need direct control over memory. -+ * But it's also a complex one, with many restrictions (documented below). -+ * Prefer using normal streaming API for an easier experience -+ ******************************************************************************/ -+ -+/*-***************************************************************************** -+ * Buffer-less streaming compression (synchronous mode) -+ * -+ * A ZSTD_CCtx object is required to track streaming operations. -+ * Use ZSTD_initCCtx() to initialize a context. -+ * ZSTD_CCtx object can be re-used multiple times within successive compression -+ * operations. -+ * -+ * Start by initializing a context. -+ * Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary -+ * compression, -+ * or ZSTD_compressBegin_advanced(), for finer parameter control. -+ * It's also possible to duplicate a reference context which has already been -+ * initialized, using ZSTD_copyCCtx() -+ * -+ * Then, consume your input using ZSTD_compressContinue(). -+ * There are some important considerations to keep in mind when using this -+ * advanced function : -+ * - ZSTD_compressContinue() has no internal buffer. It uses externally provided -+ * buffer only. -+ * - Interface is synchronous : input is consumed entirely and produce 1+ -+ * (or more) compressed blocks. -+ * - Caller must ensure there is enough space in `dst` to store compressed data -+ * under worst case scenario. Worst case evaluation is provided by -+ * ZSTD_compressBound(). -+ * ZSTD_compressContinue() doesn't guarantee recover after a failed -+ * compression. -+ * - ZSTD_compressContinue() presumes prior input ***is still accessible and -+ * unmodified*** (up to maximum distance size, see WindowLog). -+ * It remembers all previous contiguous blocks, plus one separated memory -+ * segment (which can itself consists of multiple contiguous blocks) -+ * - ZSTD_compressContinue() detects that prior input has been overwritten when -+ * `src` buffer overlaps. In which case, it will "discard" the relevant memory -+ * section from its history. -+ * -+ * Finish a frame with ZSTD_compressEnd(), which will write the last block(s) -+ * and optional checksum. It's possible to use srcSize==0, in which case, it -+ * will write a final empty block to end the frame. Without last block mark, -+ * frames will be considered unfinished (corrupted) by decoders. -+ * -+ * `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new -+ * frame. -+ ******************************************************************************/ -+ -+/*===== Buffer-less streaming compression functions =====*/ -+size_t ZSTD_compressBegin(ZSTD_CCtx *cctx, int compressionLevel); -+size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict, -+ size_t dictSize, int compressionLevel); -+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx *cctx, const void *dict, -+ size_t dictSize, ZSTD_parameters params, -+ unsigned long long pledgedSrcSize); -+size_t ZSTD_copyCCtx(ZSTD_CCtx *cctx, const ZSTD_CCtx *preparedCCtx, -+ unsigned long long pledgedSrcSize); -+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, -+ unsigned long long pledgedSrcSize); -+size_t ZSTD_compressContinue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, -+ const void *src, size_t srcSize); -+size_t ZSTD_compressEnd(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, -+ const void *src, size_t srcSize); -+ -+ -+ -+/*-***************************************************************************** -+ * Buffer-less streaming decompression (synchronous mode) -+ * -+ * A ZSTD_DCtx object is required to track streaming operations. -+ * Use ZSTD_initDCtx() to initialize a context. -+ * A ZSTD_DCtx object can be re-used multiple times. -+ * -+ * First typical operation is to retrieve frame parameters, using -+ * ZSTD_getFrameParams(). It fills a ZSTD_frameParams structure which provide -+ * important information to correctly decode the frame, such as the minimum -+ * rolling buffer size to allocate to decompress data (`windowSize`), and the -+ * dictionary ID used. -+ * Note: content size is optional, it may not be present. 0 means unknown. -+ * Note that these values could be wrong, either because of data malformation, -+ * or because an attacker is spoofing deliberate false information. As a -+ * consequence, check that values remain within valid application range, -+ * especially `windowSize`, before allocation. Each application can set its own -+ * limit, depending on local restrictions. For extended interoperability, it is -+ * recommended to support at least 8 MB. -+ * Frame parameters are extracted from the beginning of the compressed frame. -+ * Data fragment must be large enough to ensure successful decoding, typically -+ * `ZSTD_frameHeaderSize_max` bytes. -+ * Result: 0: successful decoding, the `ZSTD_frameParams` structure is filled. -+ * >0: `srcSize` is too small, provide at least this many bytes. -+ * errorCode, which can be tested using ZSTD_isError(). -+ * -+ * Start decompression, with ZSTD_decompressBegin() or -+ * ZSTD_decompressBegin_usingDict(). Alternatively, you can copy a prepared -+ * context, using ZSTD_copyDCtx(). -+ * -+ * Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() -+ * alternatively. -+ * ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' -+ * to ZSTD_decompressContinue(). -+ * ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will -+ * fail. -+ * -+ * The result of ZSTD_decompressContinue() is the number of bytes regenerated -+ * within 'dst' (necessarily <= dstCapacity). It can be zero, which is not an -+ * error; it just means ZSTD_decompressContinue() has decoded some metadata -+ * item. It can also be an error code, which can be tested with ZSTD_isError(). -+ * -+ * ZSTD_decompressContinue() needs previous data blocks during decompression, up -+ * to `windowSize`. They should preferably be located contiguously, prior to -+ * current block. Alternatively, a round buffer of sufficient size is also -+ * possible. Sufficient size is determined by frame parameters. -+ * ZSTD_decompressContinue() is very sensitive to contiguity, if 2 blocks don't -+ * follow each other, make sure that either the compressor breaks contiguity at -+ * the same place, or that previous contiguous segment is large enough to -+ * properly handle maximum back-reference. -+ * -+ * A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. -+ * Context can then be reset to start a new decompression. -+ * -+ * Note: it's possible to know if next input to present is a header or a block, -+ * using ZSTD_nextInputType(). This information is not required to properly -+ * decode a frame. -+ * -+ * == Special case: skippable frames == -+ * -+ * Skippable frames allow integration of user-defined data into a flow of -+ * concatenated frames. Skippable frames will be ignored (skipped) by a -+ * decompressor. The format of skippable frames is as follows: -+ * a) Skippable frame ID - 4 Bytes, Little endian format, any value from -+ * 0x184D2A50 to 0x184D2A5F -+ * b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits -+ * c) Frame Content - any content (User Data) of length equal to Frame Size -+ * For skippable frames ZSTD_decompressContinue() always returns 0. -+ * For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 -+ * what means that a frame is skippable. -+ * Note: If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might -+ * actually be a zstd encoded frame with no content. For purposes of -+ * decompression, it is valid in both cases to skip the frame using -+ * ZSTD_findFrameCompressedSize() to find its size in bytes. -+ * It also returns frame size as fparamsPtr->frameContentSize. -+ ******************************************************************************/ -+ -+/*===== Buffer-less streaming decompression functions =====*/ -+size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx); -+size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx *dctx, const void *dict, -+ size_t dictSize); -+void ZSTD_copyDCtx(ZSTD_DCtx *dctx, const ZSTD_DCtx *preparedDCtx); -+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx *dctx); -+size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, -+ const void *src, size_t srcSize); -+typedef enum { -+ ZSTDnit_frameHeader, -+ ZSTDnit_blockHeader, -+ ZSTDnit_block, -+ ZSTDnit_lastBlock, -+ ZSTDnit_checksum, -+ ZSTDnit_skippableFrame -+} ZSTD_nextInputType_e; -+ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx *dctx); -+ -+/*-***************************************************************************** -+ * Block functions -+ * -+ * Block functions produce and decode raw zstd blocks, without frame metadata. -+ * Frame metadata cost is typically ~18 bytes, which can be non-negligible for -+ * very small blocks (< 100 bytes). User will have to take in charge required -+ * information to regenerate data, such as compressed and content sizes. -+ * -+ * A few rules to respect: -+ * - Compressing and decompressing require a context structure -+ * + Use ZSTD_initCCtx() and ZSTD_initDCtx() -+ * - It is necessary to init context before starting -+ * + compression : ZSTD_compressBegin() -+ * + decompression : ZSTD_decompressBegin() -+ * + variants _usingDict() are also allowed -+ * + copyCCtx() and copyDCtx() work too -+ * - Block size is limited, it must be <= ZSTD_getBlockSizeMax() -+ * + If you need to compress more, cut data into multiple blocks -+ * + Consider using the regular ZSTD_compress() instead, as frame metadata -+ * costs become negligible when source size is large. -+ * - When a block is considered not compressible enough, ZSTD_compressBlock() -+ * result will be zero. In which case, nothing is produced into `dst`. -+ * + User must test for such outcome and deal directly with uncompressed data -+ * + ZSTD_decompressBlock() doesn't accept uncompressed data as input!!! -+ * + In case of multiple successive blocks, decoder must be informed of -+ * uncompressed block existence to follow proper history. Use -+ * ZSTD_insertBlock() in such a case. -+ ******************************************************************************/ -+ -+/* Define for static allocation */ -+#define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) -+/*===== Raw zstd block functions =====*/ -+size_t ZSTD_getBlockSizeMax(ZSTD_CCtx *cctx); -+size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, -+ const void *src, size_t srcSize); -+size_t ZSTD_decompressBlock(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, -+ const void *src, size_t srcSize); -+size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void *blockStart, -+ size_t blockSize); -+ -+#endif /* ZSTD_H */ -diff --git a/lib/Kconfig b/lib/Kconfig -index 5e7541f..0d49ed0 100644 ---- a/lib/Kconfig -+++ b/lib/Kconfig -@@ -249,6 +249,14 @@ config LZ4HC_COMPRESS - config LZ4_DECOMPRESS - tristate - -+config ZSTD_COMPRESS -+ select XXHASH -+ tristate -+ -+config ZSTD_DECOMPRESS -+ select XXHASH -+ tristate -+ - source "lib/xz/Kconfig" - - # -diff --git a/lib/Makefile b/lib/Makefile -index d06b68a..d5c8a4f 100644 ---- a/lib/Makefile -+++ b/lib/Makefile -@@ -116,6 +116,8 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/ - obj-$(CONFIG_LZ4_COMPRESS) += lz4/ - obj-$(CONFIG_LZ4HC_COMPRESS) += lz4/ - obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/ -+obj-$(CONFIG_ZSTD_COMPRESS) += zstd/ -+obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd/ - obj-$(CONFIG_XZ_DEC) += xz/ - obj-$(CONFIG_RAID6_PQ) += raid6/ - -diff --git a/lib/zstd/Makefile b/lib/zstd/Makefile -new file mode 100644 -index 0000000..dd0a359 ---- /dev/null -+++ b/lib/zstd/Makefile -@@ -0,0 +1,18 @@ -+obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o -+obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o -+ -+ccflags-y += -O3 -+ -+# Object files unique to zstd_compress and zstd_decompress -+zstd_compress-y := fse_compress.o huf_compress.o compress.o -+zstd_decompress-y := huf_decompress.o decompress.o -+ -+# These object files are shared between the modules. -+# Always add them to zstd_compress. -+# Unless both zstd_compress and zstd_decompress are built in -+# then also add them to zstd_decompress. -+zstd_compress-y += entropy_common.o fse_decompress.o zstd_common.o -+ -+ifneq ($(CONFIG_ZSTD_COMPRESS)$(CONFIG_ZSTD_DECOMPRESS),yy) -+ zstd_decompress-y += entropy_common.o fse_decompress.o zstd_common.o -+endif -diff --git a/lib/zstd/bitstream.h b/lib/zstd/bitstream.h -new file mode 100644 -index 0000000..a826b99 ---- /dev/null -+++ b/lib/zstd/bitstream.h -@@ -0,0 +1,374 @@ -+/* -+ * bitstream -+ * Part of FSE library -+ * header file (to include) -+ * Copyright (C) 2013-2016, Yann Collet. -+ * -+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above -+ * copyright notice, this list of conditions and the following disclaimer -+ * in the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ * -+ * You can contact the author at : -+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -+ */ -+#ifndef BITSTREAM_H_MODULE -+#define BITSTREAM_H_MODULE -+ -+/* -+* This API consists of small unitary functions, which must be inlined for best performance. -+* Since link-time-optimization is not available for all compilers, -+* these functions are defined into a .h to be included. -+*/ -+ -+/*-**************************************** -+* Dependencies -+******************************************/ -+#include "error_private.h" /* error codes and messages */ -+#include "mem.h" /* unaligned access routines */ -+ -+/*========================================= -+* Target specific -+=========================================*/ -+#define STREAM_ACCUMULATOR_MIN_32 25 -+#define STREAM_ACCUMULATOR_MIN_64 57 -+#define STREAM_ACCUMULATOR_MIN ((U32)(ZSTD_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) -+ -+/*-****************************************** -+* bitStream encoding API (write forward) -+********************************************/ -+/* bitStream can mix input from multiple sources. -+* A critical property of these streams is that they encode and decode in **reverse** direction. -+* So the first bit sequence you add will be the last to be read, like a LIFO stack. -+*/ -+typedef struct { -+ size_t bitContainer; -+ int bitPos; -+ char *startPtr; -+ char *ptr; -+ char *endPtr; -+} BIT_CStream_t; -+ -+ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *dstBuffer, size_t dstCapacity); -+ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits); -+ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC); -+ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC); -+ -+/* Start with initCStream, providing the size of buffer to write into. -+* bitStream will never write outside of this buffer. -+* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. -+* -+* bits are first added to a local register. -+* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. -+* Writing data into memory is an explicit operation, performed by the flushBits function. -+* Hence keep track how many bits are potentially stored into local register to avoid register overflow. -+* After a flushBits, a maximum of 7 bits might still be stored into local register. -+* -+* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. -+* -+* Last operation is to close the bitStream. -+* The function returns the final size of CStream in bytes. -+* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) -+*/ -+ -+/*-******************************************** -+* bitStream decoding API (read backward) -+**********************************************/ -+typedef struct { -+ size_t bitContainer; -+ unsigned bitsConsumed; -+ const char *ptr; -+ const char *start; -+} BIT_DStream_t; -+ -+typedef enum { -+ BIT_DStream_unfinished = 0, -+ BIT_DStream_endOfBuffer = 1, -+ BIT_DStream_completed = 2, -+ BIT_DStream_overflow = 3 -+} BIT_DStream_status; /* result of BIT_reloadDStream() */ -+/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ -+ -+ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize); -+ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, unsigned nbBits); -+ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD); -+ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *bitD); -+ -+/* Start by invoking BIT_initDStream(). -+* A chunk of the bitStream is then stored into a local register. -+* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). -+* You can then retrieve bitFields stored into the local register, **in reverse order**. -+* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. -+* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. -+* Otherwise, it can be less than that, so proceed accordingly. -+* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). -+*/ -+ -+/*-**************************************** -+* unsafe API -+******************************************/ -+ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits); -+/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ -+ -+ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC); -+/* unsafe version; does not check buffer overflow */ -+ -+ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, unsigned nbBits); -+/* faster, but works only if nbBits >= 1 */ -+ -+/*-************************************************************** -+* Internal functions -+****************************************************************/ -+ZSTD_STATIC unsigned BIT_highbit32(register U32 val) { return 31 - __builtin_clz(val); } -+ -+/*===== Local Constants =====*/ -+static const unsigned BIT_mask[] = {0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -+ 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, -+ 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF}; /* up to 26 bits */ -+ -+/*-************************************************************** -+* bitStream encoding -+****************************************************************/ -+/*! BIT_initCStream() : -+ * `dstCapacity` must be > sizeof(void*) -+ * @return : 0 if success, -+ otherwise an error code (can be tested using ERR_isError() ) */ -+ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *startPtr, size_t dstCapacity) -+{ -+ bitC->bitContainer = 0; -+ bitC->bitPos = 0; -+ bitC->startPtr = (char *)startPtr; -+ bitC->ptr = bitC->startPtr; -+ bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr); -+ if (dstCapacity <= sizeof(bitC->ptr)) -+ return ERROR(dstSize_tooSmall); -+ return 0; -+} -+ -+/*! BIT_addBits() : -+ can add up to 26 bits into `bitC`. -+ Does not check for register overflow ! */ -+ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits) -+{ -+ bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; -+ bitC->bitPos += nbBits; -+} -+ -+/*! BIT_addBitsFast() : -+ * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ -+ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits) -+{ -+ bitC->bitContainer |= value << bitC->bitPos; -+ bitC->bitPos += nbBits; -+} -+ -+/*! BIT_flushBitsFast() : -+ * unsafe version; does not check buffer overflow */ -+ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC) -+{ -+ size_t const nbBytes = bitC->bitPos >> 3; -+ ZSTD_writeLEST(bitC->ptr, bitC->bitContainer); -+ bitC->ptr += nbBytes; -+ bitC->bitPos &= 7; -+ bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ -+} -+ -+/*! BIT_flushBits() : -+ * safe version; check for buffer overflow, and prevents it. -+ * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */ -+ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC) -+{ -+ size_t const nbBytes = bitC->bitPos >> 3; -+ ZSTD_writeLEST(bitC->ptr, bitC->bitContainer); -+ bitC->ptr += nbBytes; -+ if (bitC->ptr > bitC->endPtr) -+ bitC->ptr = bitC->endPtr; -+ bitC->bitPos &= 7; -+ bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ -+} -+ -+/*! BIT_closeCStream() : -+ * @return : size of CStream, in bytes, -+ or 0 if it could not fit into dstBuffer */ -+ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC) -+{ -+ BIT_addBitsFast(bitC, 1, 1); /* endMark */ -+ BIT_flushBits(bitC); -+ -+ if (bitC->ptr >= bitC->endPtr) -+ return 0; /* doesn't fit within authorized budget : cancel */ -+ -+ return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); -+} -+ -+/*-******************************************************** -+* bitStream decoding -+**********************************************************/ -+/*! BIT_initDStream() : -+* Initialize a BIT_DStream_t. -+* `bitD` : a pointer to an already allocated BIT_DStream_t structure. -+* `srcSize` must be the *exact* size of the bitStream, in bytes. -+* @return : size of stream (== srcSize) or an errorCode if a problem is detected -+*/ -+ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize) -+{ -+ if (srcSize < 1) { -+ memset(bitD, 0, sizeof(*bitD)); -+ return ERROR(srcSize_wrong); -+ } -+ -+ if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ -+ bitD->start = (const char *)srcBuffer; -+ bitD->ptr = (const char *)srcBuffer + srcSize - sizeof(bitD->bitContainer); -+ bitD->bitContainer = ZSTD_readLEST(bitD->ptr); -+ { -+ BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1]; -+ bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ -+ if (lastByte == 0) -+ return ERROR(GENERIC); /* endMark not present */ -+ } -+ } else { -+ bitD->start = (const char *)srcBuffer; -+ bitD->ptr = bitD->start; -+ bitD->bitContainer = *(const BYTE *)(bitD->start); -+ switch (srcSize) { -+ case 7: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[6]) << (sizeof(bitD->bitContainer) * 8 - 16); -+ case 6: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[5]) << (sizeof(bitD->bitContainer) * 8 - 24); -+ case 5: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[4]) << (sizeof(bitD->bitContainer) * 8 - 32); -+ case 4: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[3]) << 24; -+ case 3: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[2]) << 16; -+ case 2: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[1]) << 8; -+ default:; -+ } -+ { -+ BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1]; -+ bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; -+ if (lastByte == 0) -+ return ERROR(GENERIC); /* endMark not present */ -+ } -+ bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize) * 8; -+ } -+ -+ return srcSize; -+} -+ -+ZSTD_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) { return bitContainer >> start; } -+ -+ZSTD_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) { return (bitContainer >> start) & BIT_mask[nbBits]; } -+ -+ZSTD_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) { return bitContainer & BIT_mask[nbBits]; } -+ -+/*! BIT_lookBits() : -+ * Provides next n bits from local register. -+ * local register is not modified. -+ * On 32-bits, maxNbBits==24. -+ * On 64-bits, maxNbBits==56. -+ * @return : value extracted -+ */ -+ZSTD_STATIC size_t BIT_lookBits(const BIT_DStream_t *bitD, U32 nbBits) -+{ -+ U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1; -+ return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask - nbBits) & bitMask); -+} -+ -+/*! BIT_lookBitsFast() : -+* unsafe version; only works only if nbBits >= 1 */ -+ZSTD_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t *bitD, U32 nbBits) -+{ -+ U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1; -+ return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask + 1) - nbBits) & bitMask); -+} -+ -+ZSTD_STATIC void BIT_skipBits(BIT_DStream_t *bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; } -+ -+/*! BIT_readBits() : -+ * Read (consume) next n bits from local register and update. -+ * Pay attention to not read more than nbBits contained into local register. -+ * @return : extracted value. -+ */ -+ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, U32 nbBits) -+{ -+ size_t const value = BIT_lookBits(bitD, nbBits); -+ BIT_skipBits(bitD, nbBits); -+ return value; -+} -+ -+/*! BIT_readBitsFast() : -+* unsafe version; only works only if nbBits >= 1 */ -+ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, U32 nbBits) -+{ -+ size_t const value = BIT_lookBitsFast(bitD, nbBits); -+ BIT_skipBits(bitD, nbBits); -+ return value; -+} -+ -+/*! BIT_reloadDStream() : -+* Refill `bitD` from buffer previously set in BIT_initDStream() . -+* This function is safe, it guarantees it will not read beyond src buffer. -+* @return : status of `BIT_DStream_t` internal register. -+ if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ -+ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD) -+{ -+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer) * 8)) /* should not happen => corruption detected */ -+ return BIT_DStream_overflow; -+ -+ if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { -+ bitD->ptr -= bitD->bitsConsumed >> 3; -+ bitD->bitsConsumed &= 7; -+ bitD->bitContainer = ZSTD_readLEST(bitD->ptr); -+ return BIT_DStream_unfinished; -+ } -+ if (bitD->ptr == bitD->start) { -+ if (bitD->bitsConsumed < sizeof(bitD->bitContainer) * 8) -+ return BIT_DStream_endOfBuffer; -+ return BIT_DStream_completed; -+ } -+ { -+ U32 nbBytes = bitD->bitsConsumed >> 3; -+ BIT_DStream_status result = BIT_DStream_unfinished; -+ if (bitD->ptr - nbBytes < bitD->start) { -+ nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ -+ result = BIT_DStream_endOfBuffer; -+ } -+ bitD->ptr -= nbBytes; -+ bitD->bitsConsumed -= nbBytes * 8; -+ bitD->bitContainer = ZSTD_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ -+ return result; -+ } -+} -+ -+/*! BIT_endOfDStream() : -+* @return Tells if DStream has exactly reached its end (all bits consumed). -+*/ -+ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *DStream) -+{ -+ return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer) * 8)); -+} -+ -+#endif /* BITSTREAM_H_MODULE */ -diff --git a/lib/zstd/compress.c b/lib/zstd/compress.c -new file mode 100644 -index 0000000..ff18ae6 ---- /dev/null -+++ b/lib/zstd/compress.c -@@ -0,0 +1,3482 @@ -+/** -+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. -+ * All rights reserved. -+ * -+ * This source code is licensed under the BSD-style license found in the -+ * LICENSE file in the root directory of https://github.com/facebook/zstd. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ */ -+ -+/*-************************************* -+* Dependencies -+***************************************/ -+#include "fse.h" -+#include "huf.h" -+#include "mem.h" -+#include "zstd_internal.h" /* includes zstd.h */ -+#include <linux/kernel.h> -+#include <linux/module.h> -+#include <linux/string.h> /* memset */ -+ -+/*-************************************* -+* Constants -+***************************************/ -+static const U32 g_searchStrength = 8; /* control skip over incompressible data */ -+#define HASH_READ_SIZE 8 -+typedef enum { ZSTDcs_created = 0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; -+ -+/*-************************************* -+* Helper functions -+***************************************/ -+size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; } -+ -+/*-************************************* -+* Sequence storage -+***************************************/ -+static void ZSTD_resetSeqStore(seqStore_t *ssPtr) -+{ -+ ssPtr->lit = ssPtr->litStart; -+ ssPtr->sequences = ssPtr->sequencesStart; -+ ssPtr->longLengthID = 0; -+} -+ -+/*-************************************* -+* Context memory management -+***************************************/ -+struct ZSTD_CCtx_s { -+ const BYTE *nextSrc; /* next block here to continue on curr prefix */ -+ const BYTE *base; /* All regular indexes relative to this position */ -+ const BYTE *dictBase; /* extDict indexes relative to this position */ -+ U32 dictLimit; /* below that point, need extDict */ -+ U32 lowLimit; /* below that point, no more data */ -+ U32 nextToUpdate; /* index from which to continue dictionary update */ -+ U32 nextToUpdate3; /* index from which to continue dictionary update */ -+ U32 hashLog3; /* dispatch table : larger == faster, more memory */ -+ U32 loadedDictEnd; /* index of end of dictionary */ -+ U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */ -+ U32 forceRawDict; /* Force loading dictionary in "content-only" mode (no header analysis) */ -+ ZSTD_compressionStage_e stage; -+ U32 rep[ZSTD_REP_NUM]; -+ U32 repToConfirm[ZSTD_REP_NUM]; -+ U32 dictID; -+ ZSTD_parameters params; -+ void *workSpace; -+ size_t workSpaceSize; -+ size_t blockSize; -+ U64 frameContentSize; -+ struct xxh64_state xxhState; -+ ZSTD_customMem customMem; -+ -+ seqStore_t seqStore; /* sequences storage ptrs */ -+ U32 *hashTable; -+ U32 *hashTable3; -+ U32 *chainTable; -+ HUF_CElt *hufTable; -+ U32 flagStaticTables; -+ HUF_repeat flagStaticHufTable; -+ FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; -+ FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; -+ FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; -+ unsigned tmpCounters[HUF_COMPRESS_WORKSPACE_SIZE_U32]; -+}; -+ -+size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams) -+{ -+ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog); -+ U32 const divider = (cParams.searchLength == 3) ? 3 : 4; -+ size_t const maxNbSeq = blockSize / divider; -+ size_t const tokenSpace = blockSize + 11 * maxNbSeq; -+ size_t const chainSize = (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog); -+ size_t const hSize = ((size_t)1) << cParams.hashLog; -+ U32 const hashLog3 = (cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog); -+ size_t const h3Size = ((size_t)1) << hashLog3; -+ size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); -+ size_t const optSpace = -+ ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) + (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); -+ size_t const workspaceSize = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace + -+ (((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); -+ -+ return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_CCtx)) + ZSTD_ALIGN(workspaceSize); -+} -+ -+static ZSTD_CCtx *ZSTD_createCCtx_advanced(ZSTD_customMem customMem) -+{ -+ ZSTD_CCtx *cctx; -+ if (!customMem.customAlloc || !customMem.customFree) -+ return NULL; -+ cctx = (ZSTD_CCtx *)ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); -+ if (!cctx) -+ return NULL; -+ memset(cctx, 0, sizeof(ZSTD_CCtx)); -+ cctx->customMem = customMem; -+ return cctx; -+} -+ -+ZSTD_CCtx *ZSTD_initCCtx(void *workspace, size_t workspaceSize) -+{ -+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); -+ ZSTD_CCtx *cctx = ZSTD_createCCtx_advanced(stackMem); -+ if (cctx) { -+ cctx->workSpace = ZSTD_stackAllocAll(cctx->customMem.opaque, &cctx->workSpaceSize); -+ } -+ return cctx; -+} -+ -+size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx) -+{ -+ if (cctx == NULL) -+ return 0; /* support free on NULL */ -+ ZSTD_free(cctx->workSpace, cctx->customMem); -+ ZSTD_free(cctx, cctx->customMem); -+ return 0; /* reserved as a potential error code in the future */ -+} -+ -+const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx) /* hidden interface */ { return &(ctx->seqStore); } -+ -+static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx *cctx) { return cctx->params; } -+ -+/** ZSTD_checkParams() : -+ ensure param values remain within authorized range. -+ @return : 0, or an error code if one value is beyond authorized range */ -+size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) -+{ -+#define CLAMPCHECK(val, min, max) \ -+ { \ -+ if ((val < min) | (val > max)) \ -+ return ERROR(compressionParameter_unsupported); \ -+ } -+ CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); -+ CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); -+ CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); -+ CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); -+ CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); -+ CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); -+ if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2) -+ return ERROR(compressionParameter_unsupported); -+ return 0; -+} -+ -+/** ZSTD_cycleLog() : -+ * condition for correct operation : hashLog > 1 */ -+static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) -+{ -+ U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); -+ return hashLog - btScale; -+} -+ -+/** ZSTD_adjustCParams() : -+ optimize `cPar` for a given input (`srcSize` and `dictSize`). -+ mostly downsizing to reduce memory consumption and initialization. -+ Both `srcSize` and `dictSize` are optional (use 0 if unknown), -+ but if both are 0, no optimization can be done. -+ Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ -+ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) -+{ -+ if (srcSize + dictSize == 0) -+ return cPar; /* no size information available : no adjustment */ -+ -+ /* resize params, to use less memory when necessary */ -+ { -+ U32 const minSrcSize = (srcSize == 0) ? 500 : 0; -+ U64 const rSize = srcSize + dictSize + minSrcSize; -+ if (rSize < ((U64)1 << ZSTD_WINDOWLOG_MAX)) { -+ U32 const srcLog = MAX(ZSTD_HASHLOG_MIN, ZSTD_highbit32((U32)(rSize)-1) + 1); -+ if (cPar.windowLog > srcLog) -+ cPar.windowLog = srcLog; -+ } -+ } -+ if (cPar.hashLog > cPar.windowLog) -+ cPar.hashLog = cPar.windowLog; -+ { -+ U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); -+ if (cycleLog > cPar.windowLog) -+ cPar.chainLog -= (cycleLog - cPar.windowLog); -+ } -+ -+ if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) -+ cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ -+ -+ return cPar; -+} -+ -+static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2) -+{ -+ return (param1.cParams.hashLog == param2.cParams.hashLog) & (param1.cParams.chainLog == param2.cParams.chainLog) & -+ (param1.cParams.strategy == param2.cParams.strategy) & ((param1.cParams.searchLength == 3) == (param2.cParams.searchLength == 3)); -+} -+ -+/*! ZSTD_continueCCtx() : -+ reuse CCtx without reset (note : requires no dictionary) */ -+static size_t ZSTD_continueCCtx(ZSTD_CCtx *cctx, ZSTD_parameters params, U64 frameContentSize) -+{ -+ U32 const end = (U32)(cctx->nextSrc - cctx->base); -+ cctx->params = params; -+ cctx->frameContentSize = frameContentSize; -+ cctx->lowLimit = end; -+ cctx->dictLimit = end; -+ cctx->nextToUpdate = end + 1; -+ cctx->stage = ZSTDcs_init; -+ cctx->dictID = 0; -+ cctx->loadedDictEnd = 0; -+ { -+ int i; -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ cctx->rep[i] = repStartValue[i]; -+ } -+ cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */ -+ xxh64_reset(&cctx->xxhState, 0); -+ return 0; -+} -+ -+typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e; -+ -+/*! ZSTD_resetCCtx_advanced() : -+ note : `params` must be validated */ -+static size_t ZSTD_resetCCtx_advanced(ZSTD_CCtx *zc, ZSTD_parameters params, U64 frameContentSize, ZSTD_compResetPolicy_e const crp) -+{ -+ if (crp == ZSTDcrp_continue) -+ if (ZSTD_equivalentParams(params, zc->params)) { -+ zc->flagStaticTables = 0; -+ zc->flagStaticHufTable = HUF_repeat_none; -+ return ZSTD_continueCCtx(zc, params, frameContentSize); -+ } -+ -+ { -+ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog); -+ U32 const divider = (params.cParams.searchLength == 3) ? 3 : 4; -+ size_t const maxNbSeq = blockSize / divider; -+ size_t const tokenSpace = blockSize + 11 * maxNbSeq; -+ size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); -+ size_t const hSize = ((size_t)1) << params.cParams.hashLog; -+ U32 const hashLog3 = (params.cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog); -+ size_t const h3Size = ((size_t)1) << hashLog3; -+ size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); -+ void *ptr; -+ -+ /* Check if workSpace is large enough, alloc a new one if needed */ -+ { -+ size_t const optSpace = ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) + -+ (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); -+ size_t const neededSpace = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace + -+ (((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); -+ if (zc->workSpaceSize < neededSpace) { -+ ZSTD_free(zc->workSpace, zc->customMem); -+ zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); -+ if (zc->workSpace == NULL) -+ return ERROR(memory_allocation); -+ zc->workSpaceSize = neededSpace; -+ } -+ } -+ -+ if (crp != ZSTDcrp_noMemset) -+ memset(zc->workSpace, 0, tableSpace); /* reset tables only */ -+ xxh64_reset(&zc->xxhState, 0); -+ zc->hashLog3 = hashLog3; -+ zc->hashTable = (U32 *)(zc->workSpace); -+ zc->chainTable = zc->hashTable + hSize; -+ zc->hashTable3 = zc->chainTable + chainSize; -+ ptr = zc->hashTable3 + h3Size; -+ zc->hufTable = (HUF_CElt *)ptr; -+ zc->flagStaticTables = 0; -+ zc->flagStaticHufTable = HUF_repeat_none; -+ ptr = ((U32 *)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */ -+ -+ zc->nextToUpdate = 1; -+ zc->nextSrc = NULL; -+ zc->base = NULL; -+ zc->dictBase = NULL; -+ zc->dictLimit = 0; -+ zc->lowLimit = 0; -+ zc->params = params; -+ zc->blockSize = blockSize; -+ zc->frameContentSize = frameContentSize; -+ { -+ int i; -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ zc->rep[i] = repStartValue[i]; -+ } -+ -+ if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) { -+ zc->seqStore.litFreq = (U32 *)ptr; -+ zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1 << Litbits); -+ zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL + 1); -+ zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML + 1); -+ ptr = zc->seqStore.offCodeFreq + (MaxOff + 1); -+ zc->seqStore.matchTable = (ZSTD_match_t *)ptr; -+ ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM + 1; -+ zc->seqStore.priceTable = (ZSTD_optimal_t *)ptr; -+ ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM + 1; -+ zc->seqStore.litLengthSum = 0; -+ } -+ zc->seqStore.sequencesStart = (seqDef *)ptr; -+ ptr = zc->seqStore.sequencesStart + maxNbSeq; -+ zc->seqStore.llCode = (BYTE *)ptr; -+ zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; -+ zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; -+ zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; -+ -+ zc->stage = ZSTDcs_init; -+ zc->dictID = 0; -+ zc->loadedDictEnd = 0; -+ -+ return 0; -+ } -+} -+ -+/* ZSTD_invalidateRepCodes() : -+ * ensures next compression will not use repcodes from previous block. -+ * Note : only works with regular variant; -+ * do not use with extDict variant ! */ -+void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx) -+{ -+ int i; -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ cctx->rep[i] = 0; -+} -+ -+/*! ZSTD_copyCCtx() : -+* Duplicate an existing context `srcCCtx` into another one `dstCCtx`. -+* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). -+* @return : 0, or an error code */ -+size_t ZSTD_copyCCtx(ZSTD_CCtx *dstCCtx, const ZSTD_CCtx *srcCCtx, unsigned long long pledgedSrcSize) -+{ -+ if (srcCCtx->stage != ZSTDcs_init) -+ return ERROR(stage_wrong); -+ -+ memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); -+ { -+ ZSTD_parameters params = srcCCtx->params; -+ params.fParams.contentSizeFlag = (pledgedSrcSize > 0); -+ ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset); -+ } -+ -+ /* copy tables */ -+ { -+ size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); -+ size_t const hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog; -+ size_t const h3Size = (size_t)1 << srcCCtx->hashLog3; -+ size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); -+ memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); -+ } -+ -+ /* copy dictionary offsets */ -+ dstCCtx->nextToUpdate = srcCCtx->nextToUpdate; -+ dstCCtx->nextToUpdate3 = srcCCtx->nextToUpdate3; -+ dstCCtx->nextSrc = srcCCtx->nextSrc; -+ dstCCtx->base = srcCCtx->base; -+ dstCCtx->dictBase = srcCCtx->dictBase; -+ dstCCtx->dictLimit = srcCCtx->dictLimit; -+ dstCCtx->lowLimit = srcCCtx->lowLimit; -+ dstCCtx->loadedDictEnd = srcCCtx->loadedDictEnd; -+ dstCCtx->dictID = srcCCtx->dictID; -+ -+ /* copy entropy tables */ -+ dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; -+ dstCCtx->flagStaticHufTable = srcCCtx->flagStaticHufTable; -+ if (srcCCtx->flagStaticTables) { -+ memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable)); -+ memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable)); -+ memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable)); -+ } -+ if (srcCCtx->flagStaticHufTable) { -+ memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256 * 4); -+ } -+ -+ return 0; -+} -+ -+/*! ZSTD_reduceTable() : -+* reduce table indexes by `reducerValue` */ -+static void ZSTD_reduceTable(U32 *const table, U32 const size, U32 const reducerValue) -+{ -+ U32 u; -+ for (u = 0; u < size; u++) { -+ if (table[u] < reducerValue) -+ table[u] = 0; -+ else -+ table[u] -= reducerValue; -+ } -+} -+ -+/*! ZSTD_reduceIndex() : -+* rescale all indexes to avoid future overflow (indexes are U32) */ -+static void ZSTD_reduceIndex(ZSTD_CCtx *zc, const U32 reducerValue) -+{ -+ { -+ U32 const hSize = 1 << zc->params.cParams.hashLog; -+ ZSTD_reduceTable(zc->hashTable, hSize, reducerValue); -+ } -+ -+ { -+ U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog); -+ ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); -+ } -+ -+ { -+ U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; -+ ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); -+ } -+} -+ -+/*-******************************************************* -+* Block entropic compression -+*********************************************************/ -+ -+/* See doc/zstd_compression_format.md for detailed format description */ -+ -+size_t ZSTD_noCompressBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ if (srcSize + ZSTD_blockHeaderSize > dstCapacity) -+ return ERROR(dstSize_tooSmall); -+ memcpy((BYTE *)dst + ZSTD_blockHeaderSize, src, srcSize); -+ ZSTD_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw); -+ return ZSTD_blockHeaderSize + srcSize; -+} -+ -+static size_t ZSTD_noCompressLiterals(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ BYTE *const ostart = (BYTE * const)dst; -+ U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095); -+ -+ if (srcSize + flSize > dstCapacity) -+ return ERROR(dstSize_tooSmall); -+ -+ switch (flSize) { -+ case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_basic + (srcSize << 3)); break; -+ case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_basic + (1 << 2) + (srcSize << 4))); break; -+ default: /*note : should not be necessary : flSize is within {1,2,3} */ -+ case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_basic + (3 << 2) + (srcSize << 4))); break; -+ } -+ -+ memcpy(ostart + flSize, src, srcSize); -+ return srcSize + flSize; -+} -+ -+static size_t ZSTD_compressRleLiteralsBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ BYTE *const ostart = (BYTE * const)dst; -+ U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095); -+ -+ (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ -+ -+ switch (flSize) { -+ case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_rle + (srcSize << 3)); break; -+ case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_rle + (1 << 2) + (srcSize << 4))); break; -+ default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */ -+ case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_rle + (3 << 2) + (srcSize << 4))); break; -+ } -+ -+ ostart[flSize] = *(const BYTE *)src; -+ return flSize + 1; -+} -+ -+static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; } -+ -+static size_t ZSTD_compressLiterals(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ size_t const minGain = ZSTD_minGain(srcSize); -+ size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); -+ BYTE *const ostart = (BYTE *)dst; -+ U32 singleStream = srcSize < 256; -+ symbolEncodingType_e hType = set_compressed; -+ size_t cLitSize; -+ -+/* small ? don't even attempt compression (speed opt) */ -+#define LITERAL_NOENTROPY 63 -+ { -+ size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY; -+ if (srcSize <= minLitSize) -+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); -+ } -+ -+ if (dstCapacity < lhSize + 1) -+ return ERROR(dstSize_tooSmall); /* not enough space for compression */ -+ { -+ HUF_repeat repeat = zc->flagStaticHufTable; -+ int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0; -+ if (repeat == HUF_repeat_valid && lhSize == 3) -+ singleStream = 1; -+ cLitSize = singleStream ? HUF_compress1X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters, -+ sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat) -+ : HUF_compress4X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters, -+ sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat); -+ if (repeat != HUF_repeat_none) { -+ hType = set_repeat; -+ } /* reused the existing table */ -+ else { -+ zc->flagStaticHufTable = HUF_repeat_check; -+ } /* now have a table to reuse */ -+ } -+ -+ if ((cLitSize == 0) | (cLitSize >= srcSize - minGain)) { -+ zc->flagStaticHufTable = HUF_repeat_none; -+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); -+ } -+ if (cLitSize == 1) { -+ zc->flagStaticHufTable = HUF_repeat_none; -+ return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); -+ } -+ -+ /* Build header */ -+ switch (lhSize) { -+ case 3: /* 2 - 2 - 10 - 10 */ -+ { -+ U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 14); -+ ZSTD_writeLE24(ostart, lhc); -+ break; -+ } -+ case 4: /* 2 - 2 - 14 - 14 */ -+ { -+ U32 const lhc = hType + (2 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 18); -+ ZSTD_writeLE32(ostart, lhc); -+ break; -+ } -+ default: /* should not be necessary, lhSize is only {3,4,5} */ -+ case 5: /* 2 - 2 - 18 - 18 */ -+ { -+ U32 const lhc = hType + (3 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 22); -+ ZSTD_writeLE32(ostart, lhc); -+ ostart[4] = (BYTE)(cLitSize >> 10); -+ break; -+ } -+ } -+ return lhSize + cLitSize; -+} -+ -+static const BYTE LL_Code[64] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, -+ 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, -+ 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24}; -+ -+static const BYTE ML_Code[128] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -+ 26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, -+ 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, -+ 40, 40, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 42, -+ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42}; -+ -+void ZSTD_seqToCodes(const seqStore_t *seqStorePtr) -+{ -+ BYTE const LL_deltaCode = 19; -+ BYTE const ML_deltaCode = 36; -+ const seqDef *const sequences = seqStorePtr->sequencesStart; -+ BYTE *const llCodeTable = seqStorePtr->llCode; -+ BYTE *const ofCodeTable = seqStorePtr->ofCode; -+ BYTE *const mlCodeTable = seqStorePtr->mlCode; -+ U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); -+ U32 u; -+ for (u = 0; u < nbSeq; u++) { -+ U32 const llv = sequences[u].litLength; -+ U32 const mlv = sequences[u].matchLength; -+ llCodeTable[u] = (llv > 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv]; -+ ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset); -+ mlCodeTable[u] = (mlv > 127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv]; -+ } -+ if (seqStorePtr->longLengthID == 1) -+ llCodeTable[seqStorePtr->longLengthPos] = MaxLL; -+ if (seqStorePtr->longLengthID == 2) -+ mlCodeTable[seqStorePtr->longLengthPos] = MaxML; -+} -+ -+ZSTD_STATIC size_t ZSTD_compressSequences_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity) -+{ -+ const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN; -+ const seqStore_t *seqStorePtr = &(zc->seqStore); -+ FSE_CTable *CTable_LitLength = zc->litlengthCTable; -+ FSE_CTable *CTable_OffsetBits = zc->offcodeCTable; -+ FSE_CTable *CTable_MatchLength = zc->matchlengthCTable; -+ U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ -+ const seqDef *const sequences = seqStorePtr->sequencesStart; -+ const BYTE *const ofCodeTable = seqStorePtr->ofCode; -+ const BYTE *const llCodeTable = seqStorePtr->llCode; -+ const BYTE *const mlCodeTable = seqStorePtr->mlCode; -+ BYTE *const ostart = (BYTE *)dst; -+ BYTE *const oend = ostart + dstCapacity; -+ BYTE *op = ostart; -+ size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; -+ BYTE *seqHead; -+ -+ U32 *count; -+ S16 *norm; -+ U32 *workspace; -+ size_t workspaceSize = sizeof(zc->tmpCounters); -+ { -+ size_t spaceUsed32 = 0; -+ count = (U32 *)zc->tmpCounters + spaceUsed32; -+ spaceUsed32 += MaxSeq + 1; -+ norm = (S16 *)((U32 *)zc->tmpCounters + spaceUsed32); -+ spaceUsed32 += ALIGN(sizeof(S16) * (MaxSeq + 1), sizeof(U32)) >> 2; -+ -+ workspace = (U32 *)zc->tmpCounters + spaceUsed32; -+ workspaceSize -= (spaceUsed32 << 2); -+ } -+ -+ /* Compress literals */ -+ { -+ const BYTE *const literals = seqStorePtr->litStart; -+ size_t const litSize = seqStorePtr->lit - literals; -+ size_t const cSize = ZSTD_compressLiterals(zc, op, dstCapacity, literals, litSize); -+ if (ZSTD_isError(cSize)) -+ return cSize; -+ op += cSize; -+ } -+ -+ /* Sequences Header */ -+ if ((oend - op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */) -+ return ERROR(dstSize_tooSmall); -+ if (nbSeq < 0x7F) -+ *op++ = (BYTE)nbSeq; -+ else if (nbSeq < LONGNBSEQ) -+ op[0] = (BYTE)((nbSeq >> 8) + 0x80), op[1] = (BYTE)nbSeq, op += 2; -+ else -+ op[0] = 0xFF, ZSTD_writeLE16(op + 1, (U16)(nbSeq - LONGNBSEQ)), op += 3; -+ if (nbSeq == 0) -+ return op - ostart; -+ -+ /* seqHead : flags for FSE encoding type */ -+ seqHead = op++; -+ -+#define MIN_SEQ_FOR_DYNAMIC_FSE 64 -+#define MAX_SEQ_FOR_STATIC_FSE 1000 -+ -+ /* convert length/distances into codes */ -+ ZSTD_seqToCodes(seqStorePtr); -+ -+ /* CTable for Literal Lengths */ -+ { -+ U32 max = MaxLL; -+ size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace); -+ if ((mostFrequent == nbSeq) && (nbSeq > 2)) { -+ *op++ = llCodeTable[0]; -+ FSE_buildCTable_rle(CTable_LitLength, (BYTE)max); -+ LLtype = set_rle; -+ } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { -+ LLtype = set_repeat; -+ } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog - 1)))) { -+ FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, workspace, workspaceSize); -+ LLtype = set_basic; -+ } else { -+ size_t nbSeq_1 = nbSeq; -+ const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max); -+ if (count[llCodeTable[nbSeq - 1]] > 1) { -+ count[llCodeTable[nbSeq - 1]]--; -+ nbSeq_1--; -+ } -+ FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); -+ { -+ size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ -+ if (FSE_isError(NCountSize)) -+ return NCountSize; -+ op += NCountSize; -+ } -+ FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, workspace, workspaceSize); -+ LLtype = set_compressed; -+ } -+ } -+ -+ /* CTable for Offsets */ -+ { -+ U32 max = MaxOff; -+ size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace); -+ if ((mostFrequent == nbSeq) && (nbSeq > 2)) { -+ *op++ = ofCodeTable[0]; -+ FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max); -+ Offtype = set_rle; -+ } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { -+ Offtype = set_repeat; -+ } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog - 1)))) { -+ FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, workspace, workspaceSize); -+ Offtype = set_basic; -+ } else { -+ size_t nbSeq_1 = nbSeq; -+ const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max); -+ if (count[ofCodeTable[nbSeq - 1]] > 1) { -+ count[ofCodeTable[nbSeq - 1]]--; -+ nbSeq_1--; -+ } -+ FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); -+ { -+ size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ -+ if (FSE_isError(NCountSize)) -+ return NCountSize; -+ op += NCountSize; -+ } -+ FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, workspace, workspaceSize); -+ Offtype = set_compressed; -+ } -+ } -+ -+ /* CTable for MatchLengths */ -+ { -+ U32 max = MaxML; -+ size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace); -+ if ((mostFrequent == nbSeq) && (nbSeq > 2)) { -+ *op++ = *mlCodeTable; -+ FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max); -+ MLtype = set_rle; -+ } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { -+ MLtype = set_repeat; -+ } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog - 1)))) { -+ FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, workspace, workspaceSize); -+ MLtype = set_basic; -+ } else { -+ size_t nbSeq_1 = nbSeq; -+ const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max); -+ if (count[mlCodeTable[nbSeq - 1]] > 1) { -+ count[mlCodeTable[nbSeq - 1]]--; -+ nbSeq_1--; -+ } -+ FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); -+ { -+ size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ -+ if (FSE_isError(NCountSize)) -+ return NCountSize; -+ op += NCountSize; -+ } -+ FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, workspace, workspaceSize); -+ MLtype = set_compressed; -+ } -+ } -+ -+ *seqHead = (BYTE)((LLtype << 6) + (Offtype << 4) + (MLtype << 2)); -+ zc->flagStaticTables = 0; -+ -+ /* Encoding Sequences */ -+ { -+ BIT_CStream_t blockStream; -+ FSE_CState_t stateMatchLength; -+ FSE_CState_t stateOffsetBits; -+ FSE_CState_t stateLitLength; -+ -+ CHECK_E(BIT_initCStream(&blockStream, op, oend - op), dstSize_tooSmall); /* not enough space remaining */ -+ -+ /* first symbols */ -+ FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq - 1]); -+ FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq - 1]); -+ FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq - 1]); -+ BIT_addBits(&blockStream, sequences[nbSeq - 1].litLength, LL_bits[llCodeTable[nbSeq - 1]]); -+ if (ZSTD_32bits()) -+ BIT_flushBits(&blockStream); -+ BIT_addBits(&blockStream, sequences[nbSeq - 1].matchLength, ML_bits[mlCodeTable[nbSeq - 1]]); -+ if (ZSTD_32bits()) -+ BIT_flushBits(&blockStream); -+ if (longOffsets) { -+ U32 const ofBits = ofCodeTable[nbSeq - 1]; -+ int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1); -+ if (extraBits) { -+ BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, extraBits); -+ BIT_flushBits(&blockStream); -+ } -+ BIT_addBits(&blockStream, sequences[nbSeq - 1].offset >> extraBits, ofBits - extraBits); -+ } else { -+ BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, ofCodeTable[nbSeq - 1]); -+ } -+ BIT_flushBits(&blockStream); -+ -+ { -+ size_t n; -+ for (n = nbSeq - 2; n < nbSeq; n--) { /* intentional underflow */ -+ BYTE const llCode = llCodeTable[n]; -+ BYTE const ofCode = ofCodeTable[n]; -+ BYTE const mlCode = mlCodeTable[n]; -+ U32 const llBits = LL_bits[llCode]; -+ U32 const ofBits = ofCode; /* 32b*/ /* 64b*/ -+ U32 const mlBits = ML_bits[mlCode]; -+ /* (7)*/ /* (7)*/ -+ FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ -+ FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */ -+ if (ZSTD_32bits()) -+ BIT_flushBits(&blockStream); /* (7)*/ -+ FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */ -+ if (ZSTD_32bits() || (ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + OffFSELog))) -+ BIT_flushBits(&blockStream); /* (7)*/ -+ BIT_addBits(&blockStream, sequences[n].litLength, llBits); -+ if (ZSTD_32bits() && ((llBits + mlBits) > 24)) -+ BIT_flushBits(&blockStream); -+ BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); -+ if (ZSTD_32bits()) -+ BIT_flushBits(&blockStream); /* (7)*/ -+ if (longOffsets) { -+ int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1); -+ if (extraBits) { -+ BIT_addBits(&blockStream, sequences[n].offset, extraBits); -+ BIT_flushBits(&blockStream); /* (7)*/ -+ } -+ BIT_addBits(&blockStream, sequences[n].offset >> extraBits, ofBits - extraBits); /* 31 */ -+ } else { -+ BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ -+ } -+ BIT_flushBits(&blockStream); /* (7)*/ -+ } -+ } -+ -+ FSE_flushCState(&blockStream, &stateMatchLength); -+ FSE_flushCState(&blockStream, &stateOffsetBits); -+ FSE_flushCState(&blockStream, &stateLitLength); -+ -+ { -+ size_t const streamSize = BIT_closeCStream(&blockStream); -+ if (streamSize == 0) -+ return ERROR(dstSize_tooSmall); /* not enough space */ -+ op += streamSize; -+ } -+ } -+ return op - ostart; -+} -+ -+ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, size_t srcSize) -+{ -+ size_t const cSize = ZSTD_compressSequences_internal(zc, dst, dstCapacity); -+ size_t const minGain = ZSTD_minGain(srcSize); -+ size_t const maxCSize = srcSize - minGain; -+ /* If the srcSize <= dstCapacity, then there is enough space to write a -+ * raw uncompressed block. Since we ran out of space, the block must not -+ * be compressible, so fall back to a raw uncompressed block. -+ */ -+ int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity; -+ int i; -+ -+ if (ZSTD_isError(cSize) && !uncompressibleError) -+ return cSize; -+ if (cSize >= maxCSize || uncompressibleError) { -+ zc->flagStaticHufTable = HUF_repeat_none; -+ return 0; -+ } -+ /* confirm repcodes */ -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ zc->rep[i] = zc->repToConfirm[i]; -+ return cSize; -+} -+ -+/*! ZSTD_storeSeq() : -+ Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. -+ `offsetCode` : distance to match, or 0 == repCode. -+ `matchCode` : matchLength - MINMATCH -+*/ -+ZSTD_STATIC void ZSTD_storeSeq(seqStore_t *seqStorePtr, size_t litLength, const void *literals, U32 offsetCode, size_t matchCode) -+{ -+ /* copy Literals */ -+ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); -+ seqStorePtr->lit += litLength; -+ -+ /* literal Length */ -+ if (litLength > 0xFFFF) { -+ seqStorePtr->longLengthID = 1; -+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); -+ } -+ seqStorePtr->sequences[0].litLength = (U16)litLength; -+ -+ /* match offset */ -+ seqStorePtr->sequences[0].offset = offsetCode + 1; -+ -+ /* match Length */ -+ if (matchCode > 0xFFFF) { -+ seqStorePtr->longLengthID = 2; -+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); -+ } -+ seqStorePtr->sequences[0].matchLength = (U16)matchCode; -+ -+ seqStorePtr->sequences++; -+} -+ -+/*-************************************* -+* Match length counter -+***************************************/ -+static unsigned ZSTD_NbCommonBytes(register size_t val) -+{ -+ if (ZSTD_isLittleEndian()) { -+ if (ZSTD_64bits()) { -+ return (__builtin_ctzll((U64)val) >> 3); -+ } else { /* 32 bits */ -+ return (__builtin_ctz((U32)val) >> 3); -+ } -+ } else { /* Big Endian CPU */ -+ if (ZSTD_64bits()) { -+ return (__builtin_clzll(val) >> 3); -+ } else { /* 32 bits */ -+ return (__builtin_clz((U32)val) >> 3); -+ } -+ } -+} -+ -+static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch, const BYTE *const pInLimit) -+{ -+ const BYTE *const pStart = pIn; -+ const BYTE *const pInLoopLimit = pInLimit - (sizeof(size_t) - 1); -+ -+ while (pIn < pInLoopLimit) { -+ size_t const diff = ZSTD_readST(pMatch) ^ ZSTD_readST(pIn); -+ if (!diff) { -+ pIn += sizeof(size_t); -+ pMatch += sizeof(size_t); -+ continue; -+ } -+ pIn += ZSTD_NbCommonBytes(diff); -+ return (size_t)(pIn - pStart); -+ } -+ if (ZSTD_64bits()) -+ if ((pIn < (pInLimit - 3)) && (ZSTD_read32(pMatch) == ZSTD_read32(pIn))) { -+ pIn += 4; -+ pMatch += 4; -+ } -+ if ((pIn < (pInLimit - 1)) && (ZSTD_read16(pMatch) == ZSTD_read16(pIn))) { -+ pIn += 2; -+ pMatch += 2; -+ } -+ if ((pIn < pInLimit) && (*pMatch == *pIn)) -+ pIn++; -+ return (size_t)(pIn - pStart); -+} -+ -+/** ZSTD_count_2segments() : -+* can count match length with `ip` & `match` in 2 different segments. -+* convention : on reaching mEnd, match count continue starting from iStart -+*/ -+static size_t ZSTD_count_2segments(const BYTE *ip, const BYTE *match, const BYTE *iEnd, const BYTE *mEnd, const BYTE *iStart) -+{ -+ const BYTE *const vEnd = MIN(ip + (mEnd - match), iEnd); -+ size_t const matchLength = ZSTD_count(ip, match, vEnd); -+ if (match + matchLength != mEnd) -+ return matchLength; -+ return matchLength + ZSTD_count(ip + matchLength, iStart, iEnd); -+} -+ -+/*-************************************* -+* Hashes -+***************************************/ -+static const U32 prime3bytes = 506832829U; -+static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32 - 24)) * prime3bytes) >> (32 - h); } -+ZSTD_STATIC size_t ZSTD_hash3Ptr(const void *ptr, U32 h) { return ZSTD_hash3(ZSTD_readLE32(ptr), h); } /* only in zstd_opt.h */ -+ -+static const U32 prime4bytes = 2654435761U; -+static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32 - h); } -+static size_t ZSTD_hash4Ptr(const void *ptr, U32 h) { return ZSTD_hash4(ZSTD_read32(ptr), h); } -+ -+static const U64 prime5bytes = 889523592379ULL; -+static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64 - 40)) * prime5bytes) >> (64 - h)); } -+static size_t ZSTD_hash5Ptr(const void *p, U32 h) { return ZSTD_hash5(ZSTD_readLE64(p), h); } -+ -+static const U64 prime6bytes = 227718039650203ULL; -+static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64 - 48)) * prime6bytes) >> (64 - h)); } -+static size_t ZSTD_hash6Ptr(const void *p, U32 h) { return ZSTD_hash6(ZSTD_readLE64(p), h); } -+ -+static const U64 prime7bytes = 58295818150454627ULL; -+static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64 - 56)) * prime7bytes) >> (64 - h)); } -+static size_t ZSTD_hash7Ptr(const void *p, U32 h) { return ZSTD_hash7(ZSTD_readLE64(p), h); } -+ -+static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; -+static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u)*prime8bytes) >> (64 - h)); } -+static size_t ZSTD_hash8Ptr(const void *p, U32 h) { return ZSTD_hash8(ZSTD_readLE64(p), h); } -+ -+static size_t ZSTD_hashPtr(const void *p, U32 hBits, U32 mls) -+{ -+ switch (mls) { -+ // case 3: return ZSTD_hash3Ptr(p, hBits); -+ default: -+ case 4: return ZSTD_hash4Ptr(p, hBits); -+ case 5: return ZSTD_hash5Ptr(p, hBits); -+ case 6: return ZSTD_hash6Ptr(p, hBits); -+ case 7: return ZSTD_hash7Ptr(p, hBits); -+ case 8: return ZSTD_hash8Ptr(p, hBits); -+ } -+} -+ -+/*-************************************* -+* Fast Scan -+***************************************/ -+static void ZSTD_fillHashTable(ZSTD_CCtx *zc, const void *end, const U32 mls) -+{ -+ U32 *const hashTable = zc->hashTable; -+ U32 const hBits = zc->params.cParams.hashLog; -+ const BYTE *const base = zc->base; -+ const BYTE *ip = base + zc->nextToUpdate; -+ const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE; -+ const size_t fastHashFillStep = 3; -+ -+ while (ip <= iend) { -+ hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base); -+ ip += fastHashFillStep; -+ } -+} -+ -+FORCE_INLINE -+void ZSTD_compressBlock_fast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls) -+{ -+ U32 *const hashTable = cctx->hashTable; -+ U32 const hBits = cctx->params.cParams.hashLog; -+ seqStore_t *seqStorePtr = &(cctx->seqStore); -+ const BYTE *const base = cctx->base; -+ const BYTE *const istart = (const BYTE *)src; -+ const BYTE *ip = istart; -+ const BYTE *anchor = istart; -+ const U32 lowestIndex = cctx->dictLimit; -+ const BYTE *const lowest = base + lowestIndex; -+ const BYTE *const iend = istart + srcSize; -+ const BYTE *const ilimit = iend - HASH_READ_SIZE; -+ U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1]; -+ U32 offsetSaved = 0; -+ -+ /* init */ -+ ip += (ip == lowest); -+ { -+ U32 const maxRep = (U32)(ip - lowest); -+ if (offset_2 > maxRep) -+ offsetSaved = offset_2, offset_2 = 0; -+ if (offset_1 > maxRep) -+ offsetSaved = offset_1, offset_1 = 0; -+ } -+ -+ /* Main Search Loop */ -+ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ -+ size_t mLength; -+ size_t const h = ZSTD_hashPtr(ip, hBits, mls); -+ U32 const curr = (U32)(ip - base); -+ U32 const matchIndex = hashTable[h]; -+ const BYTE *match = base + matchIndex; -+ hashTable[h] = curr; /* update hash table */ -+ -+ if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) { -+ mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4; -+ ip++; -+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH); -+ } else { -+ U32 offset; -+ if ((matchIndex <= lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) { -+ ip += ((ip - anchor) >> g_searchStrength) + 1; -+ continue; -+ } -+ mLength = ZSTD_count(ip + 4, match + 4, iend) + 4; -+ offset = (U32)(ip - match); -+ while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) { -+ ip--; -+ match--; -+ mLength++; -+ } /* catch up */ -+ offset_2 = offset_1; -+ offset_1 = offset; -+ -+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); -+ } -+ -+ /* match found */ -+ ip += mLength; -+ anchor = ip; -+ -+ if (ip <= ilimit) { -+ /* Fill Table */ -+ hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2; /* here because curr+2 could be > iend-8 */ -+ hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base); -+ /* check immediate repcode */ -+ while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) { -+ /* store sequence */ -+ size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4; -+ { -+ U32 const tmpOff = offset_2; -+ offset_2 = offset_1; -+ offset_1 = tmpOff; -+ } /* swap offset_2 <=> offset_1 */ -+ hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base); -+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH); -+ ip += rLength; -+ anchor = ip; -+ continue; /* faster when present ... (?) */ -+ } -+ } -+ } -+ -+ /* save reps for next block */ -+ cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; -+ cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; -+ -+ /* Last Literals */ -+ { -+ size_t const lastLLSize = iend - anchor; -+ memcpy(seqStorePtr->lit, anchor, lastLLSize); -+ seqStorePtr->lit += lastLLSize; -+ } -+} -+ -+static void ZSTD_compressBlock_fast(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -+{ -+ const U32 mls = ctx->params.cParams.searchLength; -+ switch (mls) { -+ default: /* includes case 3 */ -+ case 4: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); return; -+ case 5: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); return; -+ case 6: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); return; -+ case 7: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); return; -+ } -+} -+ -+static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls) -+{ -+ U32 *hashTable = ctx->hashTable; -+ const U32 hBits = ctx->params.cParams.hashLog; -+ seqStore_t *seqStorePtr = &(ctx->seqStore); -+ const BYTE *const base = ctx->base; -+ const BYTE *const dictBase = ctx->dictBase; -+ const BYTE *const istart = (const BYTE *)src; -+ const BYTE *ip = istart; -+ const BYTE *anchor = istart; -+ const U32 lowestIndex = ctx->lowLimit; -+ const BYTE *const dictStart = dictBase + lowestIndex; -+ const U32 dictLimit = ctx->dictLimit; -+ const BYTE *const lowPrefixPtr = base + dictLimit; -+ const BYTE *const dictEnd = dictBase + dictLimit; -+ const BYTE *const iend = istart + srcSize; -+ const BYTE *const ilimit = iend - 8; -+ U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1]; -+ -+ /* Search Loop */ -+ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ -+ const size_t h = ZSTD_hashPtr(ip, hBits, mls); -+ const U32 matchIndex = hashTable[h]; -+ const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base; -+ const BYTE *match = matchBase + matchIndex; -+ const U32 curr = (U32)(ip - base); -+ const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ -+ const BYTE *repBase = repIndex < dictLimit ? dictBase : base; -+ const BYTE *repMatch = repBase + repIndex; -+ size_t mLength; -+ hashTable[h] = curr; /* update hash table */ -+ -+ if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) && -+ (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) { -+ const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend; -+ mLength = ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32; -+ ip++; -+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH); -+ } else { -+ if ((matchIndex < lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) { -+ ip += ((ip - anchor) >> g_searchStrength) + 1; -+ continue; -+ } -+ { -+ const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend; -+ const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; -+ U32 offset; -+ mLength = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32; -+ while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) { -+ ip--; -+ match--; -+ mLength++; -+ } /* catch up */ -+ offset = curr - matchIndex; -+ offset_2 = offset_1; -+ offset_1 = offset; -+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); -+ } -+ } -+ -+ /* found a match : store it */ -+ ip += mLength; -+ anchor = ip; -+ -+ if (ip <= ilimit) { -+ /* Fill Table */ -+ hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2; -+ hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base); -+ /* check immediate repcode */ -+ while (ip <= ilimit) { -+ U32 const curr2 = (U32)(ip - base); -+ U32 const repIndex2 = curr2 - offset_2; -+ const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; -+ if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ -+ && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) { -+ const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; -+ size_t repLength2 = -+ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; -+ U32 tmpOffset = offset_2; -+ offset_2 = offset_1; -+ offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ -+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH); -+ hashTable[ZSTD_hashPtr(ip, hBits, mls)] = curr2; -+ ip += repLength2; -+ anchor = ip; -+ continue; -+ } -+ break; -+ } -+ } -+ } -+ -+ /* save reps for next block */ -+ ctx->repToConfirm[0] = offset_1; -+ ctx->repToConfirm[1] = offset_2; -+ -+ /* Last Literals */ -+ { -+ size_t const lastLLSize = iend - anchor; -+ memcpy(seqStorePtr->lit, anchor, lastLLSize); -+ seqStorePtr->lit += lastLLSize; -+ } -+} -+ -+static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -+{ -+ U32 const mls = ctx->params.cParams.searchLength; -+ switch (mls) { -+ default: /* includes case 3 */ -+ case 4: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); return; -+ case 5: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); return; -+ case 6: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); return; -+ case 7: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); return; -+ } -+} -+ -+/*-************************************* -+* Double Fast -+***************************************/ -+static void ZSTD_fillDoubleHashTable(ZSTD_CCtx *cctx, const void *end, const U32 mls) -+{ -+ U32 *const hashLarge = cctx->hashTable; -+ U32 const hBitsL = cctx->params.cParams.hashLog; -+ U32 *const hashSmall = cctx->chainTable; -+ U32 const hBitsS = cctx->params.cParams.chainLog; -+ const BYTE *const base = cctx->base; -+ const BYTE *ip = base + cctx->nextToUpdate; -+ const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE; -+ const size_t fastHashFillStep = 3; -+ -+ while (ip <= iend) { -+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base); -+ hashLarge[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base); -+ ip += fastHashFillStep; -+ } -+} -+ -+FORCE_INLINE -+void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls) -+{ -+ U32 *const hashLong = cctx->hashTable; -+ const U32 hBitsL = cctx->params.cParams.hashLog; -+ U32 *const hashSmall = cctx->chainTable; -+ const U32 hBitsS = cctx->params.cParams.chainLog; -+ seqStore_t *seqStorePtr = &(cctx->seqStore); -+ const BYTE *const base = cctx->base; -+ const BYTE *const istart = (const BYTE *)src; -+ const BYTE *ip = istart; -+ const BYTE *anchor = istart; -+ const U32 lowestIndex = cctx->dictLimit; -+ const BYTE *const lowest = base + lowestIndex; -+ const BYTE *const iend = istart + srcSize; -+ const BYTE *const ilimit = iend - HASH_READ_SIZE; -+ U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1]; -+ U32 offsetSaved = 0; -+ -+ /* init */ -+ ip += (ip == lowest); -+ { -+ U32 const maxRep = (U32)(ip - lowest); -+ if (offset_2 > maxRep) -+ offsetSaved = offset_2, offset_2 = 0; -+ if (offset_1 > maxRep) -+ offsetSaved = offset_1, offset_1 = 0; -+ } -+ -+ /* Main Search Loop */ -+ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ -+ size_t mLength; -+ size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); -+ size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); -+ U32 const curr = (U32)(ip - base); -+ U32 const matchIndexL = hashLong[h2]; -+ U32 const matchIndexS = hashSmall[h]; -+ const BYTE *matchLong = base + matchIndexL; -+ const BYTE *match = base + matchIndexS; -+ hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ -+ -+ if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) { /* note : by construction, offset_1 <= curr */ -+ mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4; -+ ip++; -+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH); -+ } else { -+ U32 offset; -+ if ((matchIndexL > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) { -+ mLength = ZSTD_count(ip + 8, matchLong + 8, iend) + 8; -+ offset = (U32)(ip - matchLong); -+ while (((ip > anchor) & (matchLong > lowest)) && (ip[-1] == matchLong[-1])) { -+ ip--; -+ matchLong--; -+ mLength++; -+ } /* catch up */ -+ } else if ((matchIndexS > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) { -+ size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8); -+ U32 const matchIndex3 = hashLong[h3]; -+ const BYTE *match3 = base + matchIndex3; -+ hashLong[h3] = curr + 1; -+ if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) { -+ mLength = ZSTD_count(ip + 9, match3 + 8, iend) + 8; -+ ip++; -+ offset = (U32)(ip - match3); -+ while (((ip > anchor) & (match3 > lowest)) && (ip[-1] == match3[-1])) { -+ ip--; -+ match3--; -+ mLength++; -+ } /* catch up */ -+ } else { -+ mLength = ZSTD_count(ip + 4, match + 4, iend) + 4; -+ offset = (U32)(ip - match); -+ while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) { -+ ip--; -+ match--; -+ mLength++; -+ } /* catch up */ -+ } -+ } else { -+ ip += ((ip - anchor) >> g_searchStrength) + 1; -+ continue; -+ } -+ -+ offset_2 = offset_1; -+ offset_1 = offset; -+ -+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); -+ } -+ -+ /* match found */ -+ ip += mLength; -+ anchor = ip; -+ -+ if (ip <= ilimit) { -+ /* Fill Table */ -+ hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] = -+ curr + 2; /* here because curr+2 could be > iend-8 */ -+ hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base); -+ -+ /* check immediate repcode */ -+ while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) { -+ /* store sequence */ -+ size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4; -+ { -+ U32 const tmpOff = offset_2; -+ offset_2 = offset_1; -+ offset_1 = tmpOff; -+ } /* swap offset_2 <=> offset_1 */ -+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base); -+ hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base); -+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH); -+ ip += rLength; -+ anchor = ip; -+ continue; /* faster when present ... (?) */ -+ } -+ } -+ } -+ -+ /* save reps for next block */ -+ cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; -+ cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; -+ -+ /* Last Literals */ -+ { -+ size_t const lastLLSize = iend - anchor; -+ memcpy(seqStorePtr->lit, anchor, lastLLSize); -+ seqStorePtr->lit += lastLLSize; -+ } -+} -+ -+static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -+{ -+ const U32 mls = ctx->params.cParams.searchLength; -+ switch (mls) { -+ default: /* includes case 3 */ -+ case 4: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return; -+ case 5: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); return; -+ case 6: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); return; -+ case 7: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); return; -+ } -+} -+ -+static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls) -+{ -+ U32 *const hashLong = ctx->hashTable; -+ U32 const hBitsL = ctx->params.cParams.hashLog; -+ U32 *const hashSmall = ctx->chainTable; -+ U32 const hBitsS = ctx->params.cParams.chainLog; -+ seqStore_t *seqStorePtr = &(ctx->seqStore); -+ const BYTE *const base = ctx->base; -+ const BYTE *const dictBase = ctx->dictBase; -+ const BYTE *const istart = (const BYTE *)src; -+ const BYTE *ip = istart; -+ const BYTE *anchor = istart; -+ const U32 lowestIndex = ctx->lowLimit; -+ const BYTE *const dictStart = dictBase + lowestIndex; -+ const U32 dictLimit = ctx->dictLimit; -+ const BYTE *const lowPrefixPtr = base + dictLimit; -+ const BYTE *const dictEnd = dictBase + dictLimit; -+ const BYTE *const iend = istart + srcSize; -+ const BYTE *const ilimit = iend - 8; -+ U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1]; -+ -+ /* Search Loop */ -+ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ -+ const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); -+ const U32 matchIndex = hashSmall[hSmall]; -+ const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base; -+ const BYTE *match = matchBase + matchIndex; -+ -+ const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); -+ const U32 matchLongIndex = hashLong[hLong]; -+ const BYTE *matchLongBase = matchLongIndex < dictLimit ? dictBase : base; -+ const BYTE *matchLong = matchLongBase + matchLongIndex; -+ -+ const U32 curr = (U32)(ip - base); -+ const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ -+ const BYTE *repBase = repIndex < dictLimit ? dictBase : base; -+ const BYTE *repMatch = repBase + repIndex; -+ size_t mLength; -+ hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ -+ -+ if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) && -+ (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) { -+ const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend; -+ mLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, lowPrefixPtr) + 4; -+ ip++; -+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH); -+ } else { -+ if ((matchLongIndex > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) { -+ const BYTE *matchEnd = matchLongIndex < dictLimit ? dictEnd : iend; -+ const BYTE *lowMatchPtr = matchLongIndex < dictLimit ? dictStart : lowPrefixPtr; -+ U32 offset; -+ mLength = ZSTD_count_2segments(ip + 8, matchLong + 8, iend, matchEnd, lowPrefixPtr) + 8; -+ offset = curr - matchLongIndex; -+ while (((ip > anchor) & (matchLong > lowMatchPtr)) && (ip[-1] == matchLong[-1])) { -+ ip--; -+ matchLong--; -+ mLength++; -+ } /* catch up */ -+ offset_2 = offset_1; -+ offset_1 = offset; -+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); -+ -+ } else if ((matchIndex > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) { -+ size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8); -+ U32 const matchIndex3 = hashLong[h3]; -+ const BYTE *const match3Base = matchIndex3 < dictLimit ? dictBase : base; -+ const BYTE *match3 = match3Base + matchIndex3; -+ U32 offset; -+ hashLong[h3] = curr + 1; -+ if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) { -+ const BYTE *matchEnd = matchIndex3 < dictLimit ? dictEnd : iend; -+ const BYTE *lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr; -+ mLength = ZSTD_count_2segments(ip + 9, match3 + 8, iend, matchEnd, lowPrefixPtr) + 8; -+ ip++; -+ offset = curr + 1 - matchIndex3; -+ while (((ip > anchor) & (match3 > lowMatchPtr)) && (ip[-1] == match3[-1])) { -+ ip--; -+ match3--; -+ mLength++; -+ } /* catch up */ -+ } else { -+ const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend; -+ const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; -+ mLength = ZSTD_count_2segments(ip + 4, match + 4, iend, matchEnd, lowPrefixPtr) + 4; -+ offset = curr - matchIndex; -+ while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) { -+ ip--; -+ match--; -+ mLength++; -+ } /* catch up */ -+ } -+ offset_2 = offset_1; -+ offset_1 = offset; -+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); -+ -+ } else { -+ ip += ((ip - anchor) >> g_searchStrength) + 1; -+ continue; -+ } -+ } -+ -+ /* found a match : store it */ -+ ip += mLength; -+ anchor = ip; -+ -+ if (ip <= ilimit) { -+ /* Fill Table */ -+ hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] = curr + 2; -+ hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = curr + 2; -+ hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base); -+ hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = (U32)(ip - 2 - base); -+ /* check immediate repcode */ -+ while (ip <= ilimit) { -+ U32 const curr2 = (U32)(ip - base); -+ U32 const repIndex2 = curr2 - offset_2; -+ const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; -+ if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ -+ && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) { -+ const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; -+ size_t const repLength2 = -+ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; -+ U32 tmpOffset = offset_2; -+ offset_2 = offset_1; -+ offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ -+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH); -+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = curr2; -+ hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = curr2; -+ ip += repLength2; -+ anchor = ip; -+ continue; -+ } -+ break; -+ } -+ } -+ } -+ -+ /* save reps for next block */ -+ ctx->repToConfirm[0] = offset_1; -+ ctx->repToConfirm[1] = offset_2; -+ -+ /* Last Literals */ -+ { -+ size_t const lastLLSize = iend - anchor; -+ memcpy(seqStorePtr->lit, anchor, lastLLSize); -+ seqStorePtr->lit += lastLLSize; -+ } -+} -+ -+static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -+{ -+ U32 const mls = ctx->params.cParams.searchLength; -+ switch (mls) { -+ default: /* includes case 3 */ -+ case 4: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return; -+ case 5: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); return; -+ case 6: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); return; -+ case 7: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); return; -+ } -+} -+ -+/*-************************************* -+* Binary Tree search -+***************************************/ -+/** ZSTD_insertBt1() : add one or multiple positions to tree. -+* ip : assumed <= iend-8 . -+* @return : nb of positions added */ -+static U32 ZSTD_insertBt1(ZSTD_CCtx *zc, const BYTE *const ip, const U32 mls, const BYTE *const iend, U32 nbCompares, U32 extDict) -+{ -+ U32 *const hashTable = zc->hashTable; -+ U32 const hashLog = zc->params.cParams.hashLog; -+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls); -+ U32 *const bt = zc->chainTable; -+ U32 const btLog = zc->params.cParams.chainLog - 1; -+ U32 const btMask = (1 << btLog) - 1; -+ U32 matchIndex = hashTable[h]; -+ size_t commonLengthSmaller = 0, commonLengthLarger = 0; -+ const BYTE *const base = zc->base; -+ const BYTE *const dictBase = zc->dictBase; -+ const U32 dictLimit = zc->dictLimit; -+ const BYTE *const dictEnd = dictBase + dictLimit; -+ const BYTE *const prefixStart = base + dictLimit; -+ const BYTE *match; -+ const U32 curr = (U32)(ip - base); -+ const U32 btLow = btMask >= curr ? 0 : curr - btMask; -+ U32 *smallerPtr = bt + 2 * (curr & btMask); -+ U32 *largerPtr = smallerPtr + 1; -+ U32 dummy32; /* to be nullified at the end */ -+ U32 const windowLow = zc->lowLimit; -+ U32 matchEndIdx = curr + 8; -+ size_t bestLength = 8; -+ -+ hashTable[h] = curr; /* Update Hash Table */ -+ -+ while (nbCompares-- && (matchIndex > windowLow)) { -+ U32 *const nextPtr = bt + 2 * (matchIndex & btMask); -+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ -+ -+ if ((!extDict) || (matchIndex + matchLength >= dictLimit)) { -+ match = base + matchIndex; -+ if (match[matchLength] == ip[matchLength]) -+ matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1; -+ } else { -+ match = dictBase + matchIndex; -+ matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart); -+ if (matchIndex + matchLength >= dictLimit) -+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ -+ } -+ -+ if (matchLength > bestLength) { -+ bestLength = matchLength; -+ if (matchLength > matchEndIdx - matchIndex) -+ matchEndIdx = matchIndex + (U32)matchLength; -+ } -+ -+ if (ip + matchLength == iend) /* equal : no way to know if inf or sup */ -+ break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ -+ -+ if (match[matchLength] < ip[matchLength]) { /* necessarily within correct buffer */ -+ /* match is smaller than curr */ -+ *smallerPtr = matchIndex; /* update smaller idx */ -+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ -+ if (matchIndex <= btLow) { -+ smallerPtr = &dummy32; -+ break; -+ } /* beyond tree size, stop the search */ -+ smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ -+ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ -+ } else { -+ /* match is larger than curr */ -+ *largerPtr = matchIndex; -+ commonLengthLarger = matchLength; -+ if (matchIndex <= btLow) { -+ largerPtr = &dummy32; -+ break; -+ } /* beyond tree size, stop the search */ -+ largerPtr = nextPtr; -+ matchIndex = nextPtr[0]; -+ } -+ } -+ -+ *smallerPtr = *largerPtr = 0; -+ if (bestLength > 384) -+ return MIN(192, (U32)(bestLength - 384)); /* speed optimization */ -+ if (matchEndIdx > curr + 8) -+ return matchEndIdx - curr - 8; -+ return 1; -+} -+ -+static size_t ZSTD_insertBtAndFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, size_t *offsetPtr, U32 nbCompares, const U32 mls, -+ U32 extDict) -+{ -+ U32 *const hashTable = zc->hashTable; -+ U32 const hashLog = zc->params.cParams.hashLog; -+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls); -+ U32 *const bt = zc->chainTable; -+ U32 const btLog = zc->params.cParams.chainLog - 1; -+ U32 const btMask = (1 << btLog) - 1; -+ U32 matchIndex = hashTable[h]; -+ size_t commonLengthSmaller = 0, commonLengthLarger = 0; -+ const BYTE *const base = zc->base; -+ const BYTE *const dictBase = zc->dictBase; -+ const U32 dictLimit = zc->dictLimit; -+ const BYTE *const dictEnd = dictBase + dictLimit; -+ const BYTE *const prefixStart = base + dictLimit; -+ const U32 curr = (U32)(ip - base); -+ const U32 btLow = btMask >= curr ? 0 : curr - btMask; -+ const U32 windowLow = zc->lowLimit; -+ U32 *smallerPtr = bt + 2 * (curr & btMask); -+ U32 *largerPtr = bt + 2 * (curr & btMask) + 1; -+ U32 matchEndIdx = curr + 8; -+ U32 dummy32; /* to be nullified at the end */ -+ size_t bestLength = 0; -+ -+ hashTable[h] = curr; /* Update Hash Table */ -+ -+ while (nbCompares-- && (matchIndex > windowLow)) { -+ U32 *const nextPtr = bt + 2 * (matchIndex & btMask); -+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ -+ const BYTE *match; -+ -+ if ((!extDict) || (matchIndex + matchLength >= dictLimit)) { -+ match = base + matchIndex; -+ if (match[matchLength] == ip[matchLength]) -+ matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1; -+ } else { -+ match = dictBase + matchIndex; -+ matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart); -+ if (matchIndex + matchLength >= dictLimit) -+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ -+ } -+ -+ if (matchLength > bestLength) { -+ if (matchLength > matchEndIdx - matchIndex) -+ matchEndIdx = matchIndex + (U32)matchLength; -+ if ((4 * (int)(matchLength - bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)offsetPtr[0] + 1))) -+ bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; -+ if (ip + matchLength == iend) /* equal : no way to know if inf or sup */ -+ break; /* drop, to guarantee consistency (miss a little bit of compression) */ -+ } -+ -+ if (match[matchLength] < ip[matchLength]) { -+ /* match is smaller than curr */ -+ *smallerPtr = matchIndex; /* update smaller idx */ -+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ -+ if (matchIndex <= btLow) { -+ smallerPtr = &dummy32; -+ break; -+ } /* beyond tree size, stop the search */ -+ smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ -+ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ -+ } else { -+ /* match is larger than curr */ -+ *largerPtr = matchIndex; -+ commonLengthLarger = matchLength; -+ if (matchIndex <= btLow) { -+ largerPtr = &dummy32; -+ break; -+ } /* beyond tree size, stop the search */ -+ largerPtr = nextPtr; -+ matchIndex = nextPtr[0]; -+ } -+ } -+ -+ *smallerPtr = *largerPtr = 0; -+ -+ zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1; -+ return bestLength; -+} -+ -+static void ZSTD_updateTree(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls) -+{ -+ const BYTE *const base = zc->base; -+ const U32 target = (U32)(ip - base); -+ U32 idx = zc->nextToUpdate; -+ -+ while (idx < target) -+ idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 0); -+} -+ -+/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ -+static size_t ZSTD_BtFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls) -+{ -+ if (ip < zc->base + zc->nextToUpdate) -+ return 0; /* skipped area */ -+ ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); -+ return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0); -+} -+ -+static size_t ZSTD_BtFindBestMatch_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */ -+ const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 matchLengthSearch) -+{ -+ switch (matchLengthSearch) { -+ default: /* includes case 3 */ -+ case 4: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); -+ case 5: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); -+ case 7: -+ case 6: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); -+ } -+} -+ -+static void ZSTD_updateTree_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls) -+{ -+ const BYTE *const base = zc->base; -+ const U32 target = (U32)(ip - base); -+ U32 idx = zc->nextToUpdate; -+ -+ while (idx < target) -+ idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 1); -+} -+ -+/** Tree updater, providing best match */ -+static size_t ZSTD_BtFindBestMatch_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, -+ const U32 mls) -+{ -+ if (ip < zc->base + zc->nextToUpdate) -+ return 0; /* skipped area */ -+ ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); -+ return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1); -+} -+ -+static size_t ZSTD_BtFindBestMatch_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */ -+ const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, -+ const U32 matchLengthSearch) -+{ -+ switch (matchLengthSearch) { -+ default: /* includes case 3 */ -+ case 4: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); -+ case 5: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); -+ case 7: -+ case 6: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); -+ } -+} -+ -+/* ********************************* -+* Hash Chain -+***********************************/ -+#define NEXT_IN_CHAIN(d, mask) chainTable[(d)&mask] -+ -+/* Update chains up to ip (excluded) -+ Assumption : always within prefix (i.e. not within extDict) */ -+FORCE_INLINE -+U32 ZSTD_insertAndFindFirstIndex(ZSTD_CCtx *zc, const BYTE *ip, U32 mls) -+{ -+ U32 *const hashTable = zc->hashTable; -+ const U32 hashLog = zc->params.cParams.hashLog; -+ U32 *const chainTable = zc->chainTable; -+ const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1; -+ const BYTE *const base = zc->base; -+ const U32 target = (U32)(ip - base); -+ U32 idx = zc->nextToUpdate; -+ -+ while (idx < target) { /* catch up */ -+ size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); -+ NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; -+ hashTable[h] = idx; -+ idx++; -+ } -+ -+ zc->nextToUpdate = target; -+ return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; -+} -+ -+/* inlining is important to hardwire a hot branch (template emulation) */ -+FORCE_INLINE -+size_t ZSTD_HcFindBestMatch_generic(ZSTD_CCtx *zc, /* Index table will be updated */ -+ const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls, -+ const U32 extDict) -+{ -+ U32 *const chainTable = zc->chainTable; -+ const U32 chainSize = (1 << zc->params.cParams.chainLog); -+ const U32 chainMask = chainSize - 1; -+ const BYTE *const base = zc->base; -+ const BYTE *const dictBase = zc->dictBase; -+ const U32 dictLimit = zc->dictLimit; -+ const BYTE *const prefixStart = base + dictLimit; -+ const BYTE *const dictEnd = dictBase + dictLimit; -+ const U32 lowLimit = zc->lowLimit; -+ const U32 curr = (U32)(ip - base); -+ const U32 minChain = curr > chainSize ? curr - chainSize : 0; -+ int nbAttempts = maxNbAttempts; -+ size_t ml = EQUAL_READ32 - 1; -+ -+ /* HC4 match finder */ -+ U32 matchIndex = ZSTD_insertAndFindFirstIndex(zc, ip, mls); -+ -+ for (; (matchIndex > lowLimit) & (nbAttempts > 0); nbAttempts--) { -+ const BYTE *match; -+ size_t currMl = 0; -+ if ((!extDict) || matchIndex >= dictLimit) { -+ match = base + matchIndex; -+ if (match[ml] == ip[ml]) /* potentially better */ -+ currMl = ZSTD_count(ip, match, iLimit); -+ } else { -+ match = dictBase + matchIndex; -+ if (ZSTD_read32(match) == ZSTD_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ -+ currMl = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iLimit, dictEnd, prefixStart) + EQUAL_READ32; -+ } -+ -+ /* save best solution */ -+ if (currMl > ml) { -+ ml = currMl; -+ *offsetPtr = curr - matchIndex + ZSTD_REP_MOVE; -+ if (ip + currMl == iLimit) -+ break; /* best possible, and avoid read overflow*/ -+ } -+ -+ if (matchIndex <= minChain) -+ break; -+ matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); -+ } -+ -+ return ml; -+} -+ -+FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, -+ const U32 matchLengthSearch) -+{ -+ switch (matchLengthSearch) { -+ default: /* includes case 3 */ -+ case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0); -+ case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0); -+ case 7: -+ case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0); -+ } -+} -+ -+FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, -+ const U32 matchLengthSearch) -+{ -+ switch (matchLengthSearch) { -+ default: /* includes case 3 */ -+ case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1); -+ case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1); -+ case 7: -+ case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1); -+ } -+} -+ -+/* ******************************* -+* Common parser - lazy strategy -+*********************************/ -+FORCE_INLINE -+void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth) -+{ -+ seqStore_t *seqStorePtr = &(ctx->seqStore); -+ const BYTE *const istart = (const BYTE *)src; -+ const BYTE *ip = istart; -+ const BYTE *anchor = istart; -+ const BYTE *const iend = istart + srcSize; -+ const BYTE *const ilimit = iend - 8; -+ const BYTE *const base = ctx->base + ctx->dictLimit; -+ -+ U32 const maxSearches = 1 << ctx->params.cParams.searchLog; -+ U32 const mls = ctx->params.cParams.searchLength; -+ -+ typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch); -+ searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; -+ U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset = 0; -+ -+ /* init */ -+ ip += (ip == base); -+ ctx->nextToUpdate3 = ctx->nextToUpdate; -+ { -+ U32 const maxRep = (U32)(ip - base); -+ if (offset_2 > maxRep) -+ savedOffset = offset_2, offset_2 = 0; -+ if (offset_1 > maxRep) -+ savedOffset = offset_1, offset_1 = 0; -+ } -+ -+ /* Match Loop */ -+ while (ip < ilimit) { -+ size_t matchLength = 0; -+ size_t offset = 0; -+ const BYTE *start = ip + 1; -+ -+ /* check repCode */ -+ if ((offset_1 > 0) & (ZSTD_read32(ip + 1) == ZSTD_read32(ip + 1 - offset_1))) { -+ /* repcode : we take it */ -+ matchLength = ZSTD_count(ip + 1 + EQUAL_READ32, ip + 1 + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32; -+ if (depth == 0) -+ goto _storeSequence; -+ } -+ -+ /* first search (depth 0) */ -+ { -+ size_t offsetFound = 99999999; -+ size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); -+ if (ml2 > matchLength) -+ matchLength = ml2, start = ip, offset = offsetFound; -+ } -+ -+ if (matchLength < EQUAL_READ32) { -+ ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ -+ continue; -+ } -+ -+ /* let's try to find a better solution */ -+ if (depth >= 1) -+ while (ip < ilimit) { -+ ip++; -+ if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) { -+ size_t const mlRep = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32; -+ int const gain2 = (int)(mlRep * 3); -+ int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1); -+ if ((mlRep >= EQUAL_READ32) && (gain2 > gain1)) -+ matchLength = mlRep, offset = 0, start = ip; -+ } -+ { -+ size_t offset2 = 99999999; -+ size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); -+ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ -+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4); -+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { -+ matchLength = ml2, offset = offset2, start = ip; -+ continue; /* search a better one */ -+ } -+ } -+ -+ /* let's find an even better one */ -+ if ((depth == 2) && (ip < ilimit)) { -+ ip++; -+ if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) { -+ size_t const ml2 = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32; -+ int const gain2 = (int)(ml2 * 4); -+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1); -+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) -+ matchLength = ml2, offset = 0, start = ip; -+ } -+ { -+ size_t offset2 = 99999999; -+ size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); -+ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ -+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7); -+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { -+ matchLength = ml2, offset = offset2, start = ip; -+ continue; -+ } -+ } -+ } -+ break; /* nothing found : store previous solution */ -+ } -+ -+ /* NOTE: -+ * start[-offset+ZSTD_REP_MOVE-1] is undefined behavior. -+ * (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which -+ * overflows the pointer, which is undefined behavior. -+ */ -+ /* catch up */ -+ if (offset) { -+ while ((start > anchor) && (start > base + offset - ZSTD_REP_MOVE) && -+ (start[-1] == (start-offset+ZSTD_REP_MOVE)[-1])) /* only search for offset within prefix */ -+ { -+ start--; -+ matchLength++; -+ } -+ offset_2 = offset_1; -+ offset_1 = (U32)(offset - ZSTD_REP_MOVE); -+ } -+ -+ /* store sequence */ -+_storeSequence: -+ { -+ size_t const litLength = start - anchor; -+ ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH); -+ anchor = ip = start + matchLength; -+ } -+ -+ /* check immediate repcode */ -+ while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) { -+ /* store sequence */ -+ matchLength = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_2, iend) + EQUAL_READ32; -+ offset = offset_2; -+ offset_2 = offset_1; -+ offset_1 = (U32)offset; /* swap repcodes */ -+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH); -+ ip += matchLength; -+ anchor = ip; -+ continue; /* faster when present ... (?) */ -+ } -+ } -+ -+ /* Save reps for next block */ -+ ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset; -+ ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset; -+ -+ /* Last Literals */ -+ { -+ size_t const lastLLSize = iend - anchor; -+ memcpy(seqStorePtr->lit, anchor, lastLLSize); -+ seqStorePtr->lit += lastLLSize; -+ } -+} -+ -+static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2); } -+ -+static void ZSTD_compressBlock_lazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2); } -+ -+static void ZSTD_compressBlock_lazy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1); } -+ -+static void ZSTD_compressBlock_greedy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0); } -+ -+FORCE_INLINE -+void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth) -+{ -+ seqStore_t *seqStorePtr = &(ctx->seqStore); -+ const BYTE *const istart = (const BYTE *)src; -+ const BYTE *ip = istart; -+ const BYTE *anchor = istart; -+ const BYTE *const iend = istart + srcSize; -+ const BYTE *const ilimit = iend - 8; -+ const BYTE *const base = ctx->base; -+ const U32 dictLimit = ctx->dictLimit; -+ const U32 lowestIndex = ctx->lowLimit; -+ const BYTE *const prefixStart = base + dictLimit; -+ const BYTE *const dictBase = ctx->dictBase; -+ const BYTE *const dictEnd = dictBase + dictLimit; -+ const BYTE *const dictStart = dictBase + ctx->lowLimit; -+ -+ const U32 maxSearches = 1 << ctx->params.cParams.searchLog; -+ const U32 mls = ctx->params.cParams.searchLength; -+ -+ typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch); -+ searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS; -+ -+ U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1]; -+ -+ /* init */ -+ ctx->nextToUpdate3 = ctx->nextToUpdate; -+ ip += (ip == prefixStart); -+ -+ /* Match Loop */ -+ while (ip < ilimit) { -+ size_t matchLength = 0; -+ size_t offset = 0; -+ const BYTE *start = ip + 1; -+ U32 curr = (U32)(ip - base); -+ -+ /* check repCode */ -+ { -+ const U32 repIndex = (U32)(curr + 1 - offset_1); -+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; -+ const BYTE *const repMatch = repBase + repIndex; -+ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ -+ if (ZSTD_read32(ip + 1) == ZSTD_read32(repMatch)) { -+ /* repcode detected we should take it */ -+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; -+ matchLength = -+ ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; -+ if (depth == 0) -+ goto _storeSequence; -+ } -+ } -+ -+ /* first search (depth 0) */ -+ { -+ size_t offsetFound = 99999999; -+ size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); -+ if (ml2 > matchLength) -+ matchLength = ml2, start = ip, offset = offsetFound; -+ } -+ -+ if (matchLength < EQUAL_READ32) { -+ ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ -+ continue; -+ } -+ -+ /* let's try to find a better solution */ -+ if (depth >= 1) -+ while (ip < ilimit) { -+ ip++; -+ curr++; -+ /* check repCode */ -+ if (offset) { -+ const U32 repIndex = (U32)(curr - offset_1); -+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; -+ const BYTE *const repMatch = repBase + repIndex; -+ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ -+ if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { -+ /* repcode detected */ -+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; -+ size_t const repLength = -+ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + -+ EQUAL_READ32; -+ int const gain2 = (int)(repLength * 3); -+ int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1); -+ if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) -+ matchLength = repLength, offset = 0, start = ip; -+ } -+ } -+ -+ /* search match, depth 1 */ -+ { -+ size_t offset2 = 99999999; -+ size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); -+ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ -+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4); -+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { -+ matchLength = ml2, offset = offset2, start = ip; -+ continue; /* search a better one */ -+ } -+ } -+ -+ /* let's find an even better one */ -+ if ((depth == 2) && (ip < ilimit)) { -+ ip++; -+ curr++; -+ /* check repCode */ -+ if (offset) { -+ const U32 repIndex = (U32)(curr - offset_1); -+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; -+ const BYTE *const repMatch = repBase + repIndex; -+ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ -+ if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { -+ /* repcode detected */ -+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; -+ size_t repLength = ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, -+ repEnd, prefixStart) + -+ EQUAL_READ32; -+ int gain2 = (int)(repLength * 4); -+ int gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1); -+ if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) -+ matchLength = repLength, offset = 0, start = ip; -+ } -+ } -+ -+ /* search match, depth 2 */ -+ { -+ size_t offset2 = 99999999; -+ size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); -+ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ -+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7); -+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { -+ matchLength = ml2, offset = offset2, start = ip; -+ continue; -+ } -+ } -+ } -+ break; /* nothing found : store previous solution */ -+ } -+ -+ /* catch up */ -+ if (offset) { -+ U32 const matchIndex = (U32)((start - base) - (offset - ZSTD_REP_MOVE)); -+ const BYTE *match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; -+ const BYTE *const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; -+ while ((start > anchor) && (match > mStart) && (start[-1] == match[-1])) { -+ start--; -+ match--; -+ matchLength++; -+ } /* catch up */ -+ offset_2 = offset_1; -+ offset_1 = (U32)(offset - ZSTD_REP_MOVE); -+ } -+ -+ /* store sequence */ -+ _storeSequence : { -+ size_t const litLength = start - anchor; -+ ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH); -+ anchor = ip = start + matchLength; -+ } -+ -+ /* check immediate repcode */ -+ while (ip <= ilimit) { -+ const U32 repIndex = (U32)((ip - base) - offset_2); -+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; -+ const BYTE *const repMatch = repBase + repIndex; -+ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ -+ if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { -+ /* repcode detected we should take it */ -+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; -+ matchLength = -+ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; -+ offset = offset_2; -+ offset_2 = offset_1; -+ offset_1 = (U32)offset; /* swap offset history */ -+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH); -+ ip += matchLength; -+ anchor = ip; -+ continue; /* faster when present ... (?) */ -+ } -+ break; -+ } -+ } -+ -+ /* Save reps for next block */ -+ ctx->repToConfirm[0] = offset_1; -+ ctx->repToConfirm[1] = offset_2; -+ -+ /* Last Literals */ -+ { -+ size_t const lastLLSize = iend - anchor; -+ memcpy(seqStorePtr->lit, anchor, lastLLSize); -+ seqStorePtr->lit += lastLLSize; -+ } -+} -+ -+void ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0); } -+ -+static void ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -+{ -+ ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 1); -+} -+ -+static void ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -+{ -+ ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 2); -+} -+ -+static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -+{ -+ ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2); -+} -+ -+/* The optimal parser */ -+#include "zstd_opt.h" -+ -+static void ZSTD_compressBlock_btopt(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -+{ -+#ifdef ZSTD_OPT_H_91842398743 -+ ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0); -+#else -+ (void)ctx; -+ (void)src; -+ (void)srcSize; -+ return; -+#endif -+} -+ -+static void ZSTD_compressBlock_btopt2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -+{ -+#ifdef ZSTD_OPT_H_91842398743 -+ ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1); -+#else -+ (void)ctx; -+ (void)src; -+ (void)srcSize; -+ return; -+#endif -+} -+ -+static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -+{ -+#ifdef ZSTD_OPT_H_91842398743 -+ ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0); -+#else -+ (void)ctx; -+ (void)src; -+ (void)srcSize; -+ return; -+#endif -+} -+ -+static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -+{ -+#ifdef ZSTD_OPT_H_91842398743 -+ ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1); -+#else -+ (void)ctx; -+ (void)src; -+ (void)srcSize; -+ return; -+#endif -+} -+ -+typedef void (*ZSTD_blockCompressor)(ZSTD_CCtx *ctx, const void *src, size_t srcSize); -+ -+static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) -+{ -+ static const ZSTD_blockCompressor blockCompressor[2][8] = { -+ {ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, -+ ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2}, -+ {ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict, -+ ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict}}; -+ -+ return blockCompressor[extDict][(U32)strat]; -+} -+ -+static size_t ZSTD_compressBlock_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit); -+ const BYTE *const base = zc->base; -+ const BYTE *const istart = (const BYTE *)src; -+ const U32 curr = (U32)(istart - base); -+ if (srcSize < MIN_CBLOCK_SIZE + ZSTD_blockHeaderSize + 1) -+ return 0; /* don't even attempt compression below a certain srcSize */ -+ ZSTD_resetSeqStore(&(zc->seqStore)); -+ if (curr > zc->nextToUpdate + 384) -+ zc->nextToUpdate = curr - MIN(192, (U32)(curr - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */ -+ blockCompressor(zc, src, srcSize); -+ return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize); -+} -+ -+/*! ZSTD_compress_generic() : -+* Compress a chunk of data into one or multiple blocks. -+* All blocks will be terminated, all input will be consumed. -+* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. -+* Frame is supposed already started (header already produced) -+* @return : compressed size, or an error code -+*/ -+static size_t ZSTD_compress_generic(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 lastFrameChunk) -+{ -+ size_t blockSize = cctx->blockSize; -+ size_t remaining = srcSize; -+ const BYTE *ip = (const BYTE *)src; -+ BYTE *const ostart = (BYTE *)dst; -+ BYTE *op = ostart; -+ U32 const maxDist = 1 << cctx->params.cParams.windowLog; -+ -+ if (cctx->params.fParams.checksumFlag && srcSize) -+ xxh64_update(&cctx->xxhState, src, srcSize); -+ -+ while (remaining) { -+ U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); -+ size_t cSize; -+ -+ if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) -+ return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ -+ if (remaining < blockSize) -+ blockSize = remaining; -+ -+ /* preemptive overflow correction */ -+ if (cctx->lowLimit > (3U << 29)) { -+ U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1; -+ U32 const curr = (U32)(ip - cctx->base); -+ U32 const newCurr = (curr & cycleMask) + (1 << cctx->params.cParams.windowLog); -+ U32 const correction = curr - newCurr; -+ ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30); -+ ZSTD_reduceIndex(cctx, correction); -+ cctx->base += correction; -+ cctx->dictBase += correction; -+ cctx->lowLimit -= correction; -+ cctx->dictLimit -= correction; -+ if (cctx->nextToUpdate < correction) -+ cctx->nextToUpdate = 0; -+ else -+ cctx->nextToUpdate -= correction; -+ } -+ -+ if ((U32)(ip + blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { -+ /* enforce maxDist */ -+ U32 const newLowLimit = (U32)(ip + blockSize - cctx->base) - maxDist; -+ if (cctx->lowLimit < newLowLimit) -+ cctx->lowLimit = newLowLimit; -+ if (cctx->dictLimit < cctx->lowLimit) -+ cctx->dictLimit = cctx->lowLimit; -+ } -+ -+ cSize = ZSTD_compressBlock_internal(cctx, op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, ip, blockSize); -+ if (ZSTD_isError(cSize)) -+ return cSize; -+ -+ if (cSize == 0) { /* block is not compressible */ -+ U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw) << 1) + (U32)(blockSize << 3); -+ if (blockSize + ZSTD_blockHeaderSize > dstCapacity) -+ return ERROR(dstSize_tooSmall); -+ ZSTD_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */ -+ memcpy(op + ZSTD_blockHeaderSize, ip, blockSize); -+ cSize = ZSTD_blockHeaderSize + blockSize; -+ } else { -+ U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed) << 1) + (U32)(cSize << 3); -+ ZSTD_writeLE24(op, cBlockHeader24); -+ cSize += ZSTD_blockHeaderSize; -+ } -+ -+ remaining -= blockSize; -+ dstCapacity -= cSize; -+ ip += blockSize; -+ op += cSize; -+ } -+ -+ if (lastFrameChunk && (op > ostart)) -+ cctx->stage = ZSTDcs_ending; -+ return op - ostart; -+} -+ -+static size_t ZSTD_writeFrameHeader(void *dst, size_t dstCapacity, ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID) -+{ -+ BYTE *const op = (BYTE *)dst; -+ U32 const dictIDSizeCode = (dictID > 0) + (dictID >= 256) + (dictID >= 65536); /* 0-3 */ -+ U32 const checksumFlag = params.fParams.checksumFlag > 0; -+ U32 const windowSize = 1U << params.cParams.windowLog; -+ U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); -+ BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); -+ U32 const fcsCode = -+ params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */ -+ BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6)); -+ size_t pos; -+ -+ if (dstCapacity < ZSTD_frameHeaderSize_max) -+ return ERROR(dstSize_tooSmall); -+ -+ ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER); -+ op[4] = frameHeaderDescriptionByte; -+ pos = 5; -+ if (!singleSegment) -+ op[pos++] = windowLogByte; -+ switch (dictIDSizeCode) { -+ default: /* impossible */ -+ case 0: break; -+ case 1: -+ op[pos] = (BYTE)(dictID); -+ pos++; -+ break; -+ case 2: -+ ZSTD_writeLE16(op + pos, (U16)dictID); -+ pos += 2; -+ break; -+ case 3: -+ ZSTD_writeLE32(op + pos, dictID); -+ pos += 4; -+ break; -+ } -+ switch (fcsCode) { -+ default: /* impossible */ -+ case 0: -+ if (singleSegment) -+ op[pos++] = (BYTE)(pledgedSrcSize); -+ break; -+ case 1: -+ ZSTD_writeLE16(op + pos, (U16)(pledgedSrcSize - 256)); -+ pos += 2; -+ break; -+ case 2: -+ ZSTD_writeLE32(op + pos, (U32)(pledgedSrcSize)); -+ pos += 4; -+ break; -+ case 3: -+ ZSTD_writeLE64(op + pos, (U64)(pledgedSrcSize)); -+ pos += 8; -+ break; -+ } -+ return pos; -+} -+ -+static size_t ZSTD_compressContinue_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 frame, U32 lastFrameChunk) -+{ -+ const BYTE *const ip = (const BYTE *)src; -+ size_t fhSize = 0; -+ -+ if (cctx->stage == ZSTDcs_created) -+ return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ -+ -+ if (frame && (cctx->stage == ZSTDcs_init)) { -+ fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID); -+ if (ZSTD_isError(fhSize)) -+ return fhSize; -+ dstCapacity -= fhSize; -+ dst = (char *)dst + fhSize; -+ cctx->stage = ZSTDcs_ongoing; -+ } -+ -+ /* Check if blocks follow each other */ -+ if (src != cctx->nextSrc) { -+ /* not contiguous */ -+ ptrdiff_t const delta = cctx->nextSrc - ip; -+ cctx->lowLimit = cctx->dictLimit; -+ cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base); -+ cctx->dictBase = cctx->base; -+ cctx->base -= delta; -+ cctx->nextToUpdate = cctx->dictLimit; -+ if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) -+ cctx->lowLimit = cctx->dictLimit; /* too small extDict */ -+ } -+ -+ /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ -+ if ((ip + srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) { -+ ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase; -+ U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx; -+ cctx->lowLimit = lowLimitMax; -+ } -+ -+ cctx->nextSrc = ip + srcSize; -+ -+ if (srcSize) { -+ size_t const cSize = frame ? ZSTD_compress_generic(cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) -+ : ZSTD_compressBlock_internal(cctx, dst, dstCapacity, src, srcSize); -+ if (ZSTD_isError(cSize)) -+ return cSize; -+ return cSize + fhSize; -+ } else -+ return fhSize; -+} -+ -+size_t ZSTD_compressContinue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0); -+} -+ -+size_t ZSTD_getBlockSizeMax(ZSTD_CCtx *cctx) { return MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog); } -+ -+size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx); -+ if (srcSize > blockSizeMax) -+ return ERROR(srcSize_wrong); -+ return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0); -+} -+ -+/*! ZSTD_loadDictionaryContent() : -+ * @return : 0, or an error code -+ */ -+static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx *zc, const void *src, size_t srcSize) -+{ -+ const BYTE *const ip = (const BYTE *)src; -+ const BYTE *const iend = ip + srcSize; -+ -+ /* input becomes curr prefix */ -+ zc->lowLimit = zc->dictLimit; -+ zc->dictLimit = (U32)(zc->nextSrc - zc->base); -+ zc->dictBase = zc->base; -+ zc->base += ip - zc->nextSrc; -+ zc->nextToUpdate = zc->dictLimit; -+ zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base); -+ -+ zc->nextSrc = iend; -+ if (srcSize <= HASH_READ_SIZE) -+ return 0; -+ -+ switch (zc->params.cParams.strategy) { -+ case ZSTD_fast: ZSTD_fillHashTable(zc, iend, zc->params.cParams.searchLength); break; -+ -+ case ZSTD_dfast: ZSTD_fillDoubleHashTable(zc, iend, zc->params.cParams.searchLength); break; -+ -+ case ZSTD_greedy: -+ case ZSTD_lazy: -+ case ZSTD_lazy2: -+ if (srcSize >= HASH_READ_SIZE) -+ ZSTD_insertAndFindFirstIndex(zc, iend - HASH_READ_SIZE, zc->params.cParams.searchLength); -+ break; -+ -+ case ZSTD_btlazy2: -+ case ZSTD_btopt: -+ case ZSTD_btopt2: -+ if (srcSize >= HASH_READ_SIZE) -+ ZSTD_updateTree(zc, iend - HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength); -+ break; -+ -+ default: -+ return ERROR(GENERIC); /* strategy doesn't exist; impossible */ -+ } -+ -+ zc->nextToUpdate = (U32)(iend - zc->base); -+ return 0; -+} -+ -+/* Dictionaries that assign zero probability to symbols that show up causes problems -+ when FSE encoding. Refuse dictionaries that assign zero probability to symbols -+ that we may encounter during compression. -+ NOTE: This behavior is not standard and could be improved in the future. */ -+static size_t ZSTD_checkDictNCount(short *normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) -+{ -+ U32 s; -+ if (dictMaxSymbolValue < maxSymbolValue) -+ return ERROR(dictionary_corrupted); -+ for (s = 0; s <= maxSymbolValue; ++s) { -+ if (normalizedCounter[s] == 0) -+ return ERROR(dictionary_corrupted); -+ } -+ return 0; -+} -+ -+/* Dictionary format : -+ * See : -+ * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format -+ */ -+/*! ZSTD_loadZstdDictionary() : -+ * @return : 0, or an error code -+ * assumptions : magic number supposed already checked -+ * dictSize supposed > 8 -+ */ -+static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize) -+{ -+ const BYTE *dictPtr = (const BYTE *)dict; -+ const BYTE *const dictEnd = dictPtr + dictSize; -+ short offcodeNCount[MaxOff + 1]; -+ unsigned offcodeMaxValue = MaxOff; -+ -+ dictPtr += 4; /* skip magic number */ -+ cctx->dictID = cctx->params.fParams.noDictIDFlag ? 0 : ZSTD_readLE32(dictPtr); -+ dictPtr += 4; -+ -+ { -+ size_t const hufHeaderSize = HUF_readCTable_wksp(cctx->hufTable, 255, dictPtr, dictEnd - dictPtr, cctx->tmpCounters, sizeof(cctx->tmpCounters)); -+ if (HUF_isError(hufHeaderSize)) -+ return ERROR(dictionary_corrupted); -+ dictPtr += hufHeaderSize; -+ } -+ -+ { -+ unsigned offcodeLog; -+ size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr); -+ if (FSE_isError(offcodeHeaderSize)) -+ return ERROR(dictionary_corrupted); -+ if (offcodeLog > OffFSELog) -+ return ERROR(dictionary_corrupted); -+ /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ -+ CHECK_E(FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)), -+ dictionary_corrupted); -+ dictPtr += offcodeHeaderSize; -+ } -+ -+ { -+ short matchlengthNCount[MaxML + 1]; -+ unsigned matchlengthMaxValue = MaxML, matchlengthLog; -+ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr); -+ if (FSE_isError(matchlengthHeaderSize)) -+ return ERROR(dictionary_corrupted); -+ if (matchlengthLog > MLFSELog) -+ return ERROR(dictionary_corrupted); -+ /* Every match length code must have non-zero probability */ -+ CHECK_F(ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); -+ CHECK_E( -+ FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)), -+ dictionary_corrupted); -+ dictPtr += matchlengthHeaderSize; -+ } -+ -+ { -+ short litlengthNCount[MaxLL + 1]; -+ unsigned litlengthMaxValue = MaxLL, litlengthLog; -+ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr); -+ if (FSE_isError(litlengthHeaderSize)) -+ return ERROR(dictionary_corrupted); -+ if (litlengthLog > LLFSELog) -+ return ERROR(dictionary_corrupted); -+ /* Every literal length code must have non-zero probability */ -+ CHECK_F(ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); -+ CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)), -+ dictionary_corrupted); -+ dictPtr += litlengthHeaderSize; -+ } -+ -+ if (dictPtr + 12 > dictEnd) -+ return ERROR(dictionary_corrupted); -+ cctx->rep[0] = ZSTD_readLE32(dictPtr + 0); -+ cctx->rep[1] = ZSTD_readLE32(dictPtr + 4); -+ cctx->rep[2] = ZSTD_readLE32(dictPtr + 8); -+ dictPtr += 12; -+ -+ { -+ size_t const dictContentSize = (size_t)(dictEnd - dictPtr); -+ U32 offcodeMax = MaxOff; -+ if (dictContentSize <= ((U32)-1) - 128 KB) { -+ U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ -+ offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ -+ } -+ /* All offset values <= dictContentSize + 128 KB must be representable */ -+ CHECK_F(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); -+ /* All repCodes must be <= dictContentSize and != 0*/ -+ { -+ U32 u; -+ for (u = 0; u < 3; u++) { -+ if (cctx->rep[u] == 0) -+ return ERROR(dictionary_corrupted); -+ if (cctx->rep[u] > dictContentSize) -+ return ERROR(dictionary_corrupted); -+ } -+ } -+ -+ cctx->flagStaticTables = 1; -+ cctx->flagStaticHufTable = HUF_repeat_valid; -+ return ZSTD_loadDictionaryContent(cctx, dictPtr, dictContentSize); -+ } -+} -+ -+/** ZSTD_compress_insertDictionary() : -+* @return : 0, or an error code */ -+static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize) -+{ -+ if ((dict == NULL) || (dictSize <= 8)) -+ return 0; -+ -+ /* dict as pure content */ -+ if ((ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) || (cctx->forceRawDict)) -+ return ZSTD_loadDictionaryContent(cctx, dict, dictSize); -+ -+ /* dict as zstd dictionary */ -+ return ZSTD_loadZstdDictionary(cctx, dict, dictSize); -+} -+ -+/*! ZSTD_compressBegin_internal() : -+* @return : 0, or an error code */ -+static size_t ZSTD_compressBegin_internal(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) -+{ -+ ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue; -+ CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp)); -+ return ZSTD_compress_insertDictionary(cctx, dict, dictSize); -+} -+ -+/*! ZSTD_compressBegin_advanced() : -+* @return : 0, or an error code */ -+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) -+{ -+ /* compression parameters verification and optimization */ -+ CHECK_F(ZSTD_checkCParams(params.cParams)); -+ return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); -+} -+ -+size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, int compressionLevel) -+{ -+ ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); -+ return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0); -+} -+ -+size_t ZSTD_compressBegin(ZSTD_CCtx *cctx, int compressionLevel) { return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); } -+ -+/*! ZSTD_writeEpilogue() : -+* Ends a frame. -+* @return : nb of bytes written into dst (or an error code) */ -+static size_t ZSTD_writeEpilogue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity) -+{ -+ BYTE *const ostart = (BYTE *)dst; -+ BYTE *op = ostart; -+ size_t fhSize = 0; -+ -+ if (cctx->stage == ZSTDcs_created) -+ return ERROR(stage_wrong); /* init missing */ -+ -+ /* special case : empty frame */ -+ if (cctx->stage == ZSTDcs_init) { -+ fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0); -+ if (ZSTD_isError(fhSize)) -+ return fhSize; -+ dstCapacity -= fhSize; -+ op += fhSize; -+ cctx->stage = ZSTDcs_ongoing; -+ } -+ -+ if (cctx->stage != ZSTDcs_ending) { -+ /* write one last empty block, make it the "last" block */ -+ U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw) << 1) + 0; -+ if (dstCapacity < 4) -+ return ERROR(dstSize_tooSmall); -+ ZSTD_writeLE32(op, cBlockHeader24); -+ op += ZSTD_blockHeaderSize; -+ dstCapacity -= ZSTD_blockHeaderSize; -+ } -+ -+ if (cctx->params.fParams.checksumFlag) { -+ U32 const checksum = (U32)xxh64_digest(&cctx->xxhState); -+ if (dstCapacity < 4) -+ return ERROR(dstSize_tooSmall); -+ ZSTD_writeLE32(op, checksum); -+ op += 4; -+ } -+ -+ cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ -+ return op - ostart; -+} -+ -+size_t ZSTD_compressEnd(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ size_t endResult; -+ size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1); -+ if (ZSTD_isError(cSize)) -+ return cSize; -+ endResult = ZSTD_writeEpilogue(cctx, (char *)dst + cSize, dstCapacity - cSize); -+ if (ZSTD_isError(endResult)) -+ return endResult; -+ return cSize + endResult; -+} -+ -+static size_t ZSTD_compress_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, -+ ZSTD_parameters params) -+{ -+ CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize)); -+ return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); -+} -+ -+size_t ZSTD_compress_usingDict(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, -+ ZSTD_parameters params) -+{ -+ return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); -+} -+ -+size_t ZSTD_compressCCtx(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, ZSTD_parameters params) -+{ -+ return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, NULL, 0, params); -+} -+ -+/* ===== Dictionary API ===== */ -+ -+struct ZSTD_CDict_s { -+ void *dictBuffer; -+ const void *dictContent; -+ size_t dictContentSize; -+ ZSTD_CCtx *refContext; -+}; /* typedef'd tp ZSTD_CDict within "zstd.h" */ -+ -+size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams) { return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CDict)); } -+ -+static ZSTD_CDict *ZSTD_createCDict_advanced(const void *dictBuffer, size_t dictSize, unsigned byReference, ZSTD_parameters params, ZSTD_customMem customMem) -+{ -+ if (!customMem.customAlloc || !customMem.customFree) -+ return NULL; -+ -+ { -+ ZSTD_CDict *const cdict = (ZSTD_CDict *)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); -+ ZSTD_CCtx *const cctx = ZSTD_createCCtx_advanced(customMem); -+ -+ if (!cdict || !cctx) { -+ ZSTD_free(cdict, customMem); -+ ZSTD_freeCCtx(cctx); -+ return NULL; -+ } -+ -+ if ((byReference) || (!dictBuffer) || (!dictSize)) { -+ cdict->dictBuffer = NULL; -+ cdict->dictContent = dictBuffer; -+ } else { -+ void *const internalBuffer = ZSTD_malloc(dictSize, customMem); -+ if (!internalBuffer) { -+ ZSTD_free(cctx, customMem); -+ ZSTD_free(cdict, customMem); -+ return NULL; -+ } -+ memcpy(internalBuffer, dictBuffer, dictSize); -+ cdict->dictBuffer = internalBuffer; -+ cdict->dictContent = internalBuffer; -+ } -+ -+ { -+ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0); -+ if (ZSTD_isError(errorCode)) { -+ ZSTD_free(cdict->dictBuffer, customMem); -+ ZSTD_free(cdict, customMem); -+ ZSTD_freeCCtx(cctx); -+ return NULL; -+ } -+ } -+ -+ cdict->refContext = cctx; -+ cdict->dictContentSize = dictSize; -+ return cdict; -+ } -+} -+ -+ZSTD_CDict *ZSTD_initCDict(const void *dict, size_t dictSize, ZSTD_parameters params, void *workspace, size_t workspaceSize) -+{ -+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); -+ return ZSTD_createCDict_advanced(dict, dictSize, 1, params, stackMem); -+} -+ -+size_t ZSTD_freeCDict(ZSTD_CDict *cdict) -+{ -+ if (cdict == NULL) -+ return 0; /* support free on NULL */ -+ { -+ ZSTD_customMem const cMem = cdict->refContext->customMem; -+ ZSTD_freeCCtx(cdict->refContext); -+ ZSTD_free(cdict->dictBuffer, cMem); -+ ZSTD_free(cdict, cMem); -+ return 0; -+ } -+} -+ -+static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict *cdict) { return ZSTD_getParamsFromCCtx(cdict->refContext); } -+ -+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize) -+{ -+ if (cdict->dictContentSize) -+ CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) -+ else { -+ ZSTD_parameters params = cdict->refContext->params; -+ params.fParams.contentSizeFlag = (pledgedSrcSize > 0); -+ CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, params, pledgedSrcSize)); -+ } -+ return 0; -+} -+ -+/*! ZSTD_compress_usingCDict() : -+* Compression using a digested Dictionary. -+* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. -+* Note that compression level is decided during dictionary creation */ -+size_t ZSTD_compress_usingCDict(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_CDict *cdict) -+{ -+ CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize)); -+ -+ if (cdict->refContext->params.fParams.contentSizeFlag == 1) { -+ cctx->params.fParams.contentSizeFlag = 1; -+ cctx->frameContentSize = srcSize; -+ } else { -+ cctx->params.fParams.contentSizeFlag = 0; -+ } -+ -+ return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); -+} -+ -+/* ****************************************************************** -+* Streaming -+********************************************************************/ -+ -+typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; -+ -+struct ZSTD_CStream_s { -+ ZSTD_CCtx *cctx; -+ ZSTD_CDict *cdictLocal; -+ const ZSTD_CDict *cdict; -+ char *inBuff; -+ size_t inBuffSize; -+ size_t inToCompress; -+ size_t inBuffPos; -+ size_t inBuffTarget; -+ size_t blockSize; -+ char *outBuff; -+ size_t outBuffSize; -+ size_t outBuffContentSize; -+ size_t outBuffFlushedSize; -+ ZSTD_cStreamStage stage; -+ U32 checksum; -+ U32 frameEnded; -+ U64 pledgedSrcSize; -+ U64 inputProcessed; -+ ZSTD_parameters params; -+ ZSTD_customMem customMem; -+}; /* typedef'd to ZSTD_CStream within "zstd.h" */ -+ -+size_t ZSTD_CStreamWorkspaceBound(ZSTD_compressionParameters cParams) -+{ -+ size_t const inBuffSize = (size_t)1 << cParams.windowLog; -+ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, inBuffSize); -+ size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; -+ -+ return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize); -+} -+ -+ZSTD_CStream *ZSTD_createCStream_advanced(ZSTD_customMem customMem) -+{ -+ ZSTD_CStream *zcs; -+ -+ if (!customMem.customAlloc || !customMem.customFree) -+ return NULL; -+ -+ zcs = (ZSTD_CStream *)ZSTD_malloc(sizeof(ZSTD_CStream), customMem); -+ if (zcs == NULL) -+ return NULL; -+ memset(zcs, 0, sizeof(ZSTD_CStream)); -+ memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); -+ zcs->cctx = ZSTD_createCCtx_advanced(customMem); -+ if (zcs->cctx == NULL) { -+ ZSTD_freeCStream(zcs); -+ return NULL; -+ } -+ return zcs; -+} -+ -+size_t ZSTD_freeCStream(ZSTD_CStream *zcs) -+{ -+ if (zcs == NULL) -+ return 0; /* support free on NULL */ -+ { -+ ZSTD_customMem const cMem = zcs->customMem; -+ ZSTD_freeCCtx(zcs->cctx); -+ zcs->cctx = NULL; -+ ZSTD_freeCDict(zcs->cdictLocal); -+ zcs->cdictLocal = NULL; -+ ZSTD_free(zcs->inBuff, cMem); -+ zcs->inBuff = NULL; -+ ZSTD_free(zcs->outBuff, cMem); -+ zcs->outBuff = NULL; -+ ZSTD_free(zcs, cMem); -+ return 0; -+ } -+} -+ -+/*====== Initialization ======*/ -+ -+size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } -+size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */; } -+ -+static size_t ZSTD_resetCStream_internal(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize) -+{ -+ if (zcs->inBuffSize == 0) -+ return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */ -+ -+ if (zcs->cdict) -+ CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)) -+ else -+ CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize)); -+ -+ zcs->inToCompress = 0; -+ zcs->inBuffPos = 0; -+ zcs->inBuffTarget = zcs->blockSize; -+ zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; -+ zcs->stage = zcss_load; -+ zcs->frameEnded = 0; -+ zcs->pledgedSrcSize = pledgedSrcSize; -+ zcs->inputProcessed = 0; -+ return 0; /* ready to go */ -+} -+ -+size_t ZSTD_resetCStream(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize) -+{ -+ -+ zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0); -+ -+ return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); -+} -+ -+static size_t ZSTD_initCStream_advanced(ZSTD_CStream *zcs, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) -+{ -+ /* allocate buffers */ -+ { -+ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; -+ if (zcs->inBuffSize < neededInBuffSize) { -+ zcs->inBuffSize = neededInBuffSize; -+ ZSTD_free(zcs->inBuff, zcs->customMem); -+ zcs->inBuff = (char *)ZSTD_malloc(neededInBuffSize, zcs->customMem); -+ if (zcs->inBuff == NULL) -+ return ERROR(memory_allocation); -+ } -+ zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); -+ } -+ if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize) + 1) { -+ zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize) + 1; -+ ZSTD_free(zcs->outBuff, zcs->customMem); -+ zcs->outBuff = (char *)ZSTD_malloc(zcs->outBuffSize, zcs->customMem); -+ if (zcs->outBuff == NULL) -+ return ERROR(memory_allocation); -+ } -+ -+ if (dict && dictSize >= 8) { -+ ZSTD_freeCDict(zcs->cdictLocal); -+ zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem); -+ if (zcs->cdictLocal == NULL) -+ return ERROR(memory_allocation); -+ zcs->cdict = zcs->cdictLocal; -+ } else -+ zcs->cdict = NULL; -+ -+ zcs->checksum = params.fParams.checksumFlag > 0; -+ zcs->params = params; -+ -+ return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); -+} -+ -+ZSTD_CStream *ZSTD_initCStream(ZSTD_parameters params, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize) -+{ -+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); -+ ZSTD_CStream *const zcs = ZSTD_createCStream_advanced(stackMem); -+ if (zcs) { -+ size_t const code = ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize); -+ if (ZSTD_isError(code)) { -+ return NULL; -+ } -+ } -+ return zcs; -+} -+ -+ZSTD_CStream *ZSTD_initCStream_usingCDict(const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize) -+{ -+ ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict); -+ ZSTD_CStream *const zcs = ZSTD_initCStream(params, pledgedSrcSize, workspace, workspaceSize); -+ if (zcs) { -+ zcs->cdict = cdict; -+ if (ZSTD_isError(ZSTD_resetCStream_internal(zcs, pledgedSrcSize))) { -+ return NULL; -+ } -+ } -+ return zcs; -+} -+ -+/*====== Compression ======*/ -+ -+typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e; -+ -+ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ size_t const length = MIN(dstCapacity, srcSize); -+ memcpy(dst, src, length); -+ return length; -+} -+ -+static size_t ZSTD_compressStream_generic(ZSTD_CStream *zcs, void *dst, size_t *dstCapacityPtr, const void *src, size_t *srcSizePtr, ZSTD_flush_e const flush) -+{ -+ U32 someMoreWork = 1; -+ const char *const istart = (const char *)src; -+ const char *const iend = istart + *srcSizePtr; -+ const char *ip = istart; -+ char *const ostart = (char *)dst; -+ char *const oend = ostart + *dstCapacityPtr; -+ char *op = ostart; -+ -+ while (someMoreWork) { -+ switch (zcs->stage) { -+ case zcss_init: -+ return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ -+ -+ case zcss_load: -+ /* complete inBuffer */ -+ { -+ size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; -+ size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend - ip); -+ zcs->inBuffPos += loaded; -+ ip += loaded; -+ if ((zcs->inBuffPos == zcs->inToCompress) || (!flush && (toLoad != loaded))) { -+ someMoreWork = 0; -+ break; /* not enough input to get a full block : stop there, wait for more */ -+ } -+ } -+ /* compress curr block (note : this stage cannot be stopped in the middle) */ -+ { -+ void *cDst; -+ size_t cSize; -+ size_t const iSize = zcs->inBuffPos - zcs->inToCompress; -+ size_t oSize = oend - op; -+ if (oSize >= ZSTD_compressBound(iSize)) -+ cDst = op; /* compress directly into output buffer (avoid flush stage) */ -+ else -+ cDst = zcs->outBuff, oSize = zcs->outBuffSize; -+ cSize = (flush == zsf_end) ? ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) -+ : ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); -+ if (ZSTD_isError(cSize)) -+ return cSize; -+ if (flush == zsf_end) -+ zcs->frameEnded = 1; -+ /* prepare next block */ -+ zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; -+ if (zcs->inBuffTarget > zcs->inBuffSize) -+ zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */ -+ zcs->inToCompress = zcs->inBuffPos; -+ if (cDst == op) { -+ op += cSize; -+ break; -+ } /* no need to flush */ -+ zcs->outBuffContentSize = cSize; -+ zcs->outBuffFlushedSize = 0; -+ zcs->stage = zcss_flush; /* pass-through to flush stage */ -+ } -+ -+ case zcss_flush: { -+ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; -+ size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); -+ op += flushed; -+ zcs->outBuffFlushedSize += flushed; -+ if (toFlush != flushed) { -+ someMoreWork = 0; -+ break; -+ } /* dst too small to store flushed data : stop there */ -+ zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; -+ zcs->stage = zcss_load; -+ break; -+ } -+ -+ case zcss_final: -+ someMoreWork = 0; /* do nothing */ -+ break; -+ -+ default: -+ return ERROR(GENERIC); /* impossible */ -+ } -+ } -+ -+ *srcSizePtr = ip - istart; -+ *dstCapacityPtr = op - ostart; -+ zcs->inputProcessed += *srcSizePtr; -+ if (zcs->frameEnded) -+ return 0; -+ { -+ size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; -+ if (hintInSize == 0) -+ hintInSize = zcs->blockSize; -+ return hintInSize; -+ } -+} -+ -+size_t ZSTD_compressStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output, ZSTD_inBuffer *input) -+{ -+ size_t sizeRead = input->size - input->pos; -+ size_t sizeWritten = output->size - output->pos; -+ size_t const result = -+ ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, (const char *)(input->src) + input->pos, &sizeRead, zsf_gather); -+ input->pos += sizeRead; -+ output->pos += sizeWritten; -+ return result; -+} -+ -+/*====== Finalize ======*/ -+ -+/*! ZSTD_flushStream() : -+* @return : amount of data remaining to flush */ -+size_t ZSTD_flushStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output) -+{ -+ size_t srcSize = 0; -+ size_t sizeWritten = output->size - output->pos; -+ size_t const result = ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, &srcSize, -+ &srcSize, /* use a valid src address instead of NULL */ -+ zsf_flush); -+ output->pos += sizeWritten; -+ if (ZSTD_isError(result)) -+ return result; -+ return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ -+} -+ -+size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output) -+{ -+ BYTE *const ostart = (BYTE *)(output->dst) + output->pos; -+ BYTE *const oend = (BYTE *)(output->dst) + output->size; -+ BYTE *op = ostart; -+ -+ if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize)) -+ return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */ -+ -+ if (zcs->stage != zcss_final) { -+ /* flush whatever remains */ -+ size_t srcSize = 0; -+ size_t sizeWritten = output->size - output->pos; -+ size_t const notEnded = -+ ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */ -+ size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; -+ op += sizeWritten; -+ if (remainingToFlush) { -+ output->pos += sizeWritten; -+ return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4); -+ } -+ /* create epilogue */ -+ zcs->stage = zcss_final; -+ zcs->outBuffContentSize = !notEnded ? 0 : ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, -+ 0); /* write epilogue, including final empty block, into outBuff */ -+ } -+ -+ /* flush epilogue */ -+ { -+ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; -+ size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); -+ op += flushed; -+ zcs->outBuffFlushedSize += flushed; -+ output->pos += op - ostart; -+ if (toFlush == flushed) -+ zcs->stage = zcss_init; /* end reached */ -+ return toFlush - flushed; -+ } -+} -+ -+/*-===== Pre-defined compression levels =====-*/ -+ -+#define ZSTD_DEFAULT_CLEVEL 1 -+#define ZSTD_MAX_CLEVEL 22 -+int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } -+ -+static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL + 1] = { -+ { -+ /* "default" */ -+ /* W, C, H, S, L, TL, strat */ -+ {18, 12, 12, 1, 7, 16, ZSTD_fast}, /* level 0 - never used */ -+ {19, 13, 14, 1, 7, 16, ZSTD_fast}, /* level 1 */ -+ {19, 15, 16, 1, 6, 16, ZSTD_fast}, /* level 2 */ -+ {20, 16, 17, 1, 5, 16, ZSTD_dfast}, /* level 3.*/ -+ {20, 18, 18, 1, 5, 16, ZSTD_dfast}, /* level 4.*/ -+ {20, 15, 18, 3, 5, 16, ZSTD_greedy}, /* level 5 */ -+ {21, 16, 19, 2, 5, 16, ZSTD_lazy}, /* level 6 */ -+ {21, 17, 20, 3, 5, 16, ZSTD_lazy}, /* level 7 */ -+ {21, 18, 20, 3, 5, 16, ZSTD_lazy2}, /* level 8 */ -+ {21, 20, 20, 3, 5, 16, ZSTD_lazy2}, /* level 9 */ -+ {21, 19, 21, 4, 5, 16, ZSTD_lazy2}, /* level 10 */ -+ {22, 20, 22, 4, 5, 16, ZSTD_lazy2}, /* level 11 */ -+ {22, 20, 22, 5, 5, 16, ZSTD_lazy2}, /* level 12 */ -+ {22, 21, 22, 5, 5, 16, ZSTD_lazy2}, /* level 13 */ -+ {22, 21, 22, 6, 5, 16, ZSTD_lazy2}, /* level 14 */ -+ {22, 21, 21, 5, 5, 16, ZSTD_btlazy2}, /* level 15 */ -+ {23, 22, 22, 5, 5, 16, ZSTD_btlazy2}, /* level 16 */ -+ {23, 21, 22, 4, 5, 24, ZSTD_btopt}, /* level 17 */ -+ {23, 23, 22, 6, 5, 32, ZSTD_btopt}, /* level 18 */ -+ {23, 23, 22, 6, 3, 48, ZSTD_btopt}, /* level 19 */ -+ {25, 25, 23, 7, 3, 64, ZSTD_btopt2}, /* level 20 */ -+ {26, 26, 23, 7, 3, 256, ZSTD_btopt2}, /* level 21 */ -+ {27, 27, 25, 9, 3, 512, ZSTD_btopt2}, /* level 22 */ -+ }, -+ { -+ /* for srcSize <= 256 KB */ -+ /* W, C, H, S, L, T, strat */ -+ {0, 0, 0, 0, 0, 0, ZSTD_fast}, /* level 0 - not used */ -+ {18, 13, 14, 1, 6, 8, ZSTD_fast}, /* level 1 */ -+ {18, 14, 13, 1, 5, 8, ZSTD_dfast}, /* level 2 */ -+ {18, 16, 15, 1, 5, 8, ZSTD_dfast}, /* level 3 */ -+ {18, 15, 17, 1, 5, 8, ZSTD_greedy}, /* level 4.*/ -+ {18, 16, 17, 4, 5, 8, ZSTD_greedy}, /* level 5.*/ -+ {18, 16, 17, 3, 5, 8, ZSTD_lazy}, /* level 6.*/ -+ {18, 17, 17, 4, 4, 8, ZSTD_lazy}, /* level 7 */ -+ {18, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */ -+ {18, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */ -+ {18, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */ -+ {18, 18, 17, 6, 4, 8, ZSTD_lazy2}, /* level 11.*/ -+ {18, 18, 17, 7, 4, 8, ZSTD_lazy2}, /* level 12.*/ -+ {18, 19, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13 */ -+ {18, 18, 18, 4, 4, 16, ZSTD_btopt}, /* level 14.*/ -+ {18, 18, 18, 4, 3, 16, ZSTD_btopt}, /* level 15.*/ -+ {18, 19, 18, 6, 3, 32, ZSTD_btopt}, /* level 16.*/ -+ {18, 19, 18, 8, 3, 64, ZSTD_btopt}, /* level 17.*/ -+ {18, 19, 18, 9, 3, 128, ZSTD_btopt}, /* level 18.*/ -+ {18, 19, 18, 10, 3, 256, ZSTD_btopt}, /* level 19.*/ -+ {18, 19, 18, 11, 3, 512, ZSTD_btopt2}, /* level 20.*/ -+ {18, 19, 18, 12, 3, 512, ZSTD_btopt2}, /* level 21.*/ -+ {18, 19, 18, 13, 3, 512, ZSTD_btopt2}, /* level 22.*/ -+ }, -+ { -+ /* for srcSize <= 128 KB */ -+ /* W, C, H, S, L, T, strat */ -+ {17, 12, 12, 1, 7, 8, ZSTD_fast}, /* level 0 - not used */ -+ {17, 12, 13, 1, 6, 8, ZSTD_fast}, /* level 1 */ -+ {17, 13, 16, 1, 5, 8, ZSTD_fast}, /* level 2 */ -+ {17, 16, 16, 2, 5, 8, ZSTD_dfast}, /* level 3 */ -+ {17, 13, 15, 3, 4, 8, ZSTD_greedy}, /* level 4 */ -+ {17, 15, 17, 4, 4, 8, ZSTD_greedy}, /* level 5 */ -+ {17, 16, 17, 3, 4, 8, ZSTD_lazy}, /* level 6 */ -+ {17, 15, 17, 4, 4, 8, ZSTD_lazy2}, /* level 7 */ -+ {17, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */ -+ {17, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */ -+ {17, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */ -+ {17, 17, 17, 7, 4, 8, ZSTD_lazy2}, /* level 11 */ -+ {17, 17, 17, 8, 4, 8, ZSTD_lazy2}, /* level 12 */ -+ {17, 18, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13.*/ -+ {17, 17, 17, 7, 3, 8, ZSTD_btopt}, /* level 14.*/ -+ {17, 17, 17, 7, 3, 16, ZSTD_btopt}, /* level 15.*/ -+ {17, 18, 17, 7, 3, 32, ZSTD_btopt}, /* level 16.*/ -+ {17, 18, 17, 7, 3, 64, ZSTD_btopt}, /* level 17.*/ -+ {17, 18, 17, 7, 3, 256, ZSTD_btopt}, /* level 18.*/ -+ {17, 18, 17, 8, 3, 256, ZSTD_btopt}, /* level 19.*/ -+ {17, 18, 17, 9, 3, 256, ZSTD_btopt2}, /* level 20.*/ -+ {17, 18, 17, 10, 3, 256, ZSTD_btopt2}, /* level 21.*/ -+ {17, 18, 17, 11, 3, 512, ZSTD_btopt2}, /* level 22.*/ -+ }, -+ { -+ /* for srcSize <= 16 KB */ -+ /* W, C, H, S, L, T, strat */ -+ {14, 12, 12, 1, 7, 6, ZSTD_fast}, /* level 0 - not used */ -+ {14, 14, 14, 1, 6, 6, ZSTD_fast}, /* level 1 */ -+ {14, 14, 14, 1, 4, 6, ZSTD_fast}, /* level 2 */ -+ {14, 14, 14, 1, 4, 6, ZSTD_dfast}, /* level 3.*/ -+ {14, 14, 14, 4, 4, 6, ZSTD_greedy}, /* level 4.*/ -+ {14, 14, 14, 3, 4, 6, ZSTD_lazy}, /* level 5.*/ -+ {14, 14, 14, 4, 4, 6, ZSTD_lazy2}, /* level 6 */ -+ {14, 14, 14, 5, 4, 6, ZSTD_lazy2}, /* level 7 */ -+ {14, 14, 14, 6, 4, 6, ZSTD_lazy2}, /* level 8.*/ -+ {14, 15, 14, 6, 4, 6, ZSTD_btlazy2}, /* level 9.*/ -+ {14, 15, 14, 3, 3, 6, ZSTD_btopt}, /* level 10.*/ -+ {14, 15, 14, 6, 3, 8, ZSTD_btopt}, /* level 11.*/ -+ {14, 15, 14, 6, 3, 16, ZSTD_btopt}, /* level 12.*/ -+ {14, 15, 14, 6, 3, 24, ZSTD_btopt}, /* level 13.*/ -+ {14, 15, 15, 6, 3, 48, ZSTD_btopt}, /* level 14.*/ -+ {14, 15, 15, 6, 3, 64, ZSTD_btopt}, /* level 15.*/ -+ {14, 15, 15, 6, 3, 96, ZSTD_btopt}, /* level 16.*/ -+ {14, 15, 15, 6, 3, 128, ZSTD_btopt}, /* level 17.*/ -+ {14, 15, 15, 6, 3, 256, ZSTD_btopt}, /* level 18.*/ -+ {14, 15, 15, 7, 3, 256, ZSTD_btopt}, /* level 19.*/ -+ {14, 15, 15, 8, 3, 256, ZSTD_btopt2}, /* level 20.*/ -+ {14, 15, 15, 9, 3, 256, ZSTD_btopt2}, /* level 21.*/ -+ {14, 15, 15, 10, 3, 256, ZSTD_btopt2}, /* level 22.*/ -+ }, -+}; -+ -+/*! ZSTD_getCParams() : -+* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`. -+* Size values are optional, provide 0 if not known or unused */ -+ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) -+{ -+ ZSTD_compressionParameters cp; -+ size_t const addedSize = srcSize ? 0 : 500; -+ U64 const rSize = srcSize + dictSize ? srcSize + dictSize + addedSize : (U64)-1; -+ U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ -+ if (compressionLevel <= 0) -+ compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */ -+ if (compressionLevel > ZSTD_MAX_CLEVEL) -+ compressionLevel = ZSTD_MAX_CLEVEL; -+ cp = ZSTD_defaultCParameters[tableID][compressionLevel]; -+ if (ZSTD_32bits()) { /* auto-correction, for 32-bits mode */ -+ if (cp.windowLog > ZSTD_WINDOWLOG_MAX) -+ cp.windowLog = ZSTD_WINDOWLOG_MAX; -+ if (cp.chainLog > ZSTD_CHAINLOG_MAX) -+ cp.chainLog = ZSTD_CHAINLOG_MAX; -+ if (cp.hashLog > ZSTD_HASHLOG_MAX) -+ cp.hashLog = ZSTD_HASHLOG_MAX; -+ } -+ cp = ZSTD_adjustCParams(cp, srcSize, dictSize); -+ return cp; -+} -+ -+/*! ZSTD_getParams() : -+* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). -+* All fields of `ZSTD_frameParameters` are set to default (0) */ -+ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) -+{ -+ ZSTD_parameters params; -+ ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); -+ memset(¶ms, 0, sizeof(params)); -+ params.cParams = cParams; -+ return params; -+} -+ -+EXPORT_SYMBOL(ZSTD_maxCLevel); -+EXPORT_SYMBOL(ZSTD_compressBound); -+ -+EXPORT_SYMBOL(ZSTD_CCtxWorkspaceBound); -+EXPORT_SYMBOL(ZSTD_initCCtx); -+EXPORT_SYMBOL(ZSTD_compressCCtx); -+EXPORT_SYMBOL(ZSTD_compress_usingDict); -+ -+EXPORT_SYMBOL(ZSTD_CDictWorkspaceBound); -+EXPORT_SYMBOL(ZSTD_initCDict); -+EXPORT_SYMBOL(ZSTD_compress_usingCDict); -+ -+EXPORT_SYMBOL(ZSTD_CStreamWorkspaceBound); -+EXPORT_SYMBOL(ZSTD_initCStream); -+EXPORT_SYMBOL(ZSTD_initCStream_usingCDict); -+EXPORT_SYMBOL(ZSTD_resetCStream); -+EXPORT_SYMBOL(ZSTD_compressStream); -+EXPORT_SYMBOL(ZSTD_flushStream); -+EXPORT_SYMBOL(ZSTD_endStream); -+EXPORT_SYMBOL(ZSTD_CStreamInSize); -+EXPORT_SYMBOL(ZSTD_CStreamOutSize); -+ -+EXPORT_SYMBOL(ZSTD_getCParams); -+EXPORT_SYMBOL(ZSTD_getParams); -+EXPORT_SYMBOL(ZSTD_checkCParams); -+EXPORT_SYMBOL(ZSTD_adjustCParams); -+ -+EXPORT_SYMBOL(ZSTD_compressBegin); -+EXPORT_SYMBOL(ZSTD_compressBegin_usingDict); -+EXPORT_SYMBOL(ZSTD_compressBegin_advanced); -+EXPORT_SYMBOL(ZSTD_copyCCtx); -+EXPORT_SYMBOL(ZSTD_compressBegin_usingCDict); -+EXPORT_SYMBOL(ZSTD_compressContinue); -+EXPORT_SYMBOL(ZSTD_compressEnd); -+ -+EXPORT_SYMBOL(ZSTD_getBlockSizeMax); -+EXPORT_SYMBOL(ZSTD_compressBlock); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("Zstd Compressor"); -diff --git a/lib/zstd/decompress.c b/lib/zstd/decompress.c -new file mode 100644 -index 0000000..72df4828 ---- /dev/null -+++ b/lib/zstd/decompress.c -@@ -0,0 +1,2526 @@ -+/** -+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. -+ * All rights reserved. -+ * -+ * This source code is licensed under the BSD-style license found in the -+ * LICENSE file in the root directory of https://github.com/facebook/zstd. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ */ -+ -+/* *************************************************************** -+* Tuning parameters -+*****************************************************************/ -+/*! -+* MAXWINDOWSIZE_DEFAULT : -+* maximum window size accepted by DStream, by default. -+* Frames requiring more memory will be rejected. -+*/ -+#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT -+#define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */ -+#endif -+ -+/*-******************************************************* -+* Dependencies -+*********************************************************/ -+#include "fse.h" -+#include "huf.h" -+#include "mem.h" /* low level memory routines */ -+#include "zstd_internal.h" -+#include <linux/kernel.h> -+#include <linux/module.h> -+#include <linux/string.h> /* memcpy, memmove, memset */ -+ -+#define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0) -+ -+/*-************************************* -+* Macros -+***************************************/ -+#define ZSTD_isError ERR_isError /* for inlining */ -+#define FSE_isError ERR_isError -+#define HUF_isError ERR_isError -+ -+/*_******************************************************* -+* Memory operations -+**********************************************************/ -+static void ZSTD_copy4(void *dst, const void *src) { memcpy(dst, src, 4); } -+ -+/*-************************************************************* -+* Context management -+***************************************************************/ -+typedef enum { -+ ZSTDds_getFrameHeaderSize, -+ ZSTDds_decodeFrameHeader, -+ ZSTDds_decodeBlockHeader, -+ ZSTDds_decompressBlock, -+ ZSTDds_decompressLastBlock, -+ ZSTDds_checkChecksum, -+ ZSTDds_decodeSkippableHeader, -+ ZSTDds_skipFrame -+} ZSTD_dStage; -+ -+typedef struct { -+ FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; -+ FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; -+ FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; -+ HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ -+ U64 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32 / 2]; -+ U32 rep[ZSTD_REP_NUM]; -+} ZSTD_entropyTables_t; -+ -+struct ZSTD_DCtx_s { -+ const FSE_DTable *LLTptr; -+ const FSE_DTable *MLTptr; -+ const FSE_DTable *OFTptr; -+ const HUF_DTable *HUFptr; -+ ZSTD_entropyTables_t entropy; -+ const void *previousDstEnd; /* detect continuity */ -+ const void *base; /* start of curr segment */ -+ const void *vBase; /* virtual start of previous segment if it was just before curr one */ -+ const void *dictEnd; /* end of previous segment */ -+ size_t expected; -+ ZSTD_frameParams fParams; -+ blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ -+ ZSTD_dStage stage; -+ U32 litEntropy; -+ U32 fseEntropy; -+ struct xxh64_state xxhState; -+ size_t headerSize; -+ U32 dictID; -+ const BYTE *litPtr; -+ ZSTD_customMem customMem; -+ size_t litSize; -+ size_t rleSize; -+ BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; -+ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; -+}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ -+ -+size_t ZSTD_DCtxWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DCtx)); } -+ -+size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx) -+{ -+ dctx->expected = ZSTD_frameHeaderSize_prefix; -+ dctx->stage = ZSTDds_getFrameHeaderSize; -+ dctx->previousDstEnd = NULL; -+ dctx->base = NULL; -+ dctx->vBase = NULL; -+ dctx->dictEnd = NULL; -+ dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ -+ dctx->litEntropy = dctx->fseEntropy = 0; -+ dctx->dictID = 0; -+ ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); -+ memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ -+ dctx->LLTptr = dctx->entropy.LLTable; -+ dctx->MLTptr = dctx->entropy.MLTable; -+ dctx->OFTptr = dctx->entropy.OFTable; -+ dctx->HUFptr = dctx->entropy.hufTable; -+ return 0; -+} -+ -+ZSTD_DCtx *ZSTD_createDCtx_advanced(ZSTD_customMem customMem) -+{ -+ ZSTD_DCtx *dctx; -+ -+ if (!customMem.customAlloc || !customMem.customFree) -+ return NULL; -+ -+ dctx = (ZSTD_DCtx *)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem); -+ if (!dctx) -+ return NULL; -+ memcpy(&dctx->customMem, &customMem, sizeof(customMem)); -+ ZSTD_decompressBegin(dctx); -+ return dctx; -+} -+ -+ZSTD_DCtx *ZSTD_initDCtx(void *workspace, size_t workspaceSize) -+{ -+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); -+ return ZSTD_createDCtx_advanced(stackMem); -+} -+ -+size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx) -+{ -+ if (dctx == NULL) -+ return 0; /* support free on NULL */ -+ ZSTD_free(dctx, dctx->customMem); -+ return 0; /* reserved as a potential error code in the future */ -+} -+ -+void ZSTD_copyDCtx(ZSTD_DCtx *dstDCtx, const ZSTD_DCtx *srcDCtx) -+{ -+ size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max; -+ memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ -+} -+ -+static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict); -+ -+/*-************************************************************* -+* Decompression section -+***************************************************************/ -+ -+/*! ZSTD_isFrame() : -+ * Tells if the content of `buffer` starts with a valid Frame Identifier. -+ * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. -+ * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. -+ * Note 3 : Skippable Frame Identifiers are considered valid. */ -+unsigned ZSTD_isFrame(const void *buffer, size_t size) -+{ -+ if (size < 4) -+ return 0; -+ { -+ U32 const magic = ZSTD_readLE32(buffer); -+ if (magic == ZSTD_MAGICNUMBER) -+ return 1; -+ if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) -+ return 1; -+ } -+ return 0; -+} -+ -+/** ZSTD_frameHeaderSize() : -+* srcSize must be >= ZSTD_frameHeaderSize_prefix. -+* @return : size of the Frame Header */ -+static size_t ZSTD_frameHeaderSize(const void *src, size_t srcSize) -+{ -+ if (srcSize < ZSTD_frameHeaderSize_prefix) -+ return ERROR(srcSize_wrong); -+ { -+ BYTE const fhd = ((const BYTE *)src)[4]; -+ U32 const dictID = fhd & 3; -+ U32 const singleSegment = (fhd >> 5) & 1; -+ U32 const fcsId = fhd >> 6; -+ return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + (singleSegment && !fcsId); -+ } -+} -+ -+/** ZSTD_getFrameParams() : -+* decode Frame Header, or require larger `srcSize`. -+* @return : 0, `fparamsPtr` is correctly filled, -+* >0, `srcSize` is too small, result is expected `srcSize`, -+* or an error code, which can be tested using ZSTD_isError() */ -+size_t ZSTD_getFrameParams(ZSTD_frameParams *fparamsPtr, const void *src, size_t srcSize) -+{ -+ const BYTE *ip = (const BYTE *)src; -+ -+ if (srcSize < ZSTD_frameHeaderSize_prefix) -+ return ZSTD_frameHeaderSize_prefix; -+ if (ZSTD_readLE32(src) != ZSTD_MAGICNUMBER) { -+ if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { -+ if (srcSize < ZSTD_skippableHeaderSize) -+ return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ -+ memset(fparamsPtr, 0, sizeof(*fparamsPtr)); -+ fparamsPtr->frameContentSize = ZSTD_readLE32((const char *)src + 4); -+ fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ -+ return 0; -+ } -+ return ERROR(prefix_unknown); -+ } -+ -+ /* ensure there is enough `srcSize` to fully read/decode frame header */ -+ { -+ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); -+ if (srcSize < fhsize) -+ return fhsize; -+ } -+ -+ { -+ BYTE const fhdByte = ip[4]; -+ size_t pos = 5; -+ U32 const dictIDSizeCode = fhdByte & 3; -+ U32 const checksumFlag = (fhdByte >> 2) & 1; -+ U32 const singleSegment = (fhdByte >> 5) & 1; -+ U32 const fcsID = fhdByte >> 6; -+ U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; -+ U32 windowSize = 0; -+ U32 dictID = 0; -+ U64 frameContentSize = 0; -+ if ((fhdByte & 0x08) != 0) -+ return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */ -+ if (!singleSegment) { -+ BYTE const wlByte = ip[pos++]; -+ U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; -+ if (windowLog > ZSTD_WINDOWLOG_MAX) -+ return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */ -+ windowSize = (1U << windowLog); -+ windowSize += (windowSize >> 3) * (wlByte & 7); -+ } -+ -+ switch (dictIDSizeCode) { -+ default: /* impossible */ -+ case 0: break; -+ case 1: -+ dictID = ip[pos]; -+ pos++; -+ break; -+ case 2: -+ dictID = ZSTD_readLE16(ip + pos); -+ pos += 2; -+ break; -+ case 3: -+ dictID = ZSTD_readLE32(ip + pos); -+ pos += 4; -+ break; -+ } -+ switch (fcsID) { -+ default: /* impossible */ -+ case 0: -+ if (singleSegment) -+ frameContentSize = ip[pos]; -+ break; -+ case 1: frameContentSize = ZSTD_readLE16(ip + pos) + 256; break; -+ case 2: frameContentSize = ZSTD_readLE32(ip + pos); break; -+ case 3: frameContentSize = ZSTD_readLE64(ip + pos); break; -+ } -+ if (!windowSize) -+ windowSize = (U32)frameContentSize; -+ if (windowSize > windowSizeMax) -+ return ERROR(frameParameter_windowTooLarge); -+ fparamsPtr->frameContentSize = frameContentSize; -+ fparamsPtr->windowSize = windowSize; -+ fparamsPtr->dictID = dictID; -+ fparamsPtr->checksumFlag = checksumFlag; -+ } -+ return 0; -+} -+ -+/** ZSTD_getFrameContentSize() : -+* compatible with legacy mode -+* @return : decompressed size of the single frame pointed to be `src` if known, otherwise -+* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined -+* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ -+unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) -+{ -+ { -+ ZSTD_frameParams fParams; -+ if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0) -+ return ZSTD_CONTENTSIZE_ERROR; -+ if (fParams.windowSize == 0) { -+ /* Either skippable or empty frame, size == 0 either way */ -+ return 0; -+ } else if (fParams.frameContentSize != 0) { -+ return fParams.frameContentSize; -+ } else { -+ return ZSTD_CONTENTSIZE_UNKNOWN; -+ } -+ } -+} -+ -+/** ZSTD_findDecompressedSize() : -+ * compatible with legacy mode -+ * `srcSize` must be the exact length of some number of ZSTD compressed and/or -+ * skippable frames -+ * @return : decompressed size of the frames contained */ -+unsigned long long ZSTD_findDecompressedSize(const void *src, size_t srcSize) -+{ -+ { -+ unsigned long long totalDstSize = 0; -+ while (srcSize >= ZSTD_frameHeaderSize_prefix) { -+ const U32 magicNumber = ZSTD_readLE32(src); -+ -+ if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { -+ size_t skippableSize; -+ if (srcSize < ZSTD_skippableHeaderSize) -+ return ERROR(srcSize_wrong); -+ skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize; -+ if (srcSize < skippableSize) { -+ return ZSTD_CONTENTSIZE_ERROR; -+ } -+ -+ src = (const BYTE *)src + skippableSize; -+ srcSize -= skippableSize; -+ continue; -+ } -+ -+ { -+ unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); -+ if (ret >= ZSTD_CONTENTSIZE_ERROR) -+ return ret; -+ -+ /* check for overflow */ -+ if (totalDstSize + ret < totalDstSize) -+ return ZSTD_CONTENTSIZE_ERROR; -+ totalDstSize += ret; -+ } -+ { -+ size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); -+ if (ZSTD_isError(frameSrcSize)) { -+ return ZSTD_CONTENTSIZE_ERROR; -+ } -+ -+ src = (const BYTE *)src + frameSrcSize; -+ srcSize -= frameSrcSize; -+ } -+ } -+ -+ if (srcSize) { -+ return ZSTD_CONTENTSIZE_ERROR; -+ } -+ -+ return totalDstSize; -+ } -+} -+ -+/** ZSTD_decodeFrameHeader() : -+* `headerSize` must be the size provided by ZSTD_frameHeaderSize(). -+* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ -+static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx *dctx, const void *src, size_t headerSize) -+{ -+ size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize); -+ if (ZSTD_isError(result)) -+ return result; /* invalid header */ -+ if (result > 0) -+ return ERROR(srcSize_wrong); /* headerSize too small */ -+ if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) -+ return ERROR(dictionary_wrong); -+ if (dctx->fParams.checksumFlag) -+ xxh64_reset(&dctx->xxhState, 0); -+ return 0; -+} -+ -+typedef struct { -+ blockType_e blockType; -+ U32 lastBlock; -+ U32 origSize; -+} blockProperties_t; -+ -+/*! ZSTD_getcBlockSize() : -+* Provides the size of compressed block from block header `src` */ -+size_t ZSTD_getcBlockSize(const void *src, size_t srcSize, blockProperties_t *bpPtr) -+{ -+ if (srcSize < ZSTD_blockHeaderSize) -+ return ERROR(srcSize_wrong); -+ { -+ U32 const cBlockHeader = ZSTD_readLE24(src); -+ U32 const cSize = cBlockHeader >> 3; -+ bpPtr->lastBlock = cBlockHeader & 1; -+ bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); -+ bpPtr->origSize = cSize; /* only useful for RLE */ -+ if (bpPtr->blockType == bt_rle) -+ return 1; -+ if (bpPtr->blockType == bt_reserved) -+ return ERROR(corruption_detected); -+ return cSize; -+ } -+} -+ -+static size_t ZSTD_copyRawBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ if (srcSize > dstCapacity) -+ return ERROR(dstSize_tooSmall); -+ memcpy(dst, src, srcSize); -+ return srcSize; -+} -+ -+static size_t ZSTD_setRleBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize, size_t regenSize) -+{ -+ if (srcSize != 1) -+ return ERROR(srcSize_wrong); -+ if (regenSize > dstCapacity) -+ return ERROR(dstSize_tooSmall); -+ memset(dst, *(const BYTE *)src, regenSize); -+ return regenSize; -+} -+ -+/*! ZSTD_decodeLiteralsBlock() : -+ @return : nb of bytes read from src (< srcSize ) */ -+size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx *dctx, const void *src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ -+{ -+ if (srcSize < MIN_CBLOCK_SIZE) -+ return ERROR(corruption_detected); -+ -+ { -+ const BYTE *const istart = (const BYTE *)src; -+ symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); -+ -+ switch (litEncType) { -+ case set_repeat: -+ if (dctx->litEntropy == 0) -+ return ERROR(dictionary_corrupted); -+ /* fall-through */ -+ case set_compressed: -+ if (srcSize < 5) -+ return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ -+ { -+ size_t lhSize, litSize, litCSize; -+ U32 singleStream = 0; -+ U32 const lhlCode = (istart[0] >> 2) & 3; -+ U32 const lhc = ZSTD_readLE32(istart); -+ switch (lhlCode) { -+ case 0: -+ case 1: -+ default: /* note : default is impossible, since lhlCode into [0..3] */ -+ /* 2 - 2 - 10 - 10 */ -+ singleStream = !lhlCode; -+ lhSize = 3; -+ litSize = (lhc >> 4) & 0x3FF; -+ litCSize = (lhc >> 14) & 0x3FF; -+ break; -+ case 2: -+ /* 2 - 2 - 14 - 14 */ -+ lhSize = 4; -+ litSize = (lhc >> 4) & 0x3FFF; -+ litCSize = lhc >> 18; -+ break; -+ case 3: -+ /* 2 - 2 - 18 - 18 */ -+ lhSize = 5; -+ litSize = (lhc >> 4) & 0x3FFFF; -+ litCSize = (lhc >> 22) + (istart[4] << 10); -+ break; -+ } -+ if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) -+ return ERROR(corruption_detected); -+ if (litCSize + lhSize > srcSize) -+ return ERROR(corruption_detected); -+ -+ if (HUF_isError( -+ (litEncType == set_repeat) -+ ? (singleStream ? HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr) -+ : HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr)) -+ : (singleStream -+ ? HUF_decompress1X2_DCtx_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize, -+ dctx->entropy.workspace, sizeof(dctx->entropy.workspace)) -+ : HUF_decompress4X_hufOnly_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize, -+ dctx->entropy.workspace, sizeof(dctx->entropy.workspace))))) -+ return ERROR(corruption_detected); -+ -+ dctx->litPtr = dctx->litBuffer; -+ dctx->litSize = litSize; -+ dctx->litEntropy = 1; -+ if (litEncType == set_compressed) -+ dctx->HUFptr = dctx->entropy.hufTable; -+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); -+ return litCSize + lhSize; -+ } -+ -+ case set_basic: { -+ size_t litSize, lhSize; -+ U32 const lhlCode = ((istart[0]) >> 2) & 3; -+ switch (lhlCode) { -+ case 0: -+ case 2: -+ default: /* note : default is impossible, since lhlCode into [0..3] */ -+ lhSize = 1; -+ litSize = istart[0] >> 3; -+ break; -+ case 1: -+ lhSize = 2; -+ litSize = ZSTD_readLE16(istart) >> 4; -+ break; -+ case 3: -+ lhSize = 3; -+ litSize = ZSTD_readLE24(istart) >> 4; -+ break; -+ } -+ -+ if (lhSize + litSize + WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ -+ if (litSize + lhSize > srcSize) -+ return ERROR(corruption_detected); -+ memcpy(dctx->litBuffer, istart + lhSize, litSize); -+ dctx->litPtr = dctx->litBuffer; -+ dctx->litSize = litSize; -+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); -+ return lhSize + litSize; -+ } -+ /* direct reference into compressed stream */ -+ dctx->litPtr = istart + lhSize; -+ dctx->litSize = litSize; -+ return lhSize + litSize; -+ } -+ -+ case set_rle: { -+ U32 const lhlCode = ((istart[0]) >> 2) & 3; -+ size_t litSize, lhSize; -+ switch (lhlCode) { -+ case 0: -+ case 2: -+ default: /* note : default is impossible, since lhlCode into [0..3] */ -+ lhSize = 1; -+ litSize = istart[0] >> 3; -+ break; -+ case 1: -+ lhSize = 2; -+ litSize = ZSTD_readLE16(istart) >> 4; -+ break; -+ case 3: -+ lhSize = 3; -+ litSize = ZSTD_readLE24(istart) >> 4; -+ if (srcSize < 4) -+ return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ -+ break; -+ } -+ if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) -+ return ERROR(corruption_detected); -+ memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); -+ dctx->litPtr = dctx->litBuffer; -+ dctx->litSize = litSize; -+ return lhSize + 1; -+ } -+ default: -+ return ERROR(corruption_detected); /* impossible */ -+ } -+ } -+} -+ -+typedef union { -+ FSE_decode_t realData; -+ U32 alignedBy4; -+} FSE_decode_t4; -+ -+static const FSE_decode_t4 LL_defaultDTable[(1 << LL_DEFAULTNORMLOG) + 1] = { -+ {{LL_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */ -+ {{0, 0, 4}}, /* 0 : base, symbol, bits */ -+ {{16, 0, 4}}, -+ {{32, 1, 5}}, -+ {{0, 3, 5}}, -+ {{0, 4, 5}}, -+ {{0, 6, 5}}, -+ {{0, 7, 5}}, -+ {{0, 9, 5}}, -+ {{0, 10, 5}}, -+ {{0, 12, 5}}, -+ {{0, 14, 6}}, -+ {{0, 16, 5}}, -+ {{0, 18, 5}}, -+ {{0, 19, 5}}, -+ {{0, 21, 5}}, -+ {{0, 22, 5}}, -+ {{0, 24, 5}}, -+ {{32, 25, 5}}, -+ {{0, 26, 5}}, -+ {{0, 27, 6}}, -+ {{0, 29, 6}}, -+ {{0, 31, 6}}, -+ {{32, 0, 4}}, -+ {{0, 1, 4}}, -+ {{0, 2, 5}}, -+ {{32, 4, 5}}, -+ {{0, 5, 5}}, -+ {{32, 7, 5}}, -+ {{0, 8, 5}}, -+ {{32, 10, 5}}, -+ {{0, 11, 5}}, -+ {{0, 13, 6}}, -+ {{32, 16, 5}}, -+ {{0, 17, 5}}, -+ {{32, 19, 5}}, -+ {{0, 20, 5}}, -+ {{32, 22, 5}}, -+ {{0, 23, 5}}, -+ {{0, 25, 4}}, -+ {{16, 25, 4}}, -+ {{32, 26, 5}}, -+ {{0, 28, 6}}, -+ {{0, 30, 6}}, -+ {{48, 0, 4}}, -+ {{16, 1, 4}}, -+ {{32, 2, 5}}, -+ {{32, 3, 5}}, -+ {{32, 5, 5}}, -+ {{32, 6, 5}}, -+ {{32, 8, 5}}, -+ {{32, 9, 5}}, -+ {{32, 11, 5}}, -+ {{32, 12, 5}}, -+ {{0, 15, 6}}, -+ {{32, 17, 5}}, -+ {{32, 18, 5}}, -+ {{32, 20, 5}}, -+ {{32, 21, 5}}, -+ {{32, 23, 5}}, -+ {{32, 24, 5}}, -+ {{0, 35, 6}}, -+ {{0, 34, 6}}, -+ {{0, 33, 6}}, -+ {{0, 32, 6}}, -+}; /* LL_defaultDTable */ -+ -+static const FSE_decode_t4 ML_defaultDTable[(1 << ML_DEFAULTNORMLOG) + 1] = { -+ {{ML_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */ -+ {{0, 0, 6}}, /* 0 : base, symbol, bits */ -+ {{0, 1, 4}}, -+ {{32, 2, 5}}, -+ {{0, 3, 5}}, -+ {{0, 5, 5}}, -+ {{0, 6, 5}}, -+ {{0, 8, 5}}, -+ {{0, 10, 6}}, -+ {{0, 13, 6}}, -+ {{0, 16, 6}}, -+ {{0, 19, 6}}, -+ {{0, 22, 6}}, -+ {{0, 25, 6}}, -+ {{0, 28, 6}}, -+ {{0, 31, 6}}, -+ {{0, 33, 6}}, -+ {{0, 35, 6}}, -+ {{0, 37, 6}}, -+ {{0, 39, 6}}, -+ {{0, 41, 6}}, -+ {{0, 43, 6}}, -+ {{0, 45, 6}}, -+ {{16, 1, 4}}, -+ {{0, 2, 4}}, -+ {{32, 3, 5}}, -+ {{0, 4, 5}}, -+ {{32, 6, 5}}, -+ {{0, 7, 5}}, -+ {{0, 9, 6}}, -+ {{0, 12, 6}}, -+ {{0, 15, 6}}, -+ {{0, 18, 6}}, -+ {{0, 21, 6}}, -+ {{0, 24, 6}}, -+ {{0, 27, 6}}, -+ {{0, 30, 6}}, -+ {{0, 32, 6}}, -+ {{0, 34, 6}}, -+ {{0, 36, 6}}, -+ {{0, 38, 6}}, -+ {{0, 40, 6}}, -+ {{0, 42, 6}}, -+ {{0, 44, 6}}, -+ {{32, 1, 4}}, -+ {{48, 1, 4}}, -+ {{16, 2, 4}}, -+ {{32, 4, 5}}, -+ {{32, 5, 5}}, -+ {{32, 7, 5}}, -+ {{32, 8, 5}}, -+ {{0, 11, 6}}, -+ {{0, 14, 6}}, -+ {{0, 17, 6}}, -+ {{0, 20, 6}}, -+ {{0, 23, 6}}, -+ {{0, 26, 6}}, -+ {{0, 29, 6}}, -+ {{0, 52, 6}}, -+ {{0, 51, 6}}, -+ {{0, 50, 6}}, -+ {{0, 49, 6}}, -+ {{0, 48, 6}}, -+ {{0, 47, 6}}, -+ {{0, 46, 6}}, -+}; /* ML_defaultDTable */ -+ -+static const FSE_decode_t4 OF_defaultDTable[(1 << OF_DEFAULTNORMLOG) + 1] = { -+ {{OF_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */ -+ {{0, 0, 5}}, /* 0 : base, symbol, bits */ -+ {{0, 6, 4}}, -+ {{0, 9, 5}}, -+ {{0, 15, 5}}, -+ {{0, 21, 5}}, -+ {{0, 3, 5}}, -+ {{0, 7, 4}}, -+ {{0, 12, 5}}, -+ {{0, 18, 5}}, -+ {{0, 23, 5}}, -+ {{0, 5, 5}}, -+ {{0, 8, 4}}, -+ {{0, 14, 5}}, -+ {{0, 20, 5}}, -+ {{0, 2, 5}}, -+ {{16, 7, 4}}, -+ {{0, 11, 5}}, -+ {{0, 17, 5}}, -+ {{0, 22, 5}}, -+ {{0, 4, 5}}, -+ {{16, 8, 4}}, -+ {{0, 13, 5}}, -+ {{0, 19, 5}}, -+ {{0, 1, 5}}, -+ {{16, 6, 4}}, -+ {{0, 10, 5}}, -+ {{0, 16, 5}}, -+ {{0, 28, 5}}, -+ {{0, 27, 5}}, -+ {{0, 26, 5}}, -+ {{0, 25, 5}}, -+ {{0, 24, 5}}, -+}; /* OF_defaultDTable */ -+ -+/*! ZSTD_buildSeqTable() : -+ @return : nb bytes read from src, -+ or an error code if it fails, testable with ZSTD_isError() -+*/ -+static size_t ZSTD_buildSeqTable(FSE_DTable *DTableSpace, const FSE_DTable **DTablePtr, symbolEncodingType_e type, U32 max, U32 maxLog, const void *src, -+ size_t srcSize, const FSE_decode_t4 *defaultTable, U32 flagRepeatTable, void *workspace, size_t workspaceSize) -+{ -+ const void *const tmpPtr = defaultTable; /* bypass strict aliasing */ -+ switch (type) { -+ case set_rle: -+ if (!srcSize) -+ return ERROR(srcSize_wrong); -+ if ((*(const BYTE *)src) > max) -+ return ERROR(corruption_detected); -+ FSE_buildDTable_rle(DTableSpace, *(const BYTE *)src); -+ *DTablePtr = DTableSpace; -+ return 1; -+ case set_basic: *DTablePtr = (const FSE_DTable *)tmpPtr; return 0; -+ case set_repeat: -+ if (!flagRepeatTable) -+ return ERROR(corruption_detected); -+ return 0; -+ default: /* impossible */ -+ case set_compressed: { -+ U32 tableLog; -+ S16 *norm = (S16 *)workspace; -+ size_t const spaceUsed32 = ALIGN(sizeof(S16) * (MaxSeq + 1), sizeof(U32)) >> 2; -+ -+ if ((spaceUsed32 << 2) > workspaceSize) -+ return ERROR(GENERIC); -+ workspace = (U32 *)workspace + spaceUsed32; -+ workspaceSize -= (spaceUsed32 << 2); -+ { -+ size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); -+ if (FSE_isError(headerSize)) -+ return ERROR(corruption_detected); -+ if (tableLog > maxLog) -+ return ERROR(corruption_detected); -+ FSE_buildDTable_wksp(DTableSpace, norm, max, tableLog, workspace, workspaceSize); -+ *DTablePtr = DTableSpace; -+ return headerSize; -+ } -+ } -+ } -+} -+ -+size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx *dctx, int *nbSeqPtr, const void *src, size_t srcSize) -+{ -+ const BYTE *const istart = (const BYTE *const)src; -+ const BYTE *const iend = istart + srcSize; -+ const BYTE *ip = istart; -+ -+ /* check */ -+ if (srcSize < MIN_SEQUENCES_SIZE) -+ return ERROR(srcSize_wrong); -+ -+ /* SeqHead */ -+ { -+ int nbSeq = *ip++; -+ if (!nbSeq) { -+ *nbSeqPtr = 0; -+ return 1; -+ } -+ if (nbSeq > 0x7F) { -+ if (nbSeq == 0xFF) { -+ if (ip + 2 > iend) -+ return ERROR(srcSize_wrong); -+ nbSeq = ZSTD_readLE16(ip) + LONGNBSEQ, ip += 2; -+ } else { -+ if (ip >= iend) -+ return ERROR(srcSize_wrong); -+ nbSeq = ((nbSeq - 0x80) << 8) + *ip++; -+ } -+ } -+ *nbSeqPtr = nbSeq; -+ } -+ -+ /* FSE table descriptors */ -+ if (ip + 4 > iend) -+ return ERROR(srcSize_wrong); /* minimum possible size */ -+ { -+ symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); -+ symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); -+ symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); -+ ip++; -+ -+ /* Build DTables */ -+ { -+ size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, LLtype, MaxLL, LLFSELog, ip, iend - ip, -+ LL_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace)); -+ if (ZSTD_isError(llhSize)) -+ return ERROR(corruption_detected); -+ ip += llhSize; -+ } -+ { -+ size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, OFtype, MaxOff, OffFSELog, ip, iend - ip, -+ OF_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace)); -+ if (ZSTD_isError(ofhSize)) -+ return ERROR(corruption_detected); -+ ip += ofhSize; -+ } -+ { -+ size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, MLtype, MaxML, MLFSELog, ip, iend - ip, -+ ML_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace)); -+ if (ZSTD_isError(mlhSize)) -+ return ERROR(corruption_detected); -+ ip += mlhSize; -+ } -+ } -+ -+ return ip - istart; -+} -+ -+typedef struct { -+ size_t litLength; -+ size_t matchLength; -+ size_t offset; -+ const BYTE *match; -+} seq_t; -+ -+typedef struct { -+ BIT_DStream_t DStream; -+ FSE_DState_t stateLL; -+ FSE_DState_t stateOffb; -+ FSE_DState_t stateML; -+ size_t prevOffset[ZSTD_REP_NUM]; -+ const BYTE *base; -+ size_t pos; -+ uPtrDiff gotoDict; -+} seqState_t; -+ -+FORCE_NOINLINE -+size_t ZSTD_execSequenceLast7(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base, -+ const BYTE *const vBase, const BYTE *const dictEnd) -+{ -+ BYTE *const oLitEnd = op + sequence.litLength; -+ size_t const sequenceLength = sequence.litLength + sequence.matchLength; -+ BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ -+ BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH; -+ const BYTE *const iLitEnd = *litPtr + sequence.litLength; -+ const BYTE *match = oLitEnd - sequence.offset; -+ -+ /* check */ -+ if (oMatchEnd > oend) -+ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ -+ if (iLitEnd > litLimit) -+ return ERROR(corruption_detected); /* over-read beyond lit buffer */ -+ if (oLitEnd <= oend_w) -+ return ERROR(GENERIC); /* Precondition */ -+ -+ /* copy literals */ -+ if (op < oend_w) { -+ ZSTD_wildcopy(op, *litPtr, oend_w - op); -+ *litPtr += oend_w - op; -+ op = oend_w; -+ } -+ while (op < oLitEnd) -+ *op++ = *(*litPtr)++; -+ -+ /* copy Match */ -+ if (sequence.offset > (size_t)(oLitEnd - base)) { -+ /* offset beyond prefix */ -+ if (sequence.offset > (size_t)(oLitEnd - vBase)) -+ return ERROR(corruption_detected); -+ match = dictEnd - (base - match); -+ if (match + sequence.matchLength <= dictEnd) { -+ memmove(oLitEnd, match, sequence.matchLength); -+ return sequenceLength; -+ } -+ /* span extDict & currPrefixSegment */ -+ { -+ size_t const length1 = dictEnd - match; -+ memmove(oLitEnd, match, length1); -+ op = oLitEnd + length1; -+ sequence.matchLength -= length1; -+ match = base; -+ } -+ } -+ while (op < oMatchEnd) -+ *op++ = *match++; -+ return sequenceLength; -+} -+ -+static seq_t ZSTD_decodeSequence(seqState_t *seqState) -+{ -+ seq_t seq; -+ -+ U32 const llCode = FSE_peekSymbol(&seqState->stateLL); -+ U32 const mlCode = FSE_peekSymbol(&seqState->stateML); -+ U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */ -+ -+ U32 const llBits = LL_bits[llCode]; -+ U32 const mlBits = ML_bits[mlCode]; -+ U32 const ofBits = ofCode; -+ U32 const totalBits = llBits + mlBits + ofBits; -+ -+ static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, -+ 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000}; -+ -+ static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, -+ 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41, -+ 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003}; -+ -+ static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD, -+ 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, -+ 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD}; -+ -+ /* sequence */ -+ { -+ size_t offset; -+ if (!ofCode) -+ offset = 0; -+ else { -+ offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ -+ if (ZSTD_32bits()) -+ BIT_reloadDStream(&seqState->DStream); -+ } -+ -+ if (ofCode <= 1) { -+ offset += (llCode == 0); -+ if (offset) { -+ size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; -+ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ -+ if (offset != 1) -+ seqState->prevOffset[2] = seqState->prevOffset[1]; -+ seqState->prevOffset[1] = seqState->prevOffset[0]; -+ seqState->prevOffset[0] = offset = temp; -+ } else { -+ offset = seqState->prevOffset[0]; -+ } -+ } else { -+ seqState->prevOffset[2] = seqState->prevOffset[1]; -+ seqState->prevOffset[1] = seqState->prevOffset[0]; -+ seqState->prevOffset[0] = offset; -+ } -+ seq.offset = offset; -+ } -+ -+ seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ -+ if (ZSTD_32bits() && (mlBits + llBits > 24)) -+ BIT_reloadDStream(&seqState->DStream); -+ -+ seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ -+ if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog))) -+ BIT_reloadDStream(&seqState->DStream); -+ -+ /* ANS state update */ -+ FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ -+ FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ -+ if (ZSTD_32bits()) -+ BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ -+ FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ -+ -+ seq.match = NULL; -+ -+ return seq; -+} -+ -+FORCE_INLINE -+size_t ZSTD_execSequence(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base, -+ const BYTE *const vBase, const BYTE *const dictEnd) -+{ -+ BYTE *const oLitEnd = op + sequence.litLength; -+ size_t const sequenceLength = sequence.litLength + sequence.matchLength; -+ BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ -+ BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH; -+ const BYTE *const iLitEnd = *litPtr + sequence.litLength; -+ const BYTE *match = oLitEnd - sequence.offset; -+ -+ /* check */ -+ if (oMatchEnd > oend) -+ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ -+ if (iLitEnd > litLimit) -+ return ERROR(corruption_detected); /* over-read beyond lit buffer */ -+ if (oLitEnd > oend_w) -+ return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); -+ -+ /* copy Literals */ -+ ZSTD_copy8(op, *litPtr); -+ if (sequence.litLength > 8) -+ ZSTD_wildcopy(op + 8, (*litPtr) + 8, -+ sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ -+ op = oLitEnd; -+ *litPtr = iLitEnd; /* update for next sequence */ -+ -+ /* copy Match */ -+ if (sequence.offset > (size_t)(oLitEnd - base)) { -+ /* offset beyond prefix */ -+ if (sequence.offset > (size_t)(oLitEnd - vBase)) -+ return ERROR(corruption_detected); -+ match = dictEnd + (match - base); -+ if (match + sequence.matchLength <= dictEnd) { -+ memmove(oLitEnd, match, sequence.matchLength); -+ return sequenceLength; -+ } -+ /* span extDict & currPrefixSegment */ -+ { -+ size_t const length1 = dictEnd - match; -+ memmove(oLitEnd, match, length1); -+ op = oLitEnd + length1; -+ sequence.matchLength -= length1; -+ match = base; -+ if (op > oend_w || sequence.matchLength < MINMATCH) { -+ U32 i; -+ for (i = 0; i < sequence.matchLength; ++i) -+ op[i] = match[i]; -+ return sequenceLength; -+ } -+ } -+ } -+ /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ -+ -+ /* match within prefix */ -+ if (sequence.offset < 8) { -+ /* close range match, overlap */ -+ static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */ -+ static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */ -+ int const sub2 = dec64table[sequence.offset]; -+ op[0] = match[0]; -+ op[1] = match[1]; -+ op[2] = match[2]; -+ op[3] = match[3]; -+ match += dec32table[sequence.offset]; -+ ZSTD_copy4(op + 4, match); -+ match -= sub2; -+ } else { -+ ZSTD_copy8(op, match); -+ } -+ op += 8; -+ match += 8; -+ -+ if (oMatchEnd > oend - (16 - MINMATCH)) { -+ if (op < oend_w) { -+ ZSTD_wildcopy(op, match, oend_w - op); -+ match += oend_w - op; -+ op = oend_w; -+ } -+ while (op < oMatchEnd) -+ *op++ = *match++; -+ } else { -+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */ -+ } -+ return sequenceLength; -+} -+ -+static size_t ZSTD_decompressSequences(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize) -+{ -+ const BYTE *ip = (const BYTE *)seqStart; -+ const BYTE *const iend = ip + seqSize; -+ BYTE *const ostart = (BYTE * const)dst; -+ BYTE *const oend = ostart + maxDstSize; -+ BYTE *op = ostart; -+ const BYTE *litPtr = dctx->litPtr; -+ const BYTE *const litEnd = litPtr + dctx->litSize; -+ const BYTE *const base = (const BYTE *)(dctx->base); -+ const BYTE *const vBase = (const BYTE *)(dctx->vBase); -+ const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd); -+ int nbSeq; -+ -+ /* Build Decoding Tables */ -+ { -+ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); -+ if (ZSTD_isError(seqHSize)) -+ return seqHSize; -+ ip += seqHSize; -+ } -+ -+ /* Regen sequences */ -+ if (nbSeq) { -+ seqState_t seqState; -+ dctx->fseEntropy = 1; -+ { -+ U32 i; -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ seqState.prevOffset[i] = dctx->entropy.rep[i]; -+ } -+ CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected); -+ FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); -+ FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); -+ FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); -+ -+ for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq;) { -+ nbSeq--; -+ { -+ seq_t const sequence = ZSTD_decodeSequence(&seqState); -+ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd); -+ if (ZSTD_isError(oneSeqSize)) -+ return oneSeqSize; -+ op += oneSeqSize; -+ } -+ } -+ -+ /* check if reached exact end */ -+ if (nbSeq) -+ return ERROR(corruption_detected); -+ /* save reps for next block */ -+ { -+ U32 i; -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); -+ } -+ } -+ -+ /* last literal segment */ -+ { -+ size_t const lastLLSize = litEnd - litPtr; -+ if (lastLLSize > (size_t)(oend - op)) -+ return ERROR(dstSize_tooSmall); -+ memcpy(op, litPtr, lastLLSize); -+ op += lastLLSize; -+ } -+ -+ return op - ostart; -+} -+ -+FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t *seqState, int const longOffsets) -+{ -+ seq_t seq; -+ -+ U32 const llCode = FSE_peekSymbol(&seqState->stateLL); -+ U32 const mlCode = FSE_peekSymbol(&seqState->stateML); -+ U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */ -+ -+ U32 const llBits = LL_bits[llCode]; -+ U32 const mlBits = ML_bits[mlCode]; -+ U32 const ofBits = ofCode; -+ U32 const totalBits = llBits + mlBits + ofBits; -+ -+ static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, -+ 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000}; -+ -+ static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, -+ 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41, -+ 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003}; -+ -+ static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD, -+ 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, -+ 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD}; -+ -+ /* sequence */ -+ { -+ size_t offset; -+ if (!ofCode) -+ offset = 0; -+ else { -+ if (longOffsets) { -+ int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN); -+ offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); -+ if (ZSTD_32bits() || extraBits) -+ BIT_reloadDStream(&seqState->DStream); -+ if (extraBits) -+ offset += BIT_readBitsFast(&seqState->DStream, extraBits); -+ } else { -+ offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ -+ if (ZSTD_32bits()) -+ BIT_reloadDStream(&seqState->DStream); -+ } -+ } -+ -+ if (ofCode <= 1) { -+ offset += (llCode == 0); -+ if (offset) { -+ size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; -+ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ -+ if (offset != 1) -+ seqState->prevOffset[2] = seqState->prevOffset[1]; -+ seqState->prevOffset[1] = seqState->prevOffset[0]; -+ seqState->prevOffset[0] = offset = temp; -+ } else { -+ offset = seqState->prevOffset[0]; -+ } -+ } else { -+ seqState->prevOffset[2] = seqState->prevOffset[1]; -+ seqState->prevOffset[1] = seqState->prevOffset[0]; -+ seqState->prevOffset[0] = offset; -+ } -+ seq.offset = offset; -+ } -+ -+ seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ -+ if (ZSTD_32bits() && (mlBits + llBits > 24)) -+ BIT_reloadDStream(&seqState->DStream); -+ -+ seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ -+ if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog))) -+ BIT_reloadDStream(&seqState->DStream); -+ -+ { -+ size_t const pos = seqState->pos + seq.litLength; -+ seq.match = seqState->base + pos - seq.offset; /* single memory segment */ -+ if (seq.offset > pos) -+ seq.match += seqState->gotoDict; /* separate memory segment */ -+ seqState->pos = pos + seq.matchLength; -+ } -+ -+ /* ANS state update */ -+ FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ -+ FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ -+ if (ZSTD_32bits()) -+ BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ -+ FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ -+ -+ return seq; -+} -+ -+static seq_t ZSTD_decodeSequenceLong(seqState_t *seqState, unsigned const windowSize) -+{ -+ if (ZSTD_highbit32(windowSize) > STREAM_ACCUMULATOR_MIN) { -+ return ZSTD_decodeSequenceLong_generic(seqState, 1); -+ } else { -+ return ZSTD_decodeSequenceLong_generic(seqState, 0); -+ } -+} -+ -+FORCE_INLINE -+size_t ZSTD_execSequenceLong(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base, -+ const BYTE *const vBase, const BYTE *const dictEnd) -+{ -+ BYTE *const oLitEnd = op + sequence.litLength; -+ size_t const sequenceLength = sequence.litLength + sequence.matchLength; -+ BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ -+ BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH; -+ const BYTE *const iLitEnd = *litPtr + sequence.litLength; -+ const BYTE *match = sequence.match; -+ -+ /* check */ -+ if (oMatchEnd > oend) -+ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ -+ if (iLitEnd > litLimit) -+ return ERROR(corruption_detected); /* over-read beyond lit buffer */ -+ if (oLitEnd > oend_w) -+ return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); -+ -+ /* copy Literals */ -+ ZSTD_copy8(op, *litPtr); -+ if (sequence.litLength > 8) -+ ZSTD_wildcopy(op + 8, (*litPtr) + 8, -+ sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ -+ op = oLitEnd; -+ *litPtr = iLitEnd; /* update for next sequence */ -+ -+ /* copy Match */ -+ if (sequence.offset > (size_t)(oLitEnd - base)) { -+ /* offset beyond prefix */ -+ if (sequence.offset > (size_t)(oLitEnd - vBase)) -+ return ERROR(corruption_detected); -+ if (match + sequence.matchLength <= dictEnd) { -+ memmove(oLitEnd, match, sequence.matchLength); -+ return sequenceLength; -+ } -+ /* span extDict & currPrefixSegment */ -+ { -+ size_t const length1 = dictEnd - match; -+ memmove(oLitEnd, match, length1); -+ op = oLitEnd + length1; -+ sequence.matchLength -= length1; -+ match = base; -+ if (op > oend_w || sequence.matchLength < MINMATCH) { -+ U32 i; -+ for (i = 0; i < sequence.matchLength; ++i) -+ op[i] = match[i]; -+ return sequenceLength; -+ } -+ } -+ } -+ /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ -+ -+ /* match within prefix */ -+ if (sequence.offset < 8) { -+ /* close range match, overlap */ -+ static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */ -+ static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */ -+ int const sub2 = dec64table[sequence.offset]; -+ op[0] = match[0]; -+ op[1] = match[1]; -+ op[2] = match[2]; -+ op[3] = match[3]; -+ match += dec32table[sequence.offset]; -+ ZSTD_copy4(op + 4, match); -+ match -= sub2; -+ } else { -+ ZSTD_copy8(op, match); -+ } -+ op += 8; -+ match += 8; -+ -+ if (oMatchEnd > oend - (16 - MINMATCH)) { -+ if (op < oend_w) { -+ ZSTD_wildcopy(op, match, oend_w - op); -+ match += oend_w - op; -+ op = oend_w; -+ } -+ while (op < oMatchEnd) -+ *op++ = *match++; -+ } else { -+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */ -+ } -+ return sequenceLength; -+} -+ -+static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize) -+{ -+ const BYTE *ip = (const BYTE *)seqStart; -+ const BYTE *const iend = ip + seqSize; -+ BYTE *const ostart = (BYTE * const)dst; -+ BYTE *const oend = ostart + maxDstSize; -+ BYTE *op = ostart; -+ const BYTE *litPtr = dctx->litPtr; -+ const BYTE *const litEnd = litPtr + dctx->litSize; -+ const BYTE *const base = (const BYTE *)(dctx->base); -+ const BYTE *const vBase = (const BYTE *)(dctx->vBase); -+ const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd); -+ unsigned const windowSize = dctx->fParams.windowSize; -+ int nbSeq; -+ -+ /* Build Decoding Tables */ -+ { -+ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); -+ if (ZSTD_isError(seqHSize)) -+ return seqHSize; -+ ip += seqHSize; -+ } -+ -+ /* Regen sequences */ -+ if (nbSeq) { -+#define STORED_SEQS 4 -+#define STOSEQ_MASK (STORED_SEQS - 1) -+#define ADVANCED_SEQS 4 -+ seq_t *sequences = (seq_t *)dctx->entropy.workspace; -+ int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); -+ seqState_t seqState; -+ int seqNb; -+ ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.workspace) >= sizeof(seq_t) * STORED_SEQS); -+ dctx->fseEntropy = 1; -+ { -+ U32 i; -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ seqState.prevOffset[i] = dctx->entropy.rep[i]; -+ } -+ seqState.base = base; -+ seqState.pos = (size_t)(op - base); -+ seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */ -+ CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected); -+ FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); -+ FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); -+ FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); -+ -+ /* prepare in advance */ -+ for (seqNb = 0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb < seqAdvance; seqNb++) { -+ sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize); -+ } -+ if (seqNb < seqAdvance) -+ return ERROR(corruption_detected); -+ -+ /* decode and decompress */ -+ for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb < nbSeq; seqNb++) { -+ seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize); -+ size_t const oneSeqSize = -+ ZSTD_execSequenceLong(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd); -+ if (ZSTD_isError(oneSeqSize)) -+ return oneSeqSize; -+ ZSTD_PREFETCH(sequence.match); -+ sequences[seqNb & STOSEQ_MASK] = sequence; -+ op += oneSeqSize; -+ } -+ if (seqNb < nbSeq) -+ return ERROR(corruption_detected); -+ -+ /* finish queue */ -+ seqNb -= seqAdvance; -+ for (; seqNb < nbSeq; seqNb++) { -+ size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd); -+ if (ZSTD_isError(oneSeqSize)) -+ return oneSeqSize; -+ op += oneSeqSize; -+ } -+ -+ /* save reps for next block */ -+ { -+ U32 i; -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); -+ } -+ } -+ -+ /* last literal segment */ -+ { -+ size_t const lastLLSize = litEnd - litPtr; -+ if (lastLLSize > (size_t)(oend - op)) -+ return ERROR(dstSize_tooSmall); -+ memcpy(op, litPtr, lastLLSize); -+ op += lastLLSize; -+ } -+ -+ return op - ostart; -+} -+ -+static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ /* blockType == blockCompressed */ -+ const BYTE *ip = (const BYTE *)src; -+ -+ if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX) -+ return ERROR(srcSize_wrong); -+ -+ /* Decode literals section */ -+ { -+ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); -+ if (ZSTD_isError(litCSize)) -+ return litCSize; -+ ip += litCSize; -+ srcSize -= litCSize; -+ } -+ if (sizeof(size_t) > 4) /* do not enable prefetching on 32-bits x86, as it's performance detrimental */ -+ /* likely because of register pressure */ -+ /* if that's the correct cause, then 32-bits ARM should be affected differently */ -+ /* it would be good to test this on ARM real hardware, to see if prefetch version improves speed */ -+ if (dctx->fParams.windowSize > (1 << 23)) -+ return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize); -+ return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize); -+} -+ -+static void ZSTD_checkContinuity(ZSTD_DCtx *dctx, const void *dst) -+{ -+ if (dst != dctx->previousDstEnd) { /* not contiguous */ -+ dctx->dictEnd = dctx->previousDstEnd; -+ dctx->vBase = (const char *)dst - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base)); -+ dctx->base = dst; -+ dctx->previousDstEnd = dst; -+ } -+} -+ -+size_t ZSTD_decompressBlock(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ size_t dSize; -+ ZSTD_checkContinuity(dctx, dst); -+ dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); -+ dctx->previousDstEnd = (char *)dst + dSize; -+ return dSize; -+} -+ -+/** ZSTD_insertBlock() : -+ insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ -+size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void *blockStart, size_t blockSize) -+{ -+ ZSTD_checkContinuity(dctx, blockStart); -+ dctx->previousDstEnd = (const char *)blockStart + blockSize; -+ return blockSize; -+} -+ -+size_t ZSTD_generateNxBytes(void *dst, size_t dstCapacity, BYTE byte, size_t length) -+{ -+ if (length > dstCapacity) -+ return ERROR(dstSize_tooSmall); -+ memset(dst, byte, length); -+ return length; -+} -+ -+/** ZSTD_findFrameCompressedSize() : -+ * compatible with legacy mode -+ * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame -+ * `srcSize` must be at least as large as the frame contained -+ * @return : the compressed size of the frame starting at `src` */ -+size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) -+{ -+ if (srcSize >= ZSTD_skippableHeaderSize && (ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { -+ return ZSTD_skippableHeaderSize + ZSTD_readLE32((const BYTE *)src + 4); -+ } else { -+ const BYTE *ip = (const BYTE *)src; -+ const BYTE *const ipstart = ip; -+ size_t remainingSize = srcSize; -+ ZSTD_frameParams fParams; -+ -+ size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize); -+ if (ZSTD_isError(headerSize)) -+ return headerSize; -+ -+ /* Frame Header */ -+ { -+ size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize); -+ if (ZSTD_isError(ret)) -+ return ret; -+ if (ret > 0) -+ return ERROR(srcSize_wrong); -+ } -+ -+ ip += headerSize; -+ remainingSize -= headerSize; -+ -+ /* Loop on each block */ -+ while (1) { -+ blockProperties_t blockProperties; -+ size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); -+ if (ZSTD_isError(cBlockSize)) -+ return cBlockSize; -+ -+ if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) -+ return ERROR(srcSize_wrong); -+ -+ ip += ZSTD_blockHeaderSize + cBlockSize; -+ remainingSize -= ZSTD_blockHeaderSize + cBlockSize; -+ -+ if (blockProperties.lastBlock) -+ break; -+ } -+ -+ if (fParams.checksumFlag) { /* Frame content checksum */ -+ if (remainingSize < 4) -+ return ERROR(srcSize_wrong); -+ ip += 4; -+ remainingSize -= 4; -+ } -+ -+ return ip - ipstart; -+ } -+} -+ -+/*! ZSTD_decompressFrame() : -+* @dctx must be properly initialized */ -+static size_t ZSTD_decompressFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void **srcPtr, size_t *srcSizePtr) -+{ -+ const BYTE *ip = (const BYTE *)(*srcPtr); -+ BYTE *const ostart = (BYTE * const)dst; -+ BYTE *const oend = ostart + dstCapacity; -+ BYTE *op = ostart; -+ size_t remainingSize = *srcSizePtr; -+ -+ /* check */ -+ if (remainingSize < ZSTD_frameHeaderSize_min + ZSTD_blockHeaderSize) -+ return ERROR(srcSize_wrong); -+ -+ /* Frame Header */ -+ { -+ size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix); -+ if (ZSTD_isError(frameHeaderSize)) -+ return frameHeaderSize; -+ if (remainingSize < frameHeaderSize + ZSTD_blockHeaderSize) -+ return ERROR(srcSize_wrong); -+ CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize)); -+ ip += frameHeaderSize; -+ remainingSize -= frameHeaderSize; -+ } -+ -+ /* Loop on each block */ -+ while (1) { -+ size_t decodedSize; -+ blockProperties_t blockProperties; -+ size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); -+ if (ZSTD_isError(cBlockSize)) -+ return cBlockSize; -+ -+ ip += ZSTD_blockHeaderSize; -+ remainingSize -= ZSTD_blockHeaderSize; -+ if (cBlockSize > remainingSize) -+ return ERROR(srcSize_wrong); -+ -+ switch (blockProperties.blockType) { -+ case bt_compressed: decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend - op, ip, cBlockSize); break; -+ case bt_raw: decodedSize = ZSTD_copyRawBlock(op, oend - op, ip, cBlockSize); break; -+ case bt_rle: decodedSize = ZSTD_generateNxBytes(op, oend - op, *ip, blockProperties.origSize); break; -+ case bt_reserved: -+ default: return ERROR(corruption_detected); -+ } -+ -+ if (ZSTD_isError(decodedSize)) -+ return decodedSize; -+ if (dctx->fParams.checksumFlag) -+ xxh64_update(&dctx->xxhState, op, decodedSize); -+ op += decodedSize; -+ ip += cBlockSize; -+ remainingSize -= cBlockSize; -+ if (blockProperties.lastBlock) -+ break; -+ } -+ -+ if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ -+ U32 const checkCalc = (U32)xxh64_digest(&dctx->xxhState); -+ U32 checkRead; -+ if (remainingSize < 4) -+ return ERROR(checksum_wrong); -+ checkRead = ZSTD_readLE32(ip); -+ if (checkRead != checkCalc) -+ return ERROR(checksum_wrong); -+ ip += 4; -+ remainingSize -= 4; -+ } -+ -+ /* Allow caller to get size read */ -+ *srcPtr = ip; -+ *srcSizePtr = remainingSize; -+ return op - ostart; -+} -+ -+static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict); -+static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict); -+ -+static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, -+ const ZSTD_DDict *ddict) -+{ -+ void *const dststart = dst; -+ -+ if (ddict) { -+ if (dict) { -+ /* programmer error, these two cases should be mutually exclusive */ -+ return ERROR(GENERIC); -+ } -+ -+ dict = ZSTD_DDictDictContent(ddict); -+ dictSize = ZSTD_DDictDictSize(ddict); -+ } -+ -+ while (srcSize >= ZSTD_frameHeaderSize_prefix) { -+ U32 magicNumber; -+ -+ magicNumber = ZSTD_readLE32(src); -+ if (magicNumber != ZSTD_MAGICNUMBER) { -+ if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { -+ size_t skippableSize; -+ if (srcSize < ZSTD_skippableHeaderSize) -+ return ERROR(srcSize_wrong); -+ skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize; -+ if (srcSize < skippableSize) { -+ return ERROR(srcSize_wrong); -+ } -+ -+ src = (const BYTE *)src + skippableSize; -+ srcSize -= skippableSize; -+ continue; -+ } else { -+ return ERROR(prefix_unknown); -+ } -+ } -+ -+ if (ddict) { -+ /* we were called from ZSTD_decompress_usingDDict */ -+ ZSTD_refDDict(dctx, ddict); -+ } else { -+ /* this will initialize correctly with no dict if dict == NULL, so -+ * use this in all cases but ddict */ -+ CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize)); -+ } -+ ZSTD_checkContinuity(dctx, dst); -+ -+ { -+ const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, &src, &srcSize); -+ if (ZSTD_isError(res)) -+ return res; -+ /* don't need to bounds check this, ZSTD_decompressFrame will have -+ * already */ -+ dst = (BYTE *)dst + res; -+ dstCapacity -= res; -+ } -+ } -+ -+ if (srcSize) -+ return ERROR(srcSize_wrong); /* input not entirely consumed */ -+ -+ return (BYTE *)dst - (BYTE *)dststart; -+} -+ -+size_t ZSTD_decompress_usingDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize) -+{ -+ return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); -+} -+ -+size_t ZSTD_decompressDCtx(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); -+} -+ -+/*-************************************** -+* Advanced Streaming Decompression API -+* Bufferless and synchronous -+****************************************/ -+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx *dctx) { return dctx->expected; } -+ -+ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx *dctx) -+{ -+ switch (dctx->stage) { -+ default: /* should not happen */ -+ case ZSTDds_getFrameHeaderSize: -+ case ZSTDds_decodeFrameHeader: return ZSTDnit_frameHeader; -+ case ZSTDds_decodeBlockHeader: return ZSTDnit_blockHeader; -+ case ZSTDds_decompressBlock: return ZSTDnit_block; -+ case ZSTDds_decompressLastBlock: return ZSTDnit_lastBlock; -+ case ZSTDds_checkChecksum: return ZSTDnit_checksum; -+ case ZSTDds_decodeSkippableHeader: -+ case ZSTDds_skipFrame: return ZSTDnit_skippableFrame; -+ } -+} -+ -+int ZSTD_isSkipFrame(ZSTD_DCtx *dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */ -+ -+/** ZSTD_decompressContinue() : -+* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) -+* or an error code, which can be tested using ZSTD_isError() */ -+size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ /* Sanity check */ -+ if (srcSize != dctx->expected) -+ return ERROR(srcSize_wrong); -+ if (dstCapacity) -+ ZSTD_checkContinuity(dctx, dst); -+ -+ switch (dctx->stage) { -+ case ZSTDds_getFrameHeaderSize: -+ if (srcSize != ZSTD_frameHeaderSize_prefix) -+ return ERROR(srcSize_wrong); /* impossible */ -+ if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ -+ memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); -+ dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */ -+ dctx->stage = ZSTDds_decodeSkippableHeader; -+ return 0; -+ } -+ dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix); -+ if (ZSTD_isError(dctx->headerSize)) -+ return dctx->headerSize; -+ memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); -+ if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) { -+ dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix; -+ dctx->stage = ZSTDds_decodeFrameHeader; -+ return 0; -+ } -+ dctx->expected = 0; /* not necessary to copy more */ -+ -+ case ZSTDds_decodeFrameHeader: -+ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); -+ CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize)); -+ dctx->expected = ZSTD_blockHeaderSize; -+ dctx->stage = ZSTDds_decodeBlockHeader; -+ return 0; -+ -+ case ZSTDds_decodeBlockHeader: { -+ blockProperties_t bp; -+ size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); -+ if (ZSTD_isError(cBlockSize)) -+ return cBlockSize; -+ dctx->expected = cBlockSize; -+ dctx->bType = bp.blockType; -+ dctx->rleSize = bp.origSize; -+ if (cBlockSize) { -+ dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; -+ return 0; -+ } -+ /* empty block */ -+ if (bp.lastBlock) { -+ if (dctx->fParams.checksumFlag) { -+ dctx->expected = 4; -+ dctx->stage = ZSTDds_checkChecksum; -+ } else { -+ dctx->expected = 0; /* end of frame */ -+ dctx->stage = ZSTDds_getFrameHeaderSize; -+ } -+ } else { -+ dctx->expected = 3; /* go directly to next header */ -+ dctx->stage = ZSTDds_decodeBlockHeader; -+ } -+ return 0; -+ } -+ case ZSTDds_decompressLastBlock: -+ case ZSTDds_decompressBlock: { -+ size_t rSize; -+ switch (dctx->bType) { -+ case bt_compressed: rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); break; -+ case bt_raw: rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); break; -+ case bt_rle: rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); break; -+ case bt_reserved: /* should never happen */ -+ default: return ERROR(corruption_detected); -+ } -+ if (ZSTD_isError(rSize)) -+ return rSize; -+ if (dctx->fParams.checksumFlag) -+ xxh64_update(&dctx->xxhState, dst, rSize); -+ -+ if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ -+ if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ -+ dctx->expected = 4; -+ dctx->stage = ZSTDds_checkChecksum; -+ } else { -+ dctx->expected = 0; /* ends here */ -+ dctx->stage = ZSTDds_getFrameHeaderSize; -+ } -+ } else { -+ dctx->stage = ZSTDds_decodeBlockHeader; -+ dctx->expected = ZSTD_blockHeaderSize; -+ dctx->previousDstEnd = (char *)dst + rSize; -+ } -+ return rSize; -+ } -+ case ZSTDds_checkChecksum: { -+ U32 const h32 = (U32)xxh64_digest(&dctx->xxhState); -+ U32 const check32 = ZSTD_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */ -+ if (check32 != h32) -+ return ERROR(checksum_wrong); -+ dctx->expected = 0; -+ dctx->stage = ZSTDds_getFrameHeaderSize; -+ return 0; -+ } -+ case ZSTDds_decodeSkippableHeader: { -+ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); -+ dctx->expected = ZSTD_readLE32(dctx->headerBuffer + 4); -+ dctx->stage = ZSTDds_skipFrame; -+ return 0; -+ } -+ case ZSTDds_skipFrame: { -+ dctx->expected = 0; -+ dctx->stage = ZSTDds_getFrameHeaderSize; -+ return 0; -+ } -+ default: -+ return ERROR(GENERIC); /* impossible */ -+ } -+} -+ -+static size_t ZSTD_refDictContent(ZSTD_DCtx *dctx, const void *dict, size_t dictSize) -+{ -+ dctx->dictEnd = dctx->previousDstEnd; -+ dctx->vBase = (const char *)dict - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base)); -+ dctx->base = dict; -+ dctx->previousDstEnd = (const char *)dict + dictSize; -+ return 0; -+} -+ -+/* ZSTD_loadEntropy() : -+ * dict : must point at beginning of a valid zstd dictionary -+ * @return : size of entropy tables read */ -+static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t *entropy, const void *const dict, size_t const dictSize) -+{ -+ const BYTE *dictPtr = (const BYTE *)dict; -+ const BYTE *const dictEnd = dictPtr + dictSize; -+ -+ if (dictSize <= 8) -+ return ERROR(dictionary_corrupted); -+ dictPtr += 8; /* skip header = magic + dictID */ -+ -+ { -+ size_t const hSize = HUF_readDTableX4_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, entropy->workspace, sizeof(entropy->workspace)); -+ if (HUF_isError(hSize)) -+ return ERROR(dictionary_corrupted); -+ dictPtr += hSize; -+ } -+ -+ { -+ short offcodeNCount[MaxOff + 1]; -+ U32 offcodeMaxValue = MaxOff, offcodeLog; -+ size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr); -+ if (FSE_isError(offcodeHeaderSize)) -+ return ERROR(dictionary_corrupted); -+ if (offcodeLog > OffFSELog) -+ return ERROR(dictionary_corrupted); -+ CHECK_E(FSE_buildDTable_wksp(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted); -+ dictPtr += offcodeHeaderSize; -+ } -+ -+ { -+ short matchlengthNCount[MaxML + 1]; -+ unsigned matchlengthMaxValue = MaxML, matchlengthLog; -+ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr); -+ if (FSE_isError(matchlengthHeaderSize)) -+ return ERROR(dictionary_corrupted); -+ if (matchlengthLog > MLFSELog) -+ return ERROR(dictionary_corrupted); -+ CHECK_E(FSE_buildDTable_wksp(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted); -+ dictPtr += matchlengthHeaderSize; -+ } -+ -+ { -+ short litlengthNCount[MaxLL + 1]; -+ unsigned litlengthMaxValue = MaxLL, litlengthLog; -+ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr); -+ if (FSE_isError(litlengthHeaderSize)) -+ return ERROR(dictionary_corrupted); -+ if (litlengthLog > LLFSELog) -+ return ERROR(dictionary_corrupted); -+ CHECK_E(FSE_buildDTable_wksp(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted); -+ dictPtr += litlengthHeaderSize; -+ } -+ -+ if (dictPtr + 12 > dictEnd) -+ return ERROR(dictionary_corrupted); -+ { -+ int i; -+ size_t const dictContentSize = (size_t)(dictEnd - (dictPtr + 12)); -+ for (i = 0; i < 3; i++) { -+ U32 const rep = ZSTD_readLE32(dictPtr); -+ dictPtr += 4; -+ if (rep == 0 || rep >= dictContentSize) -+ return ERROR(dictionary_corrupted); -+ entropy->rep[i] = rep; -+ } -+ } -+ -+ return dictPtr - (const BYTE *)dict; -+} -+ -+static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx *dctx, const void *dict, size_t dictSize) -+{ -+ if (dictSize < 8) -+ return ZSTD_refDictContent(dctx, dict, dictSize); -+ { -+ U32 const magic = ZSTD_readLE32(dict); -+ if (magic != ZSTD_DICT_MAGIC) { -+ return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ -+ } -+ } -+ dctx->dictID = ZSTD_readLE32((const char *)dict + 4); -+ -+ /* load entropy tables */ -+ { -+ size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize); -+ if (ZSTD_isError(eSize)) -+ return ERROR(dictionary_corrupted); -+ dict = (const char *)dict + eSize; -+ dictSize -= eSize; -+ } -+ dctx->litEntropy = dctx->fseEntropy = 1; -+ -+ /* reference dictionary content */ -+ return ZSTD_refDictContent(dctx, dict, dictSize); -+} -+ -+size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx *dctx, const void *dict, size_t dictSize) -+{ -+ CHECK_F(ZSTD_decompressBegin(dctx)); -+ if (dict && dictSize) -+ CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted); -+ return 0; -+} -+ -+/* ====== ZSTD_DDict ====== */ -+ -+struct ZSTD_DDict_s { -+ void *dictBuffer; -+ const void *dictContent; -+ size_t dictSize; -+ ZSTD_entropyTables_t entropy; -+ U32 dictID; -+ U32 entropyPresent; -+ ZSTD_customMem cMem; -+}; /* typedef'd to ZSTD_DDict within "zstd.h" */ -+ -+size_t ZSTD_DDictWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DDict)); } -+ -+static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict) { return ddict->dictContent; } -+ -+static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict) { return ddict->dictSize; } -+ -+static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict) -+{ -+ ZSTD_decompressBegin(dstDCtx); /* init */ -+ if (ddict) { /* support refDDict on NULL */ -+ dstDCtx->dictID = ddict->dictID; -+ dstDCtx->base = ddict->dictContent; -+ dstDCtx->vBase = ddict->dictContent; -+ dstDCtx->dictEnd = (const BYTE *)ddict->dictContent + ddict->dictSize; -+ dstDCtx->previousDstEnd = dstDCtx->dictEnd; -+ if (ddict->entropyPresent) { -+ dstDCtx->litEntropy = 1; -+ dstDCtx->fseEntropy = 1; -+ dstDCtx->LLTptr = ddict->entropy.LLTable; -+ dstDCtx->MLTptr = ddict->entropy.MLTable; -+ dstDCtx->OFTptr = ddict->entropy.OFTable; -+ dstDCtx->HUFptr = ddict->entropy.hufTable; -+ dstDCtx->entropy.rep[0] = ddict->entropy.rep[0]; -+ dstDCtx->entropy.rep[1] = ddict->entropy.rep[1]; -+ dstDCtx->entropy.rep[2] = ddict->entropy.rep[2]; -+ } else { -+ dstDCtx->litEntropy = 0; -+ dstDCtx->fseEntropy = 0; -+ } -+ } -+} -+ -+static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict *ddict) -+{ -+ ddict->dictID = 0; -+ ddict->entropyPresent = 0; -+ if (ddict->dictSize < 8) -+ return 0; -+ { -+ U32 const magic = ZSTD_readLE32(ddict->dictContent); -+ if (magic != ZSTD_DICT_MAGIC) -+ return 0; /* pure content mode */ -+ } -+ ddict->dictID = ZSTD_readLE32((const char *)ddict->dictContent + 4); -+ -+ /* load entropy tables */ -+ CHECK_E(ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted); -+ ddict->entropyPresent = 1; -+ return 0; -+} -+ -+static ZSTD_DDict *ZSTD_createDDict_advanced(const void *dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem) -+{ -+ if (!customMem.customAlloc || !customMem.customFree) -+ return NULL; -+ -+ { -+ ZSTD_DDict *const ddict = (ZSTD_DDict *)ZSTD_malloc(sizeof(ZSTD_DDict), customMem); -+ if (!ddict) -+ return NULL; -+ ddict->cMem = customMem; -+ -+ if ((byReference) || (!dict) || (!dictSize)) { -+ ddict->dictBuffer = NULL; -+ ddict->dictContent = dict; -+ } else { -+ void *const internalBuffer = ZSTD_malloc(dictSize, customMem); -+ if (!internalBuffer) { -+ ZSTD_freeDDict(ddict); -+ return NULL; -+ } -+ memcpy(internalBuffer, dict, dictSize); -+ ddict->dictBuffer = internalBuffer; -+ ddict->dictContent = internalBuffer; -+ } -+ ddict->dictSize = dictSize; -+ ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ -+ /* parse dictionary content */ -+ { -+ size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict); -+ if (ZSTD_isError(errorCode)) { -+ ZSTD_freeDDict(ddict); -+ return NULL; -+ } -+ } -+ -+ return ddict; -+ } -+} -+ -+/*! ZSTD_initDDict() : -+* Create a digested dictionary, to start decompression without startup delay. -+* `dict` content is copied inside DDict. -+* Consequently, `dict` can be released after `ZSTD_DDict` creation */ -+ZSTD_DDict *ZSTD_initDDict(const void *dict, size_t dictSize, void *workspace, size_t workspaceSize) -+{ -+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); -+ return ZSTD_createDDict_advanced(dict, dictSize, 1, stackMem); -+} -+ -+size_t ZSTD_freeDDict(ZSTD_DDict *ddict) -+{ -+ if (ddict == NULL) -+ return 0; /* support free on NULL */ -+ { -+ ZSTD_customMem const cMem = ddict->cMem; -+ ZSTD_free(ddict->dictBuffer, cMem); -+ ZSTD_free(ddict, cMem); -+ return 0; -+ } -+} -+ -+/*! ZSTD_getDictID_fromDict() : -+ * Provides the dictID stored within dictionary. -+ * if @return == 0, the dictionary is not conformant with Zstandard specification. -+ * It can still be loaded, but as a content-only dictionary. */ -+unsigned ZSTD_getDictID_fromDict(const void *dict, size_t dictSize) -+{ -+ if (dictSize < 8) -+ return 0; -+ if (ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) -+ return 0; -+ return ZSTD_readLE32((const char *)dict + 4); -+} -+ -+/*! ZSTD_getDictID_fromDDict() : -+ * Provides the dictID of the dictionary loaded into `ddict`. -+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. -+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -+unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict) -+{ -+ if (ddict == NULL) -+ return 0; -+ return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); -+} -+ -+/*! ZSTD_getDictID_fromFrame() : -+ * Provides the dictID required to decompressed the frame stored within `src`. -+ * If @return == 0, the dictID could not be decoded. -+ * This could for one of the following reasons : -+ * - The frame does not require a dictionary to be decoded (most common case). -+ * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. -+ * Note : this use case also happens when using a non-conformant dictionary. -+ * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). -+ * - This is not a Zstandard frame. -+ * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */ -+unsigned ZSTD_getDictID_fromFrame(const void *src, size_t srcSize) -+{ -+ ZSTD_frameParams zfp = {0, 0, 0, 0}; -+ size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize); -+ if (ZSTD_isError(hError)) -+ return 0; -+ return zfp.dictID; -+} -+ -+/*! ZSTD_decompress_usingDDict() : -+* Decompression using a pre-digested Dictionary -+* Use dictionary without significant overhead. */ -+size_t ZSTD_decompress_usingDDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_DDict *ddict) -+{ -+ /* pass content and size in case legacy frames are encountered */ -+ return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, NULL, 0, ddict); -+} -+ -+/*===================================== -+* Streaming decompression -+*====================================*/ -+ -+typedef enum { zdss_init, zdss_loadHeader, zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; -+ -+/* *** Resource management *** */ -+struct ZSTD_DStream_s { -+ ZSTD_DCtx *dctx; -+ ZSTD_DDict *ddictLocal; -+ const ZSTD_DDict *ddict; -+ ZSTD_frameParams fParams; -+ ZSTD_dStreamStage stage; -+ char *inBuff; -+ size_t inBuffSize; -+ size_t inPos; -+ size_t maxWindowSize; -+ char *outBuff; -+ size_t outBuffSize; -+ size_t outStart; -+ size_t outEnd; -+ size_t blockSize; -+ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */ -+ size_t lhSize; -+ ZSTD_customMem customMem; -+ void *legacyContext; -+ U32 previousLegacyVersion; -+ U32 legacyVersion; -+ U32 hostageByte; -+}; /* typedef'd to ZSTD_DStream within "zstd.h" */ -+ -+size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize) -+{ -+ size_t const blockSize = MIN(maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); -+ size_t const inBuffSize = blockSize; -+ size_t const outBuffSize = maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; -+ return ZSTD_DCtxWorkspaceBound() + ZSTD_ALIGN(sizeof(ZSTD_DStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize); -+} -+ -+static ZSTD_DStream *ZSTD_createDStream_advanced(ZSTD_customMem customMem) -+{ -+ ZSTD_DStream *zds; -+ -+ if (!customMem.customAlloc || !customMem.customFree) -+ return NULL; -+ -+ zds = (ZSTD_DStream *)ZSTD_malloc(sizeof(ZSTD_DStream), customMem); -+ if (zds == NULL) -+ return NULL; -+ memset(zds, 0, sizeof(ZSTD_DStream)); -+ memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); -+ zds->dctx = ZSTD_createDCtx_advanced(customMem); -+ if (zds->dctx == NULL) { -+ ZSTD_freeDStream(zds); -+ return NULL; -+ } -+ zds->stage = zdss_init; -+ zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; -+ return zds; -+} -+ -+ZSTD_DStream *ZSTD_initDStream(size_t maxWindowSize, void *workspace, size_t workspaceSize) -+{ -+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); -+ ZSTD_DStream *zds = ZSTD_createDStream_advanced(stackMem); -+ if (!zds) { -+ return NULL; -+ } -+ -+ zds->maxWindowSize = maxWindowSize; -+ zds->stage = zdss_loadHeader; -+ zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; -+ ZSTD_freeDDict(zds->ddictLocal); -+ zds->ddictLocal = NULL; -+ zds->ddict = zds->ddictLocal; -+ zds->legacyVersion = 0; -+ zds->hostageByte = 0; -+ -+ { -+ size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); -+ size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; -+ -+ zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem); -+ zds->inBuffSize = blockSize; -+ zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem); -+ zds->outBuffSize = neededOutSize; -+ if (zds->inBuff == NULL || zds->outBuff == NULL) { -+ ZSTD_freeDStream(zds); -+ return NULL; -+ } -+ } -+ return zds; -+} -+ -+ZSTD_DStream *ZSTD_initDStream_usingDDict(size_t maxWindowSize, const ZSTD_DDict *ddict, void *workspace, size_t workspaceSize) -+{ -+ ZSTD_DStream *zds = ZSTD_initDStream(maxWindowSize, workspace, workspaceSize); -+ if (zds) { -+ zds->ddict = ddict; -+ } -+ return zds; -+} -+ -+size_t ZSTD_freeDStream(ZSTD_DStream *zds) -+{ -+ if (zds == NULL) -+ return 0; /* support free on null */ -+ { -+ ZSTD_customMem const cMem = zds->customMem; -+ ZSTD_freeDCtx(zds->dctx); -+ zds->dctx = NULL; -+ ZSTD_freeDDict(zds->ddictLocal); -+ zds->ddictLocal = NULL; -+ ZSTD_free(zds->inBuff, cMem); -+ zds->inBuff = NULL; -+ ZSTD_free(zds->outBuff, cMem); -+ zds->outBuff = NULL; -+ ZSTD_free(zds, cMem); -+ return 0; -+ } -+} -+ -+/* *** Initialization *** */ -+ -+size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; } -+size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } -+ -+size_t ZSTD_resetDStream(ZSTD_DStream *zds) -+{ -+ zds->stage = zdss_loadHeader; -+ zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; -+ zds->legacyVersion = 0; -+ zds->hostageByte = 0; -+ return ZSTD_frameHeaderSize_prefix; -+} -+ -+/* ***** Decompression ***** */ -+ -+ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -+{ -+ size_t const length = MIN(dstCapacity, srcSize); -+ memcpy(dst, src, length); -+ return length; -+} -+ -+size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inBuffer *input) -+{ -+ const char *const istart = (const char *)(input->src) + input->pos; -+ const char *const iend = (const char *)(input->src) + input->size; -+ const char *ip = istart; -+ char *const ostart = (char *)(output->dst) + output->pos; -+ char *const oend = (char *)(output->dst) + output->size; -+ char *op = ostart; -+ U32 someMoreWork = 1; -+ -+ while (someMoreWork) { -+ switch (zds->stage) { -+ case zdss_init: -+ ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */ -+ /* fall-through */ -+ -+ case zdss_loadHeader: { -+ size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize); -+ if (ZSTD_isError(hSize)) -+ return hSize; -+ if (hSize != 0) { /* need more input */ -+ size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ -+ if (toLoad > (size_t)(iend - ip)) { /* not enough input to load full header */ -+ memcpy(zds->headerBuffer + zds->lhSize, ip, iend - ip); -+ zds->lhSize += iend - ip; -+ input->pos = input->size; -+ return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + -+ ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ -+ } -+ memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); -+ zds->lhSize = hSize; -+ ip += toLoad; -+ break; -+ } -+ -+ /* check for single-pass mode opportunity */ -+ if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ -+ && (U64)(size_t)(oend - op) >= zds->fParams.frameContentSize) { -+ size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend - istart); -+ if (cSize <= (size_t)(iend - istart)) { -+ size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend - op, istart, cSize, zds->ddict); -+ if (ZSTD_isError(decompressedSize)) -+ return decompressedSize; -+ ip = istart + cSize; -+ op += decompressedSize; -+ zds->dctx->expected = 0; -+ zds->stage = zdss_init; -+ someMoreWork = 0; -+ break; -+ } -+ } -+ -+ /* Consume header */ -+ ZSTD_refDDict(zds->dctx, zds->ddict); -+ { -+ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ -+ CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); -+ { -+ size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); -+ CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer + h1Size, h2Size)); -+ } -+ } -+ -+ zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); -+ if (zds->fParams.windowSize > zds->maxWindowSize) -+ return ERROR(frameParameter_windowTooLarge); -+ -+ /* Buffers are preallocated, but double check */ -+ { -+ size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); -+ size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; -+ if (zds->inBuffSize < blockSize) { -+ return ERROR(GENERIC); -+ } -+ if (zds->outBuffSize < neededOutSize) { -+ return ERROR(GENERIC); -+ } -+ zds->blockSize = blockSize; -+ } -+ zds->stage = zdss_read; -+ } -+ /* pass-through */ -+ -+ case zdss_read: { -+ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); -+ if (neededInSize == 0) { /* end of frame */ -+ zds->stage = zdss_init; -+ someMoreWork = 0; -+ break; -+ } -+ if ((size_t)(iend - ip) >= neededInSize) { /* decode directly from src */ -+ const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); -+ size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, -+ (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), ip, neededInSize); -+ if (ZSTD_isError(decodedSize)) -+ return decodedSize; -+ ip += neededInSize; -+ if (!decodedSize && !isSkipFrame) -+ break; /* this was just a header */ -+ zds->outEnd = zds->outStart + decodedSize; -+ zds->stage = zdss_flush; -+ break; -+ } -+ if (ip == iend) { -+ someMoreWork = 0; -+ break; -+ } /* no more input */ -+ zds->stage = zdss_load; -+ /* pass-through */ -+ } -+ -+ case zdss_load: { -+ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); -+ size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ -+ size_t loadedSize; -+ if (toLoad > zds->inBuffSize - zds->inPos) -+ return ERROR(corruption_detected); /* should never happen */ -+ loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend - ip); -+ ip += loadedSize; -+ zds->inPos += loadedSize; -+ if (loadedSize < toLoad) { -+ someMoreWork = 0; -+ break; -+ } /* not enough input, wait for more */ -+ -+ /* decode loaded input */ -+ { -+ const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); -+ size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, -+ zds->inBuff, neededInSize); -+ if (ZSTD_isError(decodedSize)) -+ return decodedSize; -+ zds->inPos = 0; /* input is consumed */ -+ if (!decodedSize && !isSkipFrame) { -+ zds->stage = zdss_read; -+ break; -+ } /* this was just a header */ -+ zds->outEnd = zds->outStart + decodedSize; -+ zds->stage = zdss_flush; -+ /* pass-through */ -+ } -+ } -+ -+ case zdss_flush: { -+ size_t const toFlushSize = zds->outEnd - zds->outStart; -+ size_t const flushedSize = ZSTD_limitCopy(op, oend - op, zds->outBuff + zds->outStart, toFlushSize); -+ op += flushedSize; -+ zds->outStart += flushedSize; -+ if (flushedSize == toFlushSize) { /* flush completed */ -+ zds->stage = zdss_read; -+ if (zds->outStart + zds->blockSize > zds->outBuffSize) -+ zds->outStart = zds->outEnd = 0; -+ break; -+ } -+ /* cannot complete flush */ -+ someMoreWork = 0; -+ break; -+ } -+ default: -+ return ERROR(GENERIC); /* impossible */ -+ } -+ } -+ -+ /* result */ -+ input->pos += (size_t)(ip - istart); -+ output->pos += (size_t)(op - ostart); -+ { -+ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx); -+ if (!nextSrcSizeHint) { /* frame fully decoded */ -+ if (zds->outEnd == zds->outStart) { /* output fully flushed */ -+ if (zds->hostageByte) { -+ if (input->pos >= input->size) { -+ zds->stage = zdss_read; -+ return 1; -+ } /* can't release hostage (not present) */ -+ input->pos++; /* release hostage */ -+ } -+ return 0; -+ } -+ if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ -+ input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ -+ zds->hostageByte = 1; -+ } -+ return 1; -+ } -+ nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */ -+ if (zds->inPos > nextSrcSizeHint) -+ return ERROR(GENERIC); /* should never happen */ -+ nextSrcSizeHint -= zds->inPos; /* already loaded*/ -+ return nextSrcSizeHint; -+ } -+} -+ -+EXPORT_SYMBOL(ZSTD_DCtxWorkspaceBound); -+EXPORT_SYMBOL(ZSTD_initDCtx); -+EXPORT_SYMBOL(ZSTD_decompressDCtx); -+EXPORT_SYMBOL(ZSTD_decompress_usingDict); -+ -+EXPORT_SYMBOL(ZSTD_DDictWorkspaceBound); -+EXPORT_SYMBOL(ZSTD_initDDict); -+EXPORT_SYMBOL(ZSTD_decompress_usingDDict); -+ -+EXPORT_SYMBOL(ZSTD_DStreamWorkspaceBound); -+EXPORT_SYMBOL(ZSTD_initDStream); -+EXPORT_SYMBOL(ZSTD_initDStream_usingDDict); -+EXPORT_SYMBOL(ZSTD_resetDStream); -+EXPORT_SYMBOL(ZSTD_decompressStream); -+EXPORT_SYMBOL(ZSTD_DStreamInSize); -+EXPORT_SYMBOL(ZSTD_DStreamOutSize); -+ -+EXPORT_SYMBOL(ZSTD_findFrameCompressedSize); -+EXPORT_SYMBOL(ZSTD_getFrameContentSize); -+EXPORT_SYMBOL(ZSTD_findDecompressedSize); -+ -+EXPORT_SYMBOL(ZSTD_isFrame); -+EXPORT_SYMBOL(ZSTD_getDictID_fromDict); -+EXPORT_SYMBOL(ZSTD_getDictID_fromDDict); -+EXPORT_SYMBOL(ZSTD_getDictID_fromFrame); -+ -+EXPORT_SYMBOL(ZSTD_getFrameParams); -+EXPORT_SYMBOL(ZSTD_decompressBegin); -+EXPORT_SYMBOL(ZSTD_decompressBegin_usingDict); -+EXPORT_SYMBOL(ZSTD_copyDCtx); -+EXPORT_SYMBOL(ZSTD_nextSrcSizeToDecompress); -+EXPORT_SYMBOL(ZSTD_decompressContinue); -+EXPORT_SYMBOL(ZSTD_nextInputType); -+ -+EXPORT_SYMBOL(ZSTD_decompressBlock); -+EXPORT_SYMBOL(ZSTD_insertBlock); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("Zstd Decompressor"); -diff --git a/lib/zstd/entropy_common.c b/lib/zstd/entropy_common.c -new file mode 100644 -index 0000000..2b0a643 ---- /dev/null -+++ b/lib/zstd/entropy_common.c -@@ -0,0 +1,243 @@ -+/* -+ * Common functions of New Generation Entropy library -+ * Copyright (C) 2016, Yann Collet. -+ * -+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above -+ * copyright notice, this list of conditions and the following disclaimer -+ * in the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ * -+ * You can contact the author at : -+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -+ */ -+ -+/* ************************************* -+* Dependencies -+***************************************/ -+#include "error_private.h" /* ERR_*, ERROR */ -+#include "fse.h" -+#include "huf.h" -+#include "mem.h" -+ -+/*=== Version ===*/ -+unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } -+ -+/*=== Error Management ===*/ -+unsigned FSE_isError(size_t code) { return ERR_isError(code); } -+ -+unsigned HUF_isError(size_t code) { return ERR_isError(code); } -+ -+/*-************************************************************** -+* FSE NCount encoding-decoding -+****************************************************************/ -+size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSVPtr, unsigned *tableLogPtr, const void *headerBuffer, size_t hbSize) -+{ -+ const BYTE *const istart = (const BYTE *)headerBuffer; -+ const BYTE *const iend = istart + hbSize; -+ const BYTE *ip = istart; -+ int nbBits; -+ int remaining; -+ int threshold; -+ U32 bitStream; -+ int bitCount; -+ unsigned charnum = 0; -+ int previous0 = 0; -+ -+ if (hbSize < 4) -+ return ERROR(srcSize_wrong); -+ bitStream = ZSTD_readLE32(ip); -+ nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ -+ if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) -+ return ERROR(tableLog_tooLarge); -+ bitStream >>= 4; -+ bitCount = 4; -+ *tableLogPtr = nbBits; -+ remaining = (1 << nbBits) + 1; -+ threshold = 1 << nbBits; -+ nbBits++; -+ -+ while ((remaining > 1) & (charnum <= *maxSVPtr)) { -+ if (previous0) { -+ unsigned n0 = charnum; -+ while ((bitStream & 0xFFFF) == 0xFFFF) { -+ n0 += 24; -+ if (ip < iend - 5) { -+ ip += 2; -+ bitStream = ZSTD_readLE32(ip) >> bitCount; -+ } else { -+ bitStream >>= 16; -+ bitCount += 16; -+ } -+ } -+ while ((bitStream & 3) == 3) { -+ n0 += 3; -+ bitStream >>= 2; -+ bitCount += 2; -+ } -+ n0 += bitStream & 3; -+ bitCount += 2; -+ if (n0 > *maxSVPtr) -+ return ERROR(maxSymbolValue_tooSmall); -+ while (charnum < n0) -+ normalizedCounter[charnum++] = 0; -+ if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) { -+ ip += bitCount >> 3; -+ bitCount &= 7; -+ bitStream = ZSTD_readLE32(ip) >> bitCount; -+ } else { -+ bitStream >>= 2; -+ } -+ } -+ { -+ int const max = (2 * threshold - 1) - remaining; -+ int count; -+ -+ if ((bitStream & (threshold - 1)) < (U32)max) { -+ count = bitStream & (threshold - 1); -+ bitCount += nbBits - 1; -+ } else { -+ count = bitStream & (2 * threshold - 1); -+ if (count >= threshold) -+ count -= max; -+ bitCount += nbBits; -+ } -+ -+ count--; /* extra accuracy */ -+ remaining -= count < 0 ? -count : count; /* -1 means +1 */ -+ normalizedCounter[charnum++] = (short)count; -+ previous0 = !count; -+ while (remaining < threshold) { -+ nbBits--; -+ threshold >>= 1; -+ } -+ -+ if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) { -+ ip += bitCount >> 3; -+ bitCount &= 7; -+ } else { -+ bitCount -= (int)(8 * (iend - 4 - ip)); -+ ip = iend - 4; -+ } -+ bitStream = ZSTD_readLE32(ip) >> (bitCount & 31); -+ } -+ } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ -+ if (remaining != 1) -+ return ERROR(corruption_detected); -+ if (bitCount > 32) -+ return ERROR(corruption_detected); -+ *maxSVPtr = charnum - 1; -+ -+ ip += (bitCount + 7) >> 3; -+ return ip - istart; -+} -+ -+/*! HUF_readStats() : -+ Read compact Huffman tree, saved by HUF_writeCTable(). -+ `huffWeight` is destination buffer. -+ `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. -+ @return : size read from `src` , or an error Code . -+ Note : Needed by HUF_readCTable() and HUF_readDTableX?() . -+*/ -+size_t HUF_readStats_wksp(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize, void *workspace, size_t workspaceSize) -+{ -+ U32 weightTotal; -+ const BYTE *ip = (const BYTE *)src; -+ size_t iSize; -+ size_t oSize; -+ -+ if (!srcSize) -+ return ERROR(srcSize_wrong); -+ iSize = ip[0]; -+ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */ -+ -+ if (iSize >= 128) { /* special header */ -+ oSize = iSize - 127; -+ iSize = ((oSize + 1) / 2); -+ if (iSize + 1 > srcSize) -+ return ERROR(srcSize_wrong); -+ if (oSize >= hwSize) -+ return ERROR(corruption_detected); -+ ip += 1; -+ { -+ U32 n; -+ for (n = 0; n < oSize; n += 2) { -+ huffWeight[n] = ip[n / 2] >> 4; -+ huffWeight[n + 1] = ip[n / 2] & 15; -+ } -+ } -+ } else { /* header compressed with FSE (normal case) */ -+ if (iSize + 1 > srcSize) -+ return ERROR(srcSize_wrong); -+ oSize = FSE_decompress_wksp(huffWeight, hwSize - 1, ip + 1, iSize, 6, workspace, workspaceSize); /* max (hwSize-1) values decoded, as last one is implied */ -+ if (FSE_isError(oSize)) -+ return oSize; -+ } -+ -+ /* collect weight stats */ -+ memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); -+ weightTotal = 0; -+ { -+ U32 n; -+ for (n = 0; n < oSize; n++) { -+ if (huffWeight[n] >= HUF_TABLELOG_MAX) -+ return ERROR(corruption_detected); -+ rankStats[huffWeight[n]]++; -+ weightTotal += (1 << huffWeight[n]) >> 1; -+ } -+ } -+ if (weightTotal == 0) -+ return ERROR(corruption_detected); -+ -+ /* get last non-null symbol weight (implied, total must be 2^n) */ -+ { -+ U32 const tableLog = BIT_highbit32(weightTotal) + 1; -+ if (tableLog > HUF_TABLELOG_MAX) -+ return ERROR(corruption_detected); -+ *tableLogPtr = tableLog; -+ /* determine last weight */ -+ { -+ U32 const total = 1 << tableLog; -+ U32 const rest = total - weightTotal; -+ U32 const verif = 1 << BIT_highbit32(rest); -+ U32 const lastWeight = BIT_highbit32(rest) + 1; -+ if (verif != rest) -+ return ERROR(corruption_detected); /* last value must be a clean power of 2 */ -+ huffWeight[oSize] = (BYTE)lastWeight; -+ rankStats[lastWeight]++; -+ } -+ } -+ -+ /* check tree construction validity */ -+ if ((rankStats[1] < 2) || (rankStats[1] & 1)) -+ return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ -+ -+ /* results */ -+ *nbSymbolsPtr = (U32)(oSize + 1); -+ return iSize + 1; -+} -diff --git a/lib/zstd/error_private.h b/lib/zstd/error_private.h -new file mode 100644 -index 0000000..2062ff0 ---- /dev/null -+++ b/lib/zstd/error_private.h -@@ -0,0 +1,51 @@ -+/** -+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. -+ * All rights reserved. -+ * -+ * This source code is licensed under the BSD-style license found in the -+ * LICENSE file in the root directory of https://github.com/facebook/zstd. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ */ -+ -+/* Note : this module is expected to remain private, do not expose it */ -+ -+#ifndef ERROR_H_MODULE -+#define ERROR_H_MODULE -+ -+/* **************************************** -+* Dependencies -+******************************************/ -+#include <linux/types.h> /* size_t */ -+#include <linux/zstd.h> /* enum list */ -+ -+/* **************************************** -+* Compiler-specific -+******************************************/ -+#define ERR_STATIC static __attribute__((unused)) -+ -+/*-**************************************** -+* Customization (error_public.h) -+******************************************/ -+typedef ZSTD_ErrorCode ERR_enum; -+#define PREFIX(name) ZSTD_error_##name -+ -+/*-**************************************** -+* Error codes handling -+******************************************/ -+#define ERROR(name) ((size_t)-PREFIX(name)) -+ -+ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } -+ -+ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) -+{ -+ if (!ERR_isError(code)) -+ return (ERR_enum)0; -+ return (ERR_enum)(0 - code); -+} -+ -+#endif /* ERROR_H_MODULE */ -diff --git a/lib/zstd/fse.h b/lib/zstd/fse.h -new file mode 100644 -index 0000000..7460ab0 ---- /dev/null -+++ b/lib/zstd/fse.h -@@ -0,0 +1,575 @@ -+/* -+ * FSE : Finite State Entropy codec -+ * Public Prototypes declaration -+ * Copyright (C) 2013-2016, Yann Collet. -+ * -+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above -+ * copyright notice, this list of conditions and the following disclaimer -+ * in the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ * -+ * You can contact the author at : -+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -+ */ -+#ifndef FSE_H -+#define FSE_H -+ -+/*-***************************************** -+* Dependencies -+******************************************/ -+#include <linux/types.h> /* size_t, ptrdiff_t */ -+ -+/*-***************************************** -+* FSE_PUBLIC_API : control library symbols visibility -+******************************************/ -+#define FSE_PUBLIC_API -+ -+/*------ Version ------*/ -+#define FSE_VERSION_MAJOR 0 -+#define FSE_VERSION_MINOR 9 -+#define FSE_VERSION_RELEASE 0 -+ -+#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE -+#define FSE_QUOTE(str) #str -+#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) -+#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) -+ -+#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR * 100 * 100 + FSE_VERSION_MINOR * 100 + FSE_VERSION_RELEASE) -+FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ -+ -+/*-***************************************** -+* Tool functions -+******************************************/ -+FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ -+ -+/* Error Management */ -+FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ -+ -+/*-***************************************** -+* FSE detailed API -+******************************************/ -+/*! -+FSE_compress() does the following: -+1. count symbol occurrence from source[] into table count[] -+2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) -+3. save normalized counters to memory buffer using writeNCount() -+4. build encoding table 'CTable' from normalized counters -+5. encode the data stream using encoding table 'CTable' -+ -+FSE_decompress() does the following: -+1. read normalized counters with readNCount() -+2. build decoding table 'DTable' from normalized counters -+3. decode the data stream using decoding table 'DTable' -+ -+The following API allows targeting specific sub-functions for advanced tasks. -+For example, it's possible to compress several blocks using the same 'CTable', -+or to save and provide normalized distribution using external method. -+*/ -+ -+/* *** COMPRESSION *** */ -+/*! FSE_optimalTableLog(): -+ dynamically downsize 'tableLog' when conditions are met. -+ It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. -+ @return : recommended tableLog (necessarily <= 'maxTableLog') */ -+FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -+ -+/*! FSE_normalizeCount(): -+ normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) -+ 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). -+ @return : tableLog, -+ or an errorCode, which can be tested using FSE_isError() */ -+FSE_PUBLIC_API size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t srcSize, unsigned maxSymbolValue); -+ -+/*! FSE_NCountWriteBound(): -+ Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. -+ Typically useful for allocation purpose. */ -+FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); -+ -+/*! FSE_writeNCount(): -+ Compactly save 'normalizedCounter' into 'buffer'. -+ @return : size of the compressed table, -+ or an errorCode, which can be tested using FSE_isError(). */ -+FSE_PUBLIC_API size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); -+ -+/*! Constructor and Destructor of FSE_CTable. -+ Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ -+typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ -+ -+/*! FSE_compress_usingCTable(): -+ Compress `src` using `ct` into `dst` which must be already allocated. -+ @return : size of compressed data (<= `dstCapacity`), -+ or 0 if compressed data could not fit into `dst`, -+ or an errorCode, which can be tested using FSE_isError() */ -+FSE_PUBLIC_API size_t FSE_compress_usingCTable(void *dst, size_t dstCapacity, const void *src, size_t srcSize, const FSE_CTable *ct); -+ -+/*! -+Tutorial : -+---------- -+The first step is to count all symbols. FSE_count() does this job very fast. -+Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. -+'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] -+maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) -+FSE_count() will return the number of occurrence of the most frequent symbol. -+This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. -+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). -+ -+The next step is to normalize the frequencies. -+FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. -+It also guarantees a minimum of 1 to any Symbol with frequency >= 1. -+You can use 'tableLog'==0 to mean "use default tableLog value". -+If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), -+which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). -+ -+The result of FSE_normalizeCount() will be saved into a table, -+called 'normalizedCounter', which is a table of signed short. -+'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. -+The return value is tableLog if everything proceeded as expected. -+It is 0 if there is a single symbol within distribution. -+If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). -+ -+'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). -+'buffer' must be already allocated. -+For guaranteed success, buffer size must be at least FSE_headerBound(). -+The result of the function is the number of bytes written into 'buffer'. -+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). -+ -+'normalizedCounter' can then be used to create the compression table 'CTable'. -+The space required by 'CTable' must be already allocated, using FSE_createCTable(). -+You can then use FSE_buildCTable() to fill 'CTable'. -+If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). -+ -+'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). -+Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' -+The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. -+If it returns '0', compressed data could not fit into 'dst'. -+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). -+*/ -+ -+/* *** DECOMPRESSION *** */ -+ -+/*! FSE_readNCount(): -+ Read compactly saved 'normalizedCounter' from 'rBuffer'. -+ @return : size read from 'rBuffer', -+ or an errorCode, which can be tested using FSE_isError(). -+ maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ -+FSE_PUBLIC_API size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSymbolValuePtr, unsigned *tableLogPtr, const void *rBuffer, size_t rBuffSize); -+ -+/*! Constructor and Destructor of FSE_DTable. -+ Note that its size depends on 'tableLog' */ -+typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ -+ -+/*! FSE_buildDTable(): -+ Builds 'dt', which must be already allocated, using FSE_createDTable(). -+ return : 0, or an errorCode, which can be tested using FSE_isError() */ -+FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize); -+ -+/*! FSE_decompress_usingDTable(): -+ Decompress compressed source `cSrc` of size `cSrcSize` using `dt` -+ into `dst` which must be already allocated. -+ @return : size of regenerated data (necessarily <= `dstCapacity`), -+ or an errorCode, which can be tested using FSE_isError() */ -+FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt); -+ -+/*! -+Tutorial : -+---------- -+(Note : these functions only decompress FSE-compressed blocks. -+ If block is uncompressed, use memcpy() instead -+ If block is a single repeated byte, use memset() instead ) -+ -+The first step is to obtain the normalized frequencies of symbols. -+This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). -+'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. -+In practice, that means it's necessary to know 'maxSymbolValue' beforehand, -+or size the table to handle worst case situations (typically 256). -+FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. -+The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. -+Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. -+If there is an error, the function will return an error code, which can be tested using FSE_isError(). -+ -+The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. -+This is performed by the function FSE_buildDTable(). -+The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). -+If there is an error, the function will return an error code, which can be tested using FSE_isError(). -+ -+`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). -+`cSrcSize` must be strictly correct, otherwise decompression will fail. -+FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). -+If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) -+*/ -+ -+/* *** Dependency *** */ -+#include "bitstream.h" -+ -+/* ***************************************** -+* Static allocation -+*******************************************/ -+/* FSE buffer bounds */ -+#define FSE_NCOUNTBOUND 512 -+#define FSE_BLOCKBOUND(size) (size + (size >> 7)) -+#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ -+ -+/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ -+#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1 << (maxTableLog - 1)) + ((maxSymbolValue + 1) * 2)) -+#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1 << maxTableLog)) -+ -+/* ***************************************** -+* FSE advanced API -+*******************************************/ -+/* FSE_count_wksp() : -+ * Same as FSE_count(), but using an externally provided scratch buffer. -+ * `workSpace` size must be table of >= `1024` unsigned -+ */ -+size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace); -+ -+/* FSE_countFast_wksp() : -+ * Same as FSE_countFast(), but using an externally provided scratch buffer. -+ * `workSpace` must be a table of minimum `1024` unsigned -+ */ -+size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize, unsigned *workSpace); -+ -+/*! FSE_count_simple -+ * Same as FSE_countFast(), but does not use any additional memory (not even on stack). -+ * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`). -+*/ -+size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize); -+ -+unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); -+/**< same as FSE_optimalTableLog(), which used `minus==2` */ -+ -+size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits); -+/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ -+ -+size_t FSE_buildCTable_rle(FSE_CTable *ct, unsigned char symbolValue); -+/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ -+ -+/* FSE_buildCTable_wksp() : -+ * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). -+ * `wkspSize` must be >= `(1<<tableLog)`. -+ */ -+size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, size_t wkspSize); -+ -+size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits); -+/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ -+ -+size_t FSE_buildDTable_rle(FSE_DTable *dt, unsigned char symbolValue); -+/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ -+ -+size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, unsigned maxLog, void *workspace, size_t workspaceSize); -+/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */ -+ -+/* ***************************************** -+* FSE symbol compression API -+*******************************************/ -+/*! -+ This API consists of small unitary functions, which highly benefit from being inlined. -+ Hence their body are included in next section. -+*/ -+typedef struct { -+ ptrdiff_t value; -+ const void *stateTable; -+ const void *symbolTT; -+ unsigned stateLog; -+} FSE_CState_t; -+ -+static void FSE_initCState(FSE_CState_t *CStatePtr, const FSE_CTable *ct); -+ -+static void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *CStatePtr, unsigned symbol); -+ -+static void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *CStatePtr); -+ -+/**< -+These functions are inner components of FSE_compress_usingCTable(). -+They allow the creation of custom streams, mixing multiple tables and bit sources. -+ -+A key property to keep in mind is that encoding and decoding are done **in reverse direction**. -+So the first symbol you will encode is the last you will decode, like a LIFO stack. -+ -+You will need a few variables to track your CStream. They are : -+ -+FSE_CTable ct; // Provided by FSE_buildCTable() -+BIT_CStream_t bitStream; // bitStream tracking structure -+FSE_CState_t state; // State tracking structure (can have several) -+ -+ -+The first thing to do is to init bitStream and state. -+ size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize); -+ FSE_initCState(&state, ct); -+ -+Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError(); -+You can then encode your input data, byte after byte. -+FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time. -+Remember decoding will be done in reverse direction. -+ FSE_encodeByte(&bitStream, &state, symbol); -+ -+At any time, you can also add any bit sequence. -+Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders -+ BIT_addBits(&bitStream, bitField, nbBits); -+ -+The above methods don't commit data to memory, they just store it into local register, for speed. -+Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). -+Writing data to memory is a manual operation, performed by the flushBits function. -+ BIT_flushBits(&bitStream); -+ -+Your last FSE encoding operation shall be to flush your last state value(s). -+ FSE_flushState(&bitStream, &state); -+ -+Finally, you must close the bitStream. -+The function returns the size of CStream in bytes. -+If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible) -+If there is an error, it returns an errorCode (which can be tested using FSE_isError()). -+ size_t size = BIT_closeCStream(&bitStream); -+*/ -+ -+/* ***************************************** -+* FSE symbol decompression API -+*******************************************/ -+typedef struct { -+ size_t state; -+ const void *table; /* precise table may vary, depending on U16 */ -+} FSE_DState_t; -+ -+static void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt); -+ -+static unsigned char FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD); -+ -+static unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr); -+ -+/**< -+Let's now decompose FSE_decompress_usingDTable() into its unitary components. -+You will decode FSE-encoded symbols from the bitStream, -+and also any other bitFields you put in, **in reverse order**. -+ -+You will need a few variables to track your bitStream. They are : -+ -+BIT_DStream_t DStream; // Stream context -+FSE_DState_t DState; // State context. Multiple ones are possible -+FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable() -+ -+The first thing to do is to init the bitStream. -+ errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); -+ -+You should then retrieve your initial state(s) -+(in reverse flushing order if you have several ones) : -+ errorCode = FSE_initDState(&DState, &DStream, DTablePtr); -+ -+You can then decode your data, symbol after symbol. -+For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'. -+Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). -+ unsigned char symbol = FSE_decodeSymbol(&DState, &DStream); -+ -+You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) -+Note : maximum allowed nbBits is 25, for 32-bits compatibility -+ size_t bitField = BIT_readBits(&DStream, nbBits); -+ -+All above operations only read from local register (which size depends on size_t). -+Refueling the register from memory is manually performed by the reload method. -+ endSignal = FSE_reloadDStream(&DStream); -+ -+BIT_reloadDStream() result tells if there is still some more data to read from DStream. -+BIT_DStream_unfinished : there is still some data left into the DStream. -+BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. -+BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. -+BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. -+ -+When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, -+to properly detect the exact end of stream. -+After each decoded symbol, check if DStream is fully consumed using this simple test : -+ BIT_reloadDStream(&DStream) >= BIT_DStream_completed -+ -+When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. -+Checking if DStream has reached its end is performed by : -+ BIT_endOfDStream(&DStream); -+Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. -+ FSE_endOfDState(&DState); -+*/ -+ -+/* ***************************************** -+* FSE unsafe API -+*******************************************/ -+static unsigned char FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD); -+/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ -+ -+/* ***************************************** -+* Implementation of inlined functions -+*******************************************/ -+typedef struct { -+ int deltaFindState; -+ U32 deltaNbBits; -+} FSE_symbolCompressionTransform; /* total 8 bytes */ -+ -+ZSTD_STATIC void FSE_initCState(FSE_CState_t *statePtr, const FSE_CTable *ct) -+{ -+ const void *ptr = ct; -+ const U16 *u16ptr = (const U16 *)ptr; -+ const U32 tableLog = ZSTD_read16(ptr); -+ statePtr->value = (ptrdiff_t)1 << tableLog; -+ statePtr->stateTable = u16ptr + 2; -+ statePtr->symbolTT = ((const U32 *)ct + 1 + (tableLog ? (1 << (tableLog - 1)) : 1)); -+ statePtr->stateLog = tableLog; -+} -+ -+/*! FSE_initCState2() : -+* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) -+* uses the smallest state value possible, saving the cost of this symbol */ -+ZSTD_STATIC void FSE_initCState2(FSE_CState_t *statePtr, const FSE_CTable *ct, U32 symbol) -+{ -+ FSE_initCState(statePtr, ct); -+ { -+ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol]; -+ const U16 *stateTable = (const U16 *)(statePtr->stateTable); -+ U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1 << 15)) >> 16); -+ statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; -+ statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; -+ } -+} -+ -+ZSTD_STATIC void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *statePtr, U32 symbol) -+{ -+ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol]; -+ const U16 *const stateTable = (const U16 *)(statePtr->stateTable); -+ U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); -+ BIT_addBits(bitC, statePtr->value, nbBitsOut); -+ statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; -+} -+ -+ZSTD_STATIC void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *statePtr) -+{ -+ BIT_addBits(bitC, statePtr->value, statePtr->stateLog); -+ BIT_flushBits(bitC); -+} -+ -+/* ====== Decompression ====== */ -+ -+typedef struct { -+ U16 tableLog; -+ U16 fastMode; -+} FSE_DTableHeader; /* sizeof U32 */ -+ -+typedef struct { -+ unsigned short newState; -+ unsigned char symbol; -+ unsigned char nbBits; -+} FSE_decode_t; /* size == U32 */ -+ -+ZSTD_STATIC void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt) -+{ -+ const void *ptr = dt; -+ const FSE_DTableHeader *const DTableH = (const FSE_DTableHeader *)ptr; -+ DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); -+ BIT_reloadDStream(bitD); -+ DStatePtr->table = dt + 1; -+} -+ -+ZSTD_STATIC BYTE FSE_peekSymbol(const FSE_DState_t *DStatePtr) -+{ -+ FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; -+ return DInfo.symbol; -+} -+ -+ZSTD_STATIC void FSE_updateState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD) -+{ -+ FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; -+ U32 const nbBits = DInfo.nbBits; -+ size_t const lowBits = BIT_readBits(bitD, nbBits); -+ DStatePtr->state = DInfo.newState + lowBits; -+} -+ -+ZSTD_STATIC BYTE FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD) -+{ -+ FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; -+ U32 const nbBits = DInfo.nbBits; -+ BYTE const symbol = DInfo.symbol; -+ size_t const lowBits = BIT_readBits(bitD, nbBits); -+ -+ DStatePtr->state = DInfo.newState + lowBits; -+ return symbol; -+} -+ -+/*! FSE_decodeSymbolFast() : -+ unsafe, only works if no symbol has a probability > 50% */ -+ZSTD_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD) -+{ -+ FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; -+ U32 const nbBits = DInfo.nbBits; -+ BYTE const symbol = DInfo.symbol; -+ size_t const lowBits = BIT_readBitsFast(bitD, nbBits); -+ -+ DStatePtr->state = DInfo.newState + lowBits; -+ return symbol; -+} -+ -+ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr) { return DStatePtr->state == 0; } -+ -+/* ************************************************************** -+* Tuning parameters -+****************************************************************/ -+/*!MEMORY_USAGE : -+* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) -+* Increasing memory usage improves compression ratio -+* Reduced memory usage can improve speed, due to cache effect -+* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -+#ifndef FSE_MAX_MEMORY_USAGE -+#define FSE_MAX_MEMORY_USAGE 14 -+#endif -+#ifndef FSE_DEFAULT_MEMORY_USAGE -+#define FSE_DEFAULT_MEMORY_USAGE 13 -+#endif -+ -+/*!FSE_MAX_SYMBOL_VALUE : -+* Maximum symbol value authorized. -+* Required for proper stack allocation */ -+#ifndef FSE_MAX_SYMBOL_VALUE -+#define FSE_MAX_SYMBOL_VALUE 255 -+#endif -+ -+/* ************************************************************** -+* template functions type & suffix -+****************************************************************/ -+#define FSE_FUNCTION_TYPE BYTE -+#define FSE_FUNCTION_EXTENSION -+#define FSE_DECODE_TYPE FSE_decode_t -+ -+/* *************************************************************** -+* Constants -+*****************************************************************/ -+#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE - 2) -+#define FSE_MAX_TABLESIZE (1U << FSE_MAX_TABLELOG) -+#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE - 1) -+#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE - 2) -+#define FSE_MIN_TABLELOG 5 -+ -+#define FSE_TABLELOG_ABSOLUTE_MAX 15 -+#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX -+#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" -+#endif -+ -+#define FSE_TABLESTEP(tableSize) ((tableSize >> 1) + (tableSize >> 3) + 3) -+ -+#endif /* FSE_H */ -diff --git a/lib/zstd/fse_compress.c b/lib/zstd/fse_compress.c -new file mode 100644 -index 0000000..ef3d174 ---- /dev/null -+++ b/lib/zstd/fse_compress.c -@@ -0,0 +1,795 @@ -+/* -+ * FSE : Finite State Entropy encoder -+ * Copyright (C) 2013-2015, Yann Collet. -+ * -+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above -+ * copyright notice, this list of conditions and the following disclaimer -+ * in the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ * -+ * You can contact the author at : -+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -+ */ -+ -+/* ************************************************************** -+* Compiler specifics -+****************************************************************/ -+#define FORCE_INLINE static __always_inline -+ -+/* ************************************************************** -+* Includes -+****************************************************************/ -+#include "bitstream.h" -+#include "fse.h" -+#include <linux/compiler.h> -+#include <linux/kernel.h> -+#include <linux/math64.h> -+#include <linux/string.h> /* memcpy, memset */ -+ -+/* ************************************************************** -+* Error Management -+****************************************************************/ -+#define FSE_STATIC_ASSERT(c) \ -+ { \ -+ enum { FSE_static_assert = 1 / (int)(!!(c)) }; \ -+ } /* use only *after* variable declarations */ -+ -+/* ************************************************************** -+* Templates -+****************************************************************/ -+/* -+ designed to be included -+ for type-specific functions (template emulation in C) -+ Objective is to write these functions only once, for improved maintenance -+*/ -+ -+/* safety checks */ -+#ifndef FSE_FUNCTION_EXTENSION -+#error "FSE_FUNCTION_EXTENSION must be defined" -+#endif -+#ifndef FSE_FUNCTION_TYPE -+#error "FSE_FUNCTION_TYPE must be defined" -+#endif -+ -+/* Function names */ -+#define FSE_CAT(X, Y) X##Y -+#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y) -+#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y) -+ -+/* Function templates */ -+ -+/* FSE_buildCTable_wksp() : -+ * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). -+ * wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)` -+ * workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements -+ */ -+size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize) -+{ -+ U32 const tableSize = 1 << tableLog; -+ U32 const tableMask = tableSize - 1; -+ void *const ptr = ct; -+ U16 *const tableU16 = ((U16 *)ptr) + 2; -+ void *const FSCT = ((U32 *)ptr) + 1 /* header */ + (tableLog ? tableSize >> 1 : 1); -+ FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT); -+ U32 const step = FSE_TABLESTEP(tableSize); -+ U32 highThreshold = tableSize - 1; -+ -+ U32 *cumul; -+ FSE_FUNCTION_TYPE *tableSymbol; -+ size_t spaceUsed32 = 0; -+ -+ cumul = (U32 *)workspace + spaceUsed32; -+ spaceUsed32 += FSE_MAX_SYMBOL_VALUE + 2; -+ tableSymbol = (FSE_FUNCTION_TYPE *)((U32 *)workspace + spaceUsed32); -+ spaceUsed32 += ALIGN(sizeof(FSE_FUNCTION_TYPE) * ((size_t)1 << tableLog), sizeof(U32)) >> 2; -+ -+ if ((spaceUsed32 << 2) > workspaceSize) -+ return ERROR(tableLog_tooLarge); -+ workspace = (U32 *)workspace + spaceUsed32; -+ workspaceSize -= (spaceUsed32 << 2); -+ -+ /* CTable header */ -+ tableU16[-2] = (U16)tableLog; -+ tableU16[-1] = (U16)maxSymbolValue; -+ -+ /* For explanations on how to distribute symbol values over the table : -+ * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ -+ -+ /* symbol start positions */ -+ { -+ U32 u; -+ cumul[0] = 0; -+ for (u = 1; u <= maxSymbolValue + 1; u++) { -+ if (normalizedCounter[u - 1] == -1) { /* Low proba symbol */ -+ cumul[u] = cumul[u - 1] + 1; -+ tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u - 1); -+ } else { -+ cumul[u] = cumul[u - 1] + normalizedCounter[u - 1]; -+ } -+ } -+ cumul[maxSymbolValue + 1] = tableSize + 1; -+ } -+ -+ /* Spread symbols */ -+ { -+ U32 position = 0; -+ U32 symbol; -+ for (symbol = 0; symbol <= maxSymbolValue; symbol++) { -+ int nbOccurrences; -+ for (nbOccurrences = 0; nbOccurrences < normalizedCounter[symbol]; nbOccurrences++) { -+ tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol; -+ position = (position + step) & tableMask; -+ while (position > highThreshold) -+ position = (position + step) & tableMask; /* Low proba area */ -+ } -+ } -+ -+ if (position != 0) -+ return ERROR(GENERIC); /* Must have gone through all positions */ -+ } -+ -+ /* Build table */ -+ { -+ U32 u; -+ for (u = 0; u < tableSize; u++) { -+ FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */ -+ tableU16[cumul[s]++] = (U16)(tableSize + u); /* TableU16 : sorted by symbol order; gives next state value */ -+ } -+ } -+ -+ /* Build Symbol Transformation Table */ -+ { -+ unsigned total = 0; -+ unsigned s; -+ for (s = 0; s <= maxSymbolValue; s++) { -+ switch (normalizedCounter[s]) { -+ case 0: break; -+ -+ case -1: -+ case 1: -+ symbolTT[s].deltaNbBits = (tableLog << 16) - (1 << tableLog); -+ symbolTT[s].deltaFindState = total - 1; -+ total++; -+ break; -+ default: { -+ U32 const maxBitsOut = tableLog - BIT_highbit32(normalizedCounter[s] - 1); -+ U32 const minStatePlus = normalizedCounter[s] << maxBitsOut; -+ symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; -+ symbolTT[s].deltaFindState = total - normalizedCounter[s]; -+ total += normalizedCounter[s]; -+ } -+ } -+ } -+ } -+ -+ return 0; -+} -+ -+/*-************************************************************** -+* FSE NCount encoding-decoding -+****************************************************************/ -+size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) -+{ -+ size_t const maxHeaderSize = (((maxSymbolValue + 1) * tableLog) >> 3) + 3; -+ return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ -+} -+ -+static size_t FSE_writeNCount_generic(void *header, size_t headerBufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, -+ unsigned writeIsSafe) -+{ -+ BYTE *const ostart = (BYTE *)header; -+ BYTE *out = ostart; -+ BYTE *const oend = ostart + headerBufferSize; -+ int nbBits; -+ const int tableSize = 1 << tableLog; -+ int remaining; -+ int threshold; -+ U32 bitStream; -+ int bitCount; -+ unsigned charnum = 0; -+ int previous0 = 0; -+ -+ bitStream = 0; -+ bitCount = 0; -+ /* Table Size */ -+ bitStream += (tableLog - FSE_MIN_TABLELOG) << bitCount; -+ bitCount += 4; -+ -+ /* Init */ -+ remaining = tableSize + 1; /* +1 for extra accuracy */ -+ threshold = tableSize; -+ nbBits = tableLog + 1; -+ -+ while (remaining > 1) { /* stops at 1 */ -+ if (previous0) { -+ unsigned start = charnum; -+ while (!normalizedCounter[charnum]) -+ charnum++; -+ while (charnum >= start + 24) { -+ start += 24; -+ bitStream += 0xFFFFU << bitCount; -+ if ((!writeIsSafe) && (out > oend - 2)) -+ return ERROR(dstSize_tooSmall); /* Buffer overflow */ -+ out[0] = (BYTE)bitStream; -+ out[1] = (BYTE)(bitStream >> 8); -+ out += 2; -+ bitStream >>= 16; -+ } -+ while (charnum >= start + 3) { -+ start += 3; -+ bitStream += 3 << bitCount; -+ bitCount += 2; -+ } -+ bitStream += (charnum - start) << bitCount; -+ bitCount += 2; -+ if (bitCount > 16) { -+ if ((!writeIsSafe) && (out > oend - 2)) -+ return ERROR(dstSize_tooSmall); /* Buffer overflow */ -+ out[0] = (BYTE)bitStream; -+ out[1] = (BYTE)(bitStream >> 8); -+ out += 2; -+ bitStream >>= 16; -+ bitCount -= 16; -+ } -+ } -+ { -+ int count = normalizedCounter[charnum++]; -+ int const max = (2 * threshold - 1) - remaining; -+ remaining -= count < 0 ? -count : count; -+ count++; /* +1 for extra accuracy */ -+ if (count >= threshold) -+ count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ -+ bitStream += count << bitCount; -+ bitCount += nbBits; -+ bitCount -= (count < max); -+ previous0 = (count == 1); -+ if (remaining < 1) -+ return ERROR(GENERIC); -+ while (remaining < threshold) -+ nbBits--, threshold >>= 1; -+ } -+ if (bitCount > 16) { -+ if ((!writeIsSafe) && (out > oend - 2)) -+ return ERROR(dstSize_tooSmall); /* Buffer overflow */ -+ out[0] = (BYTE)bitStream; -+ out[1] = (BYTE)(bitStream >> 8); -+ out += 2; -+ bitStream >>= 16; -+ bitCount -= 16; -+ } -+ } -+ -+ /* flush remaining bitStream */ -+ if ((!writeIsSafe) && (out > oend - 2)) -+ return ERROR(dstSize_tooSmall); /* Buffer overflow */ -+ out[0] = (BYTE)bitStream; -+ out[1] = (BYTE)(bitStream >> 8); -+ out += (bitCount + 7) / 8; -+ -+ if (charnum > maxSymbolValue + 1) -+ return ERROR(GENERIC); -+ -+ return (out - ostart); -+} -+ -+size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -+{ -+ if (tableLog > FSE_MAX_TABLELOG) -+ return ERROR(tableLog_tooLarge); /* Unsupported */ -+ if (tableLog < FSE_MIN_TABLELOG) -+ return ERROR(GENERIC); /* Unsupported */ -+ -+ if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) -+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); -+ -+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); -+} -+ -+/*-************************************************************** -+* Counting histogram -+****************************************************************/ -+/*! FSE_count_simple -+ This function counts byte values within `src`, and store the histogram into table `count`. -+ It doesn't use any additional memory. -+ But this function is unsafe : it doesn't check that all values within `src` can fit into `count`. -+ For this reason, prefer using a table `count` with 256 elements. -+ @return : count of most numerous element -+*/ -+size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize) -+{ -+ const BYTE *ip = (const BYTE *)src; -+ const BYTE *const end = ip + srcSize; -+ unsigned maxSymbolValue = *maxSymbolValuePtr; -+ unsigned max = 0; -+ -+ memset(count, 0, (maxSymbolValue + 1) * sizeof(*count)); -+ if (srcSize == 0) { -+ *maxSymbolValuePtr = 0; -+ return 0; -+ } -+ -+ while (ip < end) -+ count[*ip++]++; -+ -+ while (!count[maxSymbolValue]) -+ maxSymbolValue--; -+ *maxSymbolValuePtr = maxSymbolValue; -+ -+ { -+ U32 s; -+ for (s = 0; s <= maxSymbolValue; s++) -+ if (count[s] > max) -+ max = count[s]; -+ } -+ -+ return (size_t)max; -+} -+ -+/* FSE_count_parallel_wksp() : -+ * Same as FSE_count_parallel(), but using an externally provided scratch buffer. -+ * `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */ -+static size_t FSE_count_parallel_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned checkMax, -+ unsigned *const workSpace) -+{ -+ const BYTE *ip = (const BYTE *)source; -+ const BYTE *const iend = ip + sourceSize; -+ unsigned maxSymbolValue = *maxSymbolValuePtr; -+ unsigned max = 0; -+ U32 *const Counting1 = workSpace; -+ U32 *const Counting2 = Counting1 + 256; -+ U32 *const Counting3 = Counting2 + 256; -+ U32 *const Counting4 = Counting3 + 256; -+ -+ memset(Counting1, 0, 4 * 256 * sizeof(unsigned)); -+ -+ /* safety checks */ -+ if (!sourceSize) { -+ memset(count, 0, maxSymbolValue + 1); -+ *maxSymbolValuePtr = 0; -+ return 0; -+ } -+ if (!maxSymbolValue) -+ maxSymbolValue = 255; /* 0 == default */ -+ -+ /* by stripes of 16 bytes */ -+ { -+ U32 cached = ZSTD_read32(ip); -+ ip += 4; -+ while (ip < iend - 15) { -+ U32 c = cached; -+ cached = ZSTD_read32(ip); -+ ip += 4; -+ Counting1[(BYTE)c]++; -+ Counting2[(BYTE)(c >> 8)]++; -+ Counting3[(BYTE)(c >> 16)]++; -+ Counting4[c >> 24]++; -+ c = cached; -+ cached = ZSTD_read32(ip); -+ ip += 4; -+ Counting1[(BYTE)c]++; -+ Counting2[(BYTE)(c >> 8)]++; -+ Counting3[(BYTE)(c >> 16)]++; -+ Counting4[c >> 24]++; -+ c = cached; -+ cached = ZSTD_read32(ip); -+ ip += 4; -+ Counting1[(BYTE)c]++; -+ Counting2[(BYTE)(c >> 8)]++; -+ Counting3[(BYTE)(c >> 16)]++; -+ Counting4[c >> 24]++; -+ c = cached; -+ cached = ZSTD_read32(ip); -+ ip += 4; -+ Counting1[(BYTE)c]++; -+ Counting2[(BYTE)(c >> 8)]++; -+ Counting3[(BYTE)(c >> 16)]++; -+ Counting4[c >> 24]++; -+ } -+ ip -= 4; -+ } -+ -+ /* finish last symbols */ -+ while (ip < iend) -+ Counting1[*ip++]++; -+ -+ if (checkMax) { /* verify stats will fit into destination table */ -+ U32 s; -+ for (s = 255; s > maxSymbolValue; s--) { -+ Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; -+ if (Counting1[s]) -+ return ERROR(maxSymbolValue_tooSmall); -+ } -+ } -+ -+ { -+ U32 s; -+ for (s = 0; s <= maxSymbolValue; s++) { -+ count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; -+ if (count[s] > max) -+ max = count[s]; -+ } -+ } -+ -+ while (!count[maxSymbolValue]) -+ maxSymbolValue--; -+ *maxSymbolValuePtr = maxSymbolValue; -+ return (size_t)max; -+} -+ -+/* FSE_countFast_wksp() : -+ * Same as FSE_countFast(), but using an externally provided scratch buffer. -+ * `workSpace` size must be table of >= `1024` unsigned */ -+size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace) -+{ -+ if (sourceSize < 1500) -+ return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize); -+ return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace); -+} -+ -+/* FSE_count_wksp() : -+ * Same as FSE_count(), but using an externally provided scratch buffer. -+ * `workSpace` size must be table of >= `1024` unsigned */ -+size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace) -+{ -+ if (*maxSymbolValuePtr < 255) -+ return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace); -+ *maxSymbolValuePtr = 255; -+ return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace); -+} -+ -+/*-************************************************************** -+* FSE Compression Code -+****************************************************************/ -+/*! FSE_sizeof_CTable() : -+ FSE_CTable is a variable size structure which contains : -+ `U16 tableLog;` -+ `U16 maxSymbolValue;` -+ `U16 nextStateNumber[1 << tableLog];` // This size is variable -+ `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable -+Allocation is manual (C standard does not support variable-size structures). -+*/ -+size_t FSE_sizeof_CTable(unsigned maxSymbolValue, unsigned tableLog) -+{ -+ if (tableLog > FSE_MAX_TABLELOG) -+ return ERROR(tableLog_tooLarge); -+ return FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue) * sizeof(U32); -+} -+ -+/* provides the minimum logSize to safely represent a distribution */ -+static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) -+{ -+ U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; -+ U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; -+ U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; -+ return minBits; -+} -+ -+unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) -+{ -+ U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; -+ U32 tableLog = maxTableLog; -+ U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); -+ if (tableLog == 0) -+ tableLog = FSE_DEFAULT_TABLELOG; -+ if (maxBitsSrc < tableLog) -+ tableLog = maxBitsSrc; /* Accuracy can be reduced */ -+ if (minBits > tableLog) -+ tableLog = minBits; /* Need a minimum to safely represent all symbol values */ -+ if (tableLog < FSE_MIN_TABLELOG) -+ tableLog = FSE_MIN_TABLELOG; -+ if (tableLog > FSE_MAX_TABLELOG) -+ tableLog = FSE_MAX_TABLELOG; -+ return tableLog; -+} -+ -+unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) -+{ -+ return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); -+} -+ -+/* Secondary normalization method. -+ To be used when primary method fails. */ -+ -+static size_t FSE_normalizeM2(short *norm, U32 tableLog, const unsigned *count, size_t total, U32 maxSymbolValue) -+{ -+ short const NOT_YET_ASSIGNED = -2; -+ U32 s; -+ U32 distributed = 0; -+ U32 ToDistribute; -+ -+ /* Init */ -+ U32 const lowThreshold = (U32)(total >> tableLog); -+ U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); -+ -+ for (s = 0; s <= maxSymbolValue; s++) { -+ if (count[s] == 0) { -+ norm[s] = 0; -+ continue; -+ } -+ if (count[s] <= lowThreshold) { -+ norm[s] = -1; -+ distributed++; -+ total -= count[s]; -+ continue; -+ } -+ if (count[s] <= lowOne) { -+ norm[s] = 1; -+ distributed++; -+ total -= count[s]; -+ continue; -+ } -+ -+ norm[s] = NOT_YET_ASSIGNED; -+ } -+ ToDistribute = (1 << tableLog) - distributed; -+ -+ if ((total / ToDistribute) > lowOne) { -+ /* risk of rounding to zero */ -+ lowOne = (U32)((total * 3) / (ToDistribute * 2)); -+ for (s = 0; s <= maxSymbolValue; s++) { -+ if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { -+ norm[s] = 1; -+ distributed++; -+ total -= count[s]; -+ continue; -+ } -+ } -+ ToDistribute = (1 << tableLog) - distributed; -+ } -+ -+ if (distributed == maxSymbolValue + 1) { -+ /* all values are pretty poor; -+ probably incompressible data (should have already been detected); -+ find max, then give all remaining points to max */ -+ U32 maxV = 0, maxC = 0; -+ for (s = 0; s <= maxSymbolValue; s++) -+ if (count[s] > maxC) -+ maxV = s, maxC = count[s]; -+ norm[maxV] += (short)ToDistribute; -+ return 0; -+ } -+ -+ if (total == 0) { -+ /* all of the symbols were low enough for the lowOne or lowThreshold */ -+ for (s = 0; ToDistribute > 0; s = (s + 1) % (maxSymbolValue + 1)) -+ if (norm[s] > 0) -+ ToDistribute--, norm[s]++; -+ return 0; -+ } -+ -+ { -+ U64 const vStepLog = 62 - tableLog; -+ U64 const mid = (1ULL << (vStepLog - 1)) - 1; -+ U64 const rStep = div_u64((((U64)1 << vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */ -+ U64 tmpTotal = mid; -+ for (s = 0; s <= maxSymbolValue; s++) { -+ if (norm[s] == NOT_YET_ASSIGNED) { -+ U64 const end = tmpTotal + (count[s] * rStep); -+ U32 const sStart = (U32)(tmpTotal >> vStepLog); -+ U32 const sEnd = (U32)(end >> vStepLog); -+ U32 const weight = sEnd - sStart; -+ if (weight < 1) -+ return ERROR(GENERIC); -+ norm[s] = (short)weight; -+ tmpTotal = end; -+ } -+ } -+ } -+ -+ return 0; -+} -+ -+size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t total, unsigned maxSymbolValue) -+{ -+ /* Sanity checks */ -+ if (tableLog == 0) -+ tableLog = FSE_DEFAULT_TABLELOG; -+ if (tableLog < FSE_MIN_TABLELOG) -+ return ERROR(GENERIC); /* Unsupported size */ -+ if (tableLog > FSE_MAX_TABLELOG) -+ return ERROR(tableLog_tooLarge); /* Unsupported size */ -+ if (tableLog < FSE_minTableLog(total, maxSymbolValue)) -+ return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ -+ -+ { -+ U32 const rtbTable[] = {0, 473195, 504333, 520860, 550000, 700000, 750000, 830000}; -+ U64 const scale = 62 - tableLog; -+ U64 const step = div_u64((U64)1 << 62, (U32)total); /* <== here, one division ! */ -+ U64 const vStep = 1ULL << (scale - 20); -+ int stillToDistribute = 1 << tableLog; -+ unsigned s; -+ unsigned largest = 0; -+ short largestP = 0; -+ U32 lowThreshold = (U32)(total >> tableLog); -+ -+ for (s = 0; s <= maxSymbolValue; s++) { -+ if (count[s] == total) -+ return 0; /* rle special case */ -+ if (count[s] == 0) { -+ normalizedCounter[s] = 0; -+ continue; -+ } -+ if (count[s] <= lowThreshold) { -+ normalizedCounter[s] = -1; -+ stillToDistribute--; -+ } else { -+ short proba = (short)((count[s] * step) >> scale); -+ if (proba < 8) { -+ U64 restToBeat = vStep * rtbTable[proba]; -+ proba += (count[s] * step) - ((U64)proba << scale) > restToBeat; -+ } -+ if (proba > largestP) -+ largestP = proba, largest = s; -+ normalizedCounter[s] = proba; -+ stillToDistribute -= proba; -+ } -+ } -+ if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { -+ /* corner case, need another normalization method */ -+ size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); -+ if (FSE_isError(errorCode)) -+ return errorCode; -+ } else -+ normalizedCounter[largest] += (short)stillToDistribute; -+ } -+ -+ return tableLog; -+} -+ -+/* fake FSE_CTable, for raw (uncompressed) input */ -+size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits) -+{ -+ const unsigned tableSize = 1 << nbBits; -+ const unsigned tableMask = tableSize - 1; -+ const unsigned maxSymbolValue = tableMask; -+ void *const ptr = ct; -+ U16 *const tableU16 = ((U16 *)ptr) + 2; -+ void *const FSCT = ((U32 *)ptr) + 1 /* header */ + (tableSize >> 1); /* assumption : tableLog >= 1 */ -+ FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT); -+ unsigned s; -+ -+ /* Sanity checks */ -+ if (nbBits < 1) -+ return ERROR(GENERIC); /* min size */ -+ -+ /* header */ -+ tableU16[-2] = (U16)nbBits; -+ tableU16[-1] = (U16)maxSymbolValue; -+ -+ /* Build table */ -+ for (s = 0; s < tableSize; s++) -+ tableU16[s] = (U16)(tableSize + s); -+ -+ /* Build Symbol Transformation Table */ -+ { -+ const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits); -+ for (s = 0; s <= maxSymbolValue; s++) { -+ symbolTT[s].deltaNbBits = deltaNbBits; -+ symbolTT[s].deltaFindState = s - 1; -+ } -+ } -+ -+ return 0; -+} -+ -+/* fake FSE_CTable, for rle input (always same symbol) */ -+size_t FSE_buildCTable_rle(FSE_CTable *ct, BYTE symbolValue) -+{ -+ void *ptr = ct; -+ U16 *tableU16 = ((U16 *)ptr) + 2; -+ void *FSCTptr = (U32 *)ptr + 2; -+ FSE_symbolCompressionTransform *symbolTT = (FSE_symbolCompressionTransform *)FSCTptr; -+ -+ /* header */ -+ tableU16[-2] = (U16)0; -+ tableU16[-1] = (U16)symbolValue; -+ -+ /* Build table */ -+ tableU16[0] = 0; -+ tableU16[1] = 0; /* just in case */ -+ -+ /* Build Symbol Transformation Table */ -+ symbolTT[symbolValue].deltaNbBits = 0; -+ symbolTT[symbolValue].deltaFindState = 0; -+ -+ return 0; -+} -+ -+static size_t FSE_compress_usingCTable_generic(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct, const unsigned fast) -+{ -+ const BYTE *const istart = (const BYTE *)src; -+ const BYTE *const iend = istart + srcSize; -+ const BYTE *ip = iend; -+ -+ BIT_CStream_t bitC; -+ FSE_CState_t CState1, CState2; -+ -+ /* init */ -+ if (srcSize <= 2) -+ return 0; -+ { -+ size_t const initError = BIT_initCStream(&bitC, dst, dstSize); -+ if (FSE_isError(initError)) -+ return 0; /* not enough space available to write a bitstream */ -+ } -+ -+#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) -+ -+ if (srcSize & 1) { -+ FSE_initCState2(&CState1, ct, *--ip); -+ FSE_initCState2(&CState2, ct, *--ip); -+ FSE_encodeSymbol(&bitC, &CState1, *--ip); -+ FSE_FLUSHBITS(&bitC); -+ } else { -+ FSE_initCState2(&CState2, ct, *--ip); -+ FSE_initCState2(&CState1, ct, *--ip); -+ } -+ -+ /* join to mod 4 */ -+ srcSize -= 2; -+ if ((sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) && (srcSize & 2)) { /* test bit 2 */ -+ FSE_encodeSymbol(&bitC, &CState2, *--ip); -+ FSE_encodeSymbol(&bitC, &CState1, *--ip); -+ FSE_FLUSHBITS(&bitC); -+ } -+ -+ /* 2 or 4 encoding per loop */ -+ while (ip > istart) { -+ -+ FSE_encodeSymbol(&bitC, &CState2, *--ip); -+ -+ if (sizeof(bitC.bitContainer) * 8 < FSE_MAX_TABLELOG * 2 + 7) /* this test must be static */ -+ FSE_FLUSHBITS(&bitC); -+ -+ FSE_encodeSymbol(&bitC, &CState1, *--ip); -+ -+ if (sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) { /* this test must be static */ -+ FSE_encodeSymbol(&bitC, &CState2, *--ip); -+ FSE_encodeSymbol(&bitC, &CState1, *--ip); -+ } -+ -+ FSE_FLUSHBITS(&bitC); -+ } -+ -+ FSE_flushCState(&bitC, &CState2); -+ FSE_flushCState(&bitC, &CState1); -+ return BIT_closeCStream(&bitC); -+} -+ -+size_t FSE_compress_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct) -+{ -+ unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); -+ -+ if (fast) -+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); -+ else -+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); -+} -+ -+size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } -diff --git a/lib/zstd/fse_decompress.c b/lib/zstd/fse_decompress.c -new file mode 100644 -index 0000000..a84300e ---- /dev/null -+++ b/lib/zstd/fse_decompress.c -@@ -0,0 +1,332 @@ -+/* -+ * FSE : Finite State Entropy decoder -+ * Copyright (C) 2013-2015, Yann Collet. -+ * -+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above -+ * copyright notice, this list of conditions and the following disclaimer -+ * in the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ * -+ * You can contact the author at : -+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -+ */ -+ -+/* ************************************************************** -+* Compiler specifics -+****************************************************************/ -+#define FORCE_INLINE static __always_inline -+ -+/* ************************************************************** -+* Includes -+****************************************************************/ -+#include "bitstream.h" -+#include "fse.h" -+#include <linux/compiler.h> -+#include <linux/kernel.h> -+#include <linux/string.h> /* memcpy, memset */ -+ -+/* ************************************************************** -+* Error Management -+****************************************************************/ -+#define FSE_isError ERR_isError -+#define FSE_STATIC_ASSERT(c) \ -+ { \ -+ enum { FSE_static_assert = 1 / (int)(!!(c)) }; \ -+ } /* use only *after* variable declarations */ -+ -+/* check and forward error code */ -+#define CHECK_F(f) \ -+ { \ -+ size_t const e = f; \ -+ if (FSE_isError(e)) \ -+ return e; \ -+ } -+ -+/* ************************************************************** -+* Templates -+****************************************************************/ -+/* -+ designed to be included -+ for type-specific functions (template emulation in C) -+ Objective is to write these functions only once, for improved maintenance -+*/ -+ -+/* safety checks */ -+#ifndef FSE_FUNCTION_EXTENSION -+#error "FSE_FUNCTION_EXTENSION must be defined" -+#endif -+#ifndef FSE_FUNCTION_TYPE -+#error "FSE_FUNCTION_TYPE must be defined" -+#endif -+ -+/* Function names */ -+#define FSE_CAT(X, Y) X##Y -+#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y) -+#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y) -+ -+/* Function templates */ -+ -+size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize) -+{ -+ void *const tdPtr = dt + 1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ -+ FSE_DECODE_TYPE *const tableDecode = (FSE_DECODE_TYPE *)(tdPtr); -+ U16 *symbolNext = (U16 *)workspace; -+ -+ U32 const maxSV1 = maxSymbolValue + 1; -+ U32 const tableSize = 1 << tableLog; -+ U32 highThreshold = tableSize - 1; -+ -+ /* Sanity Checks */ -+ if (workspaceSize < sizeof(U16) * (FSE_MAX_SYMBOL_VALUE + 1)) -+ return ERROR(tableLog_tooLarge); -+ if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) -+ return ERROR(maxSymbolValue_tooLarge); -+ if (tableLog > FSE_MAX_TABLELOG) -+ return ERROR(tableLog_tooLarge); -+ -+ /* Init, lay down lowprob symbols */ -+ { -+ FSE_DTableHeader DTableH; -+ DTableH.tableLog = (U16)tableLog; -+ DTableH.fastMode = 1; -+ { -+ S16 const largeLimit = (S16)(1 << (tableLog - 1)); -+ U32 s; -+ for (s = 0; s < maxSV1; s++) { -+ if (normalizedCounter[s] == -1) { -+ tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; -+ symbolNext[s] = 1; -+ } else { -+ if (normalizedCounter[s] >= largeLimit) -+ DTableH.fastMode = 0; -+ symbolNext[s] = normalizedCounter[s]; -+ } -+ } -+ } -+ memcpy(dt, &DTableH, sizeof(DTableH)); -+ } -+ -+ /* Spread symbols */ -+ { -+ U32 const tableMask = tableSize - 1; -+ U32 const step = FSE_TABLESTEP(tableSize); -+ U32 s, position = 0; -+ for (s = 0; s < maxSV1; s++) { -+ int i; -+ for (i = 0; i < normalizedCounter[s]; i++) { -+ tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s; -+ position = (position + step) & tableMask; -+ while (position > highThreshold) -+ position = (position + step) & tableMask; /* lowprob area */ -+ } -+ } -+ if (position != 0) -+ return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ -+ } -+ -+ /* Build Decoding table */ -+ { -+ U32 u; -+ for (u = 0; u < tableSize; u++) { -+ FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol); -+ U16 nextState = symbolNext[symbol]++; -+ tableDecode[u].nbBits = (BYTE)(tableLog - BIT_highbit32((U32)nextState)); -+ tableDecode[u].newState = (U16)((nextState << tableDecode[u].nbBits) - tableSize); -+ } -+ } -+ -+ return 0; -+} -+ -+/*-******************************************************* -+* Decompression (Byte symbols) -+*********************************************************/ -+size_t FSE_buildDTable_rle(FSE_DTable *dt, BYTE symbolValue) -+{ -+ void *ptr = dt; -+ FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr; -+ void *dPtr = dt + 1; -+ FSE_decode_t *const cell = (FSE_decode_t *)dPtr; -+ -+ DTableH->tableLog = 0; -+ DTableH->fastMode = 0; -+ -+ cell->newState = 0; -+ cell->symbol = symbolValue; -+ cell->nbBits = 0; -+ -+ return 0; -+} -+ -+size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits) -+{ -+ void *ptr = dt; -+ FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr; -+ void *dPtr = dt + 1; -+ FSE_decode_t *const dinfo = (FSE_decode_t *)dPtr; -+ const unsigned tableSize = 1 << nbBits; -+ const unsigned tableMask = tableSize - 1; -+ const unsigned maxSV1 = tableMask + 1; -+ unsigned s; -+ -+ /* Sanity checks */ -+ if (nbBits < 1) -+ return ERROR(GENERIC); /* min size */ -+ -+ /* Build Decoding Table */ -+ DTableH->tableLog = (U16)nbBits; -+ DTableH->fastMode = 1; -+ for (s = 0; s < maxSV1; s++) { -+ dinfo[s].newState = 0; -+ dinfo[s].symbol = (BYTE)s; -+ dinfo[s].nbBits = (BYTE)nbBits; -+ } -+ -+ return 0; -+} -+ -+FORCE_INLINE size_t FSE_decompress_usingDTable_generic(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt, -+ const unsigned fast) -+{ -+ BYTE *const ostart = (BYTE *)dst; -+ BYTE *op = ostart; -+ BYTE *const omax = op + maxDstSize; -+ BYTE *const olimit = omax - 3; -+ -+ BIT_DStream_t bitD; -+ FSE_DState_t state1; -+ FSE_DState_t state2; -+ -+ /* Init */ -+ CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize)); -+ -+ FSE_initDState(&state1, &bitD, dt); -+ FSE_initDState(&state2, &bitD, dt); -+ -+#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) -+ -+ /* 4 symbols per loop */ -+ for (; (BIT_reloadDStream(&bitD) == BIT_DStream_unfinished) & (op < olimit); op += 4) { -+ op[0] = FSE_GETSYMBOL(&state1); -+ -+ if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */ -+ BIT_reloadDStream(&bitD); -+ -+ op[1] = FSE_GETSYMBOL(&state2); -+ -+ if (FSE_MAX_TABLELOG * 4 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */ -+ { -+ if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { -+ op += 2; -+ break; -+ } -+ } -+ -+ op[2] = FSE_GETSYMBOL(&state1); -+ -+ if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */ -+ BIT_reloadDStream(&bitD); -+ -+ op[3] = FSE_GETSYMBOL(&state2); -+ } -+ -+ /* tail */ -+ /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ -+ while (1) { -+ if (op > (omax - 2)) -+ return ERROR(dstSize_tooSmall); -+ *op++ = FSE_GETSYMBOL(&state1); -+ if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) { -+ *op++ = FSE_GETSYMBOL(&state2); -+ break; -+ } -+ -+ if (op > (omax - 2)) -+ return ERROR(dstSize_tooSmall); -+ *op++ = FSE_GETSYMBOL(&state2); -+ if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) { -+ *op++ = FSE_GETSYMBOL(&state1); -+ break; -+ } -+ } -+ -+ return op - ostart; -+} -+ -+size_t FSE_decompress_usingDTable(void *dst, size_t originalSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt) -+{ -+ const void *ptr = dt; -+ const FSE_DTableHeader *DTableH = (const FSE_DTableHeader *)ptr; -+ const U32 fastMode = DTableH->fastMode; -+ -+ /* select fast mode (static) */ -+ if (fastMode) -+ return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); -+ return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); -+} -+ -+size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, unsigned maxLog, void *workspace, size_t workspaceSize) -+{ -+ const BYTE *const istart = (const BYTE *)cSrc; -+ const BYTE *ip = istart; -+ unsigned tableLog; -+ unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; -+ size_t NCountLength; -+ -+ FSE_DTable *dt; -+ short *counting; -+ size_t spaceUsed32 = 0; -+ -+ FSE_STATIC_ASSERT(sizeof(FSE_DTable) == sizeof(U32)); -+ -+ dt = (FSE_DTable *)((U32 *)workspace + spaceUsed32); -+ spaceUsed32 += FSE_DTABLE_SIZE_U32(maxLog); -+ counting = (short *)((U32 *)workspace + spaceUsed32); -+ spaceUsed32 += ALIGN(sizeof(short) * (FSE_MAX_SYMBOL_VALUE + 1), sizeof(U32)) >> 2; -+ -+ if ((spaceUsed32 << 2) > workspaceSize) -+ return ERROR(tableLog_tooLarge); -+ workspace = (U32 *)workspace + spaceUsed32; -+ workspaceSize -= (spaceUsed32 << 2); -+ -+ /* normal FSE decoding mode */ -+ NCountLength = FSE_readNCount(counting, &maxSymbolValue, &tableLog, istart, cSrcSize); -+ if (FSE_isError(NCountLength)) -+ return NCountLength; -+ // if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining -+ // case : NCountLength==cSrcSize */ -+ if (tableLog > maxLog) -+ return ERROR(tableLog_tooLarge); -+ ip += NCountLength; -+ cSrcSize -= NCountLength; -+ -+ CHECK_F(FSE_buildDTable_wksp(dt, counting, maxSymbolValue, tableLog, workspace, workspaceSize)); -+ -+ return FSE_decompress_usingDTable(dst, dstCapacity, ip, cSrcSize, dt); /* always return, even if it is an error code */ -+} -diff --git a/lib/zstd/huf.h b/lib/zstd/huf.h -new file mode 100644 -index 0000000..2143da2 ---- /dev/null -+++ b/lib/zstd/huf.h -@@ -0,0 +1,212 @@ -+/* -+ * Huffman coder, part of New Generation Entropy library -+ * header file -+ * Copyright (C) 2013-2016, Yann Collet. -+ * -+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above -+ * copyright notice, this list of conditions and the following disclaimer -+ * in the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ * -+ * You can contact the author at : -+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -+ */ -+#ifndef HUF_H_298734234 -+#define HUF_H_298734234 -+ -+/* *** Dependencies *** */ -+#include <linux/types.h> /* size_t */ -+ -+/* *** Tool functions *** */ -+#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ -+size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ -+ -+/* Error Management */ -+unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ -+ -+/* *** Advanced function *** */ -+ -+/** HUF_compress4X_wksp() : -+* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */ -+size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, -+ size_t wkspSize); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */ -+ -+/* *** Dependencies *** */ -+#include "mem.h" /* U32 */ -+ -+/* *** Constants *** */ -+#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ -+#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ -+#define HUF_SYMBOLVALUE_MAX 255 -+ -+#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ -+#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) -+#error "HUF_TABLELOG_MAX is too large !" -+#endif -+ -+/* **************************************** -+* Static allocation -+******************************************/ -+/* HUF buffer bounds */ -+#define HUF_CTABLEBOUND 129 -+#define HUF_BLOCKBOUND(size) (size + (size >> 8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ -+#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ -+ -+/* static allocation of HUF's Compression Table */ -+#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ -+ U32 name##hb[maxSymbolValue + 1]; \ -+ void *name##hv = &(name##hb); \ -+ HUF_CElt *name = (HUF_CElt *)(name##hv) /* no final ; */ -+ -+/* static allocation of HUF's DTable */ -+typedef U32 HUF_DTable; -+#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1 << (maxTableLog))) -+#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = {((U32)((maxTableLog)-1) * 0x01000001)} -+#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = {((U32)(maxTableLog)*0x01000001)} -+ -+/* The workspace must have alignment at least 4 and be at least this large */ -+#define HUF_COMPRESS_WORKSPACE_SIZE (6 << 10) -+#define HUF_COMPRESS_WORKSPACE_SIZE_U32 (HUF_COMPRESS_WORKSPACE_SIZE / sizeof(U32)) -+ -+/* The workspace must have alignment at least 4 and be at least this large */ -+#define HUF_DECOMPRESS_WORKSPACE_SIZE (3 << 10) -+#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) -+ -+/* **************************************** -+* Advanced decompression functions -+******************************************/ -+size_t HUF_decompress4X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize); /**< decodes RLE and uncompressed */ -+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, -+ size_t workspaceSize); /**< considers RLE and uncompressed as errors */ -+size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, -+ size_t workspaceSize); /**< single-symbol decoder */ -+size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, -+ size_t workspaceSize); /**< double-symbols decoder */ -+ -+/* **************************************** -+* HUF detailed API -+******************************************/ -+/*! -+HUF_compress() does the following: -+1. count symbol occurrence from source[] into table count[] using FSE_count() -+2. (optional) refine tableLog using HUF_optimalTableLog() -+3. build Huffman table from count using HUF_buildCTable() -+4. save Huffman table to memory buffer using HUF_writeCTable_wksp() -+5. encode the data stream using HUF_compress4X_usingCTable() -+ -+The following API allows targeting specific sub-functions for advanced tasks. -+For example, it's possible to compress several blocks using the same 'CTable', -+or to save and regenerate 'CTable' using external methods. -+*/ -+/* FSE_count() : find it within "fse.h" */ -+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -+typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ -+size_t HUF_writeCTable_wksp(void *dst, size_t maxDstSize, const HUF_CElt *CTable, unsigned maxSymbolValue, unsigned huffLog, void *workspace, size_t workspaceSize); -+size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable); -+ -+typedef enum { -+ HUF_repeat_none, /**< Cannot use the previous table */ -+ HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, -+ 4}X_repeat */ -+ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ -+} HUF_repeat; -+/** HUF_compress4X_repeat() : -+* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. -+* If it uses hufTable it does not modify hufTable or repeat. -+* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. -+* If preferRepeat then the old table will always be used if valid. */ -+size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, -+ size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, -+ int preferRepeat); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */ -+ -+/** HUF_buildCTable_wksp() : -+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer. -+ * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. -+ */ -+size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize); -+ -+/*! HUF_readStats() : -+ Read compact Huffman tree, saved by HUF_writeCTable(). -+ `huffWeight` is destination buffer. -+ @return : size read from `src` , or an error Code . -+ Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ -+size_t HUF_readStats_wksp(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize, -+ void *workspace, size_t workspaceSize); -+ -+/** HUF_readCTable() : -+* Loading a CTable saved with HUF_writeCTable() */ -+size_t HUF_readCTable_wksp(HUF_CElt *CTable, unsigned maxSymbolValue, const void *src, size_t srcSize, void *workspace, size_t workspaceSize); -+ -+/* -+HUF_decompress() does the following: -+1. select the decompression algorithm (X2, X4) based on pre-computed heuristics -+2. build Huffman table from save, using HUF_readDTableXn() -+3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable -+*/ -+ -+/** HUF_selectDecoder() : -+* Tells which decoder is likely to decode faster, -+* based on a set of pre-determined metrics. -+* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . -+* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -+U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize); -+ -+size_t HUF_readDTableX2_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize); -+size_t HUF_readDTableX4_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize); -+ -+size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); -+size_t HUF_decompress4X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); -+size_t HUF_decompress4X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); -+ -+/* single stream variants */ -+ -+size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, -+ size_t wkspSize); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */ -+size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable); -+/** HUF_compress1X_repeat() : -+* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. -+* If it uses hufTable it does not modify hufTable or repeat. -+* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. -+* If preferRepeat then the old table will always be used if valid. */ -+size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, -+ size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, -+ int preferRepeat); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */ -+ -+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize); -+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, -+ size_t workspaceSize); /**< single-symbol decoder */ -+size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, -+ size_t workspaceSize); /**< double-symbols decoder */ -+ -+size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, -+ const HUF_DTable *DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ -+size_t HUF_decompress1X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); -+size_t HUF_decompress1X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); -+ -+#endif /* HUF_H_298734234 */ -diff --git a/lib/zstd/huf_compress.c b/lib/zstd/huf_compress.c -new file mode 100644 -index 0000000..40055a7 ---- /dev/null -+++ b/lib/zstd/huf_compress.c -@@ -0,0 +1,770 @@ -+/* -+ * Huffman encoder, part of New Generation Entropy library -+ * Copyright (C) 2013-2016, Yann Collet. -+ * -+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above -+ * copyright notice, this list of conditions and the following disclaimer -+ * in the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ * -+ * You can contact the author at : -+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -+ */ -+ -+/* ************************************************************** -+* Includes -+****************************************************************/ -+#include "bitstream.h" -+#include "fse.h" /* header compression */ -+#include "huf.h" -+#include <linux/kernel.h> -+#include <linux/string.h> /* memcpy, memset */ -+ -+/* ************************************************************** -+* Error Management -+****************************************************************/ -+#define HUF_STATIC_ASSERT(c) \ -+ { \ -+ enum { HUF_static_assert = 1 / (int)(!!(c)) }; \ -+ } /* use only *after* variable declarations */ -+#define CHECK_V_F(e, f) \ -+ size_t const e = f; \ -+ if (ERR_isError(e)) \ -+ return f -+#define CHECK_F(f) \ -+ { \ -+ CHECK_V_F(_var_err__, f); \ -+ } -+ -+/* ************************************************************** -+* Utils -+****************************************************************/ -+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) -+{ -+ return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); -+} -+ -+/* ******************************************************* -+* HUF : Huffman block compression -+*********************************************************/ -+/* HUF_compressWeights() : -+ * Same as FSE_compress(), but dedicated to huff0's weights compression. -+ * The use case needs much less stack memory. -+ * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. -+ */ -+#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 -+size_t HUF_compressWeights_wksp(void *dst, size_t dstSize, const void *weightTable, size_t wtSize, void *workspace, size_t workspaceSize) -+{ -+ BYTE *const ostart = (BYTE *)dst; -+ BYTE *op = ostart; -+ BYTE *const oend = ostart + dstSize; -+ -+ U32 maxSymbolValue = HUF_TABLELOG_MAX; -+ U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; -+ -+ FSE_CTable *CTable; -+ U32 *count; -+ S16 *norm; -+ size_t spaceUsed32 = 0; -+ -+ HUF_STATIC_ASSERT(sizeof(FSE_CTable) == sizeof(U32)); -+ -+ CTable = (FSE_CTable *)((U32 *)workspace + spaceUsed32); -+ spaceUsed32 += FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX); -+ count = (U32 *)workspace + spaceUsed32; -+ spaceUsed32 += HUF_TABLELOG_MAX + 1; -+ norm = (S16 *)((U32 *)workspace + spaceUsed32); -+ spaceUsed32 += ALIGN(sizeof(S16) * (HUF_TABLELOG_MAX + 1), sizeof(U32)) >> 2; -+ -+ if ((spaceUsed32 << 2) > workspaceSize) -+ return ERROR(tableLog_tooLarge); -+ workspace = (U32 *)workspace + spaceUsed32; -+ workspaceSize -= (spaceUsed32 << 2); -+ -+ /* init conditions */ -+ if (wtSize <= 1) -+ return 0; /* Not compressible */ -+ -+ /* Scan input and build symbol stats */ -+ { -+ CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize)); -+ if (maxCount == wtSize) -+ return 1; /* only a single symbol in src : rle */ -+ if (maxCount == 1) -+ return 0; /* each symbol present maximum once => not compressible */ -+ } -+ -+ tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); -+ CHECK_F(FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue)); -+ -+ /* Write table description header */ -+ { -+ CHECK_V_F(hSize, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog)); -+ op += hSize; -+ } -+ -+ /* Compress */ -+ CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, workspace, workspaceSize)); -+ { -+ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable)); -+ if (cSize == 0) -+ return 0; /* not enough space for compressed data */ -+ op += cSize; -+ } -+ -+ return op - ostart; -+} -+ -+struct HUF_CElt_s { -+ U16 val; -+ BYTE nbBits; -+}; /* typedef'd to HUF_CElt within "huf.h" */ -+ -+/*! HUF_writeCTable_wksp() : -+ `CTable` : Huffman tree to save, using huf representation. -+ @return : size of saved CTable */ -+size_t HUF_writeCTable_wksp(void *dst, size_t maxDstSize, const HUF_CElt *CTable, U32 maxSymbolValue, U32 huffLog, void *workspace, size_t workspaceSize) -+{ -+ BYTE *op = (BYTE *)dst; -+ U32 n; -+ -+ BYTE *bitsToWeight; -+ BYTE *huffWeight; -+ size_t spaceUsed32 = 0; -+ -+ bitsToWeight = (BYTE *)((U32 *)workspace + spaceUsed32); -+ spaceUsed32 += ALIGN(HUF_TABLELOG_MAX + 1, sizeof(U32)) >> 2; -+ huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32); -+ spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX, sizeof(U32)) >> 2; -+ -+ if ((spaceUsed32 << 2) > workspaceSize) -+ return ERROR(tableLog_tooLarge); -+ workspace = (U32 *)workspace + spaceUsed32; -+ workspaceSize -= (spaceUsed32 << 2); -+ -+ /* check conditions */ -+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) -+ return ERROR(maxSymbolValue_tooLarge); -+ -+ /* convert to weight */ -+ bitsToWeight[0] = 0; -+ for (n = 1; n < huffLog + 1; n++) -+ bitsToWeight[n] = (BYTE)(huffLog + 1 - n); -+ for (n = 0; n < maxSymbolValue; n++) -+ huffWeight[n] = bitsToWeight[CTable[n].nbBits]; -+ -+ /* attempt weights compression by FSE */ -+ { -+ CHECK_V_F(hSize, HUF_compressWeights_wksp(op + 1, maxDstSize - 1, huffWeight, maxSymbolValue, workspace, workspaceSize)); -+ if ((hSize > 1) & (hSize < maxSymbolValue / 2)) { /* FSE compressed */ -+ op[0] = (BYTE)hSize; -+ return hSize + 1; -+ } -+ } -+ -+ /* write raw values as 4-bits (max : 15) */ -+ if (maxSymbolValue > (256 - 128)) -+ return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ -+ if (((maxSymbolValue + 1) / 2) + 1 > maxDstSize) -+ return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ -+ op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue - 1)); -+ huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ -+ for (n = 0; n < maxSymbolValue; n += 2) -+ op[(n / 2) + 1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n + 1]); -+ return ((maxSymbolValue + 1) / 2) + 1; -+} -+ -+size_t HUF_readCTable_wksp(HUF_CElt *CTable, U32 maxSymbolValue, const void *src, size_t srcSize, void *workspace, size_t workspaceSize) -+{ -+ U32 *rankVal; -+ BYTE *huffWeight; -+ U32 tableLog = 0; -+ U32 nbSymbols = 0; -+ size_t readSize; -+ size_t spaceUsed32 = 0; -+ -+ rankVal = (U32 *)workspace + spaceUsed32; -+ spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1; -+ huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32); -+ spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; -+ -+ if ((spaceUsed32 << 2) > workspaceSize) -+ return ERROR(tableLog_tooLarge); -+ workspace = (U32 *)workspace + spaceUsed32; -+ workspaceSize -= (spaceUsed32 << 2); -+ -+ /* get symbol weights */ -+ readSize = HUF_readStats_wksp(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize); -+ if (ERR_isError(readSize)) -+ return readSize; -+ -+ /* check result */ -+ if (tableLog > HUF_TABLELOG_MAX) -+ return ERROR(tableLog_tooLarge); -+ if (nbSymbols > maxSymbolValue + 1) -+ return ERROR(maxSymbolValue_tooSmall); -+ -+ /* Prepare base value per rank */ -+ { -+ U32 n, nextRankStart = 0; -+ for (n = 1; n <= tableLog; n++) { -+ U32 curr = nextRankStart; -+ nextRankStart += (rankVal[n] << (n - 1)); -+ rankVal[n] = curr; -+ } -+ } -+ -+ /* fill nbBits */ -+ { -+ U32 n; -+ for (n = 0; n < nbSymbols; n++) { -+ const U32 w = huffWeight[n]; -+ CTable[n].nbBits = (BYTE)(tableLog + 1 - w); -+ } -+ } -+ -+ /* fill val */ -+ { -+ U16 nbPerRank[HUF_TABLELOG_MAX + 2] = {0}; /* support w=0=>n=tableLog+1 */ -+ U16 valPerRank[HUF_TABLELOG_MAX + 2] = {0}; -+ { -+ U32 n; -+ for (n = 0; n < nbSymbols; n++) -+ nbPerRank[CTable[n].nbBits]++; -+ } -+ /* determine stating value per rank */ -+ valPerRank[tableLog + 1] = 0; /* for w==0 */ -+ { -+ U16 min = 0; -+ U32 n; -+ for (n = tableLog; n > 0; n--) { /* start at n=tablelog <-> w=1 */ -+ valPerRank[n] = min; /* get starting value within each rank */ -+ min += nbPerRank[n]; -+ min >>= 1; -+ } -+ } -+ /* assign value within rank, symbol order */ -+ { -+ U32 n; -+ for (n = 0; n <= maxSymbolValue; n++) -+ CTable[n].val = valPerRank[CTable[n].nbBits]++; -+ } -+ } -+ -+ return readSize; -+} -+ -+typedef struct nodeElt_s { -+ U32 count; -+ U16 parent; -+ BYTE byte; -+ BYTE nbBits; -+} nodeElt; -+ -+static U32 HUF_setMaxHeight(nodeElt *huffNode, U32 lastNonNull, U32 maxNbBits) -+{ -+ const U32 largestBits = huffNode[lastNonNull].nbBits; -+ if (largestBits <= maxNbBits) -+ return largestBits; /* early exit : no elt > maxNbBits */ -+ -+ /* there are several too large elements (at least >= 2) */ -+ { -+ int totalCost = 0; -+ const U32 baseCost = 1 << (largestBits - maxNbBits); -+ U32 n = lastNonNull; -+ -+ while (huffNode[n].nbBits > maxNbBits) { -+ totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); -+ huffNode[n].nbBits = (BYTE)maxNbBits; -+ n--; -+ } /* n stops at huffNode[n].nbBits <= maxNbBits */ -+ while (huffNode[n].nbBits == maxNbBits) -+ n--; /* n end at index of smallest symbol using < maxNbBits */ -+ -+ /* renorm totalCost */ -+ totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ -+ -+ /* repay normalized cost */ -+ { -+ U32 const noSymbol = 0xF0F0F0F0; -+ U32 rankLast[HUF_TABLELOG_MAX + 2]; -+ int pos; -+ -+ /* Get pos of last (smallest) symbol per rank */ -+ memset(rankLast, 0xF0, sizeof(rankLast)); -+ { -+ U32 currNbBits = maxNbBits; -+ for (pos = n; pos >= 0; pos--) { -+ if (huffNode[pos].nbBits >= currNbBits) -+ continue; -+ currNbBits = huffNode[pos].nbBits; /* < maxNbBits */ -+ rankLast[maxNbBits - currNbBits] = pos; -+ } -+ } -+ -+ while (totalCost > 0) { -+ U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; -+ for (; nBitsToDecrease > 1; nBitsToDecrease--) { -+ U32 highPos = rankLast[nBitsToDecrease]; -+ U32 lowPos = rankLast[nBitsToDecrease - 1]; -+ if (highPos == noSymbol) -+ continue; -+ if (lowPos == noSymbol) -+ break; -+ { -+ U32 const highTotal = huffNode[highPos].count; -+ U32 const lowTotal = 2 * huffNode[lowPos].count; -+ if (highTotal <= lowTotal) -+ break; -+ } -+ } -+ /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ -+ /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ -+ while ((nBitsToDecrease <= HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) -+ nBitsToDecrease++; -+ totalCost -= 1 << (nBitsToDecrease - 1); -+ if (rankLast[nBitsToDecrease - 1] == noSymbol) -+ rankLast[nBitsToDecrease - 1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ -+ huffNode[rankLast[nBitsToDecrease]].nbBits++; -+ if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ -+ rankLast[nBitsToDecrease] = noSymbol; -+ else { -+ rankLast[nBitsToDecrease]--; -+ if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits - nBitsToDecrease) -+ rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ -+ } -+ } /* while (totalCost > 0) */ -+ -+ while (totalCost < 0) { /* Sometimes, cost correction overshoot */ -+ if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 -+ (using maxNbBits) */ -+ while (huffNode[n].nbBits == maxNbBits) -+ n--; -+ huffNode[n + 1].nbBits--; -+ rankLast[1] = n + 1; -+ totalCost++; -+ continue; -+ } -+ huffNode[rankLast[1] + 1].nbBits--; -+ rankLast[1]++; -+ totalCost++; -+ } -+ } -+ } /* there are several too large elements (at least >= 2) */ -+ -+ return maxNbBits; -+} -+ -+typedef struct { -+ U32 base; -+ U32 curr; -+} rankPos; -+ -+static void HUF_sort(nodeElt *huffNode, const U32 *count, U32 maxSymbolValue) -+{ -+ rankPos rank[32]; -+ U32 n; -+ -+ memset(rank, 0, sizeof(rank)); -+ for (n = 0; n <= maxSymbolValue; n++) { -+ U32 r = BIT_highbit32(count[n] + 1); -+ rank[r].base++; -+ } -+ for (n = 30; n > 0; n--) -+ rank[n - 1].base += rank[n].base; -+ for (n = 0; n < 32; n++) -+ rank[n].curr = rank[n].base; -+ for (n = 0; n <= maxSymbolValue; n++) { -+ U32 const c = count[n]; -+ U32 const r = BIT_highbit32(c + 1) + 1; -+ U32 pos = rank[r].curr++; -+ while ((pos > rank[r].base) && (c > huffNode[pos - 1].count)) -+ huffNode[pos] = huffNode[pos - 1], pos--; -+ huffNode[pos].count = c; -+ huffNode[pos].byte = (BYTE)n; -+ } -+} -+ -+/** HUF_buildCTable_wksp() : -+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer. -+ * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. -+ */ -+#define STARTNODE (HUF_SYMBOLVALUE_MAX + 1) -+typedef nodeElt huffNodeTable[2 * HUF_SYMBOLVALUE_MAX + 1 + 1]; -+size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize) -+{ -+ nodeElt *const huffNode0 = (nodeElt *)workSpace; -+ nodeElt *const huffNode = huffNode0 + 1; -+ U32 n, nonNullRank; -+ int lowS, lowN; -+ U16 nodeNb = STARTNODE; -+ U32 nodeRoot; -+ -+ /* safety checks */ -+ if (wkspSize < sizeof(huffNodeTable)) -+ return ERROR(GENERIC); /* workSpace is not large enough */ -+ if (maxNbBits == 0) -+ maxNbBits = HUF_TABLELOG_DEFAULT; -+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) -+ return ERROR(GENERIC); -+ memset(huffNode0, 0, sizeof(huffNodeTable)); -+ -+ /* sort, decreasing order */ -+ HUF_sort(huffNode, count, maxSymbolValue); -+ -+ /* init for parents */ -+ nonNullRank = maxSymbolValue; -+ while (huffNode[nonNullRank].count == 0) -+ nonNullRank--; -+ lowS = nonNullRank; -+ nodeRoot = nodeNb + lowS - 1; -+ lowN = nodeNb; -+ huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS - 1].count; -+ huffNode[lowS].parent = huffNode[lowS - 1].parent = nodeNb; -+ nodeNb++; -+ lowS -= 2; -+ for (n = nodeNb; n <= nodeRoot; n++) -+ huffNode[n].count = (U32)(1U << 30); -+ huffNode0[0].count = (U32)(1U << 31); /* fake entry, strong barrier */ -+ -+ /* create parents */ -+ while (nodeNb <= nodeRoot) { -+ U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; -+ U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; -+ huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; -+ huffNode[n1].parent = huffNode[n2].parent = nodeNb; -+ nodeNb++; -+ } -+ -+ /* distribute weights (unlimited tree height) */ -+ huffNode[nodeRoot].nbBits = 0; -+ for (n = nodeRoot - 1; n >= STARTNODE; n--) -+ huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1; -+ for (n = 0; n <= nonNullRank; n++) -+ huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1; -+ -+ /* enforce maxTableLog */ -+ maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); -+ -+ /* fill result into tree (val, nbBits) */ -+ { -+ U16 nbPerRank[HUF_TABLELOG_MAX + 1] = {0}; -+ U16 valPerRank[HUF_TABLELOG_MAX + 1] = {0}; -+ if (maxNbBits > HUF_TABLELOG_MAX) -+ return ERROR(GENERIC); /* check fit into table */ -+ for (n = 0; n <= nonNullRank; n++) -+ nbPerRank[huffNode[n].nbBits]++; -+ /* determine stating value per rank */ -+ { -+ U16 min = 0; -+ for (n = maxNbBits; n > 0; n--) { -+ valPerRank[n] = min; /* get starting value within each rank */ -+ min += nbPerRank[n]; -+ min >>= 1; -+ } -+ } -+ for (n = 0; n <= maxSymbolValue; n++) -+ tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ -+ for (n = 0; n <= maxSymbolValue; n++) -+ tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ -+ } -+ -+ return maxNbBits; -+} -+ -+static size_t HUF_estimateCompressedSize(HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue) -+{ -+ size_t nbBits = 0; -+ int s; -+ for (s = 0; s <= (int)maxSymbolValue; ++s) { -+ nbBits += CTable[s].nbBits * count[s]; -+ } -+ return nbBits >> 3; -+} -+ -+static int HUF_validateCTable(const HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue) -+{ -+ int bad = 0; -+ int s; -+ for (s = 0; s <= (int)maxSymbolValue; ++s) { -+ bad |= (count[s] != 0) & (CTable[s].nbBits == 0); -+ } -+ return !bad; -+} -+ -+static void HUF_encodeSymbol(BIT_CStream_t *bitCPtr, U32 symbol, const HUF_CElt *CTable) -+{ -+ BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); -+} -+ -+size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } -+ -+#define HUF_FLUSHBITS(s) BIT_flushBits(s) -+ -+#define HUF_FLUSHBITS_1(stream) \ -+ if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \ -+ HUF_FLUSHBITS(stream) -+ -+#define HUF_FLUSHBITS_2(stream) \ -+ if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 4 + 7) \ -+ HUF_FLUSHBITS(stream) -+ -+size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable) -+{ -+ const BYTE *ip = (const BYTE *)src; -+ BYTE *const ostart = (BYTE *)dst; -+ BYTE *const oend = ostart + dstSize; -+ BYTE *op = ostart; -+ size_t n; -+ BIT_CStream_t bitC; -+ -+ /* init */ -+ if (dstSize < 8) -+ return 0; /* not enough space to compress */ -+ { -+ size_t const initErr = BIT_initCStream(&bitC, op, oend - op); -+ if (HUF_isError(initErr)) -+ return 0; -+ } -+ -+ n = srcSize & ~3; /* join to mod 4 */ -+ switch (srcSize & 3) { -+ case 3: HUF_encodeSymbol(&bitC, ip[n + 2], CTable); HUF_FLUSHBITS_2(&bitC); -+ case 2: HUF_encodeSymbol(&bitC, ip[n + 1], CTable); HUF_FLUSHBITS_1(&bitC); -+ case 1: HUF_encodeSymbol(&bitC, ip[n + 0], CTable); HUF_FLUSHBITS(&bitC); -+ case 0: -+ default:; -+ } -+ -+ for (; n > 0; n -= 4) { /* note : n&3==0 at this stage */ -+ HUF_encodeSymbol(&bitC, ip[n - 1], CTable); -+ HUF_FLUSHBITS_1(&bitC); -+ HUF_encodeSymbol(&bitC, ip[n - 2], CTable); -+ HUF_FLUSHBITS_2(&bitC); -+ HUF_encodeSymbol(&bitC, ip[n - 3], CTable); -+ HUF_FLUSHBITS_1(&bitC); -+ HUF_encodeSymbol(&bitC, ip[n - 4], CTable); -+ HUF_FLUSHBITS(&bitC); -+ } -+ -+ return BIT_closeCStream(&bitC); -+} -+ -+size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable) -+{ -+ size_t const segmentSize = (srcSize + 3) / 4; /* first 3 segments */ -+ const BYTE *ip = (const BYTE *)src; -+ const BYTE *const iend = ip + srcSize; -+ BYTE *const ostart = (BYTE *)dst; -+ BYTE *const oend = ostart + dstSize; -+ BYTE *op = ostart; -+ -+ if (dstSize < 6 + 1 + 1 + 1 + 8) -+ return 0; /* minimum space to compress successfully */ -+ if (srcSize < 12) -+ return 0; /* no saving possible : too small input */ -+ op += 6; /* jumpTable */ -+ -+ { -+ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable)); -+ if (cSize == 0) -+ return 0; -+ ZSTD_writeLE16(ostart, (U16)cSize); -+ op += cSize; -+ } -+ -+ ip += segmentSize; -+ { -+ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable)); -+ if (cSize == 0) -+ return 0; -+ ZSTD_writeLE16(ostart + 2, (U16)cSize); -+ op += cSize; -+ } -+ -+ ip += segmentSize; -+ { -+ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable)); -+ if (cSize == 0) -+ return 0; -+ ZSTD_writeLE16(ostart + 4, (U16)cSize); -+ op += cSize; -+ } -+ -+ ip += segmentSize; -+ { -+ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, iend - ip, CTable)); -+ if (cSize == 0) -+ return 0; -+ op += cSize; -+ } -+ -+ return op - ostart; -+} -+ -+static size_t HUF_compressCTable_internal(BYTE *const ostart, BYTE *op, BYTE *const oend, const void *src, size_t srcSize, unsigned singleStream, -+ const HUF_CElt *CTable) -+{ -+ size_t const cSize = -+ singleStream ? HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); -+ if (HUF_isError(cSize)) { -+ return cSize; -+ } -+ if (cSize == 0) { -+ return 0; -+ } /* uncompressible */ -+ op += cSize; -+ /* check compressibility */ -+ if ((size_t)(op - ostart) >= srcSize - 1) { -+ return 0; -+ } -+ return op - ostart; -+} -+ -+/* `workSpace` must a table of at least 1024 unsigned */ -+static size_t HUF_compress_internal(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, -+ unsigned singleStream, void *workSpace, size_t wkspSize, HUF_CElt *oldHufTable, HUF_repeat *repeat, int preferRepeat) -+{ -+ BYTE *const ostart = (BYTE *)dst; -+ BYTE *const oend = ostart + dstSize; -+ BYTE *op = ostart; -+ -+ U32 *count; -+ size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1); -+ HUF_CElt *CTable; -+ size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1); -+ -+ /* checks & inits */ -+ if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize) -+ return ERROR(GENERIC); -+ if (!srcSize) -+ return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */ -+ if (!dstSize) -+ return 0; /* cannot fit within dst budget */ -+ if (srcSize > HUF_BLOCKSIZE_MAX) -+ return ERROR(srcSize_wrong); /* curr block size limit */ -+ if (huffLog > HUF_TABLELOG_MAX) -+ return ERROR(tableLog_tooLarge); -+ if (!maxSymbolValue) -+ maxSymbolValue = HUF_SYMBOLVALUE_MAX; -+ if (!huffLog) -+ huffLog = HUF_TABLELOG_DEFAULT; -+ -+ count = (U32 *)workSpace; -+ workSpace = (BYTE *)workSpace + countSize; -+ wkspSize -= countSize; -+ CTable = (HUF_CElt *)workSpace; -+ workSpace = (BYTE *)workSpace + CTableSize; -+ wkspSize -= CTableSize; -+ -+ /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */ -+ if (preferRepeat && repeat && *repeat == HUF_repeat_valid) { -+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable); -+ } -+ -+ /* Scan input and build symbol stats */ -+ { -+ CHECK_V_F(largest, FSE_count_wksp(count, &maxSymbolValue, (const BYTE *)src, srcSize, (U32 *)workSpace)); -+ if (largest == srcSize) { -+ *ostart = ((const BYTE *)src)[0]; -+ return 1; -+ } /* single symbol, rle */ -+ if (largest <= (srcSize >> 7) + 1) -+ return 0; /* Fast heuristic : not compressible enough */ -+ } -+ -+ /* Check validity of previous table */ -+ if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) { -+ *repeat = HUF_repeat_none; -+ } -+ /* Heuristic : use existing table for small inputs */ -+ if (preferRepeat && repeat && *repeat != HUF_repeat_none) { -+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable); -+ } -+ -+ /* Build Huffman Tree */ -+ huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); -+ { -+ CHECK_V_F(maxBits, HUF_buildCTable_wksp(CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize)); -+ huffLog = (U32)maxBits; -+ /* Zero the unused symbols so we can check it for validity */ -+ memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt)); -+ } -+ -+ /* Write table description header */ -+ { -+ CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, CTable, maxSymbolValue, huffLog, workSpace, wkspSize)); -+ /* Check if using the previous table will be beneficial */ -+ if (repeat && *repeat != HUF_repeat_none) { -+ size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue); -+ size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue); -+ if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { -+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable); -+ } -+ } -+ /* Use the new table */ -+ if (hSize + 12ul >= srcSize) { -+ return 0; -+ } -+ op += hSize; -+ if (repeat) { -+ *repeat = HUF_repeat_none; -+ } -+ if (oldHufTable) { -+ memcpy(oldHufTable, CTable, CTableSize); -+ } /* Save the new table */ -+ } -+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable); -+} -+ -+size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace, -+ size_t wkspSize) -+{ -+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0); -+} -+ -+size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace, -+ size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat) -+{ -+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat, -+ preferRepeat); -+} -+ -+size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace, -+ size_t wkspSize) -+{ -+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0); -+} -+ -+size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace, -+ size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat) -+{ -+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat, -+ preferRepeat); -+} -diff --git a/lib/zstd/huf_decompress.c b/lib/zstd/huf_decompress.c -new file mode 100644 -index 0000000..6526482 ---- /dev/null -+++ b/lib/zstd/huf_decompress.c -@@ -0,0 +1,960 @@ -+/* -+ * Huffman decoder, part of New Generation Entropy library -+ * Copyright (C) 2013-2016, Yann Collet. -+ * -+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above -+ * copyright notice, this list of conditions and the following disclaimer -+ * in the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ * -+ * You can contact the author at : -+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -+ */ -+ -+/* ************************************************************** -+* Compiler specifics -+****************************************************************/ -+#define FORCE_INLINE static __always_inline -+ -+/* ************************************************************** -+* Dependencies -+****************************************************************/ -+#include "bitstream.h" /* BIT_* */ -+#include "fse.h" /* header compression */ -+#include "huf.h" -+#include <linux/compiler.h> -+#include <linux/kernel.h> -+#include <linux/string.h> /* memcpy, memset */ -+ -+/* ************************************************************** -+* Error Management -+****************************************************************/ -+#define HUF_STATIC_ASSERT(c) \ -+ { \ -+ enum { HUF_static_assert = 1 / (int)(!!(c)) }; \ -+ } /* use only *after* variable declarations */ -+ -+/*-***************************/ -+/* generic DTableDesc */ -+/*-***************************/ -+ -+typedef struct { -+ BYTE maxTableLog; -+ BYTE tableType; -+ BYTE tableLog; -+ BYTE reserved; -+} DTableDesc; -+ -+static DTableDesc HUF_getDTableDesc(const HUF_DTable *table) -+{ -+ DTableDesc dtd; -+ memcpy(&dtd, table, sizeof(dtd)); -+ return dtd; -+} -+ -+/*-***************************/ -+/* single-symbol decoding */ -+/*-***************************/ -+ -+typedef struct { -+ BYTE byte; -+ BYTE nbBits; -+} HUF_DEltX2; /* single-symbol decoding */ -+ -+size_t HUF_readDTableX2_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize) -+{ -+ U32 tableLog = 0; -+ U32 nbSymbols = 0; -+ size_t iSize; -+ void *const dtPtr = DTable + 1; -+ HUF_DEltX2 *const dt = (HUF_DEltX2 *)dtPtr; -+ -+ U32 *rankVal; -+ BYTE *huffWeight; -+ size_t spaceUsed32 = 0; -+ -+ rankVal = (U32 *)workspace + spaceUsed32; -+ spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1; -+ huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32); -+ spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; -+ -+ if ((spaceUsed32 << 2) > workspaceSize) -+ return ERROR(tableLog_tooLarge); -+ workspace = (U32 *)workspace + spaceUsed32; -+ workspaceSize -= (spaceUsed32 << 2); -+ -+ HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); -+ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ -+ -+ iSize = HUF_readStats_wksp(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize); -+ if (HUF_isError(iSize)) -+ return iSize; -+ -+ /* Table header */ -+ { -+ DTableDesc dtd = HUF_getDTableDesc(DTable); -+ if (tableLog > (U32)(dtd.maxTableLog + 1)) -+ return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ -+ dtd.tableType = 0; -+ dtd.tableLog = (BYTE)tableLog; -+ memcpy(DTable, &dtd, sizeof(dtd)); -+ } -+ -+ /* Calculate starting value for each rank */ -+ { -+ U32 n, nextRankStart = 0; -+ for (n = 1; n < tableLog + 1; n++) { -+ U32 const curr = nextRankStart; -+ nextRankStart += (rankVal[n] << (n - 1)); -+ rankVal[n] = curr; -+ } -+ } -+ -+ /* fill DTable */ -+ { -+ U32 n; -+ for (n = 0; n < nbSymbols; n++) { -+ U32 const w = huffWeight[n]; -+ U32 const length = (1 << w) >> 1; -+ U32 u; -+ HUF_DEltX2 D; -+ D.byte = (BYTE)n; -+ D.nbBits = (BYTE)(tableLog + 1 - w); -+ for (u = rankVal[w]; u < rankVal[w] + length; u++) -+ dt[u] = D; -+ rankVal[w] += length; -+ } -+ } -+ -+ return iSize; -+} -+ -+static BYTE HUF_decodeSymbolX2(BIT_DStream_t *Dstream, const HUF_DEltX2 *dt, const U32 dtLog) -+{ -+ size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ -+ BYTE const c = dt[val].byte; -+ BIT_skipBits(Dstream, dt[val].nbBits); -+ return c; -+} -+ -+#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog) -+ -+#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ -+ if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \ -+ HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) -+ -+#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ -+ if (ZSTD_64bits()) \ -+ HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) -+ -+FORCE_INLINE size_t HUF_decodeStreamX2(BYTE *p, BIT_DStream_t *const bitDPtr, BYTE *const pEnd, const HUF_DEltX2 *const dt, const U32 dtLog) -+{ -+ BYTE *const pStart = p; -+ -+ /* up to 4 symbols at a time */ -+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd - 4)) { -+ HUF_DECODE_SYMBOLX2_2(p, bitDPtr); -+ HUF_DECODE_SYMBOLX2_1(p, bitDPtr); -+ HUF_DECODE_SYMBOLX2_2(p, bitDPtr); -+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr); -+ } -+ -+ /* closer to the end */ -+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) -+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr); -+ -+ /* no more data to retrieve from bitstream, hence no need to reload */ -+ while (p < pEnd) -+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr); -+ -+ return pEnd - pStart; -+} -+ -+static size_t HUF_decompress1X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -+{ -+ BYTE *op = (BYTE *)dst; -+ BYTE *const oend = op + dstSize; -+ const void *dtPtr = DTable + 1; -+ const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr; -+ BIT_DStream_t bitD; -+ DTableDesc const dtd = HUF_getDTableDesc(DTable); -+ U32 const dtLog = dtd.tableLog; -+ -+ { -+ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); -+ if (HUF_isError(errorCode)) -+ return errorCode; -+ } -+ -+ HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); -+ -+ /* check */ -+ if (!BIT_endOfDStream(&bitD)) -+ return ERROR(corruption_detected); -+ -+ return dstSize; -+} -+ -+size_t HUF_decompress1X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -+{ -+ DTableDesc dtd = HUF_getDTableDesc(DTable); -+ if (dtd.tableType != 0) -+ return ERROR(GENERIC); -+ return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -+} -+ -+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -+{ -+ const BYTE *ip = (const BYTE *)cSrc; -+ -+ size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, workspace, workspaceSize); -+ if (HUF_isError(hSize)) -+ return hSize; -+ if (hSize >= cSrcSize) -+ return ERROR(srcSize_wrong); -+ ip += hSize; -+ cSrcSize -= hSize; -+ -+ return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx); -+} -+ -+static size_t HUF_decompress4X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -+{ -+ /* Check */ -+ if (cSrcSize < 10) -+ return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ -+ -+ { -+ const BYTE *const istart = (const BYTE *)cSrc; -+ BYTE *const ostart = (BYTE *)dst; -+ BYTE *const oend = ostart + dstSize; -+ const void *const dtPtr = DTable + 1; -+ const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr; -+ -+ /* Init */ -+ BIT_DStream_t bitD1; -+ BIT_DStream_t bitD2; -+ BIT_DStream_t bitD3; -+ BIT_DStream_t bitD4; -+ size_t const length1 = ZSTD_readLE16(istart); -+ size_t const length2 = ZSTD_readLE16(istart + 2); -+ size_t const length3 = ZSTD_readLE16(istart + 4); -+ size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); -+ const BYTE *const istart1 = istart + 6; /* jumpTable */ -+ const BYTE *const istart2 = istart1 + length1; -+ const BYTE *const istart3 = istart2 + length2; -+ const BYTE *const istart4 = istart3 + length3; -+ const size_t segmentSize = (dstSize + 3) / 4; -+ BYTE *const opStart2 = ostart + segmentSize; -+ BYTE *const opStart3 = opStart2 + segmentSize; -+ BYTE *const opStart4 = opStart3 + segmentSize; -+ BYTE *op1 = ostart; -+ BYTE *op2 = opStart2; -+ BYTE *op3 = opStart3; -+ BYTE *op4 = opStart4; -+ U32 endSignal; -+ DTableDesc const dtd = HUF_getDTableDesc(DTable); -+ U32 const dtLog = dtd.tableLog; -+ -+ if (length4 > cSrcSize) -+ return ERROR(corruption_detected); /* overflow */ -+ { -+ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); -+ if (HUF_isError(errorCode)) -+ return errorCode; -+ } -+ { -+ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); -+ if (HUF_isError(errorCode)) -+ return errorCode; -+ } -+ { -+ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); -+ if (HUF_isError(errorCode)) -+ return errorCode; -+ } -+ { -+ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); -+ if (HUF_isError(errorCode)) -+ return errorCode; -+ } -+ -+ /* 16-32 symbols per loop (4-8 symbols per stream) */ -+ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); -+ for (; (endSignal == BIT_DStream_unfinished) && (op4 < (oend - 7));) { -+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1); -+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2); -+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3); -+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4); -+ HUF_DECODE_SYMBOLX2_1(op1, &bitD1); -+ HUF_DECODE_SYMBOLX2_1(op2, &bitD2); -+ HUF_DECODE_SYMBOLX2_1(op3, &bitD3); -+ HUF_DECODE_SYMBOLX2_1(op4, &bitD4); -+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1); -+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2); -+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3); -+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4); -+ HUF_DECODE_SYMBOLX2_0(op1, &bitD1); -+ HUF_DECODE_SYMBOLX2_0(op2, &bitD2); -+ HUF_DECODE_SYMBOLX2_0(op3, &bitD3); -+ HUF_DECODE_SYMBOLX2_0(op4, &bitD4); -+ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); -+ } -+ -+ /* check corruption */ -+ if (op1 > opStart2) -+ return ERROR(corruption_detected); -+ if (op2 > opStart3) -+ return ERROR(corruption_detected); -+ if (op3 > opStart4) -+ return ERROR(corruption_detected); -+ /* note : op4 supposed already verified within main loop */ -+ -+ /* finish bitStreams one by one */ -+ HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); -+ HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); -+ HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); -+ HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); -+ -+ /* check */ -+ endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); -+ if (!endSignal) -+ return ERROR(corruption_detected); -+ -+ /* decoded size */ -+ return dstSize; -+ } -+} -+ -+size_t HUF_decompress4X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -+{ -+ DTableDesc dtd = HUF_getDTableDesc(DTable); -+ if (dtd.tableType != 0) -+ return ERROR(GENERIC); -+ return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -+} -+ -+size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -+{ -+ const BYTE *ip = (const BYTE *)cSrc; -+ -+ size_t const hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, workspace, workspaceSize); -+ if (HUF_isError(hSize)) -+ return hSize; -+ if (hSize >= cSrcSize) -+ return ERROR(srcSize_wrong); -+ ip += hSize; -+ cSrcSize -= hSize; -+ -+ return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); -+} -+ -+/* *************************/ -+/* double-symbols decoding */ -+/* *************************/ -+typedef struct { -+ U16 sequence; -+ BYTE nbBits; -+ BYTE length; -+} HUF_DEltX4; /* double-symbols decoding */ -+ -+typedef struct { -+ BYTE symbol; -+ BYTE weight; -+} sortedSymbol_t; -+ -+/* HUF_fillDTableX4Level2() : -+ * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ -+static void HUF_fillDTableX4Level2(HUF_DEltX4 *DTable, U32 sizeLog, const U32 consumed, const U32 *rankValOrigin, const int minWeight, -+ const sortedSymbol_t *sortedSymbols, const U32 sortedListSize, U32 nbBitsBaseline, U16 baseSeq) -+{ -+ HUF_DEltX4 DElt; -+ U32 rankVal[HUF_TABLELOG_MAX + 1]; -+ -+ /* get pre-calculated rankVal */ -+ memcpy(rankVal, rankValOrigin, sizeof(rankVal)); -+ -+ /* fill skipped values */ -+ if (minWeight > 1) { -+ U32 i, skipSize = rankVal[minWeight]; -+ ZSTD_writeLE16(&(DElt.sequence), baseSeq); -+ DElt.nbBits = (BYTE)(consumed); -+ DElt.length = 1; -+ for (i = 0; i < skipSize; i++) -+ DTable[i] = DElt; -+ } -+ -+ /* fill DTable */ -+ { -+ U32 s; -+ for (s = 0; s < sortedListSize; s++) { /* note : sortedSymbols already skipped */ -+ const U32 symbol = sortedSymbols[s].symbol; -+ const U32 weight = sortedSymbols[s].weight; -+ const U32 nbBits = nbBitsBaseline - weight; -+ const U32 length = 1 << (sizeLog - nbBits); -+ const U32 start = rankVal[weight]; -+ U32 i = start; -+ const U32 end = start + length; -+ -+ ZSTD_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); -+ DElt.nbBits = (BYTE)(nbBits + consumed); -+ DElt.length = 2; -+ do { -+ DTable[i++] = DElt; -+ } while (i < end); /* since length >= 1 */ -+ -+ rankVal[weight] += length; -+ } -+ } -+} -+ -+typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1]; -+typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; -+ -+static void HUF_fillDTableX4(HUF_DEltX4 *DTable, const U32 targetLog, const sortedSymbol_t *sortedList, const U32 sortedListSize, const U32 *rankStart, -+ rankVal_t rankValOrigin, const U32 maxWeight, const U32 nbBitsBaseline) -+{ -+ U32 rankVal[HUF_TABLELOG_MAX + 1]; -+ const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ -+ const U32 minBits = nbBitsBaseline - maxWeight; -+ U32 s; -+ -+ memcpy(rankVal, rankValOrigin, sizeof(rankVal)); -+ -+ /* fill DTable */ -+ for (s = 0; s < sortedListSize; s++) { -+ const U16 symbol = sortedList[s].symbol; -+ const U32 weight = sortedList[s].weight; -+ const U32 nbBits = nbBitsBaseline - weight; -+ const U32 start = rankVal[weight]; -+ const U32 length = 1 << (targetLog - nbBits); -+ -+ if (targetLog - nbBits >= minBits) { /* enough room for a second symbol */ -+ U32 sortedRank; -+ int minWeight = nbBits + scaleLog; -+ if (minWeight < 1) -+ minWeight = 1; -+ sortedRank = rankStart[minWeight]; -+ HUF_fillDTableX4Level2(DTable + start, targetLog - nbBits, nbBits, rankValOrigin[nbBits], minWeight, sortedList + sortedRank, -+ sortedListSize - sortedRank, nbBitsBaseline, symbol); -+ } else { -+ HUF_DEltX4 DElt; -+ ZSTD_writeLE16(&(DElt.sequence), symbol); -+ DElt.nbBits = (BYTE)(nbBits); -+ DElt.length = 1; -+ { -+ U32 const end = start + length; -+ U32 u; -+ for (u = start; u < end; u++) -+ DTable[u] = DElt; -+ } -+ } -+ rankVal[weight] += length; -+ } -+} -+ -+size_t HUF_readDTableX4_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize) -+{ -+ U32 tableLog, maxW, sizeOfSort, nbSymbols; -+ DTableDesc dtd = HUF_getDTableDesc(DTable); -+ U32 const maxTableLog = dtd.maxTableLog; -+ size_t iSize; -+ void *dtPtr = DTable + 1; /* force compiler to avoid strict-aliasing */ -+ HUF_DEltX4 *const dt = (HUF_DEltX4 *)dtPtr; -+ U32 *rankStart; -+ -+ rankValCol_t *rankVal; -+ U32 *rankStats; -+ U32 *rankStart0; -+ sortedSymbol_t *sortedSymbol; -+ BYTE *weightList; -+ size_t spaceUsed32 = 0; -+ -+ HUF_STATIC_ASSERT((sizeof(rankValCol_t) & 3) == 0); -+ -+ rankVal = (rankValCol_t *)((U32 *)workspace + spaceUsed32); -+ spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2; -+ rankStats = (U32 *)workspace + spaceUsed32; -+ spaceUsed32 += HUF_TABLELOG_MAX + 1; -+ rankStart0 = (U32 *)workspace + spaceUsed32; -+ spaceUsed32 += HUF_TABLELOG_MAX + 2; -+ sortedSymbol = (sortedSymbol_t *)((U32 *)workspace + spaceUsed32); -+ spaceUsed32 += ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2; -+ weightList = (BYTE *)((U32 *)workspace + spaceUsed32); -+ spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; -+ -+ if ((spaceUsed32 << 2) > workspaceSize) -+ return ERROR(tableLog_tooLarge); -+ workspace = (U32 *)workspace + spaceUsed32; -+ workspaceSize -= (spaceUsed32 << 2); -+ -+ rankStart = rankStart0 + 1; -+ memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1)); -+ -+ HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ -+ if (maxTableLog > HUF_TABLELOG_MAX) -+ return ERROR(tableLog_tooLarge); -+ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ -+ -+ iSize = HUF_readStats_wksp(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize); -+ if (HUF_isError(iSize)) -+ return iSize; -+ -+ /* check result */ -+ if (tableLog > maxTableLog) -+ return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ -+ -+ /* find maxWeight */ -+ for (maxW = tableLog; rankStats[maxW] == 0; maxW--) { -+ } /* necessarily finds a solution before 0 */ -+ -+ /* Get start index of each weight */ -+ { -+ U32 w, nextRankStart = 0; -+ for (w = 1; w < maxW + 1; w++) { -+ U32 curr = nextRankStart; -+ nextRankStart += rankStats[w]; -+ rankStart[w] = curr; -+ } -+ rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ -+ sizeOfSort = nextRankStart; -+ } -+ -+ /* sort symbols by weight */ -+ { -+ U32 s; -+ for (s = 0; s < nbSymbols; s++) { -+ U32 const w = weightList[s]; -+ U32 const r = rankStart[w]++; -+ sortedSymbol[r].symbol = (BYTE)s; -+ sortedSymbol[r].weight = (BYTE)w; -+ } -+ rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ -+ } -+ -+ /* Build rankVal */ -+ { -+ U32 *const rankVal0 = rankVal[0]; -+ { -+ int const rescale = (maxTableLog - tableLog) - 1; /* tableLog <= maxTableLog */ -+ U32 nextRankVal = 0; -+ U32 w; -+ for (w = 1; w < maxW + 1; w++) { -+ U32 curr = nextRankVal; -+ nextRankVal += rankStats[w] << (w + rescale); -+ rankVal0[w] = curr; -+ } -+ } -+ { -+ U32 const minBits = tableLog + 1 - maxW; -+ U32 consumed; -+ for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { -+ U32 *const rankValPtr = rankVal[consumed]; -+ U32 w; -+ for (w = 1; w < maxW + 1; w++) { -+ rankValPtr[w] = rankVal0[w] >> consumed; -+ } -+ } -+ } -+ } -+ -+ HUF_fillDTableX4(dt, maxTableLog, sortedSymbol, sizeOfSort, rankStart0, rankVal, maxW, tableLog + 1); -+ -+ dtd.tableLog = (BYTE)maxTableLog; -+ dtd.tableType = 1; -+ memcpy(DTable, &dtd, sizeof(dtd)); -+ return iSize; -+} -+ -+static U32 HUF_decodeSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog) -+{ -+ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ -+ memcpy(op, dt + val, 2); -+ BIT_skipBits(DStream, dt[val].nbBits); -+ return dt[val].length; -+} -+ -+static U32 HUF_decodeLastSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog) -+{ -+ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ -+ memcpy(op, dt + val, 1); -+ if (dt[val].length == 1) -+ BIT_skipBits(DStream, dt[val].nbBits); -+ else { -+ if (DStream->bitsConsumed < (sizeof(DStream->bitContainer) * 8)) { -+ BIT_skipBits(DStream, dt[val].nbBits); -+ if (DStream->bitsConsumed > (sizeof(DStream->bitContainer) * 8)) -+ /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ -+ DStream->bitsConsumed = (sizeof(DStream->bitContainer) * 8); -+ } -+ } -+ return 1; -+} -+ -+#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) -+ -+#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ -+ if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \ -+ ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) -+ -+#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ -+ if (ZSTD_64bits()) \ -+ ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) -+ -+FORCE_INLINE size_t HUF_decodeStreamX4(BYTE *p, BIT_DStream_t *bitDPtr, BYTE *const pEnd, const HUF_DEltX4 *const dt, const U32 dtLog) -+{ -+ BYTE *const pStart = p; -+ -+ /* up to 8 symbols at a time */ -+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd - (sizeof(bitDPtr->bitContainer) - 1))) { -+ HUF_DECODE_SYMBOLX4_2(p, bitDPtr); -+ HUF_DECODE_SYMBOLX4_1(p, bitDPtr); -+ HUF_DECODE_SYMBOLX4_2(p, bitDPtr); -+ HUF_DECODE_SYMBOLX4_0(p, bitDPtr); -+ } -+ -+ /* closer to end : up to 2 symbols at a time */ -+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd - 2)) -+ HUF_DECODE_SYMBOLX4_0(p, bitDPtr); -+ -+ while (p <= pEnd - 2) -+ HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ -+ -+ if (p < pEnd) -+ p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); -+ -+ return p - pStart; -+} -+ -+static size_t HUF_decompress1X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -+{ -+ BIT_DStream_t bitD; -+ -+ /* Init */ -+ { -+ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); -+ if (HUF_isError(errorCode)) -+ return errorCode; -+ } -+ -+ /* decode */ -+ { -+ BYTE *const ostart = (BYTE *)dst; -+ BYTE *const oend = ostart + dstSize; -+ const void *const dtPtr = DTable + 1; /* force compiler to not use strict-aliasing */ -+ const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr; -+ DTableDesc const dtd = HUF_getDTableDesc(DTable); -+ HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); -+ } -+ -+ /* check */ -+ if (!BIT_endOfDStream(&bitD)) -+ return ERROR(corruption_detected); -+ -+ /* decoded size */ -+ return dstSize; -+} -+ -+size_t HUF_decompress1X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -+{ -+ DTableDesc dtd = HUF_getDTableDesc(DTable); -+ if (dtd.tableType != 1) -+ return ERROR(GENERIC); -+ return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -+} -+ -+size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -+{ -+ const BYTE *ip = (const BYTE *)cSrc; -+ -+ size_t const hSize = HUF_readDTableX4_wksp(DCtx, cSrc, cSrcSize, workspace, workspaceSize); -+ if (HUF_isError(hSize)) -+ return hSize; -+ if (hSize >= cSrcSize) -+ return ERROR(srcSize_wrong); -+ ip += hSize; -+ cSrcSize -= hSize; -+ -+ return HUF_decompress1X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx); -+} -+ -+static size_t HUF_decompress4X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -+{ -+ if (cSrcSize < 10) -+ return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ -+ -+ { -+ const BYTE *const istart = (const BYTE *)cSrc; -+ BYTE *const ostart = (BYTE *)dst; -+ BYTE *const oend = ostart + dstSize; -+ const void *const dtPtr = DTable + 1; -+ const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr; -+ -+ /* Init */ -+ BIT_DStream_t bitD1; -+ BIT_DStream_t bitD2; -+ BIT_DStream_t bitD3; -+ BIT_DStream_t bitD4; -+ size_t const length1 = ZSTD_readLE16(istart); -+ size_t const length2 = ZSTD_readLE16(istart + 2); -+ size_t const length3 = ZSTD_readLE16(istart + 4); -+ size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); -+ const BYTE *const istart1 = istart + 6; /* jumpTable */ -+ const BYTE *const istart2 = istart1 + length1; -+ const BYTE *const istart3 = istart2 + length2; -+ const BYTE *const istart4 = istart3 + length3; -+ size_t const segmentSize = (dstSize + 3) / 4; -+ BYTE *const opStart2 = ostart + segmentSize; -+ BYTE *const opStart3 = opStart2 + segmentSize; -+ BYTE *const opStart4 = opStart3 + segmentSize; -+ BYTE *op1 = ostart; -+ BYTE *op2 = opStart2; -+ BYTE *op3 = opStart3; -+ BYTE *op4 = opStart4; -+ U32 endSignal; -+ DTableDesc const dtd = HUF_getDTableDesc(DTable); -+ U32 const dtLog = dtd.tableLog; -+ -+ if (length4 > cSrcSize) -+ return ERROR(corruption_detected); /* overflow */ -+ { -+ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); -+ if (HUF_isError(errorCode)) -+ return errorCode; -+ } -+ { -+ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); -+ if (HUF_isError(errorCode)) -+ return errorCode; -+ } -+ { -+ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); -+ if (HUF_isError(errorCode)) -+ return errorCode; -+ } -+ { -+ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); -+ if (HUF_isError(errorCode)) -+ return errorCode; -+ } -+ -+ /* 16-32 symbols per loop (4-8 symbols per stream) */ -+ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); -+ for (; (endSignal == BIT_DStream_unfinished) & (op4 < (oend - (sizeof(bitD4.bitContainer) - 1)));) { -+ HUF_DECODE_SYMBOLX4_2(op1, &bitD1); -+ HUF_DECODE_SYMBOLX4_2(op2, &bitD2); -+ HUF_DECODE_SYMBOLX4_2(op3, &bitD3); -+ HUF_DECODE_SYMBOLX4_2(op4, &bitD4); -+ HUF_DECODE_SYMBOLX4_1(op1, &bitD1); -+ HUF_DECODE_SYMBOLX4_1(op2, &bitD2); -+ HUF_DECODE_SYMBOLX4_1(op3, &bitD3); -+ HUF_DECODE_SYMBOLX4_1(op4, &bitD4); -+ HUF_DECODE_SYMBOLX4_2(op1, &bitD1); -+ HUF_DECODE_SYMBOLX4_2(op2, &bitD2); -+ HUF_DECODE_SYMBOLX4_2(op3, &bitD3); -+ HUF_DECODE_SYMBOLX4_2(op4, &bitD4); -+ HUF_DECODE_SYMBOLX4_0(op1, &bitD1); -+ HUF_DECODE_SYMBOLX4_0(op2, &bitD2); -+ HUF_DECODE_SYMBOLX4_0(op3, &bitD3); -+ HUF_DECODE_SYMBOLX4_0(op4, &bitD4); -+ -+ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); -+ } -+ -+ /* check corruption */ -+ if (op1 > opStart2) -+ return ERROR(corruption_detected); -+ if (op2 > opStart3) -+ return ERROR(corruption_detected); -+ if (op3 > opStart4) -+ return ERROR(corruption_detected); -+ /* note : op4 already verified within main loop */ -+ -+ /* finish bitStreams one by one */ -+ HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); -+ HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); -+ HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); -+ HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); -+ -+ /* check */ -+ { -+ U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); -+ if (!endCheck) -+ return ERROR(corruption_detected); -+ } -+ -+ /* decoded size */ -+ return dstSize; -+ } -+} -+ -+size_t HUF_decompress4X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -+{ -+ DTableDesc dtd = HUF_getDTableDesc(DTable); -+ if (dtd.tableType != 1) -+ return ERROR(GENERIC); -+ return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -+} -+ -+size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -+{ -+ const BYTE *ip = (const BYTE *)cSrc; -+ -+ size_t hSize = HUF_readDTableX4_wksp(dctx, cSrc, cSrcSize, workspace, workspaceSize); -+ if (HUF_isError(hSize)) -+ return hSize; -+ if (hSize >= cSrcSize) -+ return ERROR(srcSize_wrong); -+ ip += hSize; -+ cSrcSize -= hSize; -+ -+ return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); -+} -+ -+/* ********************************/ -+/* Generic decompression selector */ -+/* ********************************/ -+ -+size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -+{ -+ DTableDesc const dtd = HUF_getDTableDesc(DTable); -+ return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) -+ : HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -+} -+ -+size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -+{ -+ DTableDesc const dtd = HUF_getDTableDesc(DTable); -+ return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) -+ : HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -+} -+ -+typedef struct { -+ U32 tableTime; -+ U32 decode256Time; -+} algo_time_t; -+static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = { -+ /* single, double, quad */ -+ {{0, 0}, {1, 1}, {2, 2}}, /* Q==0 : impossible */ -+ {{0, 0}, {1, 1}, {2, 2}}, /* Q==1 : impossible */ -+ {{38, 130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ -+ {{448, 128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ -+ {{556, 128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ -+ {{714, 128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ -+ {{883, 128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ -+ {{897, 128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ -+ {{926, 128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ -+ {{947, 128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ -+ {{1107, 128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ -+ {{1177, 128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ -+ {{1242, 128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ -+ {{1349, 128}, {2644, 106}, {5260, 106}}, /* Q ==13 : 81-87% */ -+ {{1455, 128}, {2422, 124}, {4174, 124}}, /* Q ==14 : 87-93% */ -+ {{722, 128}, {1891, 145}, {1936, 146}}, /* Q ==15 : 93-99% */ -+}; -+ -+/** HUF_selectDecoder() : -+* Tells which decoder is likely to decode faster, -+* based on a set of pre-determined metrics. -+* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . -+* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -+U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize) -+{ -+ /* decoder timing evaluation */ -+ U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ -+ U32 const D256 = (U32)(dstSize >> 8); -+ U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); -+ U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); -+ DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */ -+ -+ return DTime1 < DTime0; -+} -+ -+typedef size_t (*decompressionAlgo)(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); -+ -+size_t HUF_decompress4X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -+{ -+ /* validation checks */ -+ if (dstSize == 0) -+ return ERROR(dstSize_tooSmall); -+ if (cSrcSize > dstSize) -+ return ERROR(corruption_detected); /* invalid */ -+ if (cSrcSize == dstSize) { -+ memcpy(dst, cSrc, dstSize); -+ return dstSize; -+ } /* not compressed */ -+ if (cSrcSize == 1) { -+ memset(dst, *(const BYTE *)cSrc, dstSize); -+ return dstSize; -+ } /* RLE */ -+ -+ { -+ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -+ return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize) -+ : HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize); -+ } -+} -+ -+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -+{ -+ /* validation checks */ -+ if (dstSize == 0) -+ return ERROR(dstSize_tooSmall); -+ if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) -+ return ERROR(corruption_detected); /* invalid */ -+ -+ { -+ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -+ return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize) -+ : HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize); -+ } -+} -+ -+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -+{ -+ /* validation checks */ -+ if (dstSize == 0) -+ return ERROR(dstSize_tooSmall); -+ if (cSrcSize > dstSize) -+ return ERROR(corruption_detected); /* invalid */ -+ if (cSrcSize == dstSize) { -+ memcpy(dst, cSrc, dstSize); -+ return dstSize; -+ } /* not compressed */ -+ if (cSrcSize == 1) { -+ memset(dst, *(const BYTE *)cSrc, dstSize); -+ return dstSize; -+ } /* RLE */ -+ -+ { -+ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -+ return algoNb ? HUF_decompress1X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize) -+ : HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize); -+ } -+} -diff --git a/lib/zstd/mem.h b/lib/zstd/mem.h -new file mode 100644 -index 0000000..42a697b ---- /dev/null -+++ b/lib/zstd/mem.h -@@ -0,0 +1,149 @@ -+/** -+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. -+ * All rights reserved. -+ * -+ * This source code is licensed under the BSD-style license found in the -+ * LICENSE file in the root directory of https://github.com/facebook/zstd. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ */ -+ -+#ifndef MEM_H_MODULE -+#define MEM_H_MODULE -+ -+/*-**************************************** -+* Dependencies -+******************************************/ -+#include <asm/unaligned.h> -+#include <linux/string.h> /* memcpy */ -+#include <linux/types.h> /* size_t, ptrdiff_t */ -+ -+/*-**************************************** -+* Compiler specifics -+******************************************/ -+#define ZSTD_STATIC static __inline __attribute__((unused)) -+ -+/*-************************************************************** -+* Basic Types -+*****************************************************************/ -+typedef uint8_t BYTE; -+typedef uint16_t U16; -+typedef int16_t S16; -+typedef uint32_t U32; -+typedef int32_t S32; -+typedef uint64_t U64; -+typedef int64_t S64; -+typedef ptrdiff_t iPtrDiff; -+typedef uintptr_t uPtrDiff; -+ -+/*-************************************************************** -+* Memory I/O -+*****************************************************************/ -+ZSTD_STATIC unsigned ZSTD_32bits(void) { return sizeof(size_t) == 4; } -+ZSTD_STATIC unsigned ZSTD_64bits(void) { return sizeof(size_t) == 8; } -+ -+#if defined(__LITTLE_ENDIAN) -+#define ZSTD_LITTLE_ENDIAN 1 -+#else -+#define ZSTD_LITTLE_ENDIAN 0 -+#endif -+ -+ZSTD_STATIC unsigned ZSTD_isLittleEndian(void) { return ZSTD_LITTLE_ENDIAN; } -+ -+ZSTD_STATIC U16 ZSTD_read16(const void *memPtr) { return get_unaligned((const U16 *)memPtr); } -+ -+ZSTD_STATIC U32 ZSTD_read32(const void *memPtr) { return get_unaligned((const U32 *)memPtr); } -+ -+ZSTD_STATIC U64 ZSTD_read64(const void *memPtr) { return get_unaligned((const U64 *)memPtr); } -+ -+ZSTD_STATIC size_t ZSTD_readST(const void *memPtr) { return get_unaligned((const size_t *)memPtr); } -+ -+ZSTD_STATIC void ZSTD_write16(void *memPtr, U16 value) { put_unaligned(value, (U16 *)memPtr); } -+ -+ZSTD_STATIC void ZSTD_write32(void *memPtr, U32 value) { put_unaligned(value, (U32 *)memPtr); } -+ -+ZSTD_STATIC void ZSTD_write64(void *memPtr, U64 value) { put_unaligned(value, (U64 *)memPtr); } -+ -+/*=== Little endian r/w ===*/ -+ -+ZSTD_STATIC U16 ZSTD_readLE16(const void *memPtr) { return get_unaligned_le16(memPtr); } -+ -+ZSTD_STATIC void ZSTD_writeLE16(void *memPtr, U16 val) { put_unaligned_le16(val, memPtr); } -+ -+ZSTD_STATIC U32 ZSTD_readLE24(const void *memPtr) { return ZSTD_readLE16(memPtr) + (((const BYTE *)memPtr)[2] << 16); } -+ -+ZSTD_STATIC void ZSTD_writeLE24(void *memPtr, U32 val) -+{ -+ ZSTD_writeLE16(memPtr, (U16)val); -+ ((BYTE *)memPtr)[2] = (BYTE)(val >> 16); -+} -+ -+ZSTD_STATIC U32 ZSTD_readLE32(const void *memPtr) { return get_unaligned_le32(memPtr); } -+ -+ZSTD_STATIC void ZSTD_writeLE32(void *memPtr, U32 val32) { put_unaligned_le32(val32, memPtr); } -+ -+ZSTD_STATIC U64 ZSTD_readLE64(const void *memPtr) { return get_unaligned_le64(memPtr); } -+ -+ZSTD_STATIC void ZSTD_writeLE64(void *memPtr, U64 val64) { put_unaligned_le64(val64, memPtr); } -+ -+ZSTD_STATIC size_t ZSTD_readLEST(const void *memPtr) -+{ -+ if (ZSTD_32bits()) -+ return (size_t)ZSTD_readLE32(memPtr); -+ else -+ return (size_t)ZSTD_readLE64(memPtr); -+} -+ -+ZSTD_STATIC void ZSTD_writeLEST(void *memPtr, size_t val) -+{ -+ if (ZSTD_32bits()) -+ ZSTD_writeLE32(memPtr, (U32)val); -+ else -+ ZSTD_writeLE64(memPtr, (U64)val); -+} -+ -+/*=== Big endian r/w ===*/ -+ -+ZSTD_STATIC U32 ZSTD_readBE32(const void *memPtr) { return get_unaligned_be32(memPtr); } -+ -+ZSTD_STATIC void ZSTD_writeBE32(void *memPtr, U32 val32) { put_unaligned_be32(val32, memPtr); } -+ -+ZSTD_STATIC U64 ZSTD_readBE64(const void *memPtr) { return get_unaligned_be64(memPtr); } -+ -+ZSTD_STATIC void ZSTD_writeBE64(void *memPtr, U64 val64) { put_unaligned_be64(val64, memPtr); } -+ -+ZSTD_STATIC size_t ZSTD_readBEST(const void *memPtr) -+{ -+ if (ZSTD_32bits()) -+ return (size_t)ZSTD_readBE32(memPtr); -+ else -+ return (size_t)ZSTD_readBE64(memPtr); -+} -+ -+ZSTD_STATIC void ZSTD_writeBEST(void *memPtr, size_t val) -+{ -+ if (ZSTD_32bits()) -+ ZSTD_writeBE32(memPtr, (U32)val); -+ else -+ ZSTD_writeBE64(memPtr, (U64)val); -+} -+ -+/* function safe only for comparisons */ -+ZSTD_STATIC U32 ZSTD_readMINMATCH(const void *memPtr, U32 length) -+{ -+ switch (length) { -+ default: -+ case 4: return ZSTD_read32(memPtr); -+ case 3: -+ if (ZSTD_isLittleEndian()) -+ return ZSTD_read32(memPtr) << 8; -+ else -+ return ZSTD_read32(memPtr) >> 8; -+ } -+} -+ -+#endif /* MEM_H_MODULE */ -diff --git a/lib/zstd/zstd_common.c b/lib/zstd/zstd_common.c -new file mode 100644 -index 0000000..e5f06d7 ---- /dev/null -+++ b/lib/zstd/zstd_common.c -@@ -0,0 +1,73 @@ -+/** -+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. -+ * All rights reserved. -+ * -+ * This source code is licensed under the BSD-style license found in the -+ * LICENSE file in the root directory of https://github.com/facebook/zstd. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ */ -+ -+/*-************************************* -+* Dependencies -+***************************************/ -+#include "error_private.h" -+#include "zstd_internal.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */ -+#include <linux/kernel.h> -+ -+/*=************************************************************** -+* Custom allocator -+****************************************************************/ -+ -+#define stack_push(stack, size) \ -+ ({ \ -+ void *const ptr = ZSTD_PTR_ALIGN((stack)->ptr); \ -+ (stack)->ptr = (char *)ptr + (size); \ -+ (stack)->ptr <= (stack)->end ? ptr : NULL; \ -+ }) -+ -+ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize) -+{ -+ ZSTD_customMem stackMem = {ZSTD_stackAlloc, ZSTD_stackFree, workspace}; -+ ZSTD_stack *stack = (ZSTD_stack *)workspace; -+ /* Verify preconditions */ -+ if (!workspace || workspaceSize < sizeof(ZSTD_stack) || workspace != ZSTD_PTR_ALIGN(workspace)) { -+ ZSTD_customMem error = {NULL, NULL, NULL}; -+ return error; -+ } -+ /* Initialize the stack */ -+ stack->ptr = workspace; -+ stack->end = (char *)workspace + workspaceSize; -+ stack_push(stack, sizeof(ZSTD_stack)); -+ return stackMem; -+} -+ -+void *ZSTD_stackAllocAll(void *opaque, size_t *size) -+{ -+ ZSTD_stack *stack = (ZSTD_stack *)opaque; -+ *size = (BYTE const *)stack->end - (BYTE *)ZSTD_PTR_ALIGN(stack->ptr); -+ return stack_push(stack, *size); -+} -+ -+void *ZSTD_stackAlloc(void *opaque, size_t size) -+{ -+ ZSTD_stack *stack = (ZSTD_stack *)opaque; -+ return stack_push(stack, size); -+} -+void ZSTD_stackFree(void *opaque, void *address) -+{ -+ (void)opaque; -+ (void)address; -+} -+ -+void *ZSTD_malloc(size_t size, ZSTD_customMem customMem) { return customMem.customAlloc(customMem.opaque, size); } -+ -+void ZSTD_free(void *ptr, ZSTD_customMem customMem) -+{ -+ if (ptr != NULL) -+ customMem.customFree(customMem.opaque, ptr); -+} -diff --git a/lib/zstd/zstd_internal.h b/lib/zstd/zstd_internal.h -new file mode 100644 -index 0000000..a0fb83e ---- /dev/null -+++ b/lib/zstd/zstd_internal.h -@@ -0,0 +1,261 @@ -+/** -+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. -+ * All rights reserved. -+ * -+ * This source code is licensed under the BSD-style license found in the -+ * LICENSE file in the root directory of https://github.com/facebook/zstd. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ */ -+ -+#ifndef ZSTD_CCOMMON_H_MODULE -+#define ZSTD_CCOMMON_H_MODULE -+ -+/*-******************************************************* -+* Compiler specifics -+*********************************************************/ -+#define FORCE_INLINE static __always_inline -+#define FORCE_NOINLINE static noinline -+ -+/*-************************************* -+* Dependencies -+***************************************/ -+#include "error_private.h" -+#include "mem.h" -+#include <linux/compiler.h> -+#include <linux/kernel.h> -+#include <linux/xxhash.h> -+#include <linux/zstd.h> -+ -+/*-************************************* -+* shared macros -+***************************************/ -+#define MIN(a, b) ((a) < (b) ? (a) : (b)) -+#define MAX(a, b) ((a) > (b) ? (a) : (b)) -+#define CHECK_F(f) \ -+ { \ -+ size_t const errcod = f; \ -+ if (ERR_isError(errcod)) \ -+ return errcod; \ -+ } /* check and Forward error code */ -+#define CHECK_E(f, e) \ -+ { \ -+ size_t const errcod = f; \ -+ if (ERR_isError(errcod)) \ -+ return ERROR(e); \ -+ } /* check and send Error code */ -+#define ZSTD_STATIC_ASSERT(c) \ -+ { \ -+ enum { ZSTD_static_assert = 1 / (int)(!!(c)) }; \ -+ } -+ -+/*-************************************* -+* Common constants -+***************************************/ -+#define ZSTD_OPT_NUM (1 << 12) -+#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */ -+ -+#define ZSTD_REP_NUM 3 /* number of repcodes */ -+#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */ -+#define ZSTD_REP_MOVE (ZSTD_REP_NUM - 1) -+#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM) -+static const U32 repStartValue[ZSTD_REP_NUM] = {1, 4, 8}; -+ -+#define KB *(1 << 10) -+#define MB *(1 << 20) -+#define GB *(1U << 30) -+ -+#define BIT7 128 -+#define BIT6 64 -+#define BIT5 32 -+#define BIT4 16 -+#define BIT1 2 -+#define BIT0 1 -+ -+#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 -+static const size_t ZSTD_fcs_fieldSize[4] = {0, 2, 4, 8}; -+static const size_t ZSTD_did_fieldSize[4] = {0, 1, 2, 4}; -+ -+#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ -+static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; -+typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; -+ -+#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ -+#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ -+ -+#define HufLog 12 -+typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; -+ -+#define LONGNBSEQ 0x7F00 -+ -+#define MINMATCH 3 -+#define EQUAL_READ32 4 -+ -+#define Litbits 8 -+#define MaxLit ((1 << Litbits) - 1) -+#define MaxML 52 -+#define MaxLL 35 -+#define MaxOff 28 -+#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */ -+#define MLFSELog 9 -+#define LLFSELog 9 -+#define OffFSELog 8 -+ -+static const U32 LL_bits[MaxLL + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; -+static const S16 LL_defaultNorm[MaxLL + 1] = {4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, -1, -1, -1, -1}; -+#define LL_DEFAULTNORMLOG 6 /* for static allocation */ -+static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG; -+ -+static const U32 ML_bits[MaxML + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; -+static const S16 ML_defaultNorm[MaxML + 1] = {1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1}; -+#define ML_DEFAULTNORMLOG 6 /* for static allocation */ -+static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG; -+ -+static const S16 OF_defaultNorm[MaxOff + 1] = {1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1}; -+#define OF_DEFAULTNORMLOG 5 /* for static allocation */ -+static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; -+ -+/*-******************************************* -+* Shared functions to include for inlining -+*********************************************/ -+ZSTD_STATIC void ZSTD_copy8(void *dst, const void *src) { -+ memcpy(dst, src, 8); -+} -+/*! ZSTD_wildcopy() : -+* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ -+#define WILDCOPY_OVERLENGTH 8 -+ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length) -+{ -+ const BYTE* ip = (const BYTE*)src; -+ BYTE* op = (BYTE*)dst; -+ BYTE* const oend = op + length; -+ /* Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388. -+ * Avoid the bad case where the loop only runs once by handling the -+ * special case separately. This doesn't trigger the bug because it -+ * doesn't involve pointer/integer overflow. -+ */ -+ if (length <= 8) -+ return ZSTD_copy8(dst, src); -+ do { -+ ZSTD_copy8(op, ip); -+ op += 8; -+ ip += 8; -+ } while (op < oend); -+} -+ -+/*-******************************************* -+* Private interfaces -+*********************************************/ -+typedef struct ZSTD_stats_s ZSTD_stats_t; -+ -+typedef struct { -+ U32 off; -+ U32 len; -+} ZSTD_match_t; -+ -+typedef struct { -+ U32 price; -+ U32 off; -+ U32 mlen; -+ U32 litlen; -+ U32 rep[ZSTD_REP_NUM]; -+} ZSTD_optimal_t; -+ -+typedef struct seqDef_s { -+ U32 offset; -+ U16 litLength; -+ U16 matchLength; -+} seqDef; -+ -+typedef struct { -+ seqDef *sequencesStart; -+ seqDef *sequences; -+ BYTE *litStart; -+ BYTE *lit; -+ BYTE *llCode; -+ BYTE *mlCode; -+ BYTE *ofCode; -+ U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ -+ U32 longLengthPos; -+ /* opt */ -+ ZSTD_optimal_t *priceTable; -+ ZSTD_match_t *matchTable; -+ U32 *matchLengthFreq; -+ U32 *litLengthFreq; -+ U32 *litFreq; -+ U32 *offCodeFreq; -+ U32 matchLengthSum; -+ U32 matchSum; -+ U32 litLengthSum; -+ U32 litSum; -+ U32 offCodeSum; -+ U32 log2matchLengthSum; -+ U32 log2matchSum; -+ U32 log2litLengthSum; -+ U32 log2litSum; -+ U32 log2offCodeSum; -+ U32 factor; -+ U32 staticPrices; -+ U32 cachedPrice; -+ U32 cachedLitLength; -+ const BYTE *cachedLiterals; -+} seqStore_t; -+ -+const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx); -+void ZSTD_seqToCodes(const seqStore_t *seqStorePtr); -+int ZSTD_isSkipFrame(ZSTD_DCtx *dctx); -+ -+/*= Custom memory allocation functions */ -+typedef void *(*ZSTD_allocFunction)(void *opaque, size_t size); -+typedef void (*ZSTD_freeFunction)(void *opaque, void *address); -+typedef struct { -+ ZSTD_allocFunction customAlloc; -+ ZSTD_freeFunction customFree; -+ void *opaque; -+} ZSTD_customMem; -+ -+void *ZSTD_malloc(size_t size, ZSTD_customMem customMem); -+void ZSTD_free(void *ptr, ZSTD_customMem customMem); -+ -+/*====== stack allocation ======*/ -+ -+typedef struct { -+ void *ptr; -+ const void *end; -+} ZSTD_stack; -+ -+#define ZSTD_ALIGN(x) ALIGN(x, sizeof(size_t)) -+#define ZSTD_PTR_ALIGN(p) PTR_ALIGN(p, sizeof(size_t)) -+ -+ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize); -+ -+void *ZSTD_stackAllocAll(void *opaque, size_t *size); -+void *ZSTD_stackAlloc(void *opaque, size_t size); -+void ZSTD_stackFree(void *opaque, void *address); -+ -+/*====== common function ======*/ -+ -+ZSTD_STATIC U32 ZSTD_highbit32(U32 val) { return 31 - __builtin_clz(val); } -+ -+/* hidden functions */ -+ -+/* ZSTD_invalidateRepCodes() : -+ * ensures next compression will not use repcodes from previous block. -+ * Note : only works with regular variant; -+ * do not use with extDict variant ! */ -+void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx); -+ -+size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx); -+size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx); -+size_t ZSTD_freeCDict(ZSTD_CDict *cdict); -+size_t ZSTD_freeDDict(ZSTD_DDict *cdict); -+size_t ZSTD_freeCStream(ZSTD_CStream *zcs); -+size_t ZSTD_freeDStream(ZSTD_DStream *zds); -+ -+#endif /* ZSTD_CCOMMON_H_MODULE */ -diff --git a/lib/zstd/zstd_opt.h b/lib/zstd/zstd_opt.h -new file mode 100644 -index 0000000..ecdd725 ---- /dev/null -+++ b/lib/zstd/zstd_opt.h -@@ -0,0 +1,1012 @@ -+/** -+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. -+ * All rights reserved. -+ * -+ * This source code is licensed under the BSD-style license found in the -+ * LICENSE file in the root directory of https://github.com/facebook/zstd. -+ * -+ * This program is free software; you can redistribute it and/or modify it under -+ * the terms of the GNU General Public License version 2 as published by the -+ * Free Software Foundation. This program is dual-licensed; you may select -+ * either version 2 of the GNU General Public License ("GPL") or BSD license -+ * ("BSD"). -+ */ -+ -+/* Note : this file is intended to be included within zstd_compress.c */ -+ -+#ifndef ZSTD_OPT_H_91842398743 -+#define ZSTD_OPT_H_91842398743 -+ -+#define ZSTD_LITFREQ_ADD 2 -+#define ZSTD_FREQ_DIV 4 -+#define ZSTD_MAX_PRICE (1 << 30) -+ -+/*-************************************* -+* Price functions for optimal parser -+***************************************/ -+FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t *ssPtr) -+{ -+ ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum + 1); -+ ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum + 1); -+ ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum + 1); -+ ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum + 1); -+ ssPtr->factor = 1 + ((ssPtr->litSum >> 5) / ssPtr->litLengthSum) + ((ssPtr->litSum << 1) / (ssPtr->litSum + ssPtr->matchSum)); -+} -+ -+ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t *ssPtr, const BYTE *src, size_t srcSize) -+{ -+ unsigned u; -+ -+ ssPtr->cachedLiterals = NULL; -+ ssPtr->cachedPrice = ssPtr->cachedLitLength = 0; -+ ssPtr->staticPrices = 0; -+ -+ if (ssPtr->litLengthSum == 0) { -+ if (srcSize <= 1024) -+ ssPtr->staticPrices = 1; -+ -+ for (u = 0; u <= MaxLit; u++) -+ ssPtr->litFreq[u] = 0; -+ for (u = 0; u < srcSize; u++) -+ ssPtr->litFreq[src[u]]++; -+ -+ ssPtr->litSum = 0; -+ ssPtr->litLengthSum = MaxLL + 1; -+ ssPtr->matchLengthSum = MaxML + 1; -+ ssPtr->offCodeSum = (MaxOff + 1); -+ ssPtr->matchSum = (ZSTD_LITFREQ_ADD << Litbits); -+ -+ for (u = 0; u <= MaxLit; u++) { -+ ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> ZSTD_FREQ_DIV); -+ ssPtr->litSum += ssPtr->litFreq[u]; -+ } -+ for (u = 0; u <= MaxLL; u++) -+ ssPtr->litLengthFreq[u] = 1; -+ for (u = 0; u <= MaxML; u++) -+ ssPtr->matchLengthFreq[u] = 1; -+ for (u = 0; u <= MaxOff; u++) -+ ssPtr->offCodeFreq[u] = 1; -+ } else { -+ ssPtr->matchLengthSum = 0; -+ ssPtr->litLengthSum = 0; -+ ssPtr->offCodeSum = 0; -+ ssPtr->matchSum = 0; -+ ssPtr->litSum = 0; -+ -+ for (u = 0; u <= MaxLit; u++) { -+ ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> (ZSTD_FREQ_DIV + 1)); -+ ssPtr->litSum += ssPtr->litFreq[u]; -+ } -+ for (u = 0; u <= MaxLL; u++) { -+ ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u] >> (ZSTD_FREQ_DIV + 1)); -+ ssPtr->litLengthSum += ssPtr->litLengthFreq[u]; -+ } -+ for (u = 0; u <= MaxML; u++) { -+ ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u] >> ZSTD_FREQ_DIV); -+ ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u]; -+ ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3); -+ } -+ ssPtr->matchSum *= ZSTD_LITFREQ_ADD; -+ for (u = 0; u <= MaxOff; u++) { -+ ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u] >> ZSTD_FREQ_DIV); -+ ssPtr->offCodeSum += ssPtr->offCodeFreq[u]; -+ } -+ } -+ -+ ZSTD_setLog2Prices(ssPtr); -+} -+ -+FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t *ssPtr, U32 litLength, const BYTE *literals) -+{ -+ U32 price, u; -+ -+ if (ssPtr->staticPrices) -+ return ZSTD_highbit32((U32)litLength + 1) + (litLength * 6); -+ -+ if (litLength == 0) -+ return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0] + 1); -+ -+ /* literals */ -+ if (ssPtr->cachedLiterals == literals) { -+ U32 const additional = litLength - ssPtr->cachedLitLength; -+ const BYTE *literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength; -+ price = ssPtr->cachedPrice + additional * ssPtr->log2litSum; -+ for (u = 0; u < additional; u++) -+ price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]] + 1); -+ ssPtr->cachedPrice = price; -+ ssPtr->cachedLitLength = litLength; -+ } else { -+ price = litLength * ssPtr->log2litSum; -+ for (u = 0; u < litLength; u++) -+ price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]] + 1); -+ -+ if (litLength >= 12) { -+ ssPtr->cachedLiterals = literals; -+ ssPtr->cachedPrice = price; -+ ssPtr->cachedLitLength = litLength; -+ } -+ } -+ -+ /* literal Length */ -+ { -+ const BYTE LL_deltaCode = 19; -+ const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; -+ price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode] + 1); -+ } -+ -+ return price; -+} -+ -+FORCE_INLINE U32 ZSTD_getPrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength, const int ultra) -+{ -+ /* offset */ -+ U32 price; -+ BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1); -+ -+ if (seqStorePtr->staticPrices) -+ return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength + 1) + 16 + offCode; -+ -+ price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode] + 1); -+ if (!ultra && offCode >= 20) -+ price += (offCode - 19) * 2; -+ -+ /* match Length */ -+ { -+ const BYTE ML_deltaCode = 36; -+ const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; -+ price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode] + 1); -+ } -+ -+ return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; -+} -+ -+ZSTD_STATIC void ZSTD_updatePrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength) -+{ -+ U32 u; -+ -+ /* literals */ -+ seqStorePtr->litSum += litLength * ZSTD_LITFREQ_ADD; -+ for (u = 0; u < litLength; u++) -+ seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; -+ -+ /* literal Length */ -+ { -+ const BYTE LL_deltaCode = 19; -+ const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; -+ seqStorePtr->litLengthFreq[llCode]++; -+ seqStorePtr->litLengthSum++; -+ } -+ -+ /* match offset */ -+ { -+ BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1); -+ seqStorePtr->offCodeSum++; -+ seqStorePtr->offCodeFreq[offCode]++; -+ } -+ -+ /* match Length */ -+ { -+ const BYTE ML_deltaCode = 36; -+ const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; -+ seqStorePtr->matchLengthFreq[mlCode]++; -+ seqStorePtr->matchLengthSum++; -+ } -+ -+ ZSTD_setLog2Prices(seqStorePtr); -+} -+ -+#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ -+ { \ -+ while (last_pos < pos) { \ -+ opt[last_pos + 1].price = ZSTD_MAX_PRICE; \ -+ last_pos++; \ -+ } \ -+ opt[pos].mlen = mlen_; \ -+ opt[pos].off = offset_; \ -+ opt[pos].litlen = litlen_; \ -+ opt[pos].price = price_; \ -+ } -+ -+/* Update hashTable3 up to ip (excluded) -+ Assumption : always within prefix (i.e. not within extDict) */ -+FORCE_INLINE -+U32 ZSTD_insertAndFindFirstIndexHash3(ZSTD_CCtx *zc, const BYTE *ip) -+{ -+ U32 *const hashTable3 = zc->hashTable3; -+ U32 const hashLog3 = zc->hashLog3; -+ const BYTE *const base = zc->base; -+ U32 idx = zc->nextToUpdate3; -+ const U32 target = zc->nextToUpdate3 = (U32)(ip - base); -+ const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3); -+ -+ while (idx < target) { -+ hashTable3[ZSTD_hash3Ptr(base + idx, hashLog3)] = idx; -+ idx++; -+ } -+ -+ return hashTable3[hash3]; -+} -+ -+/*-************************************* -+* Binary Tree search -+***************************************/ -+static U32 ZSTD_insertBtAndGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, U32 nbCompares, const U32 mls, U32 extDict, -+ ZSTD_match_t *matches, const U32 minMatchLen) -+{ -+ const BYTE *const base = zc->base; -+ const U32 curr = (U32)(ip - base); -+ const U32 hashLog = zc->params.cParams.hashLog; -+ const size_t h = ZSTD_hashPtr(ip, hashLog, mls); -+ U32 *const hashTable = zc->hashTable; -+ U32 matchIndex = hashTable[h]; -+ U32 *const bt = zc->chainTable; -+ const U32 btLog = zc->params.cParams.chainLog - 1; -+ const U32 btMask = (1U << btLog) - 1; -+ size_t commonLengthSmaller = 0, commonLengthLarger = 0; -+ const BYTE *const dictBase = zc->dictBase; -+ const U32 dictLimit = zc->dictLimit; -+ const BYTE *const dictEnd = dictBase + dictLimit; -+ const BYTE *const prefixStart = base + dictLimit; -+ const U32 btLow = btMask >= curr ? 0 : curr - btMask; -+ const U32 windowLow = zc->lowLimit; -+ U32 *smallerPtr = bt + 2 * (curr & btMask); -+ U32 *largerPtr = bt + 2 * (curr & btMask) + 1; -+ U32 matchEndIdx = curr + 8; -+ U32 dummy32; /* to be nullified at the end */ -+ U32 mnum = 0; -+ -+ const U32 minMatch = (mls == 3) ? 3 : 4; -+ size_t bestLength = minMatchLen - 1; -+ -+ if (minMatch == 3) { /* HC3 match finder */ -+ U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(zc, ip); -+ if (matchIndex3 > windowLow && (curr - matchIndex3 < (1 << 18))) { -+ const BYTE *match; -+ size_t currMl = 0; -+ if ((!extDict) || matchIndex3 >= dictLimit) { -+ match = base + matchIndex3; -+ if (match[bestLength] == ip[bestLength]) -+ currMl = ZSTD_count(ip, match, iLimit); -+ } else { -+ match = dictBase + matchIndex3; -+ if (ZSTD_readMINMATCH(match, MINMATCH) == -+ ZSTD_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ -+ currMl = ZSTD_count_2segments(ip + MINMATCH, match + MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; -+ } -+ -+ /* save best solution */ -+ if (currMl > bestLength) { -+ bestLength = currMl; -+ matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex3; -+ matches[mnum].len = (U32)currMl; -+ mnum++; -+ if (currMl > ZSTD_OPT_NUM) -+ goto update; -+ if (ip + currMl == iLimit) -+ goto update; /* best possible, and avoid read overflow*/ -+ } -+ } -+ } -+ -+ hashTable[h] = curr; /* Update Hash Table */ -+ -+ while (nbCompares-- && (matchIndex > windowLow)) { -+ U32 *nextPtr = bt + 2 * (matchIndex & btMask); -+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ -+ const BYTE *match; -+ -+ if ((!extDict) || (matchIndex + matchLength >= dictLimit)) { -+ match = base + matchIndex; -+ if (match[matchLength] == ip[matchLength]) { -+ matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iLimit) + 1; -+ } -+ } else { -+ match = dictBase + matchIndex; -+ matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iLimit, dictEnd, prefixStart); -+ if (matchIndex + matchLength >= dictLimit) -+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ -+ } -+ -+ if (matchLength > bestLength) { -+ if (matchLength > matchEndIdx - matchIndex) -+ matchEndIdx = matchIndex + (U32)matchLength; -+ bestLength = matchLength; -+ matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex; -+ matches[mnum].len = (U32)matchLength; -+ mnum++; -+ if (matchLength > ZSTD_OPT_NUM) -+ break; -+ if (ip + matchLength == iLimit) /* equal : no way to know if inf or sup */ -+ break; /* drop, to guarantee consistency (miss a little bit of compression) */ -+ } -+ -+ if (match[matchLength] < ip[matchLength]) { -+ /* match is smaller than curr */ -+ *smallerPtr = matchIndex; /* update smaller idx */ -+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ -+ if (matchIndex <= btLow) { -+ smallerPtr = &dummy32; -+ break; -+ } /* beyond tree size, stop the search */ -+ smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ -+ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ -+ } else { -+ /* match is larger than curr */ -+ *largerPtr = matchIndex; -+ commonLengthLarger = matchLength; -+ if (matchIndex <= btLow) { -+ largerPtr = &dummy32; -+ break; -+ } /* beyond tree size, stop the search */ -+ largerPtr = nextPtr; -+ matchIndex = nextPtr[0]; -+ } -+ } -+ -+ *smallerPtr = *largerPtr = 0; -+ -+update: -+ zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1; -+ return mnum; -+} -+ -+/** Tree updater, providing best match */ -+static U32 ZSTD_BtGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls, ZSTD_match_t *matches, -+ const U32 minMatchLen) -+{ -+ if (ip < zc->base + zc->nextToUpdate) -+ return 0; /* skipped area */ -+ ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); -+ return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen); -+} -+ -+static U32 ZSTD_BtGetAllMatches_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */ -+ const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, -+ ZSTD_match_t *matches, const U32 minMatchLen) -+{ -+ switch (matchLengthSearch) { -+ case 3: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); -+ default: -+ case 4: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); -+ case 5: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); -+ case 7: -+ case 6: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); -+ } -+} -+ -+/** Tree updater, providing best match */ -+static U32 ZSTD_BtGetAllMatches_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls, -+ ZSTD_match_t *matches, const U32 minMatchLen) -+{ -+ if (ip < zc->base + zc->nextToUpdate) -+ return 0; /* skipped area */ -+ ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); -+ return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen); -+} -+ -+static U32 ZSTD_BtGetAllMatches_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */ -+ const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, -+ ZSTD_match_t *matches, const U32 minMatchLen) -+{ -+ switch (matchLengthSearch) { -+ case 3: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); -+ default: -+ case 4: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); -+ case 5: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); -+ case 7: -+ case 6: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); -+ } -+} -+ -+/*-******************************* -+* Optimal parser -+*********************************/ -+FORCE_INLINE -+void ZSTD_compressBlock_opt_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra) -+{ -+ seqStore_t *seqStorePtr = &(ctx->seqStore); -+ const BYTE *const istart = (const BYTE *)src; -+ const BYTE *ip = istart; -+ const BYTE *anchor = istart; -+ const BYTE *const iend = istart + srcSize; -+ const BYTE *const ilimit = iend - 8; -+ const BYTE *const base = ctx->base; -+ const BYTE *const prefixStart = base + ctx->dictLimit; -+ -+ const U32 maxSearches = 1U << ctx->params.cParams.searchLog; -+ const U32 sufficient_len = ctx->params.cParams.targetLength; -+ const U32 mls = ctx->params.cParams.searchLength; -+ const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; -+ -+ ZSTD_optimal_t *opt = seqStorePtr->priceTable; -+ ZSTD_match_t *matches = seqStorePtr->matchTable; -+ const BYTE *inr; -+ U32 offset, rep[ZSTD_REP_NUM]; -+ -+ /* init */ -+ ctx->nextToUpdate3 = ctx->nextToUpdate; -+ ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize); -+ ip += (ip == prefixStart); -+ { -+ U32 i; -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ rep[i] = ctx->rep[i]; -+ } -+ -+ /* Match Loop */ -+ while (ip < ilimit) { -+ U32 cur, match_num, last_pos, litlen, price; -+ U32 u, mlen, best_mlen, best_off, litLength; -+ memset(opt, 0, sizeof(ZSTD_optimal_t)); -+ last_pos = 0; -+ litlen = (U32)(ip - anchor); -+ -+ /* check repCode */ -+ { -+ U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor); -+ for (i = (ip == anchor); i < last_i; i++) { -+ const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i]; -+ if ((repCur > 0) && (repCur < (S32)(ip - prefixStart)) && -+ (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) { -+ mlen = (U32)ZSTD_count(ip + minMatch, ip + minMatch - repCur, iend) + minMatch; -+ if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { -+ best_mlen = mlen; -+ best_off = i; -+ cur = 0; -+ last_pos = 1; -+ goto _storeSequence; -+ } -+ best_off = i - (ip == anchor); -+ do { -+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); -+ if (mlen > last_pos || price < opt[mlen].price) -+ SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ -+ mlen--; -+ } while (mlen >= minMatch); -+ } -+ } -+ } -+ -+ match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch); -+ -+ if (!last_pos && !match_num) { -+ ip++; -+ continue; -+ } -+ -+ if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) { -+ best_mlen = matches[match_num - 1].len; -+ best_off = matches[match_num - 1].off; -+ cur = 0; -+ last_pos = 1; -+ goto _storeSequence; -+ } -+ -+ /* set prices using matches at position = 0 */ -+ best_mlen = (last_pos) ? last_pos : minMatch; -+ for (u = 0; u < match_num; u++) { -+ mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen; -+ best_mlen = matches[u].len; -+ while (mlen <= best_mlen) { -+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); -+ if (mlen > last_pos || price < opt[mlen].price) -+ SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ -+ mlen++; -+ } -+ } -+ -+ if (last_pos < minMatch) { -+ ip++; -+ continue; -+ } -+ -+ /* initialize opt[0] */ -+ { -+ U32 i; -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ opt[0].rep[i] = rep[i]; -+ } -+ opt[0].mlen = 1; -+ opt[0].litlen = litlen; -+ -+ /* check further positions */ -+ for (cur = 1; cur <= last_pos; cur++) { -+ inr = ip + cur; -+ -+ if (opt[cur - 1].mlen == 1) { -+ litlen = opt[cur - 1].litlen + 1; -+ if (cur > litlen) { -+ price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen); -+ } else -+ price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); -+ } else { -+ litlen = 1; -+ price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1); -+ } -+ -+ if (cur > last_pos || price <= opt[cur].price) -+ SET_PRICE(cur, 1, 0, litlen, price); -+ -+ if (cur == last_pos) -+ break; -+ -+ if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ -+ continue; -+ -+ mlen = opt[cur].mlen; -+ if (opt[cur].off > ZSTD_REP_MOVE_OPT) { -+ opt[cur].rep[2] = opt[cur - mlen].rep[1]; -+ opt[cur].rep[1] = opt[cur - mlen].rep[0]; -+ opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; -+ } else { -+ opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2]; -+ opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1]; -+ opt[cur].rep[0] = -+ ((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]); -+ } -+ -+ best_mlen = minMatch; -+ { -+ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); -+ for (i = (opt[cur].mlen != 1); i < last_i; i++) { /* check rep */ -+ const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; -+ if ((repCur > 0) && (repCur < (S32)(inr - prefixStart)) && -+ (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) { -+ mlen = (U32)ZSTD_count(inr + minMatch, inr + minMatch - repCur, iend) + minMatch; -+ -+ if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { -+ best_mlen = mlen; -+ best_off = i; -+ last_pos = cur + 1; -+ goto _storeSequence; -+ } -+ -+ best_off = i - (opt[cur].mlen != 1); -+ if (mlen > best_mlen) -+ best_mlen = mlen; -+ -+ do { -+ if (opt[cur].mlen == 1) { -+ litlen = opt[cur].litlen; -+ if (cur > litlen) { -+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen, -+ best_off, mlen - MINMATCH, ultra); -+ } else -+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); -+ } else { -+ litlen = 0; -+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra); -+ } -+ -+ if (cur + mlen > last_pos || price <= opt[cur + mlen].price) -+ SET_PRICE(cur + mlen, mlen, i, litlen, price); -+ mlen--; -+ } while (mlen >= minMatch); -+ } -+ } -+ } -+ -+ match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen); -+ -+ if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) { -+ best_mlen = matches[match_num - 1].len; -+ best_off = matches[match_num - 1].off; -+ last_pos = cur + 1; -+ goto _storeSequence; -+ } -+ -+ /* set prices using matches at position = cur */ -+ for (u = 0; u < match_num; u++) { -+ mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen; -+ best_mlen = matches[u].len; -+ -+ while (mlen <= best_mlen) { -+ if (opt[cur].mlen == 1) { -+ litlen = opt[cur].litlen; -+ if (cur > litlen) -+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen, -+ matches[u].off - 1, mlen - MINMATCH, ultra); -+ else -+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); -+ } else { -+ litlen = 0; -+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra); -+ } -+ -+ if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) -+ SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); -+ -+ mlen++; -+ } -+ } -+ } -+ -+ best_mlen = opt[last_pos].mlen; -+ best_off = opt[last_pos].off; -+ cur = last_pos - best_mlen; -+ -+ /* store sequence */ -+_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ -+ opt[0].mlen = 1; -+ -+ while (1) { -+ mlen = opt[cur].mlen; -+ offset = opt[cur].off; -+ opt[cur].mlen = best_mlen; -+ opt[cur].off = best_off; -+ best_mlen = mlen; -+ best_off = offset; -+ if (mlen > cur) -+ break; -+ cur -= mlen; -+ } -+ -+ for (u = 0; u <= last_pos;) { -+ u += opt[u].mlen; -+ } -+ -+ for (cur = 0; cur < last_pos;) { -+ mlen = opt[cur].mlen; -+ if (mlen == 1) { -+ ip++; -+ cur++; -+ continue; -+ } -+ offset = opt[cur].off; -+ cur += mlen; -+ litLength = (U32)(ip - anchor); -+ -+ if (offset > ZSTD_REP_MOVE_OPT) { -+ rep[2] = rep[1]; -+ rep[1] = rep[0]; -+ rep[0] = offset - ZSTD_REP_MOVE_OPT; -+ offset--; -+ } else { -+ if (offset != 0) { -+ best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); -+ if (offset != 1) -+ rep[2] = rep[1]; -+ rep[1] = rep[0]; -+ rep[0] = best_off; -+ } -+ if (litLength == 0) -+ offset--; -+ } -+ -+ ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); -+ ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); -+ anchor = ip = ip + mlen; -+ } -+ } /* for (cur=0; cur < last_pos; ) */ -+ -+ /* Save reps for next block */ -+ { -+ int i; -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ ctx->repToConfirm[i] = rep[i]; -+ } -+ -+ /* Last Literals */ -+ { -+ size_t const lastLLSize = iend - anchor; -+ memcpy(seqStorePtr->lit, anchor, lastLLSize); -+ seqStorePtr->lit += lastLLSize; -+ } -+} -+ -+FORCE_INLINE -+void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra) -+{ -+ seqStore_t *seqStorePtr = &(ctx->seqStore); -+ const BYTE *const istart = (const BYTE *)src; -+ const BYTE *ip = istart; -+ const BYTE *anchor = istart; -+ const BYTE *const iend = istart + srcSize; -+ const BYTE *const ilimit = iend - 8; -+ const BYTE *const base = ctx->base; -+ const U32 lowestIndex = ctx->lowLimit; -+ const U32 dictLimit = ctx->dictLimit; -+ const BYTE *const prefixStart = base + dictLimit; -+ const BYTE *const dictBase = ctx->dictBase; -+ const BYTE *const dictEnd = dictBase + dictLimit; -+ -+ const U32 maxSearches = 1U << ctx->params.cParams.searchLog; -+ const U32 sufficient_len = ctx->params.cParams.targetLength; -+ const U32 mls = ctx->params.cParams.searchLength; -+ const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; -+ -+ ZSTD_optimal_t *opt = seqStorePtr->priceTable; -+ ZSTD_match_t *matches = seqStorePtr->matchTable; -+ const BYTE *inr; -+ -+ /* init */ -+ U32 offset, rep[ZSTD_REP_NUM]; -+ { -+ U32 i; -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ rep[i] = ctx->rep[i]; -+ } -+ -+ ctx->nextToUpdate3 = ctx->nextToUpdate; -+ ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize); -+ ip += (ip == prefixStart); -+ -+ /* Match Loop */ -+ while (ip < ilimit) { -+ U32 cur, match_num, last_pos, litlen, price; -+ U32 u, mlen, best_mlen, best_off, litLength; -+ U32 curr = (U32)(ip - base); -+ memset(opt, 0, sizeof(ZSTD_optimal_t)); -+ last_pos = 0; -+ opt[0].litlen = (U32)(ip - anchor); -+ -+ /* check repCode */ -+ { -+ U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor); -+ for (i = (ip == anchor); i < last_i; i++) { -+ const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i]; -+ const U32 repIndex = (U32)(curr - repCur); -+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; -+ const BYTE *const repMatch = repBase + repIndex; -+ if ((repCur > 0 && repCur <= (S32)curr) && -+ (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ -+ && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) { -+ /* repcode detected we should take it */ -+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; -+ mlen = (U32)ZSTD_count_2segments(ip + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch; -+ -+ if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { -+ best_mlen = mlen; -+ best_off = i; -+ cur = 0; -+ last_pos = 1; -+ goto _storeSequence; -+ } -+ -+ best_off = i - (ip == anchor); -+ litlen = opt[0].litlen; -+ do { -+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); -+ if (mlen > last_pos || price < opt[mlen].price) -+ SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ -+ mlen--; -+ } while (mlen >= minMatch); -+ } -+ } -+ } -+ -+ match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */ -+ -+ if (!last_pos && !match_num) { -+ ip++; -+ continue; -+ } -+ -+ { -+ U32 i; -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ opt[0].rep[i] = rep[i]; -+ } -+ opt[0].mlen = 1; -+ -+ if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) { -+ best_mlen = matches[match_num - 1].len; -+ best_off = matches[match_num - 1].off; -+ cur = 0; -+ last_pos = 1; -+ goto _storeSequence; -+ } -+ -+ best_mlen = (last_pos) ? last_pos : minMatch; -+ -+ /* set prices using matches at position = 0 */ -+ for (u = 0; u < match_num; u++) { -+ mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen; -+ best_mlen = matches[u].len; -+ litlen = opt[0].litlen; -+ while (mlen <= best_mlen) { -+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); -+ if (mlen > last_pos || price < opt[mlen].price) -+ SET_PRICE(mlen, mlen, matches[u].off, litlen, price); -+ mlen++; -+ } -+ } -+ -+ if (last_pos < minMatch) { -+ ip++; -+ continue; -+ } -+ -+ /* check further positions */ -+ for (cur = 1; cur <= last_pos; cur++) { -+ inr = ip + cur; -+ -+ if (opt[cur - 1].mlen == 1) { -+ litlen = opt[cur - 1].litlen + 1; -+ if (cur > litlen) { -+ price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen); -+ } else -+ price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); -+ } else { -+ litlen = 1; -+ price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1); -+ } -+ -+ if (cur > last_pos || price <= opt[cur].price) -+ SET_PRICE(cur, 1, 0, litlen, price); -+ -+ if (cur == last_pos) -+ break; -+ -+ if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ -+ continue; -+ -+ mlen = opt[cur].mlen; -+ if (opt[cur].off > ZSTD_REP_MOVE_OPT) { -+ opt[cur].rep[2] = opt[cur - mlen].rep[1]; -+ opt[cur].rep[1] = opt[cur - mlen].rep[0]; -+ opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; -+ } else { -+ opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2]; -+ opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1]; -+ opt[cur].rep[0] = -+ ((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]); -+ } -+ -+ best_mlen = minMatch; -+ { -+ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); -+ for (i = (mlen != 1); i < last_i; i++) { -+ const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; -+ const U32 repIndex = (U32)(curr + cur - repCur); -+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; -+ const BYTE *const repMatch = repBase + repIndex; -+ if ((repCur > 0 && repCur <= (S32)(curr + cur)) && -+ (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ -+ && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) { -+ /* repcode detected */ -+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; -+ mlen = (U32)ZSTD_count_2segments(inr + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch; -+ -+ if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { -+ best_mlen = mlen; -+ best_off = i; -+ last_pos = cur + 1; -+ goto _storeSequence; -+ } -+ -+ best_off = i - (opt[cur].mlen != 1); -+ if (mlen > best_mlen) -+ best_mlen = mlen; -+ -+ do { -+ if (opt[cur].mlen == 1) { -+ litlen = opt[cur].litlen; -+ if (cur > litlen) { -+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen, -+ best_off, mlen - MINMATCH, ultra); -+ } else -+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); -+ } else { -+ litlen = 0; -+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra); -+ } -+ -+ if (cur + mlen > last_pos || price <= opt[cur + mlen].price) -+ SET_PRICE(cur + mlen, mlen, i, litlen, price); -+ mlen--; -+ } while (mlen >= minMatch); -+ } -+ } -+ } -+ -+ match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); -+ -+ if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) { -+ best_mlen = matches[match_num - 1].len; -+ best_off = matches[match_num - 1].off; -+ last_pos = cur + 1; -+ goto _storeSequence; -+ } -+ -+ /* set prices using matches at position = cur */ -+ for (u = 0; u < match_num; u++) { -+ mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen; -+ best_mlen = matches[u].len; -+ -+ while (mlen <= best_mlen) { -+ if (opt[cur].mlen == 1) { -+ litlen = opt[cur].litlen; -+ if (cur > litlen) -+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen, -+ matches[u].off - 1, mlen - MINMATCH, ultra); -+ else -+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); -+ } else { -+ litlen = 0; -+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra); -+ } -+ -+ if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) -+ SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); -+ -+ mlen++; -+ } -+ } -+ } /* for (cur = 1; cur <= last_pos; cur++) */ -+ -+ best_mlen = opt[last_pos].mlen; -+ best_off = opt[last_pos].off; -+ cur = last_pos - best_mlen; -+ -+ /* store sequence */ -+_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ -+ opt[0].mlen = 1; -+ -+ while (1) { -+ mlen = opt[cur].mlen; -+ offset = opt[cur].off; -+ opt[cur].mlen = best_mlen; -+ opt[cur].off = best_off; -+ best_mlen = mlen; -+ best_off = offset; -+ if (mlen > cur) -+ break; -+ cur -= mlen; -+ } -+ -+ for (u = 0; u <= last_pos;) { -+ u += opt[u].mlen; -+ } -+ -+ for (cur = 0; cur < last_pos;) { -+ mlen = opt[cur].mlen; -+ if (mlen == 1) { -+ ip++; -+ cur++; -+ continue; -+ } -+ offset = opt[cur].off; -+ cur += mlen; -+ litLength = (U32)(ip - anchor); -+ -+ if (offset > ZSTD_REP_MOVE_OPT) { -+ rep[2] = rep[1]; -+ rep[1] = rep[0]; -+ rep[0] = offset - ZSTD_REP_MOVE_OPT; -+ offset--; -+ } else { -+ if (offset != 0) { -+ best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); -+ if (offset != 1) -+ rep[2] = rep[1]; -+ rep[1] = rep[0]; -+ rep[0] = best_off; -+ } -+ -+ if (litLength == 0) -+ offset--; -+ } -+ -+ ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); -+ ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); -+ anchor = ip = ip + mlen; -+ } -+ } /* for (cur=0; cur < last_pos; ) */ -+ -+ /* Save reps for next block */ -+ { -+ int i; -+ for (i = 0; i < ZSTD_REP_NUM; i++) -+ ctx->repToConfirm[i] = rep[i]; -+ } -+ -+ /* Last Literals */ -+ { -+ size_t lastLLSize = iend - anchor; -+ memcpy(seqStorePtr->lit, anchor, lastLLSize); -+ seqStorePtr->lit += lastLLSize; -+ } -+} -+ -+#endif /* ZSTD_OPT_H_91842398743 */ --- -2.9.5 diff --git a/contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch b/contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch deleted file mode 100644 index edc7839a..00000000 --- a/contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch +++ /dev/null @@ -1,740 +0,0 @@ -From 8a9dddfbf6551afea73911e367dd4be64d62b9fd Mon Sep 17 00:00:00 2001 -From: Nick Terrell <terrelln@fb.com> -Date: Mon, 17 Jul 2017 17:08:39 -0700 -Subject: [PATCH v5 3/5] btrfs: Add zstd support - -Add zstd compression and decompression support to BtrFS. zstd at its -fastest level compresses almost as well as zlib, while offering much -faster compression and decompression, approaching lzo speeds. - -I benchmarked btrfs with zstd compression against no compression, lzo -compression, and zlib compression. I benchmarked two scenarios. Copying -a set of files to btrfs, and then reading the files. Copying a tarball -to btrfs, extracting it to btrfs, and then reading the extracted files. -After every operation, I call `sync` and include the sync time. -Between every pair of operations I unmount and remount the filesystem -to avoid caching. The benchmark files can be found in the upstream -zstd source repository under -`contrib/linux-kernel/{btrfs-benchmark.sh,btrfs-extract-benchmark.sh}` -[1] [2]. - -I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM. -The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor, -16 GB of RAM, and a SSD. - -The first compression benchmark is copying 10 copies of the unzipped -Silesia corpus [3] into a BtrFS filesystem mounted with -`-o compress-force=Method`. The decompression benchmark times how long -it takes to `tar` all 10 copies into `/dev/null`. The compression ratio is -measured by comparing the output of `df` and `du`. See the benchmark file -[1] for details. I benchmarked multiple zstd compression levels, although -the patch uses zstd level 1. - -| Method | Ratio | Compression MB/s | Decompression speed | -|---------|-------|------------------|---------------------| -| None | 0.99 | 504 | 686 | -| lzo | 1.66 | 398 | 442 | -| zlib | 2.58 | 65 | 241 | -| zstd 1 | 2.57 | 260 | 383 | -| zstd 3 | 2.71 | 174 | 408 | -| zstd 6 | 2.87 | 70 | 398 | -| zstd 9 | 2.92 | 43 | 406 | -| zstd 12 | 2.93 | 21 | 408 | -| zstd 15 | 3.01 | 11 | 354 | - -The next benchmark first copies `linux-4.11.6.tar` [4] to btrfs. Then it -measures the compression ratio, extracts the tar, and deletes the tar. -Then it measures the compression ratio again, and `tar`s the extracted -files into `/dev/null`. See the benchmark file [2] for details. - -| Method | Tar Ratio | Extract Ratio | Copy (s) | Extract (s)| Read (s) | -|--------|-----------|---------------|----------|------------|----------| -| None | 0.97 | 0.78 | 0.981 | 5.501 | 8.807 | -| lzo | 2.06 | 1.38 | 1.631 | 8.458 | 8.585 | -| zlib | 3.40 | 1.86 | 7.750 | 21.544 | 11.744 | -| zstd 1 | 3.57 | 1.85 | 2.579 | 11.479 | 9.389 | - -[1] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-benchmark.sh -[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-extract-benchmark.sh -[3] http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia -[4] https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.11.6.tar.xz - -zstd source repository: https://github.com/facebook/zstd - -Signed-off-by: Nick Terrell <terrelln@fb.com> ---- -v2 -> v3: -- Port upstream BtrFS commits e1ddce71d6, 389a6cfc2a, and 6acafd1eff -- Change default compression level for BtrFS to 3 - -v3 -> v4: -- Add missing includes, which fixes the aarch64 build -- Fix minor linter warnings - - fs/btrfs/Kconfig | 2 + - fs/btrfs/Makefile | 2 +- - fs/btrfs/compression.c | 1 + - fs/btrfs/compression.h | 6 +- - fs/btrfs/ctree.h | 1 + - fs/btrfs/disk-io.c | 2 + - fs/btrfs/ioctl.c | 6 +- - fs/btrfs/props.c | 6 + - fs/btrfs/super.c | 12 +- - fs/btrfs/sysfs.c | 2 + - fs/btrfs/zstd.c | 432 +++++++++++++++++++++++++++++++++++++++++++++ - include/uapi/linux/btrfs.h | 8 +- - 12 files changed, 468 insertions(+), 12 deletions(-) - create mode 100644 fs/btrfs/zstd.c - -diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig -index 80e9c18..a26c63b 100644 ---- a/fs/btrfs/Kconfig -+++ b/fs/btrfs/Kconfig -@@ -6,6 +6,8 @@ config BTRFS_FS - select ZLIB_DEFLATE - select LZO_COMPRESS - select LZO_DECOMPRESS -+ select ZSTD_COMPRESS -+ select ZSTD_DECOMPRESS - select RAID6_PQ - select XOR_BLOCKS - select SRCU -diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile -index 128ce17..962a95a 100644 ---- a/fs/btrfs/Makefile -+++ b/fs/btrfs/Makefile -@@ -6,7 +6,7 @@ btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \ - transaction.o inode.o file.o tree-defrag.o \ - extent_map.o sysfs.o struct-funcs.o xattr.o ordered-data.o \ - extent_io.o volumes.o async-thread.o ioctl.o locking.o orphan.o \ -- export.o tree-log.o free-space-cache.o zlib.o lzo.o \ -+ export.o tree-log.o free-space-cache.o zlib.o lzo.o zstd.o \ - compression.o delayed-ref.o relocation.o delayed-inode.o scrub.o \ - reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \ - uuid-tree.o props.o hash.o free-space-tree.o -diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c -index d2ef9ac..4ff42d1 100644 ---- a/fs/btrfs/compression.c -+++ b/fs/btrfs/compression.c -@@ -704,6 +704,7 @@ static struct { - static const struct btrfs_compress_op * const btrfs_compress_op[] = { - &btrfs_zlib_compress, - &btrfs_lzo_compress, -+ &btrfs_zstd_compress, - }; - - void __init btrfs_init_compress(void) -diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h -index 87f6d33..2269e00 100644 ---- a/fs/btrfs/compression.h -+++ b/fs/btrfs/compression.h -@@ -99,8 +99,9 @@ enum btrfs_compression_type { - BTRFS_COMPRESS_NONE = 0, - BTRFS_COMPRESS_ZLIB = 1, - BTRFS_COMPRESS_LZO = 2, -- BTRFS_COMPRESS_TYPES = 2, -- BTRFS_COMPRESS_LAST = 3, -+ BTRFS_COMPRESS_ZSTD = 3, -+ BTRFS_COMPRESS_TYPES = 3, -+ BTRFS_COMPRESS_LAST = 4, - }; - - struct btrfs_compress_op { -@@ -128,5 +129,6 @@ struct btrfs_compress_op { - - extern const struct btrfs_compress_op btrfs_zlib_compress; - extern const struct btrfs_compress_op btrfs_lzo_compress; -+extern const struct btrfs_compress_op btrfs_zstd_compress; - - #endif -diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h -index 3f3eb7b..845d77c 100644 ---- a/fs/btrfs/ctree.h -+++ b/fs/btrfs/ctree.h -@@ -270,6 +270,7 @@ struct btrfs_super_block { - BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \ - BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \ - BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \ -+ BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD | \ - BTRFS_FEATURE_INCOMPAT_RAID56 | \ - BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \ - BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \ -diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c -index 080e2eb..04632f4 100644 ---- a/fs/btrfs/disk-io.c -+++ b/fs/btrfs/disk-io.c -@@ -2828,6 +2828,8 @@ int open_ctree(struct super_block *sb, - features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF; - if (fs_info->compress_type == BTRFS_COMPRESS_LZO) - features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO; -+ else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD) -+ features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD; - - if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA) - btrfs_info(fs_info, "has skinny extents"); -diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c -index fa1b78c..b9963d9 100644 ---- a/fs/btrfs/ioctl.c -+++ b/fs/btrfs/ioctl.c -@@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg) - - if (fs_info->compress_type == BTRFS_COMPRESS_LZO) - comp = "lzo"; -- else -+ else if (fs_info->compress_type == BTRFS_COMPRESS_ZLIB) - comp = "zlib"; -+ else -+ comp = "zstd"; - ret = btrfs_set_prop(inode, "btrfs.compression", - comp, strlen(comp), 0); - if (ret) -@@ -1466,6 +1468,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file, - - if (range->compress_type == BTRFS_COMPRESS_LZO) { - btrfs_set_fs_incompat(fs_info, COMPRESS_LZO); -+ } else if (range->compress_type == BTRFS_COMPRESS_ZSTD) { -+ btrfs_set_fs_incompat(fs_info, COMPRESS_ZSTD); - } - - ret = defrag_count; -diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c -index 4b23ae5..20631e9 100644 ---- a/fs/btrfs/props.c -+++ b/fs/btrfs/props.c -@@ -390,6 +390,8 @@ static int prop_compression_validate(const char *value, size_t len) - return 0; - else if (!strncmp("zlib", value, len)) - return 0; -+ else if (!strncmp("zstd", value, len)) -+ return 0; - - return -EINVAL; - } -@@ -412,6 +414,8 @@ static int prop_compression_apply(struct inode *inode, - type = BTRFS_COMPRESS_LZO; - else if (!strncmp("zlib", value, len)) - type = BTRFS_COMPRESS_ZLIB; -+ else if (!strncmp("zstd", value, len)) -+ type = BTRFS_COMPRESS_ZSTD; - else - return -EINVAL; - -@@ -429,6 +433,8 @@ static const char *prop_compression_extract(struct inode *inode) - return "zlib"; - case BTRFS_COMPRESS_LZO: - return "lzo"; -+ case BTRFS_COMPRESS_ZSTD: -+ return "zstd"; - } - - return NULL; -diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c -index 12540b6..c370dea 100644 ---- a/fs/btrfs/super.c -+++ b/fs/btrfs/super.c -@@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options, - btrfs_clear_opt(info->mount_opt, NODATASUM); - btrfs_set_fs_incompat(info, COMPRESS_LZO); - no_compress = 0; -+ } else if (strcmp(args[0].from, "zstd") == 0) { -+ compress_type = "zstd"; -+ info->compress_type = BTRFS_COMPRESS_ZSTD; -+ btrfs_set_opt(info->mount_opt, COMPRESS); -+ btrfs_clear_opt(info->mount_opt, NODATACOW); -+ btrfs_clear_opt(info->mount_opt, NODATASUM); -+ btrfs_set_fs_incompat(info, COMPRESS_ZSTD); -+ no_compress = 0; - } else if (strncmp(args[0].from, "no", 2) == 0) { - compress_type = "no"; - btrfs_clear_opt(info->mount_opt, COMPRESS); -@@ -1227,8 +1235,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry) - if (btrfs_test_opt(info, COMPRESS)) { - if (info->compress_type == BTRFS_COMPRESS_ZLIB) - compress_type = "zlib"; -- else -+ else if (info->compress_type == BTRFS_COMPRESS_LZO) - compress_type = "lzo"; -+ else -+ compress_type = "zstd"; - if (btrfs_test_opt(info, FORCE_COMPRESS)) - seq_printf(seq, ",compress-force=%s", compress_type); - else -diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c -index c2d5f35..2b6d37c 100644 ---- a/fs/btrfs/sysfs.c -+++ b/fs/btrfs/sysfs.c -@@ -200,6 +200,7 @@ BTRFS_FEAT_ATTR_INCOMPAT(mixed_backref, MIXED_BACKREF); - BTRFS_FEAT_ATTR_INCOMPAT(default_subvol, DEFAULT_SUBVOL); - BTRFS_FEAT_ATTR_INCOMPAT(mixed_groups, MIXED_GROUPS); - BTRFS_FEAT_ATTR_INCOMPAT(compress_lzo, COMPRESS_LZO); -+BTRFS_FEAT_ATTR_INCOMPAT(compress_zstd, COMPRESS_ZSTD); - BTRFS_FEAT_ATTR_INCOMPAT(big_metadata, BIG_METADATA); - BTRFS_FEAT_ATTR_INCOMPAT(extended_iref, EXTENDED_IREF); - BTRFS_FEAT_ATTR_INCOMPAT(raid56, RAID56); -@@ -212,6 +213,7 @@ static struct attribute *btrfs_supported_feature_attrs[] = { - BTRFS_FEAT_ATTR_PTR(default_subvol), - BTRFS_FEAT_ATTR_PTR(mixed_groups), - BTRFS_FEAT_ATTR_PTR(compress_lzo), -+ BTRFS_FEAT_ATTR_PTR(compress_zstd), - BTRFS_FEAT_ATTR_PTR(big_metadata), - BTRFS_FEAT_ATTR_PTR(extended_iref), - BTRFS_FEAT_ATTR_PTR(raid56), -diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c -new file mode 100644 -index 0000000..607ce47 ---- /dev/null -+++ b/fs/btrfs/zstd.c -@@ -0,0 +1,432 @@ -+/* -+ * Copyright (c) 2016-present, Facebook, Inc. -+ * All rights reserved. -+ * -+ * This program is free software; you can redistribute it and/or -+ * modify it under the terms of the GNU General Public -+ * License v2 as published by the Free Software Foundation. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ * General Public License for more details. -+ */ -+#include <linux/bio.h> -+#include <linux/err.h> -+#include <linux/init.h> -+#include <linux/kernel.h> -+#include <linux/mm.h> -+#include <linux/pagemap.h> -+#include <linux/refcount.h> -+#include <linux/sched.h> -+#include <linux/slab.h> -+#include <linux/zstd.h> -+#include "compression.h" -+ -+#define ZSTD_BTRFS_MAX_WINDOWLOG 17 -+#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG) -+#define ZSTD_BTRFS_DEFAULT_LEVEL 3 -+ -+static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len) -+{ -+ ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL, -+ src_len, 0); -+ -+ if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG) -+ params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG; -+ WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT); -+ return params; -+} -+ -+struct workspace { -+ void *mem; -+ size_t size; -+ char *buf; -+ struct list_head list; -+}; -+ -+static void zstd_free_workspace(struct list_head *ws) -+{ -+ struct workspace *workspace = list_entry(ws, struct workspace, list); -+ -+ kvfree(workspace->mem); -+ kfree(workspace->buf); -+ kfree(workspace); -+} -+ -+static struct list_head *zstd_alloc_workspace(void) -+{ -+ ZSTD_parameters params = -+ zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT); -+ struct workspace *workspace; -+ -+ workspace = kzalloc(sizeof(*workspace), GFP_KERNEL); -+ if (!workspace) -+ return ERR_PTR(-ENOMEM); -+ -+ workspace->size = max_t(size_t, -+ ZSTD_CStreamWorkspaceBound(params.cParams), -+ ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT)); -+ workspace->mem = kvmalloc(workspace->size, GFP_KERNEL); -+ workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL); -+ if (!workspace->mem || !workspace->buf) -+ goto fail; -+ -+ INIT_LIST_HEAD(&workspace->list); -+ -+ return &workspace->list; -+fail: -+ zstd_free_workspace(&workspace->list); -+ return ERR_PTR(-ENOMEM); -+} -+ -+static int zstd_compress_pages(struct list_head *ws, -+ struct address_space *mapping, -+ u64 start, -+ struct page **pages, -+ unsigned long *out_pages, -+ unsigned long *total_in, -+ unsigned long *total_out) -+{ -+ struct workspace *workspace = list_entry(ws, struct workspace, list); -+ ZSTD_CStream *stream; -+ int ret = 0; -+ int nr_pages = 0; -+ struct page *in_page = NULL; /* The current page to read */ -+ struct page *out_page = NULL; /* The current page to write to */ -+ ZSTD_inBuffer in_buf = { NULL, 0, 0 }; -+ ZSTD_outBuffer out_buf = { NULL, 0, 0 }; -+ unsigned long tot_in = 0; -+ unsigned long tot_out = 0; -+ unsigned long len = *total_out; -+ const unsigned long nr_dest_pages = *out_pages; -+ unsigned long max_out = nr_dest_pages * PAGE_SIZE; -+ ZSTD_parameters params = zstd_get_btrfs_parameters(len); -+ -+ *out_pages = 0; -+ *total_out = 0; -+ *total_in = 0; -+ -+ /* Initialize the stream */ -+ stream = ZSTD_initCStream(params, len, workspace->mem, -+ workspace->size); -+ if (!stream) { -+ pr_warn("BTRFS: ZSTD_initCStream failed\n"); -+ ret = -EIO; -+ goto out; -+ } -+ -+ /* map in the first page of input data */ -+ in_page = find_get_page(mapping, start >> PAGE_SHIFT); -+ in_buf.src = kmap(in_page); -+ in_buf.pos = 0; -+ in_buf.size = min_t(size_t, len, PAGE_SIZE); -+ -+ -+ /* Allocate and map in the output buffer */ -+ out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); -+ if (out_page == NULL) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ pages[nr_pages++] = out_page; -+ out_buf.dst = kmap(out_page); -+ out_buf.pos = 0; -+ out_buf.size = min_t(size_t, max_out, PAGE_SIZE); -+ -+ while (1) { -+ size_t ret2; -+ -+ ret2 = ZSTD_compressStream(stream, &out_buf, &in_buf); -+ if (ZSTD_isError(ret2)) { -+ pr_debug("BTRFS: ZSTD_compressStream returned %d\n", -+ ZSTD_getErrorCode(ret2)); -+ ret = -EIO; -+ goto out; -+ } -+ -+ /* Check to see if we are making it bigger */ -+ if (tot_in + in_buf.pos > 8192 && -+ tot_in + in_buf.pos < -+ tot_out + out_buf.pos) { -+ ret = -E2BIG; -+ goto out; -+ } -+ -+ /* We've reached the end of our output range */ -+ if (out_buf.pos >= max_out) { -+ tot_out += out_buf.pos; -+ ret = -E2BIG; -+ goto out; -+ } -+ -+ /* Check if we need more output space */ -+ if (out_buf.pos == out_buf.size) { -+ tot_out += PAGE_SIZE; -+ max_out -= PAGE_SIZE; -+ kunmap(out_page); -+ if (nr_pages == nr_dest_pages) { -+ out_page = NULL; -+ ret = -E2BIG; -+ goto out; -+ } -+ out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); -+ if (out_page == NULL) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ pages[nr_pages++] = out_page; -+ out_buf.dst = kmap(out_page); -+ out_buf.pos = 0; -+ out_buf.size = min_t(size_t, max_out, PAGE_SIZE); -+ } -+ -+ /* We've reached the end of the input */ -+ if (in_buf.pos >= len) { -+ tot_in += in_buf.pos; -+ break; -+ } -+ -+ /* Check if we need more input */ -+ if (in_buf.pos == in_buf.size) { -+ tot_in += PAGE_SIZE; -+ kunmap(in_page); -+ put_page(in_page); -+ -+ start += PAGE_SIZE; -+ len -= PAGE_SIZE; -+ in_page = find_get_page(mapping, start >> PAGE_SHIFT); -+ in_buf.src = kmap(in_page); -+ in_buf.pos = 0; -+ in_buf.size = min_t(size_t, len, PAGE_SIZE); -+ } -+ } -+ while (1) { -+ size_t ret2; -+ -+ ret2 = ZSTD_endStream(stream, &out_buf); -+ if (ZSTD_isError(ret2)) { -+ pr_debug("BTRFS: ZSTD_endStream returned %d\n", -+ ZSTD_getErrorCode(ret2)); -+ ret = -EIO; -+ goto out; -+ } -+ if (ret2 == 0) { -+ tot_out += out_buf.pos; -+ break; -+ } -+ if (out_buf.pos >= max_out) { -+ tot_out += out_buf.pos; -+ ret = -E2BIG; -+ goto out; -+ } -+ -+ tot_out += PAGE_SIZE; -+ max_out -= PAGE_SIZE; -+ kunmap(out_page); -+ if (nr_pages == nr_dest_pages) { -+ out_page = NULL; -+ ret = -E2BIG; -+ goto out; -+ } -+ out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); -+ if (out_page == NULL) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ pages[nr_pages++] = out_page; -+ out_buf.dst = kmap(out_page); -+ out_buf.pos = 0; -+ out_buf.size = min_t(size_t, max_out, PAGE_SIZE); -+ } -+ -+ if (tot_out >= tot_in) { -+ ret = -E2BIG; -+ goto out; -+ } -+ -+ ret = 0; -+ *total_in = tot_in; -+ *total_out = tot_out; -+out: -+ *out_pages = nr_pages; -+ /* Cleanup */ -+ if (in_page) { -+ kunmap(in_page); -+ put_page(in_page); -+ } -+ if (out_page) -+ kunmap(out_page); -+ return ret; -+} -+ -+static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) -+{ -+ struct workspace *workspace = list_entry(ws, struct workspace, list); -+ struct page **pages_in = cb->compressed_pages; -+ u64 disk_start = cb->start; -+ struct bio *orig_bio = cb->orig_bio; -+ size_t srclen = cb->compressed_len; -+ ZSTD_DStream *stream; -+ int ret = 0; -+ unsigned long page_in_index = 0; -+ unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE); -+ unsigned long buf_start; -+ unsigned long total_out = 0; -+ ZSTD_inBuffer in_buf = { NULL, 0, 0 }; -+ ZSTD_outBuffer out_buf = { NULL, 0, 0 }; -+ -+ stream = ZSTD_initDStream( -+ ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size); -+ if (!stream) { -+ pr_debug("BTRFS: ZSTD_initDStream failed\n"); -+ ret = -EIO; -+ goto done; -+ } -+ -+ in_buf.src = kmap(pages_in[page_in_index]); -+ in_buf.pos = 0; -+ in_buf.size = min_t(size_t, srclen, PAGE_SIZE); -+ -+ out_buf.dst = workspace->buf; -+ out_buf.pos = 0; -+ out_buf.size = PAGE_SIZE; -+ -+ while (1) { -+ size_t ret2; -+ -+ ret2 = ZSTD_decompressStream(stream, &out_buf, &in_buf); -+ if (ZSTD_isError(ret2)) { -+ pr_debug("BTRFS: ZSTD_decompressStream returned %d\n", -+ ZSTD_getErrorCode(ret2)); -+ ret = -EIO; -+ goto done; -+ } -+ buf_start = total_out; -+ total_out += out_buf.pos; -+ out_buf.pos = 0; -+ -+ ret = btrfs_decompress_buf2page(out_buf.dst, buf_start, -+ total_out, disk_start, orig_bio); -+ if (ret == 0) -+ break; -+ -+ if (in_buf.pos >= srclen) -+ break; -+ -+ /* Check if we've hit the end of a frame */ -+ if (ret2 == 0) -+ break; -+ -+ if (in_buf.pos == in_buf.size) { -+ kunmap(pages_in[page_in_index++]); -+ if (page_in_index >= total_pages_in) { -+ in_buf.src = NULL; -+ ret = -EIO; -+ goto done; -+ } -+ srclen -= PAGE_SIZE; -+ in_buf.src = kmap(pages_in[page_in_index]); -+ in_buf.pos = 0; -+ in_buf.size = min_t(size_t, srclen, PAGE_SIZE); -+ } -+ } -+ ret = 0; -+ zero_fill_bio(orig_bio); -+done: -+ if (in_buf.src) -+ kunmap(pages_in[page_in_index]); -+ return ret; -+} -+ -+static int zstd_decompress(struct list_head *ws, unsigned char *data_in, -+ struct page *dest_page, -+ unsigned long start_byte, -+ size_t srclen, size_t destlen) -+{ -+ struct workspace *workspace = list_entry(ws, struct workspace, list); -+ ZSTD_DStream *stream; -+ int ret = 0; -+ size_t ret2; -+ ZSTD_inBuffer in_buf = { NULL, 0, 0 }; -+ ZSTD_outBuffer out_buf = { NULL, 0, 0 }; -+ unsigned long total_out = 0; -+ unsigned long pg_offset = 0; -+ char *kaddr; -+ -+ stream = ZSTD_initDStream( -+ ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size); -+ if (!stream) { -+ pr_warn("BTRFS: ZSTD_initDStream failed\n"); -+ ret = -EIO; -+ goto finish; -+ } -+ -+ destlen = min_t(size_t, destlen, PAGE_SIZE); -+ -+ in_buf.src = data_in; -+ in_buf.pos = 0; -+ in_buf.size = srclen; -+ -+ out_buf.dst = workspace->buf; -+ out_buf.pos = 0; -+ out_buf.size = PAGE_SIZE; -+ -+ ret2 = 1; -+ while (pg_offset < destlen && in_buf.pos < in_buf.size) { -+ unsigned long buf_start; -+ unsigned long buf_offset; -+ unsigned long bytes; -+ -+ /* Check if the frame is over and we still need more input */ -+ if (ret2 == 0) { -+ pr_debug("BTRFS: ZSTD_decompressStream ended early\n"); -+ ret = -EIO; -+ goto finish; -+ } -+ ret2 = ZSTD_decompressStream(stream, &out_buf, &in_buf); -+ if (ZSTD_isError(ret2)) { -+ pr_debug("BTRFS: ZSTD_decompressStream returned %d\n", -+ ZSTD_getErrorCode(ret2)); -+ ret = -EIO; -+ goto finish; -+ } -+ -+ buf_start = total_out; -+ total_out += out_buf.pos; -+ out_buf.pos = 0; -+ -+ if (total_out <= start_byte) -+ continue; -+ -+ if (total_out > start_byte && buf_start < start_byte) -+ buf_offset = start_byte - buf_start; -+ else -+ buf_offset = 0; -+ -+ bytes = min_t(unsigned long, destlen - pg_offset, -+ out_buf.size - buf_offset); -+ -+ kaddr = kmap_atomic(dest_page); -+ memcpy(kaddr + pg_offset, out_buf.dst + buf_offset, bytes); -+ kunmap_atomic(kaddr); -+ -+ pg_offset += bytes; -+ } -+ ret = 0; -+finish: -+ if (pg_offset < destlen) { -+ kaddr = kmap_atomic(dest_page); -+ memset(kaddr + pg_offset, 0, destlen - pg_offset); -+ kunmap_atomic(kaddr); -+ } -+ return ret; -+} -+ -+const struct btrfs_compress_op btrfs_zstd_compress = { -+ .alloc_workspace = zstd_alloc_workspace, -+ .free_workspace = zstd_free_workspace, -+ .compress_pages = zstd_compress_pages, -+ .decompress_bio = zstd_decompress_bio, -+ .decompress = zstd_decompress, -+}; -diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h -index 9aa74f3..378230c 100644 ---- a/include/uapi/linux/btrfs.h -+++ b/include/uapi/linux/btrfs.h -@@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args { - #define BTRFS_FEATURE_INCOMPAT_DEFAULT_SUBVOL (1ULL << 1) - #define BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS (1ULL << 2) - #define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO (1ULL << 3) --/* -- * some patches floated around with a second compression method -- * lets save that incompat here for when they do get in -- * Note we don't actually support it, we're just reserving the -- * number -- */ --#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZOv2 (1ULL << 4) -+#define BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD (1ULL << 4) - - /* - * older kernels tried to do bigger metadata blocks, but the --- -2.9.3 diff --git a/contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch b/contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch deleted file mode 100644 index 36cdf71d..00000000 --- a/contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch +++ /dev/null @@ -1,306 +0,0 @@ -From 46bf8f6d30d6ddf2446c110f122482b5e5e16933 Mon Sep 17 00:00:00 2001 -From: Sean Purcell <me@seanp.xyz> -Date: Mon, 17 Jul 2017 17:08:59 -0700 -Subject: [PATCH v5 4/5] squashfs: Add zstd support - -Add zstd compression and decompression support to SquashFS. zstd is a -great fit for SquashFS because it can compress at ratios approaching xz, -while decompressing twice as fast as zlib. For SquashFS in particular, -it can decompress as fast as lzo and lz4. It also has the flexibility -to turn down the compression ratio for faster compression times. - -The compression benchmark is run on the file tree from the SquashFS archive -found in ubuntu-16.10-desktop-amd64.iso [1]. It uses `mksquashfs` with the -default block size (128 KB) and and various compression algorithms/levels. -xz and zstd are also benchmarked with 256 KB blocks. The decompression -benchmark times how long it takes to `tar` the file tree into `/dev/null`. -See the benchmark file in the upstream zstd source repository located under -`contrib/linux-kernel/squashfs-benchmark.sh` [2] for details. - -I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM. -The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor, -16 GB of RAM, and a SSD. - -| Method | Ratio | Compression MB/s | Decompression MB/s | -|----------------|-------|------------------|--------------------| -| gzip | 2.92 | 15 | 128 | -| lzo | 2.64 | 9.5 | 217 | -| lz4 | 2.12 | 94 | 218 | -| xz | 3.43 | 5.5 | 35 | -| xz 256 KB | 3.53 | 5.4 | 40 | -| zstd 1 | 2.71 | 96 | 210 | -| zstd 5 | 2.93 | 69 | 198 | -| zstd 10 | 3.01 | 41 | 225 | -| zstd 15 | 3.13 | 11.4 | 224 | -| zstd 16 256 KB | 3.24 | 8.1 | 210 | - -This patch was written by Sean Purcell <me@seanp.xyz>, but I will be -taking over the submission process. - -[1] http://releases.ubuntu.com/16.10/ -[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/squashfs-benchmark.sh - -zstd source repository: https://github.com/facebook/zstd - -Signed-off-by: Sean Purcell <me@seanp.xyz> -Signed-off-by: Nick Terrell <terrelln@fb.com> ---- -v3 -> v4: -- Fix minor linter warnings - -v4 -> v5: -- Fix ZSTD_DStream initialization code in squashfs -- Fix patch documentation to reflect that Sean Purcell is the author - - fs/squashfs/Kconfig | 14 +++++ - fs/squashfs/Makefile | 1 + - fs/squashfs/decompressor.c | 7 +++ - fs/squashfs/decompressor.h | 4 ++ - fs/squashfs/squashfs_fs.h | 1 + - fs/squashfs/zstd_wrapper.c | 151 +++++++++++++++++++++++++++++++++++++++++++++ - 6 files changed, 178 insertions(+) - create mode 100644 fs/squashfs/zstd_wrapper.c - -diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig -index ffb093e..1adb334 100644 ---- a/fs/squashfs/Kconfig -+++ b/fs/squashfs/Kconfig -@@ -165,6 +165,20 @@ config SQUASHFS_XZ - - If unsure, say N. - -+config SQUASHFS_ZSTD -+ bool "Include support for ZSTD compressed file systems" -+ depends on SQUASHFS -+ select ZSTD_DECOMPRESS -+ help -+ Saying Y here includes support for reading Squashfs file systems -+ compressed with ZSTD compression. ZSTD gives better compression than -+ the default ZLIB compression, while using less CPU. -+ -+ ZSTD is not the standard compression used in Squashfs and so most -+ file systems will be readable without selecting this option. -+ -+ If unsure, say N. -+ - config SQUASHFS_4K_DEVBLK_SIZE - bool "Use 4K device block size?" - depends on SQUASHFS -diff --git a/fs/squashfs/Makefile b/fs/squashfs/Makefile -index 246a6f3..6655631 100644 ---- a/fs/squashfs/Makefile -+++ b/fs/squashfs/Makefile -@@ -15,3 +15,4 @@ squashfs-$(CONFIG_SQUASHFS_LZ4) += lz4_wrapper.o - squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o - squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o - squashfs-$(CONFIG_SQUASHFS_ZLIB) += zlib_wrapper.o -+squashfs-$(CONFIG_SQUASHFS_ZSTD) += zstd_wrapper.o -diff --git a/fs/squashfs/decompressor.c b/fs/squashfs/decompressor.c -index d2bc136..8366398 100644 ---- a/fs/squashfs/decompressor.c -+++ b/fs/squashfs/decompressor.c -@@ -65,6 +65,12 @@ static const struct squashfs_decompressor squashfs_zlib_comp_ops = { - }; - #endif - -+#ifndef CONFIG_SQUASHFS_ZSTD -+static const struct squashfs_decompressor squashfs_zstd_comp_ops = { -+ NULL, NULL, NULL, NULL, ZSTD_COMPRESSION, "zstd", 0 -+}; -+#endif -+ - static const struct squashfs_decompressor squashfs_unknown_comp_ops = { - NULL, NULL, NULL, NULL, 0, "unknown", 0 - }; -@@ -75,6 +81,7 @@ static const struct squashfs_decompressor *decompressor[] = { - &squashfs_lzo_comp_ops, - &squashfs_xz_comp_ops, - &squashfs_lzma_unsupported_comp_ops, -+ &squashfs_zstd_comp_ops, - &squashfs_unknown_comp_ops - }; - -diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h -index a25713c..0f5a8e4 100644 ---- a/fs/squashfs/decompressor.h -+++ b/fs/squashfs/decompressor.h -@@ -58,4 +58,8 @@ extern const struct squashfs_decompressor squashfs_lzo_comp_ops; - extern const struct squashfs_decompressor squashfs_zlib_comp_ops; - #endif - -+#ifdef CONFIG_SQUASHFS_ZSTD -+extern const struct squashfs_decompressor squashfs_zstd_comp_ops; -+#endif -+ - #endif -diff --git a/fs/squashfs/squashfs_fs.h b/fs/squashfs/squashfs_fs.h -index 506f4ba..24d12fd 100644 ---- a/fs/squashfs/squashfs_fs.h -+++ b/fs/squashfs/squashfs_fs.h -@@ -241,6 +241,7 @@ struct meta_index { - #define LZO_COMPRESSION 3 - #define XZ_COMPRESSION 4 - #define LZ4_COMPRESSION 5 -+#define ZSTD_COMPRESSION 6 - - struct squashfs_super_block { - __le32 s_magic; -diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c -new file mode 100644 -index 0000000..eeaabf8 ---- /dev/null -+++ b/fs/squashfs/zstd_wrapper.c -@@ -0,0 +1,151 @@ -+/* -+ * Squashfs - a compressed read only filesystem for Linux -+ * -+ * Copyright (c) 2016-present, Facebook, Inc. -+ * All rights reserved. -+ * -+ * This program is free software; you can redistribute it and/or -+ * modify it under the terms of the GNU General Public License -+ * as published by the Free Software Foundation; either version 2, -+ * or (at your option) any later version. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * zstd_wrapper.c -+ */ -+ -+#include <linux/mutex.h> -+#include <linux/buffer_head.h> -+#include <linux/slab.h> -+#include <linux/zstd.h> -+#include <linux/vmalloc.h> -+ -+#include "squashfs_fs.h" -+#include "squashfs_fs_sb.h" -+#include "squashfs.h" -+#include "decompressor.h" -+#include "page_actor.h" -+ -+struct workspace { -+ void *mem; -+ size_t mem_size; -+ size_t window_size; -+}; -+ -+static void *zstd_init(struct squashfs_sb_info *msblk, void *buff) -+{ -+ struct workspace *wksp = kmalloc(sizeof(*wksp), GFP_KERNEL); -+ -+ if (wksp == NULL) -+ goto failed; -+ wksp->window_size = max_t(size_t, -+ msblk->block_size, SQUASHFS_METADATA_SIZE); -+ wksp->mem_size = ZSTD_DStreamWorkspaceBound(wksp->window_size); -+ wksp->mem = vmalloc(wksp->mem_size); -+ if (wksp->mem == NULL) -+ goto failed; -+ -+ return wksp; -+ -+failed: -+ ERROR("Failed to allocate zstd workspace\n"); -+ kfree(wksp); -+ return ERR_PTR(-ENOMEM); -+} -+ -+ -+static void zstd_free(void *strm) -+{ -+ struct workspace *wksp = strm; -+ -+ if (wksp) -+ vfree(wksp->mem); -+ kfree(wksp); -+} -+ -+ -+static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, -+ struct buffer_head **bh, int b, int offset, int length, -+ struct squashfs_page_actor *output) -+{ -+ struct workspace *wksp = strm; -+ ZSTD_DStream *stream; -+ size_t total_out = 0; -+ size_t zstd_err; -+ int k = 0; -+ ZSTD_inBuffer in_buf = { NULL, 0, 0 }; -+ ZSTD_outBuffer out_buf = { NULL, 0, 0 }; -+ -+ stream = ZSTD_initDStream(wksp->window_size, wksp->mem, wksp->mem_size); -+ -+ if (!stream) { -+ ERROR("Failed to initialize zstd decompressor\n"); -+ goto out; -+ } -+ -+ out_buf.size = PAGE_SIZE; -+ out_buf.dst = squashfs_first_page(output); -+ -+ do { -+ if (in_buf.pos == in_buf.size && k < b) { -+ int avail = min(length, msblk->devblksize - offset); -+ -+ length -= avail; -+ in_buf.src = bh[k]->b_data + offset; -+ in_buf.size = avail; -+ in_buf.pos = 0; -+ offset = 0; -+ } -+ -+ if (out_buf.pos == out_buf.size) { -+ out_buf.dst = squashfs_next_page(output); -+ if (out_buf.dst == NULL) { -+ /* Shouldn't run out of pages -+ * before stream is done. -+ */ -+ squashfs_finish_page(output); -+ goto out; -+ } -+ out_buf.pos = 0; -+ out_buf.size = PAGE_SIZE; -+ } -+ -+ total_out -= out_buf.pos; -+ zstd_err = ZSTD_decompressStream(stream, &out_buf, &in_buf); -+ total_out += out_buf.pos; /* add the additional data produced */ -+ -+ if (in_buf.pos == in_buf.size && k < b) -+ put_bh(bh[k++]); -+ } while (zstd_err != 0 && !ZSTD_isError(zstd_err)); -+ -+ squashfs_finish_page(output); -+ -+ if (ZSTD_isError(zstd_err)) { -+ ERROR("zstd decompression error: %d\n", -+ (int)ZSTD_getErrorCode(zstd_err)); -+ goto out; -+ } -+ -+ if (k < b) -+ goto out; -+ -+ return (int)total_out; -+ -+out: -+ for (; k < b; k++) -+ put_bh(bh[k]); -+ -+ return -EIO; -+} -+ -+const struct squashfs_decompressor squashfs_zstd_comp_ops = { -+ .init = zstd_init, -+ .free = zstd_free, -+ .decompress = zstd_uncompress, -+ .id = ZSTD_COMPRESSION, -+ .name = "zstd", -+ .supported = 1 -+}; --- -2.9.3 diff --git a/contrib/linux-kernel/0005-crypto-Add-zstd-support.patch b/contrib/linux-kernel/0005-crypto-Add-zstd-support.patch deleted file mode 100644 index 971b0634..00000000 --- a/contrib/linux-kernel/0005-crypto-Add-zstd-support.patch +++ /dev/null @@ -1,424 +0,0 @@ -From 308795a7713ca6fcd468b60fba9a2fca99cee6a0 Mon Sep 17 00:00:00 2001 -From: Nick Terrell <terrelln@fb.com> -Date: Wed, 2 Aug 2017 18:02:13 -0700 -Subject: [PATCH v5 5/5] crypto: Add zstd support - -Adds zstd support to crypto and scompress. Only supports the default -level. - -Signed-off-by: Nick Terrell <terrelln@fb.com> ---- - crypto/Kconfig | 9 ++ - crypto/Makefile | 1 + - crypto/testmgr.c | 10 +++ - crypto/testmgr.h | 71 +++++++++++++++ - crypto/zstd.c | 265 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ - 5 files changed, 356 insertions(+) - create mode 100644 crypto/zstd.c - -diff --git a/crypto/Kconfig b/crypto/Kconfig -index caa770e..4fc3936 100644 ---- a/crypto/Kconfig -+++ b/crypto/Kconfig -@@ -1662,6 +1662,15 @@ config CRYPTO_LZ4HC - help - This is the LZ4 high compression mode algorithm. - -+config CRYPTO_ZSTD -+ tristate "Zstd compression algorithm" -+ select CRYPTO_ALGAPI -+ select CRYPTO_ACOMP2 -+ select ZSTD_COMPRESS -+ select ZSTD_DECOMPRESS -+ help -+ This is the zstd algorithm. -+ - comment "Random Number Generation" - - config CRYPTO_ANSI_CPRNG -diff --git a/crypto/Makefile b/crypto/Makefile -index d41f033..b22e1e8 100644 ---- a/crypto/Makefile -+++ b/crypto/Makefile -@@ -133,6 +133,7 @@ obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o - obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o - obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o - obj-$(CONFIG_CRYPTO_USER_API_AEAD) += algif_aead.o -+obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o - - ecdh_generic-y := ecc.o - ecdh_generic-y += ecdh.o -diff --git a/crypto/testmgr.c b/crypto/testmgr.c -index 7125ba3..8a124d3 100644 ---- a/crypto/testmgr.c -+++ b/crypto/testmgr.c -@@ -3603,6 +3603,16 @@ static const struct alg_test_desc alg_test_descs[] = { - .decomp = __VECS(zlib_deflate_decomp_tv_template) - } - } -+ }, { -+ .alg = "zstd", -+ .test = alg_test_comp, -+ .fips_allowed = 1, -+ .suite = { -+ .comp = { -+ .comp = __VECS(zstd_comp_tv_template), -+ .decomp = __VECS(zstd_decomp_tv_template) -+ } -+ } - } - }; - -diff --git a/crypto/testmgr.h b/crypto/testmgr.h -index 6ceb0e2..e6b5920 100644 ---- a/crypto/testmgr.h -+++ b/crypto/testmgr.h -@@ -34631,4 +34631,75 @@ static const struct comp_testvec lz4hc_decomp_tv_template[] = { - }, - }; - -+static const struct comp_testvec zstd_comp_tv_template[] = { -+ { -+ .inlen = 68, -+ .outlen = 39, -+ .input = "The algorithm is zstd. " -+ "The algorithm is zstd. " -+ "The algorithm is zstd.", -+ .output = "\x28\xb5\x2f\xfd\x00\x50\xf5\x00\x00\xb8\x54\x68\x65" -+ "\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x20\x69\x73" -+ "\x20\x7a\x73\x74\x64\x2e\x20\x01\x00\x55\x73\x36\x01" -+ , -+ }, -+ { -+ .inlen = 244, -+ .outlen = 151, -+ .input = "zstd, short for Zstandard, is a fast lossless " -+ "compression algorithm, targeting real-time " -+ "compression scenarios at zlib-level and better " -+ "compression ratios. The zstd compression library " -+ "provides in-memory compression and decompression " -+ "functions.", -+ .output = "\x28\xb5\x2f\xfd\x00\x50\x75\x04\x00\x42\x4b\x1e\x17" -+ "\x90\x81\x31\x00\xf2\x2f\xe4\x36\xc9\xef\x92\x88\x32" -+ "\xc9\xf2\x24\x94\xd8\x68\x9a\x0f\x00\x0c\xc4\x31\x6f" -+ "\x0d\x0c\x38\xac\x5c\x48\x03\xcd\x63\x67\xc0\xf3\xad" -+ "\x4e\x90\xaa\x78\xa0\xa4\xc5\x99\xda\x2f\xb6\x24\x60" -+ "\xe2\x79\x4b\xaa\xb6\x6b\x85\x0b\xc9\xc6\x04\x66\x86" -+ "\xe2\xcc\xe2\x25\x3f\x4f\x09\xcd\xb8\x9d\xdb\xc1\x90" -+ "\xa9\x11\xbc\x35\x44\x69\x2d\x9c\x64\x4f\x13\x31\x64" -+ "\xcc\xfb\x4d\x95\x93\x86\x7f\x33\x7f\x1a\xef\xe9\x30" -+ "\xf9\x67\xa1\x94\x0a\x69\x0f\x60\xcd\xc3\xab\x99\xdc" -+ "\x42\xed\x97\x05\x00\x33\xc3\x15\x95\x3a\x06\xa0\x0e" -+ "\x20\xa9\x0e\x82\xb9\x43\x45\x01", -+ }, -+}; -+ -+static const struct comp_testvec zstd_decomp_tv_template[] = { -+ { -+ .inlen = 43, -+ .outlen = 68, -+ .input = "\x28\xb5\x2f\xfd\x04\x50\xf5\x00\x00\xb8\x54\x68\x65" -+ "\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x20\x69\x73" -+ "\x20\x7a\x73\x74\x64\x2e\x20\x01\x00\x55\x73\x36\x01" -+ "\x6b\xf4\x13\x35", -+ .output = "The algorithm is zstd. " -+ "The algorithm is zstd. " -+ "The algorithm is zstd.", -+ }, -+ { -+ .inlen = 155, -+ .outlen = 244, -+ .input = "\x28\xb5\x2f\xfd\x04\x50\x75\x04\x00\x42\x4b\x1e\x17" -+ "\x90\x81\x31\x00\xf2\x2f\xe4\x36\xc9\xef\x92\x88\x32" -+ "\xc9\xf2\x24\x94\xd8\x68\x9a\x0f\x00\x0c\xc4\x31\x6f" -+ "\x0d\x0c\x38\xac\x5c\x48\x03\xcd\x63\x67\xc0\xf3\xad" -+ "\x4e\x90\xaa\x78\xa0\xa4\xc5\x99\xda\x2f\xb6\x24\x60" -+ "\xe2\x79\x4b\xaa\xb6\x6b\x85\x0b\xc9\xc6\x04\x66\x86" -+ "\xe2\xcc\xe2\x25\x3f\x4f\x09\xcd\xb8\x9d\xdb\xc1\x90" -+ "\xa9\x11\xbc\x35\x44\x69\x2d\x9c\x64\x4f\x13\x31\x64" -+ "\xcc\xfb\x4d\x95\x93\x86\x7f\x33\x7f\x1a\xef\xe9\x30" -+ "\xf9\x67\xa1\x94\x0a\x69\x0f\x60\xcd\xc3\xab\x99\xdc" -+ "\x42\xed\x97\x05\x00\x33\xc3\x15\x95\x3a\x06\xa0\x0e" -+ "\x20\xa9\x0e\x82\xb9\x43\x45\x01\xaa\x6d\xda\x0d", -+ .output = "zstd, short for Zstandard, is a fast lossless " -+ "compression algorithm, targeting real-time " -+ "compression scenarios at zlib-level and better " -+ "compression ratios. The zstd compression library " -+ "provides in-memory compression and decompression " -+ "functions.", -+ }, -+}; - #endif /* _CRYPTO_TESTMGR_H */ -diff --git a/crypto/zstd.c b/crypto/zstd.c -new file mode 100644 -index 0000000..9a76b3e ---- /dev/null -+++ b/crypto/zstd.c -@@ -0,0 +1,265 @@ -+/* -+ * Cryptographic API. -+ * -+ * Copyright (c) 2017-present, Facebook, Inc. -+ * -+ * This program is free software; you can redistribute it and/or modify it -+ * under the terms of the GNU General Public License version 2 as published by -+ * the Free Software Foundation. -+ * -+ * This program is distributed in the hope that it will be useful, but WITHOUT -+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -+ * more details. -+ */ -+#include <linux/crypto.h> -+#include <linux/init.h> -+#include <linux/interrupt.h> -+#include <linux/mm.h> -+#include <linux/module.h> -+#include <linux/net.h> -+#include <linux/vmalloc.h> -+#include <linux/zstd.h> -+#include <crypto/internal/scompress.h> -+ -+ -+#define ZSTD_DEF_LEVEL 3 -+ -+struct zstd_ctx { -+ ZSTD_CCtx *cctx; -+ ZSTD_DCtx *dctx; -+ void *cwksp; -+ void *dwksp; -+}; -+ -+static ZSTD_parameters zstd_params(void) -+{ -+ return ZSTD_getParams(ZSTD_DEF_LEVEL, 0, 0); -+} -+ -+static int zstd_comp_init(struct zstd_ctx *ctx) -+{ -+ int ret = 0; -+ const ZSTD_parameters params = zstd_params(); -+ const size_t wksp_size = ZSTD_CCtxWorkspaceBound(params.cParams); -+ -+ ctx->cwksp = vzalloc(wksp_size); -+ if (!ctx->cwksp) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ ctx->cctx = ZSTD_initCCtx(ctx->cwksp, wksp_size); -+ if (!ctx->cctx) { -+ ret = -EINVAL; -+ goto out_free; -+ } -+out: -+ return ret; -+out_free: -+ vfree(ctx->cwksp); -+ goto out; -+} -+ -+static int zstd_decomp_init(struct zstd_ctx *ctx) -+{ -+ int ret = 0; -+ const size_t wksp_size = ZSTD_DCtxWorkspaceBound(); -+ -+ ctx->dwksp = vzalloc(wksp_size); -+ if (!ctx->dwksp) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ ctx->dctx = ZSTD_initDCtx(ctx->dwksp, wksp_size); -+ if (!ctx->dctx) { -+ ret = -EINVAL; -+ goto out_free; -+ } -+out: -+ return ret; -+out_free: -+ vfree(ctx->dwksp); -+ goto out; -+} -+ -+static void zstd_comp_exit(struct zstd_ctx *ctx) -+{ -+ vfree(ctx->cwksp); -+ ctx->cwksp = NULL; -+ ctx->cctx = NULL; -+} -+ -+static void zstd_decomp_exit(struct zstd_ctx *ctx) -+{ -+ vfree(ctx->dwksp); -+ ctx->dwksp = NULL; -+ ctx->dctx = NULL; -+} -+ -+static int __zstd_init(void *ctx) -+{ -+ int ret; -+ -+ ret = zstd_comp_init(ctx); -+ if (ret) -+ return ret; -+ ret = zstd_decomp_init(ctx); -+ if (ret) -+ zstd_comp_exit(ctx); -+ return ret; -+} -+ -+static void *zstd_alloc_ctx(struct crypto_scomp *tfm) -+{ -+ int ret; -+ struct zstd_ctx *ctx; -+ -+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); -+ if (!ctx) -+ return ERR_PTR(-ENOMEM); -+ -+ ret = __zstd_init(ctx); -+ if (ret) { -+ kfree(ctx); -+ return ERR_PTR(ret); -+ } -+ -+ return ctx; -+} -+ -+static int zstd_init(struct crypto_tfm *tfm) -+{ -+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm); -+ -+ return __zstd_init(ctx); -+} -+ -+static void __zstd_exit(void *ctx) -+{ -+ zstd_comp_exit(ctx); -+ zstd_decomp_exit(ctx); -+} -+ -+static void zstd_free_ctx(struct crypto_scomp *tfm, void *ctx) -+{ -+ __zstd_exit(ctx); -+ kzfree(ctx); -+} -+ -+static void zstd_exit(struct crypto_tfm *tfm) -+{ -+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm); -+ -+ __zstd_exit(ctx); -+} -+ -+static int __zstd_compress(const u8 *src, unsigned int slen, -+ u8 *dst, unsigned int *dlen, void *ctx) -+{ -+ size_t out_len; -+ struct zstd_ctx *zctx = ctx; -+ const ZSTD_parameters params = zstd_params(); -+ -+ out_len = ZSTD_compressCCtx(zctx->cctx, dst, *dlen, src, slen, params); -+ if (ZSTD_isError(out_len)) -+ return -EINVAL; -+ *dlen = out_len; -+ return 0; -+} -+ -+static int zstd_compress(struct crypto_tfm *tfm, const u8 *src, -+ unsigned int slen, u8 *dst, unsigned int *dlen) -+{ -+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm); -+ -+ return __zstd_compress(src, slen, dst, dlen, ctx); -+} -+ -+static int zstd_scompress(struct crypto_scomp *tfm, const u8 *src, -+ unsigned int slen, u8 *dst, unsigned int *dlen, -+ void *ctx) -+{ -+ return __zstd_compress(src, slen, dst, dlen, ctx); -+} -+ -+static int __zstd_decompress(const u8 *src, unsigned int slen, -+ u8 *dst, unsigned int *dlen, void *ctx) -+{ -+ size_t out_len; -+ struct zstd_ctx *zctx = ctx; -+ -+ out_len = ZSTD_decompressDCtx(zctx->dctx, dst, *dlen, src, slen); -+ if (ZSTD_isError(out_len)) -+ return -EINVAL; -+ *dlen = out_len; -+ return 0; -+} -+ -+static int zstd_decompress(struct crypto_tfm *tfm, const u8 *src, -+ unsigned int slen, u8 *dst, unsigned int *dlen) -+{ -+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm); -+ -+ return __zstd_decompress(src, slen, dst, dlen, ctx); -+} -+ -+static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src, -+ unsigned int slen, u8 *dst, unsigned int *dlen, -+ void *ctx) -+{ -+ return __zstd_decompress(src, slen, dst, dlen, ctx); -+} -+ -+static struct crypto_alg alg = { -+ .cra_name = "zstd", -+ .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, -+ .cra_ctxsize = sizeof(struct zstd_ctx), -+ .cra_module = THIS_MODULE, -+ .cra_init = zstd_init, -+ .cra_exit = zstd_exit, -+ .cra_u = { .compress = { -+ .coa_compress = zstd_compress, -+ .coa_decompress = zstd_decompress } } -+}; -+ -+static struct scomp_alg scomp = { -+ .alloc_ctx = zstd_alloc_ctx, -+ .free_ctx = zstd_free_ctx, -+ .compress = zstd_scompress, -+ .decompress = zstd_sdecompress, -+ .base = { -+ .cra_name = "zstd", -+ .cra_driver_name = "zstd-scomp", -+ .cra_module = THIS_MODULE, -+ } -+}; -+ -+static int __init zstd_mod_init(void) -+{ -+ int ret; -+ -+ ret = crypto_register_alg(&alg); -+ if (ret) -+ return ret; -+ -+ ret = crypto_register_scomp(&scomp); -+ if (ret) -+ crypto_unregister_alg(&alg); -+ -+ return ret; -+} -+ -+static void __exit zstd_mod_fini(void) -+{ -+ crypto_unregister_alg(&alg); -+ crypto_unregister_scomp(&scomp); -+} -+ -+module_init(zstd_mod_init); -+module_exit(zstd_mod_fini); -+ -+MODULE_LICENSE("GPL"); -+MODULE_DESCRIPTION("Zstd Compression Algorithm"); -+MODULE_ALIAS_CRYPTO("zstd"); --- -2.9.3 diff --git a/contrib/linux-kernel/0006-squashfs-tools-Add-zstd-support.patch b/contrib/linux-kernel/0006-squashfs-tools-Add-zstd-support.patch deleted file mode 100644 index 00d24e2b..00000000 --- a/contrib/linux-kernel/0006-squashfs-tools-Add-zstd-support.patch +++ /dev/null @@ -1,420 +0,0 @@ -From 57a3cf95b276946559f9e044c7352c11303bb9c1 Mon Sep 17 00:00:00 2001 -From: Sean Purcell <me@seanp.xyz> -Date: Thu, 3 Aug 2017 17:47:03 -0700 -Subject: [PATCH v6] squashfs-tools: Add zstd support - -This patch adds zstd support to squashfs-tools. It works with zstd -versions >= 1.0.0. It was originally written by Sean Purcell. - -Signed-off-by: Sean Purcell <me@seanp.xyz> -Signed-off-by: Nick Terrell <terrelln@fb.com> ---- -v4 -> v5: -- Fix patch documentation to reflect that Sean Purcell is the author -- Don't strip trailing whitespace of unrelated code -- Make zstd_display_options() static - -v5 -> v6: -- Fix build instructions in Makefile - - squashfs-tools/Makefile | 20 ++++ - squashfs-tools/compressor.c | 8 ++ - squashfs-tools/squashfs_fs.h | 1 + - squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++++++++++ - squashfs-tools/zstd_wrapper.h | 48 ++++++++ - 5 files changed, 331 insertions(+) - create mode 100644 squashfs-tools/zstd_wrapper.c - create mode 100644 squashfs-tools/zstd_wrapper.h - -diff --git a/squashfs-tools/Makefile b/squashfs-tools/Makefile -index 52d2582..22fc559 100644 ---- a/squashfs-tools/Makefile -+++ b/squashfs-tools/Makefile -@@ -75,6 +75,18 @@ GZIP_SUPPORT = 1 - #LZMA_SUPPORT = 1 - #LZMA_DIR = ../../../../LZMA/lzma465 - -+ -+########### Building ZSTD support ############ -+# -+# The ZSTD library is supported -+# ZSTD homepage: http://zstd.net -+# ZSTD source repository: https://github.com/facebook/zstd -+# -+# To build using the ZSTD library - install the library and uncomment the -+# ZSTD_SUPPORT line below. -+# -+#ZSTD_SUPPORT = 1 -+ - ######## Specifying default compression ######## - # - # The next line specifies which compression algorithm is used by default -@@ -177,6 +189,14 @@ LIBS += -llz4 - COMPRESSORS += lz4 - endif - -+ifeq ($(ZSTD_SUPPORT),1) -+CFLAGS += -DZSTD_SUPPORT -+MKSQUASHFS_OBJS += zstd_wrapper.o -+UNSQUASHFS_OBJS += zstd_wrapper.o -+LIBS += -lzstd -+COMPRESSORS += zstd -+endif -+ - ifeq ($(XATTR_SUPPORT),1) - ifeq ($(XATTR_DEFAULT),1) - CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT -diff --git a/squashfs-tools/compressor.c b/squashfs-tools/compressor.c -index 525e316..02b5e90 100644 ---- a/squashfs-tools/compressor.c -+++ b/squashfs-tools/compressor.c -@@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = { - extern struct compressor xz_comp_ops; - #endif - -+#ifndef ZSTD_SUPPORT -+static struct compressor zstd_comp_ops = { -+ ZSTD_COMPRESSION, "zstd" -+}; -+#else -+extern struct compressor zstd_comp_ops; -+#endif - - static struct compressor unknown_comp_ops = { - 0, "unknown" -@@ -77,6 +84,7 @@ struct compressor *compressor[] = { - &lzo_comp_ops, - &lz4_comp_ops, - &xz_comp_ops, -+ &zstd_comp_ops, - &unknown_comp_ops - }; - -diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h -index 791fe12..afca918 100644 ---- a/squashfs-tools/squashfs_fs.h -+++ b/squashfs-tools/squashfs_fs.h -@@ -277,6 +277,7 @@ typedef long long squashfs_inode; - #define LZO_COMPRESSION 3 - #define XZ_COMPRESSION 4 - #define LZ4_COMPRESSION 5 -+#define ZSTD_COMPRESSION 6 - - struct squashfs_super_block { - unsigned int s_magic; -diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c -new file mode 100644 -index 0000000..dcab75a ---- /dev/null -+++ b/squashfs-tools/zstd_wrapper.c -@@ -0,0 +1,254 @@ -+/* -+ * Copyright (c) 2017 -+ * Phillip Lougher <phillip@squashfs.org.uk> -+ * -+ * This program is free software; you can redistribute it and/or -+ * modify it under the terms of the GNU General Public License -+ * as published by the Free Software Foundation; either version 2, -+ * or (at your option) any later version. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * zstd_wrapper.c -+ * -+ * Support for ZSTD compression http://zstd.net -+ */ -+ -+#include <stdio.h> -+#include <string.h> -+#include <stdlib.h> -+#include <zstd.h> -+#include <zstd_errors.h> -+ -+#include "squashfs_fs.h" -+#include "zstd_wrapper.h" -+#include "compressor.h" -+ -+static int compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL; -+ -+/* -+ * This function is called by the options parsing code in mksquashfs.c -+ * to parse any -X compressor option. -+ * -+ * This function returns: -+ * >=0 (number of additional args parsed) on success -+ * -1 if the option was unrecognised, or -+ * -2 if the option was recognised, but otherwise bad in -+ * some way (e.g. invalid parameter) -+ * -+ * Note: this function sets internal compressor state, but does not -+ * pass back the results of the parsing other than success/failure. -+ * The zstd_dump_options() function is called later to get the options in -+ * a format suitable for writing to the filesystem. -+ */ -+static int zstd_options(char *argv[], int argc) -+{ -+ if (strcmp(argv[0], "-Xcompression-level") == 0) { -+ if (argc < 2) { -+ fprintf(stderr, "zstd: -Xcompression-level missing " -+ "compression level\n"); -+ fprintf(stderr, "zstd: -Xcompression-level it should " -+ "be 1 <= n <= %d\n", ZSTD_maxCLevel()); -+ goto failed; -+ } -+ -+ compression_level = atoi(argv[1]); -+ if (compression_level < 1 || -+ compression_level > ZSTD_maxCLevel()) { -+ fprintf(stderr, "zstd: -Xcompression-level invalid, it " -+ "should be 1 <= n <= %d\n", ZSTD_maxCLevel()); -+ goto failed; -+ } -+ -+ return 1; -+ } -+ -+ return -1; -+failed: -+ return -2; -+} -+ -+/* -+ * This function is called by mksquashfs to dump the parsed -+ * compressor options in a format suitable for writing to the -+ * compressor options field in the filesystem (stored immediately -+ * after the superblock). -+ * -+ * This function returns a pointer to the compression options structure -+ * to be stored (and the size), or NULL if there are no compression -+ * options. -+ */ -+static void *zstd_dump_options(int block_size, int *size) -+{ -+ static struct zstd_comp_opts comp_opts; -+ -+ /* don't return anything if the options are all default */ -+ if (compression_level == ZSTD_DEFAULT_COMPRESSION_LEVEL) -+ return NULL; -+ -+ comp_opts.compression_level = compression_level; -+ -+ SQUASHFS_INSWAP_COMP_OPTS(&comp_opts); -+ -+ *size = sizeof(comp_opts); -+ return &comp_opts; -+} -+ -+/* -+ * This function is a helper specifically for the append mode of -+ * mksquashfs. Its purpose is to set the internal compressor state -+ * to the stored compressor options in the passed compressor options -+ * structure. -+ * -+ * In effect this function sets up the compressor options -+ * to the same state they were when the filesystem was originally -+ * generated, this is to ensure on appending, the compressor uses -+ * the same compression options that were used to generate the -+ * original filesystem. -+ * -+ * Note, even if there are no compressor options, this function is still -+ * called with an empty compressor structure (size == 0), to explicitly -+ * set the default options, this is to ensure any user supplied -+ * -X options on the appending mksquashfs command line are over-ridden. -+ * -+ * This function returns 0 on successful extraction of options, and -1 on error. -+ */ -+static int zstd_extract_options(int block_size, void *buffer, int size) -+{ -+ struct zstd_comp_opts *comp_opts = buffer; -+ -+ if (size == 0) { -+ /* Set default values */ -+ compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL; -+ return 0; -+ } -+ -+ /* we expect a comp_opts structure of sufficient size to be present */ -+ if (size < sizeof(*comp_opts)) -+ goto failed; -+ -+ SQUASHFS_INSWAP_COMP_OPTS(comp_opts); -+ -+ if (comp_opts->compression_level < 1 || -+ comp_opts->compression_level > ZSTD_maxCLevel()) { -+ fprintf(stderr, "zstd: bad compression level in compression " -+ "options structure\n"); -+ goto failed; -+ } -+ -+ compression_level = comp_opts->compression_level; -+ -+ return 0; -+ -+failed: -+ fprintf(stderr, "zstd: error reading stored compressor options from " -+ "filesystem!\n"); -+ -+ return -1; -+} -+ -+static void zstd_display_options(void *buffer, int size) -+{ -+ struct zstd_comp_opts *comp_opts = buffer; -+ -+ /* we expect a comp_opts structure of sufficient size to be present */ -+ if (size < sizeof(*comp_opts)) -+ goto failed; -+ -+ SQUASHFS_INSWAP_COMP_OPTS(comp_opts); -+ -+ if (comp_opts->compression_level < 1 || -+ comp_opts->compression_level > ZSTD_maxCLevel()) { -+ fprintf(stderr, "zstd: bad compression level in compression " -+ "options structure\n"); -+ goto failed; -+ } -+ -+ printf("\tcompression-level %d\n", comp_opts->compression_level); -+ -+ return; -+ -+failed: -+ fprintf(stderr, "zstd: error reading stored compressor options from " -+ "filesystem!\n"); -+} -+ -+/* -+ * This function is called by mksquashfs to initialise the -+ * compressor, before compress() is called. -+ * -+ * This function returns 0 on success, and -1 on error. -+ */ -+static int zstd_init(void **strm, int block_size, int datablock) -+{ -+ ZSTD_CCtx *cctx = ZSTD_createCCtx(); -+ -+ if (!cctx) { -+ fprintf(stderr, "zstd: failed to allocate compression " -+ "context!\n"); -+ return -1; -+ } -+ -+ *strm = cctx; -+ return 0; -+} -+ -+static int zstd_compress(void *strm, void *dest, void *src, int size, -+ int block_size, int *error) -+{ -+ const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size, -+ src, size, compression_level); -+ -+ if (ZSTD_isError(res)) { -+ /* FIXME: -+ * zstd does not expose stable error codes. The error enum may -+ * change between versions. Until upstream zstd stablizes the -+ * error codes, we have no way of knowing why the error occurs. -+ * zstd shouldn't fail to compress any input unless there isn't -+ * enough output space. We assume that is the cause and return -+ * the special error code for not enough output space. -+ */ -+ return 0; -+ } -+ -+ return (int)res; -+} -+ -+static int zstd_uncompress(void *dest, void *src, int size, int outsize, -+ int *error) -+{ -+ const size_t res = ZSTD_decompress(dest, outsize, src, size); -+ -+ if (ZSTD_isError(res)) { -+ fprintf(stderr, "\t%d %d\n", outsize, size); -+ -+ *error = (int)ZSTD_getErrorCode(res); -+ return -1; -+ } -+ -+ return (int)res; -+} -+ -+static void zstd_usage(void) -+{ -+ fprintf(stderr, "\t -Xcompression-level <compression-level>\n"); -+ fprintf(stderr, "\t\t<compression-level> should be 1 .. %d (default " -+ "%d)\n", ZSTD_maxCLevel(), ZSTD_DEFAULT_COMPRESSION_LEVEL); -+} -+ -+struct compressor zstd_comp_ops = { -+ .init = zstd_init, -+ .compress = zstd_compress, -+ .uncompress = zstd_uncompress, -+ .options = zstd_options, -+ .dump_options = zstd_dump_options, -+ .extract_options = zstd_extract_options, -+ .display_options = zstd_display_options, -+ .usage = zstd_usage, -+ .id = ZSTD_COMPRESSION, -+ .name = "zstd", -+ .supported = 1 -+}; -diff --git a/squashfs-tools/zstd_wrapper.h b/squashfs-tools/zstd_wrapper.h -new file mode 100644 -index 0000000..4fbef0a ---- /dev/null -+++ b/squashfs-tools/zstd_wrapper.h -@@ -0,0 +1,48 @@ -+#ifndef ZSTD_WRAPPER_H -+#define ZSTD_WRAPPER_H -+/* -+ * Squashfs -+ * -+ * Copyright (c) 2017 -+ * Phillip Lougher <phillip@squashfs.org.uk> -+ * -+ * This program is free software; you can redistribute it and/or -+ * modify it under the terms of the GNU General Public License -+ * as published by the Free Software Foundation; either version 2, -+ * or (at your option) any later version. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * zstd_wrapper.h -+ * -+ */ -+ -+#ifndef linux -+#define __BYTE_ORDER BYTE_ORDER -+#define __BIG_ENDIAN BIG_ENDIAN -+#define __LITTLE_ENDIAN LITTLE_ENDIAN -+#else -+#include <endian.h> -+#endif -+ -+#if __BYTE_ORDER == __BIG_ENDIAN -+extern unsigned int inswap_le16(unsigned short); -+extern unsigned int inswap_le32(unsigned int); -+ -+#define SQUASHFS_INSWAP_COMP_OPTS(s) { \ -+ (s)->compression_level = inswap_le32((s)->compression_level); \ -+} -+#else -+#define SQUASHFS_INSWAP_COMP_OPTS(s) -+#endif -+ -+/* Default compression */ -+#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15 -+ -+struct zstd_comp_opts { -+ int compression_level; -+}; -+#endif --- -2.9.5 diff --git a/contrib/linux-kernel/COPYING b/contrib/linux-kernel/COPYING deleted file mode 100644 index ecbc0593..00000000 --- a/contrib/linux-kernel/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - <one line to give the program's name and a brief idea of what it does.> - Copyright (C) <year> <name of author> - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - <signature of Ty Coon>, 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License.
\ No newline at end of file diff --git a/contrib/linux-kernel/Makefile b/contrib/linux-kernel/Makefile new file mode 100644 index 00000000..b8a65e91 --- /dev/null +++ b/contrib/linux-kernel/Makefile @@ -0,0 +1,83 @@ +# ################################################################ +# Copyright (c) 2015-2020, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ + +.PHONY: libzstd +libzstd: + rm -rf linux + mkdir -p linux + mkdir -p linux/include/linux + mkdir -p linux/lib/zstd + ../freestanding_lib/freestanding.py \ + --source-lib ../../lib \ + --output-lib linux/lib/zstd \ + --xxhash '<linux/xxhash.h>' \ + --xxh64-state 'struct xxh64_state' \ + --xxh64-prefix 'xxh64' \ + --rewrite-include '<limits\.h>=<linux/limits.h>' \ + --rewrite-include '<stddef\.h>=<linux/types.h>' \ + -DZSTD_NO_INTRINSICS \ + -DZSTD_NO_UNUSED_FUNCTIONS \ + -DZSTD_LEGACY_SUPPORT=0 \ + -DZSTD_STATIC_LINKING_ONLY \ + -DFSE_STATIC_LINKING_ONLY \ + -DHUF_STATIC_LINKING_ONLY \ + -DXXH_STATIC_LINKING_ONLY \ + -DMEM_FORCE_MEMORY_ACCESS=0 \ + -D__GNUC__ \ + -DSTATIC_BMI2=0 \ + -DZSTD_ADDRESS_SANITIZER=0 \ + -DZSTD_MEMORY_SANITIZER=0 \ + -DZSTD_COMPRESS_HEAPMODE=1 \ + -UZSTD_NO_INLINE \ + -UNO_PREFETCH \ + -U__cplusplus \ + -UZSTD_DLL_EXPORT \ + -UZSTD_DLL_IMPORT \ + -U__ICCARM__ \ + -UZSTD_MULTITHREAD \ + -U_MSC_VER \ + -U_WIN32 \ + -RZSTDLIB_VISIBILITY= \ + -RZSTDERRORLIB_VISIBILITY= + cp linux_zstd.h linux/include/linux/zstd.h + cp zstd_compress_module.c linux/lib/zstd + cp zstd_decompress_module.c linux/lib/zstd + cp decompress_sources.h linux/lib/zstd + cp linux.mk linux/lib/zstd/Makefile + +LINUX ?= $(HOME)/repos/linux + +.PHONY: import +import: libzstd + rm -f $(LINUX)/include/linux/zstd.h + rm -f $(LINUX)/include/linux/zstd_errors.h + rm -rf $(LINUX)/lib/zstd + cp linux/include/linux/zstd.h $(LINUX)/include/linux + cp -r linux/lib/zstd $(LINUX)/lib + +import-upstream: + rm -rf $(LINUX)/lib/zstd + mkdir $(LINUX)/lib/zstd + cp ../../lib/zstd.h $(LINUX)/lib/zstd + cp -r ../../lib/common $(LINUX)/lib/zstd + cp -r ../../lib/compress $(LINUX)/lib/zstd + cp -r ../../lib/decompress $(LINUX)/lib/zstd + rm $(LINUX)/lib/zstd/common/threading.* + rm $(LINUX)/lib/zstd/common/pool.* + rm $(LINUX)/lib/zstd/common/xxhash.* + rm $(LINUX)/lib/zstd/compress/zstdmt_* + +.PHONY: test +test: libzstd + $(MAKE) -C test run-test CFLAGS="-O3 $(CFLAGS)" -j + +.PHONY: clean +clean: + $(RM) -rf linux diff --git a/contrib/linux-kernel/README.md b/contrib/linux-kernel/README.md index 86552b8b..bfa070d1 100644 --- a/contrib/linux-kernel/README.md +++ b/contrib/linux-kernel/README.md @@ -1,101 +1,14 @@ -# Linux Kernel Patch +# Zstd in the Linux Kernel -There are four pieces, the `xxhash` kernel module, the `zstd_compress` and `zstd_decompress` kernel modules, the BtrFS patch, and the SquashFS patch. -The patches are based off of the linux kernel master branch. +This directory contains the scripts needed to transform upstream zstd into the version imported into the kernel. All the transforms are automated and tested by our continuous integration. -## xxHash kernel module +## Upgrading Zstd in the Linux Kernel -* The patch is located in `xxhash.diff`. -* The header is in `include/linux/xxhash.h`. -* The source is in `lib/xxhash.c`. -* `test/XXHashUserLandTest.cpp` contains tests for the patch in userland by mocking the kernel headers. - I tested the tests by commenting a line of of each branch in `xxhash.c` one line at a time, and made sure the tests failed. - It can be run with the following commands: - ``` - cd test && make googletest && make XXHashUserLandTest && ./XXHashUserLandTest - ``` -* I also benchmarked the `xxhash` module against upstream xxHash, and made sure that they ran at the same speed. - -## Zstd Kernel modules - -* The (large) patch is located in `zstd.diff`, which depends on `xxhash.diff`. -* The header is in `include/linux/zstd.h`. -* It is split up into `zstd_compress` and `zstd_decompress`, which can be loaded independently. -* Source files are in `lib/zstd/`. -* `lib/Kconfig` and `lib/Makefile` need to be modified by applying `lib/Kconfig.diff` and `lib/Makefile.diff` respectively. - These changes are also included in the `zstd.diff`. -* `test/UserlandTest.cpp` contains tests for the patch in userland by mocking the kernel headers. - It can be run with the following commands: - ``` - cd test && make googletest && make UserlandTest && ./UserlandTest - ``` - -## BtrFS - -* The patch is located in `btrfs.diff`. -* Additionally `fs/btrfs/zstd.c` is provided as a source for convenience. -* The patch seems to be working, it doesn't crash the kernel, and compresses at speeds and ratios that are expected. - It could still use some more testing for fringe features, like printing options. - -### Benchmarks - -Benchmarks run on a Ubuntu 14.04 with 2 cores and 4 GiB of RAM. -The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor, -16 GB of ram, and a SSD. -The kernel running was built from the master branch with the patch. - -The compression benchmark is copying 10 copies of the -unzipped [silesia corpus](http://mattmahoney.net/dc/silesia.html) into a BtrFS -filesystem mounted with `-o compress-force={none, lzo, zlib, zstd}`. -The decompression benchmark is timing how long it takes to `tar` all 10 copies -into `/dev/null`. -The compression ratio is measured by comparing the output of `df` and `du`. -See `btrfs-benchmark.sh` for details. - -| Algorithm | Compression ratio | Compression speed | Decompression speed | -|-----------|-------------------|-------------------|---------------------| -| None | 0.99 | 504 MB/s | 686 MB/s | -| lzo | 1.66 | 398 MB/s | 442 MB/s | -| zlib | 2.58 | 65 MB/s | 241 MB/s | -| zstd 1 | 2.57 | 260 MB/s | 383 MB/s | -| zstd 3 | 2.71 | 174 MB/s | 408 MB/s | -| zstd 6 | 2.87 | 70 MB/s | 398 MB/s | -| zstd 9 | 2.92 | 43 MB/s | 406 MB/s | -| zstd 12 | 2.93 | 21 MB/s | 408 MB/s | -| zstd 15 | 3.01 | 11 MB/s | 354 MB/s | - - -## SquashFS - -* The patch is located in `squashfs.diff` -* Additionally `fs/squashfs/zstd_wrapper.c` is provided as a source for convenience. -* The patch has been tested on the master branch of the kernel. - -### Benchmarks - -Benchmarks run on a Ubuntu 14.04 with 2 cores and 4 GiB of RAM. -The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor, -16 GB of ram, and a SSD. -The kernel running was built from the master branch with the patch. - -The compression benchmark is the file tree from the SquashFS archive found in the -Ubuntu 16.10 desktop image (ubuntu-16.10-desktop-amd64.iso). -The compression benchmark uses mksquashfs with the default block size (128 KB) -and various compression algorithms/compression levels. -`xz` and `zstd` are also benchmarked with 256 KB blocks. -The decompression benchmark is timing how long it takes to `tar` the file tree -into `/dev/null`. -See `squashfs-benchmark.sh` for details. - -| Algorithm | Compression ratio | Compression speed | Decompression speed | -|----------------|-------------------|-------------------|---------------------| -| gzip | 2.92 | 15 MB/s | 128 MB/s | -| lzo | 2.64 | 9.5 MB/s | 217 MB/s | -| lz4 | 2.12 | 94 MB/s | 218 MB/s | -| xz | 3.43 | 5.5 MB/s | 35 MB/s | -| xz 256 KB | 3.53 | 5.4 MB/s | 40 MB/s | -| zstd 1 | 2.71 | 96 MB/s | 210 MB/s | -| zstd 5 | 2.93 | 69 MB/s | 198 MB/s | -| zstd 10 | 3.01 | 41 MB/s | 225 MB/s | -| zstd 15 | 3.13 | 11.4 MB/s | 224 MB/s | -| zstd 16 256 KB | 3.24 | 8.1 MB/s | 210 MB/s | +1. `cd` into this directory. +2. Run `make libzstd` and read the output. Make sure that all the diffs printed and changes made by the script are correct. +3. Run `make test` and ensure that it passes. +4. Import zstd into the Linux Kernel `make import LINUX=/path/to/linux/repo` +5. Inspect the diff for sanity. +6. Check the Linux Kernel history for zstd. If any patches were made to the kernel version of zstd, but not to upstream zstd, then port them upstream if necessary. +7. Test the diff. Benchmark if necessary. Make sure to test multiple architectures: At least x86, i386, and arm. +8. Submit the patch to the LKML. diff --git a/contrib/linux-kernel/decompress_sources.h b/contrib/linux-kernel/decompress_sources.h new file mode 100644 index 00000000..907753e1 --- /dev/null +++ b/contrib/linux-kernel/decompress_sources.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +/* + * This file includes every .c file needed for decompression. + * It is used by lib/decompress_unzstd.c to include the decompression + * source into the translation-unit, so it can be used for kernel + * decompression. + */ + +#include "common/debug.c" +#include "common/entropy_common.c" +#include "common/error_private.c" +#include "common/fse_decompress.c" +#include "common/zstd_common.c" +#include "decompress/huf_decompress.c" +#include "decompress/zstd_ddict.c" +#include "decompress/zstd_decompress.c" +#include "decompress/zstd_decompress_block.c" +#include "zstd_decompress_module.c" diff --git a/contrib/linux-kernel/fs/btrfs/zstd.c b/contrib/linux-kernel/fs/btrfs/zstd.c deleted file mode 100644 index 607ce47b..00000000 --- a/contrib/linux-kernel/fs/btrfs/zstd.c +++ /dev/null @@ -1,432 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include <linux/bio.h> -#include <linux/err.h> -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/pagemap.h> -#include <linux/refcount.h> -#include <linux/sched.h> -#include <linux/slab.h> -#include <linux/zstd.h> -#include "compression.h" - -#define ZSTD_BTRFS_MAX_WINDOWLOG 17 -#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG) -#define ZSTD_BTRFS_DEFAULT_LEVEL 3 - -static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len) -{ - ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL, - src_len, 0); - - if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG) - params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG; - WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT); - return params; -} - -struct workspace { - void *mem; - size_t size; - char *buf; - struct list_head list; -}; - -static void zstd_free_workspace(struct list_head *ws) -{ - struct workspace *workspace = list_entry(ws, struct workspace, list); - - kvfree(workspace->mem); - kfree(workspace->buf); - kfree(workspace); -} - -static struct list_head *zstd_alloc_workspace(void) -{ - ZSTD_parameters params = - zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT); - struct workspace *workspace; - - workspace = kzalloc(sizeof(*workspace), GFP_KERNEL); - if (!workspace) - return ERR_PTR(-ENOMEM); - - workspace->size = max_t(size_t, - ZSTD_CStreamWorkspaceBound(params.cParams), - ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT)); - workspace->mem = kvmalloc(workspace->size, GFP_KERNEL); - workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!workspace->mem || !workspace->buf) - goto fail; - - INIT_LIST_HEAD(&workspace->list); - - return &workspace->list; -fail: - zstd_free_workspace(&workspace->list); - return ERR_PTR(-ENOMEM); -} - -static int zstd_compress_pages(struct list_head *ws, - struct address_space *mapping, - u64 start, - struct page **pages, - unsigned long *out_pages, - unsigned long *total_in, - unsigned long *total_out) -{ - struct workspace *workspace = list_entry(ws, struct workspace, list); - ZSTD_CStream *stream; - int ret = 0; - int nr_pages = 0; - struct page *in_page = NULL; /* The current page to read */ - struct page *out_page = NULL; /* The current page to write to */ - ZSTD_inBuffer in_buf = { NULL, 0, 0 }; - ZSTD_outBuffer out_buf = { NULL, 0, 0 }; - unsigned long tot_in = 0; - unsigned long tot_out = 0; - unsigned long len = *total_out; - const unsigned long nr_dest_pages = *out_pages; - unsigned long max_out = nr_dest_pages * PAGE_SIZE; - ZSTD_parameters params = zstd_get_btrfs_parameters(len); - - *out_pages = 0; - *total_out = 0; - *total_in = 0; - - /* Initialize the stream */ - stream = ZSTD_initCStream(params, len, workspace->mem, - workspace->size); - if (!stream) { - pr_warn("BTRFS: ZSTD_initCStream failed\n"); - ret = -EIO; - goto out; - } - - /* map in the first page of input data */ - in_page = find_get_page(mapping, start >> PAGE_SHIFT); - in_buf.src = kmap(in_page); - in_buf.pos = 0; - in_buf.size = min_t(size_t, len, PAGE_SIZE); - - - /* Allocate and map in the output buffer */ - out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); - if (out_page == NULL) { - ret = -ENOMEM; - goto out; - } - pages[nr_pages++] = out_page; - out_buf.dst = kmap(out_page); - out_buf.pos = 0; - out_buf.size = min_t(size_t, max_out, PAGE_SIZE); - - while (1) { - size_t ret2; - - ret2 = ZSTD_compressStream(stream, &out_buf, &in_buf); - if (ZSTD_isError(ret2)) { - pr_debug("BTRFS: ZSTD_compressStream returned %d\n", - ZSTD_getErrorCode(ret2)); - ret = -EIO; - goto out; - } - - /* Check to see if we are making it bigger */ - if (tot_in + in_buf.pos > 8192 && - tot_in + in_buf.pos < - tot_out + out_buf.pos) { - ret = -E2BIG; - goto out; - } - - /* We've reached the end of our output range */ - if (out_buf.pos >= max_out) { - tot_out += out_buf.pos; - ret = -E2BIG; - goto out; - } - - /* Check if we need more output space */ - if (out_buf.pos == out_buf.size) { - tot_out += PAGE_SIZE; - max_out -= PAGE_SIZE; - kunmap(out_page); - if (nr_pages == nr_dest_pages) { - out_page = NULL; - ret = -E2BIG; - goto out; - } - out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); - if (out_page == NULL) { - ret = -ENOMEM; - goto out; - } - pages[nr_pages++] = out_page; - out_buf.dst = kmap(out_page); - out_buf.pos = 0; - out_buf.size = min_t(size_t, max_out, PAGE_SIZE); - } - - /* We've reached the end of the input */ - if (in_buf.pos >= len) { - tot_in += in_buf.pos; - break; - } - - /* Check if we need more input */ - if (in_buf.pos == in_buf.size) { - tot_in += PAGE_SIZE; - kunmap(in_page); - put_page(in_page); - - start += PAGE_SIZE; - len -= PAGE_SIZE; - in_page = find_get_page(mapping, start >> PAGE_SHIFT); - in_buf.src = kmap(in_page); - in_buf.pos = 0; - in_buf.size = min_t(size_t, len, PAGE_SIZE); - } - } - while (1) { - size_t ret2; - - ret2 = ZSTD_endStream(stream, &out_buf); - if (ZSTD_isError(ret2)) { - pr_debug("BTRFS: ZSTD_endStream returned %d\n", - ZSTD_getErrorCode(ret2)); - ret = -EIO; - goto out; - } - if (ret2 == 0) { - tot_out += out_buf.pos; - break; - } - if (out_buf.pos >= max_out) { - tot_out += out_buf.pos; - ret = -E2BIG; - goto out; - } - - tot_out += PAGE_SIZE; - max_out -= PAGE_SIZE; - kunmap(out_page); - if (nr_pages == nr_dest_pages) { - out_page = NULL; - ret = -E2BIG; - goto out; - } - out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); - if (out_page == NULL) { - ret = -ENOMEM; - goto out; - } - pages[nr_pages++] = out_page; - out_buf.dst = kmap(out_page); - out_buf.pos = 0; - out_buf.size = min_t(size_t, max_out, PAGE_SIZE); - } - - if (tot_out >= tot_in) { - ret = -E2BIG; - goto out; - } - - ret = 0; - *total_in = tot_in; - *total_out = tot_out; -out: - *out_pages = nr_pages; - /* Cleanup */ - if (in_page) { - kunmap(in_page); - put_page(in_page); - } - if (out_page) - kunmap(out_page); - return ret; -} - -static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) -{ - struct workspace *workspace = list_entry(ws, struct workspace, list); - struct page **pages_in = cb->compressed_pages; - u64 disk_start = cb->start; - struct bio *orig_bio = cb->orig_bio; - size_t srclen = cb->compressed_len; - ZSTD_DStream *stream; - int ret = 0; - unsigned long page_in_index = 0; - unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE); - unsigned long buf_start; - unsigned long total_out = 0; - ZSTD_inBuffer in_buf = { NULL, 0, 0 }; - ZSTD_outBuffer out_buf = { NULL, 0, 0 }; - - stream = ZSTD_initDStream( - ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size); - if (!stream) { - pr_debug("BTRFS: ZSTD_initDStream failed\n"); - ret = -EIO; - goto done; - } - - in_buf.src = kmap(pages_in[page_in_index]); - in_buf.pos = 0; - in_buf.size = min_t(size_t, srclen, PAGE_SIZE); - - out_buf.dst = workspace->buf; - out_buf.pos = 0; - out_buf.size = PAGE_SIZE; - - while (1) { - size_t ret2; - - ret2 = ZSTD_decompressStream(stream, &out_buf, &in_buf); - if (ZSTD_isError(ret2)) { - pr_debug("BTRFS: ZSTD_decompressStream returned %d\n", - ZSTD_getErrorCode(ret2)); - ret = -EIO; - goto done; - } - buf_start = total_out; - total_out += out_buf.pos; - out_buf.pos = 0; - - ret = btrfs_decompress_buf2page(out_buf.dst, buf_start, - total_out, disk_start, orig_bio); - if (ret == 0) - break; - - if (in_buf.pos >= srclen) - break; - - /* Check if we've hit the end of a frame */ - if (ret2 == 0) - break; - - if (in_buf.pos == in_buf.size) { - kunmap(pages_in[page_in_index++]); - if (page_in_index >= total_pages_in) { - in_buf.src = NULL; - ret = -EIO; - goto done; - } - srclen -= PAGE_SIZE; - in_buf.src = kmap(pages_in[page_in_index]); - in_buf.pos = 0; - in_buf.size = min_t(size_t, srclen, PAGE_SIZE); - } - } - ret = 0; - zero_fill_bio(orig_bio); -done: - if (in_buf.src) - kunmap(pages_in[page_in_index]); - return ret; -} - -static int zstd_decompress(struct list_head *ws, unsigned char *data_in, - struct page *dest_page, - unsigned long start_byte, - size_t srclen, size_t destlen) -{ - struct workspace *workspace = list_entry(ws, struct workspace, list); - ZSTD_DStream *stream; - int ret = 0; - size_t ret2; - ZSTD_inBuffer in_buf = { NULL, 0, 0 }; - ZSTD_outBuffer out_buf = { NULL, 0, 0 }; - unsigned long total_out = 0; - unsigned long pg_offset = 0; - char *kaddr; - - stream = ZSTD_initDStream( - ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size); - if (!stream) { - pr_warn("BTRFS: ZSTD_initDStream failed\n"); - ret = -EIO; - goto finish; - } - - destlen = min_t(size_t, destlen, PAGE_SIZE); - - in_buf.src = data_in; - in_buf.pos = 0; - in_buf.size = srclen; - - out_buf.dst = workspace->buf; - out_buf.pos = 0; - out_buf.size = PAGE_SIZE; - - ret2 = 1; - while (pg_offset < destlen && in_buf.pos < in_buf.size) { - unsigned long buf_start; - unsigned long buf_offset; - unsigned long bytes; - - /* Check if the frame is over and we still need more input */ - if (ret2 == 0) { - pr_debug("BTRFS: ZSTD_decompressStream ended early\n"); - ret = -EIO; - goto finish; - } - ret2 = ZSTD_decompressStream(stream, &out_buf, &in_buf); - if (ZSTD_isError(ret2)) { - pr_debug("BTRFS: ZSTD_decompressStream returned %d\n", - ZSTD_getErrorCode(ret2)); - ret = -EIO; - goto finish; - } - - buf_start = total_out; - total_out += out_buf.pos; - out_buf.pos = 0; - - if (total_out <= start_byte) - continue; - - if (total_out > start_byte && buf_start < start_byte) - buf_offset = start_byte - buf_start; - else - buf_offset = 0; - - bytes = min_t(unsigned long, destlen - pg_offset, - out_buf.size - buf_offset); - - kaddr = kmap_atomic(dest_page); - memcpy(kaddr + pg_offset, out_buf.dst + buf_offset, bytes); - kunmap_atomic(kaddr); - - pg_offset += bytes; - } - ret = 0; -finish: - if (pg_offset < destlen) { - kaddr = kmap_atomic(dest_page); - memset(kaddr + pg_offset, 0, destlen - pg_offset); - kunmap_atomic(kaddr); - } - return ret; -} - -const struct btrfs_compress_op btrfs_zstd_compress = { - .alloc_workspace = zstd_alloc_workspace, - .free_workspace = zstd_free_workspace, - .compress_pages = zstd_compress_pages, - .decompress_bio = zstd_decompress_bio, - .decompress = zstd_decompress, -}; diff --git a/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c b/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c deleted file mode 100644 index eeaabf88..00000000 --- a/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Squashfs - a compressed read only filesystem for Linux - * - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2, - * or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * zstd_wrapper.c - */ - -#include <linux/mutex.h> -#include <linux/buffer_head.h> -#include <linux/slab.h> -#include <linux/zstd.h> -#include <linux/vmalloc.h> - -#include "squashfs_fs.h" -#include "squashfs_fs_sb.h" -#include "squashfs.h" -#include "decompressor.h" -#include "page_actor.h" - -struct workspace { - void *mem; - size_t mem_size; - size_t window_size; -}; - -static void *zstd_init(struct squashfs_sb_info *msblk, void *buff) -{ - struct workspace *wksp = kmalloc(sizeof(*wksp), GFP_KERNEL); - - if (wksp == NULL) - goto failed; - wksp->window_size = max_t(size_t, - msblk->block_size, SQUASHFS_METADATA_SIZE); - wksp->mem_size = ZSTD_DStreamWorkspaceBound(wksp->window_size); - wksp->mem = vmalloc(wksp->mem_size); - if (wksp->mem == NULL) - goto failed; - - return wksp; - -failed: - ERROR("Failed to allocate zstd workspace\n"); - kfree(wksp); - return ERR_PTR(-ENOMEM); -} - - -static void zstd_free(void *strm) -{ - struct workspace *wksp = strm; - - if (wksp) - vfree(wksp->mem); - kfree(wksp); -} - - -static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, - struct buffer_head **bh, int b, int offset, int length, - struct squashfs_page_actor *output) -{ - struct workspace *wksp = strm; - ZSTD_DStream *stream; - size_t total_out = 0; - size_t zstd_err; - int k = 0; - ZSTD_inBuffer in_buf = { NULL, 0, 0 }; - ZSTD_outBuffer out_buf = { NULL, 0, 0 }; - - stream = ZSTD_initDStream(wksp->window_size, wksp->mem, wksp->mem_size); - - if (!stream) { - ERROR("Failed to initialize zstd decompressor\n"); - goto out; - } - - out_buf.size = PAGE_SIZE; - out_buf.dst = squashfs_first_page(output); - - do { - if (in_buf.pos == in_buf.size && k < b) { - int avail = min(length, msblk->devblksize - offset); - - length -= avail; - in_buf.src = bh[k]->b_data + offset; - in_buf.size = avail; - in_buf.pos = 0; - offset = 0; - } - - if (out_buf.pos == out_buf.size) { - out_buf.dst = squashfs_next_page(output); - if (out_buf.dst == NULL) { - /* Shouldn't run out of pages - * before stream is done. - */ - squashfs_finish_page(output); - goto out; - } - out_buf.pos = 0; - out_buf.size = PAGE_SIZE; - } - - total_out -= out_buf.pos; - zstd_err = ZSTD_decompressStream(stream, &out_buf, &in_buf); - total_out += out_buf.pos; /* add the additional data produced */ - - if (in_buf.pos == in_buf.size && k < b) - put_bh(bh[k++]); - } while (zstd_err != 0 && !ZSTD_isError(zstd_err)); - - squashfs_finish_page(output); - - if (ZSTD_isError(zstd_err)) { - ERROR("zstd decompression error: %d\n", - (int)ZSTD_getErrorCode(zstd_err)); - goto out; - } - - if (k < b) - goto out; - - return (int)total_out; - -out: - for (; k < b; k++) - put_bh(bh[k]); - - return -EIO; -} - -const struct squashfs_decompressor squashfs_zstd_comp_ops = { - .init = zstd_init, - .free = zstd_free, - .decompress = zstd_uncompress, - .id = ZSTD_COMPRESSION, - .name = "zstd", - .supported = 1 -}; diff --git a/contrib/linux-kernel/include/linux/xxhash.h b/contrib/linux-kernel/include/linux/xxhash.h deleted file mode 100644 index 9e1f42cb..00000000 --- a/contrib/linux-kernel/include/linux/xxhash.h +++ /dev/null @@ -1,236 +0,0 @@ -/* - * xxHash - Extremely Fast Hash algorithm - * Copyright (C) 2012-2016, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at: - * - xxHash homepage: http://cyan4973.github.io/xxHash/ - * - xxHash source repository: https://github.com/Cyan4973/xxHash - */ - -/* - * Notice extracted from xxHash homepage: - * - * xxHash is an extremely fast Hash algorithm, running at RAM speed limits. - * It also successfully passes all tests from the SMHasher suite. - * - * Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 - * Duo @3GHz) - * - * Name Speed Q.Score Author - * xxHash 5.4 GB/s 10 - * CrapWow 3.2 GB/s 2 Andrew - * MumurHash 3a 2.7 GB/s 10 Austin Appleby - * SpookyHash 2.0 GB/s 10 Bob Jenkins - * SBox 1.4 GB/s 9 Bret Mulvey - * Lookup3 1.2 GB/s 9 Bob Jenkins - * SuperFastHash 1.2 GB/s 1 Paul Hsieh - * CityHash64 1.05 GB/s 10 Pike & Alakuijala - * FNV 0.55 GB/s 5 Fowler, Noll, Vo - * CRC32 0.43 GB/s 9 - * MD5-32 0.33 GB/s 10 Ronald L. Rivest - * SHA1-32 0.28 GB/s 10 - * - * Q.Score is a measure of quality of the hash function. - * It depends on successfully passing SMHasher test set. - * 10 is a perfect score. - * - * A 64-bits version, named xxh64 offers much better speed, - * but for 64-bits applications only. - * Name Speed on 64 bits Speed on 32 bits - * xxh64 13.8 GB/s 1.9 GB/s - * xxh32 6.8 GB/s 6.0 GB/s - */ - -#ifndef XXHASH_H -#define XXHASH_H - -#include <linux/types.h> - -/*-**************************** - * Simple Hash Functions - *****************************/ - -/** - * xxh32() - calculate the 32-bit hash of the input with a given seed. - * - * @input: The data to hash. - * @length: The length of the data to hash. - * @seed: The seed can be used to alter the result predictably. - * - * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s - * - * Return: The 32-bit hash of the data. - */ -uint32_t xxh32(const void *input, size_t length, uint32_t seed); - -/** - * xxh64() - calculate the 64-bit hash of the input with a given seed. - * - * @input: The data to hash. - * @length: The length of the data to hash. - * @seed: The seed can be used to alter the result predictably. - * - * This function runs 2x faster on 64-bit systems, but slower on 32-bit systems. - * - * Return: The 64-bit hash of the data. - */ -uint64_t xxh64(const void *input, size_t length, uint64_t seed); - -/*-**************************** - * Streaming Hash Functions - *****************************/ - -/* - * These definitions are only meant to allow allocation of XXH state - * statically, on stack, or in a struct for example. - * Do not use members directly. - */ - -/** - * struct xxh32_state - private xxh32 state, do not use members directly - */ -struct xxh32_state { - uint32_t total_len_32; - uint32_t large_len; - uint32_t v1; - uint32_t v2; - uint32_t v3; - uint32_t v4; - uint32_t mem32[4]; - uint32_t memsize; -}; - -/** - * struct xxh32_state - private xxh64 state, do not use members directly - */ -struct xxh64_state { - uint64_t total_len; - uint64_t v1; - uint64_t v2; - uint64_t v3; - uint64_t v4; - uint64_t mem64[4]; - uint32_t memsize; -}; - -/** - * xxh32_reset() - reset the xxh32 state to start a new hashing operation - * - * @state: The xxh32 state to reset. - * @seed: Initialize the hash state with this seed. - * - * Call this function on any xxh32_state to prepare for a new hashing operation. - */ -void xxh32_reset(struct xxh32_state *state, uint32_t seed); - -/** - * xxh32_update() - hash the data given and update the xxh32 state - * - * @state: The xxh32 state to update. - * @input: The data to hash. - * @length: The length of the data to hash. - * - * After calling xxh32_reset() call xxh32_update() as many times as necessary. - * - * Return: Zero on success, otherwise an error code. - */ -int xxh32_update(struct xxh32_state *state, const void *input, size_t length); - -/** - * xxh32_digest() - produce the current xxh32 hash - * - * @state: Produce the current xxh32 hash of this state. - * - * A hash value can be produced at any time. It is still possible to continue - * inserting input into the hash state after a call to xxh32_digest(), and - * generate new hashes later on, by calling xxh32_digest() again. - * - * Return: The xxh32 hash stored in the state. - */ -uint32_t xxh32_digest(const struct xxh32_state *state); - -/** - * xxh64_reset() - reset the xxh64 state to start a new hashing operation - * - * @state: The xxh64 state to reset. - * @seed: Initialize the hash state with this seed. - */ -void xxh64_reset(struct xxh64_state *state, uint64_t seed); - -/** - * xxh64_update() - hash the data given and update the xxh64 state - * @state: The xxh64 state to update. - * @input: The data to hash. - * @length: The length of the data to hash. - * - * After calling xxh64_reset() call xxh64_update() as many times as necessary. - * - * Return: Zero on success, otherwise an error code. - */ -int xxh64_update(struct xxh64_state *state, const void *input, size_t length); - -/** - * xxh64_digest() - produce the current xxh64 hash - * - * @state: Produce the current xxh64 hash of this state. - * - * A hash value can be produced at any time. It is still possible to continue - * inserting input into the hash state after a call to xxh64_digest(), and - * generate new hashes later on, by calling xxh64_digest() again. - * - * Return: The xxh64 hash stored in the state. - */ -uint64_t xxh64_digest(const struct xxh64_state *state); - -/*-************************** - * Utils - ***************************/ - -/** - * xxh32_copy_state() - copy the source state into the destination state - * - * @src: The source xxh32 state. - * @dst: The destination xxh32 state. - */ -void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src); - -/** - * xxh64_copy_state() - copy the source state into the destination state - * - * @src: The source xxh64 state. - * @dst: The destination xxh64 state. - */ -void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src); - -#endif /* XXHASH_H */ diff --git a/contrib/linux-kernel/include/linux/zstd.h b/contrib/linux-kernel/include/linux/zstd.h deleted file mode 100644 index 305efd09..00000000 --- a/contrib/linux-kernel/include/linux/zstd.h +++ /dev/null @@ -1,1155 +0,0 @@ -/* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -#ifndef ZSTD_H -#define ZSTD_H - -/* ====== Dependency ======*/ -#include <linux/types.h> /* size_t */ - - -/*-***************************************************************************** - * Introduction - * - * zstd, short for Zstandard, is a fast lossless compression algorithm, - * targeting real-time compression scenarios at zlib-level and better - * compression ratios. The zstd compression library provides in-memory - * compression and decompression functions. The library supports compression - * levels from 1 up to ZSTD_maxCLevel() which is 22. Levels >= 20, labeled - * ultra, should be used with caution, as they require more memory. - * Compression can be done in: - * - a single step, reusing a context (described as Explicit memory management) - * - unbounded multiple steps (described as Streaming compression) - * The compression ratio achievable on small data can be highly improved using - * compression with a dictionary in: - * - a single step (described as Simple dictionary API) - * - a single step, reusing a dictionary (described as Fast dictionary API) - ******************************************************************************/ - -/*====== Helper functions ======*/ - -/** - * enum ZSTD_ErrorCode - zstd error codes - * - * Functions that return size_t can be checked for errors using ZSTD_isError() - * and the ZSTD_ErrorCode can be extracted using ZSTD_getErrorCode(). - */ -typedef enum { - ZSTD_error_no_error, - ZSTD_error_GENERIC, - ZSTD_error_prefix_unknown, - ZSTD_error_version_unsupported, - ZSTD_error_parameter_unknown, - ZSTD_error_frameParameter_unsupported, - ZSTD_error_frameParameter_unsupportedBy32bits, - ZSTD_error_frameParameter_windowTooLarge, - ZSTD_error_compressionParameter_unsupported, - ZSTD_error_init_missing, - ZSTD_error_memory_allocation, - ZSTD_error_stage_wrong, - ZSTD_error_dstSize_tooSmall, - ZSTD_error_srcSize_wrong, - ZSTD_error_corruption_detected, - ZSTD_error_checksum_wrong, - ZSTD_error_tableLog_tooLarge, - ZSTD_error_maxSymbolValue_tooLarge, - ZSTD_error_maxSymbolValue_tooSmall, - ZSTD_error_dictionary_corrupted, - ZSTD_error_dictionary_wrong, - ZSTD_error_dictionaryCreation_failed, - ZSTD_error_maxCode -} ZSTD_ErrorCode; - -/** - * ZSTD_maxCLevel() - maximum compression level available - * - * Return: Maximum compression level available. - */ -int ZSTD_maxCLevel(void); -/** - * ZSTD_compressBound() - maximum compressed size in worst case scenario - * @srcSize: The size of the data to compress. - * - * Return: The maximum compressed size in the worst case scenario. - */ -size_t ZSTD_compressBound(size_t srcSize); -/** - * ZSTD_isError() - tells if a size_t function result is an error code - * @code: The function result to check for error. - * - * Return: Non-zero iff the code is an error. - */ -static __attribute__((unused)) unsigned int ZSTD_isError(size_t code) -{ - return code > (size_t)-ZSTD_error_maxCode; -} -/** - * ZSTD_getErrorCode() - translates an error function result to a ZSTD_ErrorCode - * @functionResult: The result of a function for which ZSTD_isError() is true. - * - * Return: The ZSTD_ErrorCode corresponding to the functionResult or 0 - * if the functionResult isn't an error. - */ -static __attribute__((unused)) ZSTD_ErrorCode ZSTD_getErrorCode( - size_t functionResult) -{ - if (!ZSTD_isError(functionResult)) - return (ZSTD_ErrorCode)0; - return (ZSTD_ErrorCode)(0 - functionResult); -} - -/** - * enum ZSTD_strategy - zstd compression search strategy - * - * From faster to stronger. - */ -typedef enum { - ZSTD_fast, - ZSTD_dfast, - ZSTD_greedy, - ZSTD_lazy, - ZSTD_lazy2, - ZSTD_btlazy2, - ZSTD_btopt, - ZSTD_btopt2 -} ZSTD_strategy; - -/** - * struct ZSTD_compressionParameters - zstd compression parameters - * @windowLog: Log of the largest match distance. Larger means more - * compression, and more memory needed during decompression. - * @chainLog: Fully searched segment. Larger means more compression, slower, - * and more memory (useless for fast). - * @hashLog: Dispatch table. Larger means more compression, - * slower, and more memory. - * @searchLog: Number of searches. Larger means more compression and slower. - * @searchLength: Match length searched. Larger means faster decompression, - * sometimes less compression. - * @targetLength: Acceptable match size for optimal parser (only). Larger means - * more compression, and slower. - * @strategy: The zstd compression strategy. - */ -typedef struct { - unsigned int windowLog; - unsigned int chainLog; - unsigned int hashLog; - unsigned int searchLog; - unsigned int searchLength; - unsigned int targetLength; - ZSTD_strategy strategy; -} ZSTD_compressionParameters; - -/** - * struct ZSTD_frameParameters - zstd frame parameters - * @contentSizeFlag: Controls whether content size will be present in the frame - * header (when known). - * @checksumFlag: Controls whether a 32-bit checksum is generated at the end - * of the frame for error detection. - * @noDictIDFlag: Controls whether dictID will be saved into the frame header - * when using dictionary compression. - * - * The default value is all fields set to 0. - */ -typedef struct { - unsigned int contentSizeFlag; - unsigned int checksumFlag; - unsigned int noDictIDFlag; -} ZSTD_frameParameters; - -/** - * struct ZSTD_parameters - zstd parameters - * @cParams: The compression parameters. - * @fParams: The frame parameters. - */ -typedef struct { - ZSTD_compressionParameters cParams; - ZSTD_frameParameters fParams; -} ZSTD_parameters; - -/** - * ZSTD_getCParams() - returns ZSTD_compressionParameters for selected level - * @compressionLevel: The compression level from 1 to ZSTD_maxCLevel(). - * @estimatedSrcSize: The estimated source size to compress or 0 if unknown. - * @dictSize: The dictionary size or 0 if a dictionary isn't being used. - * - * Return: The selected ZSTD_compressionParameters. - */ -ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, - unsigned long long estimatedSrcSize, size_t dictSize); - -/** - * ZSTD_getParams() - returns ZSTD_parameters for selected level - * @compressionLevel: The compression level from 1 to ZSTD_maxCLevel(). - * @estimatedSrcSize: The estimated source size to compress or 0 if unknown. - * @dictSize: The dictionary size or 0 if a dictionary isn't being used. - * - * The same as ZSTD_getCParams() except also selects the default frame - * parameters (all zero). - * - * Return: The selected ZSTD_parameters. - */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, - unsigned long long estimatedSrcSize, size_t dictSize); - -/*-************************************* - * Explicit memory management - **************************************/ - -/** - * ZSTD_CCtxWorkspaceBound() - amount of memory needed to initialize a ZSTD_CCtx - * @cParams: The compression parameters to be used for compression. - * - * If multiple compression parameters might be used, the caller must call - * ZSTD_CCtxWorkspaceBound() for each set of parameters and use the maximum - * size. - * - * Return: A lower bound on the size of the workspace that is passed to - * ZSTD_initCCtx(). - */ -size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams); - -/** - * struct ZSTD_CCtx - the zstd compression context - * - * When compressing many times it is recommended to allocate a context just once - * and reuse it for each successive compression operation. - */ -typedef struct ZSTD_CCtx_s ZSTD_CCtx; -/** - * ZSTD_initCCtx() - initialize a zstd compression context - * @workspace: The workspace to emplace the context into. It must outlive - * the returned context. - * @workspaceSize: The size of workspace. Use ZSTD_CCtxWorkspaceBound() to - * determine how large the workspace must be. - * - * Return: A compression context emplaced into workspace. - */ -ZSTD_CCtx *ZSTD_initCCtx(void *workspace, size_t workspaceSize); - -/** - * ZSTD_compressCCtx() - compress src into dst - * @ctx: The context. Must have been initialized with a workspace at - * least as large as ZSTD_CCtxWorkspaceBound(params.cParams). - * @dst: The buffer to compress src into. - * @dstCapacity: The size of the destination buffer. May be any size, but - * ZSTD_compressBound(srcSize) is guaranteed to be large enough. - * @src: The data to compress. - * @srcSize: The size of the data to compress. - * @params: The parameters to use for compression. See ZSTD_getParams(). - * - * Return: The compressed size or an error, which can be checked using - * ZSTD_isError(). - */ -size_t ZSTD_compressCCtx(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize, ZSTD_parameters params); - -/** - * ZSTD_DCtxWorkspaceBound() - amount of memory needed to initialize a ZSTD_DCtx - * - * Return: A lower bound on the size of the workspace that is passed to - * ZSTD_initDCtx(). - */ -size_t ZSTD_DCtxWorkspaceBound(void); - -/** - * struct ZSTD_DCtx - the zstd decompression context - * - * When decompressing many times it is recommended to allocate a context just - * once and reuse it for each successive decompression operation. - */ -typedef struct ZSTD_DCtx_s ZSTD_DCtx; -/** - * ZSTD_initDCtx() - initialize a zstd decompression context - * @workspace: The workspace to emplace the context into. It must outlive - * the returned context. - * @workspaceSize: The size of workspace. Use ZSTD_DCtxWorkspaceBound() to - * determine how large the workspace must be. - * - * Return: A decompression context emplaced into workspace. - */ -ZSTD_DCtx *ZSTD_initDCtx(void *workspace, size_t workspaceSize); - -/** - * ZSTD_decompressDCtx() - decompress zstd compressed src into dst - * @ctx: The decompression context. - * @dst: The buffer to decompress src into. - * @dstCapacity: The size of the destination buffer. Must be at least as large - * as the decompressed size. If the caller cannot upper bound the - * decompressed size, then it's better to use the streaming API. - * @src: The zstd compressed data to decompress. Multiple concatenated - * frames and skippable frames are allowed. - * @srcSize: The exact size of the data to decompress. - * - * Return: The decompressed size or an error, which can be checked using - * ZSTD_isError(). - */ -size_t ZSTD_decompressDCtx(ZSTD_DCtx *ctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize); - -/*-************************ - * Simple dictionary API - **************************/ - -/** - * ZSTD_compress_usingDict() - compress src into dst using a dictionary - * @ctx: The context. Must have been initialized with a workspace at - * least as large as ZSTD_CCtxWorkspaceBound(params.cParams). - * @dst: The buffer to compress src into. - * @dstCapacity: The size of the destination buffer. May be any size, but - * ZSTD_compressBound(srcSize) is guaranteed to be large enough. - * @src: The data to compress. - * @srcSize: The size of the data to compress. - * @dict: The dictionary to use for compression. - * @dictSize: The size of the dictionary. - * @params: The parameters to use for compression. See ZSTD_getParams(). - * - * Compression using a predefined dictionary. The same dictionary must be used - * during decompression. - * - * Return: The compressed size or an error, which can be checked using - * ZSTD_isError(). - */ -size_t ZSTD_compress_usingDict(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize, const void *dict, size_t dictSize, - ZSTD_parameters params); - -/** - * ZSTD_decompress_usingDict() - decompress src into dst using a dictionary - * @ctx: The decompression context. - * @dst: The buffer to decompress src into. - * @dstCapacity: The size of the destination buffer. Must be at least as large - * as the decompressed size. If the caller cannot upper bound the - * decompressed size, then it's better to use the streaming API. - * @src: The zstd compressed data to decompress. Multiple concatenated - * frames and skippable frames are allowed. - * @srcSize: The exact size of the data to decompress. - * @dict: The dictionary to use for decompression. The same dictionary - * must've been used to compress the data. - * @dictSize: The size of the dictionary. - * - * Return: The decompressed size or an error, which can be checked using - * ZSTD_isError(). - */ -size_t ZSTD_decompress_usingDict(ZSTD_DCtx *ctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize, const void *dict, size_t dictSize); - -/*-************************** - * Fast dictionary API - ***************************/ - -/** - * ZSTD_CDictWorkspaceBound() - memory needed to initialize a ZSTD_CDict - * @cParams: The compression parameters to be used for compression. - * - * Return: A lower bound on the size of the workspace that is passed to - * ZSTD_initCDict(). - */ -size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams); - -/** - * struct ZSTD_CDict - a digested dictionary to be used for compression - */ -typedef struct ZSTD_CDict_s ZSTD_CDict; - -/** - * ZSTD_initCDict() - initialize a digested dictionary for compression - * @dictBuffer: The dictionary to digest. The buffer is referenced by the - * ZSTD_CDict so it must outlive the returned ZSTD_CDict. - * @dictSize: The size of the dictionary. - * @params: The parameters to use for compression. See ZSTD_getParams(). - * @workspace: The workspace. It must outlive the returned ZSTD_CDict. - * @workspaceSize: The workspace size. Must be at least - * ZSTD_CDictWorkspaceBound(params.cParams). - * - * When compressing multiple messages / blocks with the same dictionary it is - * recommended to load it just once. The ZSTD_CDict merely references the - * dictBuffer, so it must outlive the returned ZSTD_CDict. - * - * Return: The digested dictionary emplaced into workspace. - */ -ZSTD_CDict *ZSTD_initCDict(const void *dictBuffer, size_t dictSize, - ZSTD_parameters params, void *workspace, size_t workspaceSize); - -/** - * ZSTD_compress_usingCDict() - compress src into dst using a ZSTD_CDict - * @ctx: The context. Must have been initialized with a workspace at - * least as large as ZSTD_CCtxWorkspaceBound(cParams) where - * cParams are the compression parameters used to initialize the - * cdict. - * @dst: The buffer to compress src into. - * @dstCapacity: The size of the destination buffer. May be any size, but - * ZSTD_compressBound(srcSize) is guaranteed to be large enough. - * @src: The data to compress. - * @srcSize: The size of the data to compress. - * @cdict: The digested dictionary to use for compression. - * @params: The parameters to use for compression. See ZSTD_getParams(). - * - * Compression using a digested dictionary. The same dictionary must be used - * during decompression. - * - * Return: The compressed size or an error, which can be checked using - * ZSTD_isError(). - */ -size_t ZSTD_compress_usingCDict(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize, const ZSTD_CDict *cdict); - - -/** - * ZSTD_DDictWorkspaceBound() - memory needed to initialize a ZSTD_DDict - * - * Return: A lower bound on the size of the workspace that is passed to - * ZSTD_initDDict(). - */ -size_t ZSTD_DDictWorkspaceBound(void); - -/** - * struct ZSTD_DDict - a digested dictionary to be used for decompression - */ -typedef struct ZSTD_DDict_s ZSTD_DDict; - -/** - * ZSTD_initDDict() - initialize a digested dictionary for decompression - * @dictBuffer: The dictionary to digest. The buffer is referenced by the - * ZSTD_DDict so it must outlive the returned ZSTD_DDict. - * @dictSize: The size of the dictionary. - * @workspace: The workspace. It must outlive the returned ZSTD_DDict. - * @workspaceSize: The workspace size. Must be at least - * ZSTD_DDictWorkspaceBound(). - * - * When decompressing multiple messages / blocks with the same dictionary it is - * recommended to load it just once. The ZSTD_DDict merely references the - * dictBuffer, so it must outlive the returned ZSTD_DDict. - * - * Return: The digested dictionary emplaced into workspace. - */ -ZSTD_DDict *ZSTD_initDDict(const void *dictBuffer, size_t dictSize, - void *workspace, size_t workspaceSize); - -/** - * ZSTD_decompress_usingDDict() - decompress src into dst using a ZSTD_DDict - * @ctx: The decompression context. - * @dst: The buffer to decompress src into. - * @dstCapacity: The size of the destination buffer. Must be at least as large - * as the decompressed size. If the caller cannot upper bound the - * decompressed size, then it's better to use the streaming API. - * @src: The zstd compressed data to decompress. Multiple concatenated - * frames and skippable frames are allowed. - * @srcSize: The exact size of the data to decompress. - * @ddict: The digested dictionary to use for decompression. The same - * dictionary must've been used to compress the data. - * - * Return: The decompressed size or an error, which can be checked using - * ZSTD_isError(). - */ -size_t ZSTD_decompress_usingDDict(ZSTD_DCtx *dctx, void *dst, - size_t dstCapacity, const void *src, size_t srcSize, - const ZSTD_DDict *ddict); - - -/*-************************** - * Streaming - ***************************/ - -/** - * struct ZSTD_inBuffer - input buffer for streaming - * @src: Start of the input buffer. - * @size: Size of the input buffer. - * @pos: Position where reading stopped. Will be updated. - * Necessarily 0 <= pos <= size. - */ -typedef struct ZSTD_inBuffer_s { - const void *src; - size_t size; - size_t pos; -} ZSTD_inBuffer; - -/** - * struct ZSTD_outBuffer - output buffer for streaming - * @dst: Start of the output buffer. - * @size: Size of the output buffer. - * @pos: Position where writing stopped. Will be updated. - * Necessarily 0 <= pos <= size. - */ -typedef struct ZSTD_outBuffer_s { - void *dst; - size_t size; - size_t pos; -} ZSTD_outBuffer; - - - -/*-***************************************************************************** - * Streaming compression - HowTo - * - * A ZSTD_CStream object is required to track streaming operation. - * Use ZSTD_initCStream() to initialize a ZSTD_CStream object. - * ZSTD_CStream objects can be reused multiple times on consecutive compression - * operations. It is recommended to re-use ZSTD_CStream in situations where many - * streaming operations will be achieved consecutively. Use one separate - * ZSTD_CStream per thread for parallel execution. - * - * Use ZSTD_compressStream() repetitively to consume input stream. - * The function will automatically update both `pos` fields. - * Note that it may not consume the entire input, in which case `pos < size`, - * and it's up to the caller to present again remaining data. - * It returns a hint for the preferred number of bytes to use as an input for - * the next function call. - * - * At any moment, it's possible to flush whatever data remains within internal - * buffer, using ZSTD_flushStream(). `output->pos` will be updated. There might - * still be some content left within the internal buffer if `output->size` is - * too small. It returns the number of bytes left in the internal buffer and - * must be called until it returns 0. - * - * ZSTD_endStream() instructs to finish a frame. It will perform a flush and - * write frame epilogue. The epilogue is required for decoders to consider a - * frame completed. Similar to ZSTD_flushStream(), it may not be able to flush - * the full content if `output->size` is too small. In which case, call again - * ZSTD_endStream() to complete the flush. It returns the number of bytes left - * in the internal buffer and must be called until it returns 0. - ******************************************************************************/ - -/** - * ZSTD_CStreamWorkspaceBound() - memory needed to initialize a ZSTD_CStream - * @cParams: The compression parameters to be used for compression. - * - * Return: A lower bound on the size of the workspace that is passed to - * ZSTD_initCStream() and ZSTD_initCStream_usingCDict(). - */ -size_t ZSTD_CStreamWorkspaceBound(ZSTD_compressionParameters cParams); - -/** - * struct ZSTD_CStream - the zstd streaming compression context - */ -typedef struct ZSTD_CStream_s ZSTD_CStream; - -/*===== ZSTD_CStream management functions =====*/ -/** - * ZSTD_initCStream() - initialize a zstd streaming compression context - * @params: The zstd compression parameters. - * @pledgedSrcSize: If params.fParams.contentSizeFlag == 1 then the caller must - * pass the source size (zero means empty source). Otherwise, - * the caller may optionally pass the source size, or zero if - * unknown. - * @workspace: The workspace to emplace the context into. It must outlive - * the returned context. - * @workspaceSize: The size of workspace. - * Use ZSTD_CStreamWorkspaceBound(params.cParams) to determine - * how large the workspace must be. - * - * Return: The zstd streaming compression context. - */ -ZSTD_CStream *ZSTD_initCStream(ZSTD_parameters params, - unsigned long long pledgedSrcSize, void *workspace, - size_t workspaceSize); - -/** - * ZSTD_initCStream_usingCDict() - initialize a streaming compression context - * @cdict: The digested dictionary to use for compression. - * @pledgedSrcSize: Optionally the source size, or zero if unknown. - * @workspace: The workspace to emplace the context into. It must outlive - * the returned context. - * @workspaceSize: The size of workspace. Call ZSTD_CStreamWorkspaceBound() - * with the cParams used to initialize the cdict to determine - * how large the workspace must be. - * - * Return: The zstd streaming compression context. - */ -ZSTD_CStream *ZSTD_initCStream_usingCDict(const ZSTD_CDict *cdict, - unsigned long long pledgedSrcSize, void *workspace, - size_t workspaceSize); - -/*===== Streaming compression functions =====*/ -/** - * ZSTD_resetCStream() - reset the context using parameters from creation - * @zcs: The zstd streaming compression context to reset. - * @pledgedSrcSize: Optionally the source size, or zero if unknown. - * - * Resets the context using the parameters from creation. Skips dictionary - * loading, since it can be reused. If `pledgedSrcSize` is non-zero the frame - * content size is always written into the frame header. - * - * Return: Zero or an error, which can be checked using ZSTD_isError(). - */ -size_t ZSTD_resetCStream(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize); -/** - * ZSTD_compressStream() - streaming compress some of input into output - * @zcs: The zstd streaming compression context. - * @output: Destination buffer. `output->pos` is updated to indicate how much - * compressed data was written. - * @input: Source buffer. `input->pos` is updated to indicate how much data was - * read. Note that it may not consume the entire input, in which case - * `input->pos < input->size`, and it's up to the caller to present - * remaining data again. - * - * The `input` and `output` buffers may be any size. Guaranteed to make some - * forward progress if `input` and `output` are not empty. - * - * Return: A hint for the number of bytes to use as the input for the next - * function call or an error, which can be checked using - * ZSTD_isError(). - */ -size_t ZSTD_compressStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output, - ZSTD_inBuffer *input); -/** - * ZSTD_flushStream() - flush internal buffers into output - * @zcs: The zstd streaming compression context. - * @output: Destination buffer. `output->pos` is updated to indicate how much - * compressed data was written. - * - * ZSTD_flushStream() must be called until it returns 0, meaning all the data - * has been flushed. Since ZSTD_flushStream() causes a block to be ended, - * calling it too often will degrade the compression ratio. - * - * Return: The number of bytes still present within internal buffers or an - * error, which can be checked using ZSTD_isError(). - */ -size_t ZSTD_flushStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output); -/** - * ZSTD_endStream() - flush internal buffers into output and end the frame - * @zcs: The zstd streaming compression context. - * @output: Destination buffer. `output->pos` is updated to indicate how much - * compressed data was written. - * - * ZSTD_endStream() must be called until it returns 0, meaning all the data has - * been flushed and the frame epilogue has been written. - * - * Return: The number of bytes still present within internal buffers or an - * error, which can be checked using ZSTD_isError(). - */ -size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output); - -/** - * ZSTD_CStreamInSize() - recommended size for the input buffer - * - * Return: The recommended size for the input buffer. - */ -size_t ZSTD_CStreamInSize(void); -/** - * ZSTD_CStreamOutSize() - recommended size for the output buffer - * - * When the output buffer is at least this large, it is guaranteed to be large - * enough to flush at least one complete compressed block. - * - * Return: The recommended size for the output buffer. - */ -size_t ZSTD_CStreamOutSize(void); - - - -/*-***************************************************************************** - * Streaming decompression - HowTo - * - * A ZSTD_DStream object is required to track streaming operations. - * Use ZSTD_initDStream() to initialize a ZSTD_DStream object. - * ZSTD_DStream objects can be re-used multiple times. - * - * Use ZSTD_decompressStream() repetitively to consume your input. - * The function will update both `pos` fields. - * If `input->pos < input->size`, some input has not been consumed. - * It's up to the caller to present again remaining data. - * If `output->pos < output->size`, decoder has flushed everything it could. - * Returns 0 iff a frame is completely decoded and fully flushed. - * Otherwise it returns a suggested next input size that will never load more - * than the current frame. - ******************************************************************************/ - -/** - * ZSTD_DStreamWorkspaceBound() - memory needed to initialize a ZSTD_DStream - * @maxWindowSize: The maximum window size allowed for compressed frames. - * - * Return: A lower bound on the size of the workspace that is passed to - * ZSTD_initDStream() and ZSTD_initDStream_usingDDict(). - */ -size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize); - -/** - * struct ZSTD_DStream - the zstd streaming decompression context - */ -typedef struct ZSTD_DStream_s ZSTD_DStream; -/*===== ZSTD_DStream management functions =====*/ -/** - * ZSTD_initDStream() - initialize a zstd streaming decompression context - * @maxWindowSize: The maximum window size allowed for compressed frames. - * @workspace: The workspace to emplace the context into. It must outlive - * the returned context. - * @workspaceSize: The size of workspace. - * Use ZSTD_DStreamWorkspaceBound(maxWindowSize) to determine - * how large the workspace must be. - * - * Return: The zstd streaming decompression context. - */ -ZSTD_DStream *ZSTD_initDStream(size_t maxWindowSize, void *workspace, - size_t workspaceSize); -/** - * ZSTD_initDStream_usingDDict() - initialize streaming decompression context - * @maxWindowSize: The maximum window size allowed for compressed frames. - * @ddict: The digested dictionary to use for decompression. - * @workspace: The workspace to emplace the context into. It must outlive - * the returned context. - * @workspaceSize: The size of workspace. - * Use ZSTD_DStreamWorkspaceBound(maxWindowSize) to determine - * how large the workspace must be. - * - * Return: The zstd streaming decompression context. - */ -ZSTD_DStream *ZSTD_initDStream_usingDDict(size_t maxWindowSize, - const ZSTD_DDict *ddict, void *workspace, size_t workspaceSize); - -/*===== Streaming decompression functions =====*/ -/** - * ZSTD_resetDStream() - reset the context using parameters from creation - * @zds: The zstd streaming decompression context to reset. - * - * Resets the context using the parameters from creation. Skips dictionary - * loading, since it can be reused. - * - * Return: Zero or an error, which can be checked using ZSTD_isError(). - */ -size_t ZSTD_resetDStream(ZSTD_DStream *zds); -/** - * ZSTD_decompressStream() - streaming decompress some of input into output - * @zds: The zstd streaming decompression context. - * @output: Destination buffer. `output.pos` is updated to indicate how much - * decompressed data was written. - * @input: Source buffer. `input.pos` is updated to indicate how much data was - * read. Note that it may not consume the entire input, in which case - * `input.pos < input.size`, and it's up to the caller to present - * remaining data again. - * - * The `input` and `output` buffers may be any size. Guaranteed to make some - * forward progress if `input` and `output` are not empty. - * ZSTD_decompressStream() will not consume the last byte of the frame until - * the entire frame is flushed. - * - * Return: Returns 0 iff a frame is completely decoded and fully flushed. - * Otherwise returns a hint for the number of bytes to use as the input - * for the next function call or an error, which can be checked using - * ZSTD_isError(). The size hint will never load more than the frame. - */ -size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, - ZSTD_inBuffer *input); - -/** - * ZSTD_DStreamInSize() - recommended size for the input buffer - * - * Return: The recommended size for the input buffer. - */ -size_t ZSTD_DStreamInSize(void); -/** - * ZSTD_DStreamOutSize() - recommended size for the output buffer - * - * When the output buffer is at least this large, it is guaranteed to be large - * enough to flush at least one complete decompressed block. - * - * Return: The recommended size for the output buffer. - */ -size_t ZSTD_DStreamOutSize(void); - - -/* --- Constants ---*/ -#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */ -#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U - -#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) -#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) - -#define ZSTD_WINDOWLOG_MAX_32 27 -#define ZSTD_WINDOWLOG_MAX_64 27 -#define ZSTD_WINDOWLOG_MAX \ - ((unsigned int)(sizeof(size_t) == 4 \ - ? ZSTD_WINDOWLOG_MAX_32 \ - : ZSTD_WINDOWLOG_MAX_64)) -#define ZSTD_WINDOWLOG_MIN 10 -#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX -#define ZSTD_HASHLOG_MIN 6 -#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) -#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN -#define ZSTD_HASHLOG3_MAX 17 -#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) -#define ZSTD_SEARCHLOG_MIN 1 -/* only for ZSTD_fast, other strategies are limited to 6 */ -#define ZSTD_SEARCHLENGTH_MAX 7 -/* only for ZSTD_btopt, other strategies are limited to 4 */ -#define ZSTD_SEARCHLENGTH_MIN 3 -#define ZSTD_TARGETLENGTH_MIN 4 -#define ZSTD_TARGETLENGTH_MAX 999 - -/* for static allocation */ -#define ZSTD_FRAMEHEADERSIZE_MAX 18 -#define ZSTD_FRAMEHEADERSIZE_MIN 6 -static const size_t ZSTD_frameHeaderSize_prefix = 5; -static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN; -static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; -/* magic number + skippable frame length */ -static const size_t ZSTD_skippableHeaderSize = 8; - - -/*-************************************* - * Compressed size functions - **************************************/ - -/** - * ZSTD_findFrameCompressedSize() - returns the size of a compressed frame - * @src: Source buffer. It should point to the start of a zstd encoded frame - * or a skippable frame. - * @srcSize: The size of the source buffer. It must be at least as large as the - * size of the frame. - * - * Return: The compressed size of the frame pointed to by `src` or an error, - * which can be check with ZSTD_isError(). - * Suitable to pass to ZSTD_decompress() or similar functions. - */ -size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize); - -/*-************************************* - * Decompressed size functions - **************************************/ -/** - * ZSTD_getFrameContentSize() - returns the content size in a zstd frame header - * @src: It should point to the start of a zstd encoded frame. - * @srcSize: The size of the source buffer. It must be at least as large as the - * frame header. `ZSTD_frameHeaderSize_max` is always large enough. - * - * Return: The frame content size stored in the frame header if known. - * `ZSTD_CONTENTSIZE_UNKNOWN` if the content size isn't stored in the - * frame header. `ZSTD_CONTENTSIZE_ERROR` on invalid input. - */ -unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); - -/** - * ZSTD_findDecompressedSize() - returns decompressed size of a series of frames - * @src: It should point to the start of a series of zstd encoded and/or - * skippable frames. - * @srcSize: The exact size of the series of frames. - * - * If any zstd encoded frame in the series doesn't have the frame content size - * set, `ZSTD_CONTENTSIZE_UNKNOWN` is returned. But frame content size is always - * set when using ZSTD_compress(). The decompressed size can be very large. - * If the source is untrusted, the decompressed size could be wrong or - * intentionally modified. Always ensure the result fits within the - * application's authorized limits. ZSTD_findDecompressedSize() handles multiple - * frames, and so it must traverse the input to read each frame header. This is - * efficient as most of the data is skipped, however it does mean that all frame - * data must be present and valid. - * - * Return: Decompressed size of all the data contained in the frames if known. - * `ZSTD_CONTENTSIZE_UNKNOWN` if the decompressed size is unknown. - * `ZSTD_CONTENTSIZE_ERROR` if an error occurred. - */ -unsigned long long ZSTD_findDecompressedSize(const void *src, size_t srcSize); - -/*-************************************* - * Advanced compression functions - **************************************/ -/** - * ZSTD_checkCParams() - ensure parameter values remain within authorized range - * @cParams: The zstd compression parameters. - * - * Return: Zero or an error, which can be checked using ZSTD_isError(). - */ -size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams); - -/** - * ZSTD_adjustCParams() - optimize parameters for a given srcSize and dictSize - * @srcSize: Optionally the estimated source size, or zero if unknown. - * @dictSize: Optionally the estimated dictionary size, or zero if unknown. - * - * Return: The optimized parameters. - */ -ZSTD_compressionParameters ZSTD_adjustCParams( - ZSTD_compressionParameters cParams, unsigned long long srcSize, - size_t dictSize); - -/*--- Advanced decompression functions ---*/ - -/** - * ZSTD_isFrame() - returns true iff the buffer starts with a valid frame - * @buffer: The source buffer to check. - * @size: The size of the source buffer, must be at least 4 bytes. - * - * Return: True iff the buffer starts with a zstd or skippable frame identifier. - */ -unsigned int ZSTD_isFrame(const void *buffer, size_t size); - -/** - * ZSTD_getDictID_fromDict() - returns the dictionary id stored in a dictionary - * @dict: The dictionary buffer. - * @dictSize: The size of the dictionary buffer. - * - * Return: The dictionary id stored within the dictionary or 0 if the - * dictionary is not a zstd dictionary. If it returns 0 the - * dictionary can still be loaded as a content-only dictionary. - */ -unsigned int ZSTD_getDictID_fromDict(const void *dict, size_t dictSize); - -/** - * ZSTD_getDictID_fromDDict() - returns the dictionary id stored in a ZSTD_DDict - * @ddict: The ddict to find the id of. - * - * Return: The dictionary id stored within `ddict` or 0 if the dictionary is not - * a zstd dictionary. If it returns 0 `ddict` will be loaded as a - * content-only dictionary. - */ -unsigned int ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict); - -/** - * ZSTD_getDictID_fromFrame() - returns the dictionary id stored in a zstd frame - * @src: Source buffer. It must be a zstd encoded frame. - * @srcSize: The size of the source buffer. It must be at least as large as the - * frame header. `ZSTD_frameHeaderSize_max` is always large enough. - * - * Return: The dictionary id required to decompress the frame stored within - * `src` or 0 if the dictionary id could not be decoded. It can return - * 0 if the frame does not require a dictionary, the dictionary id - * wasn't stored in the frame, `src` is not a zstd frame, or `srcSize` - * is too small. - */ -unsigned int ZSTD_getDictID_fromFrame(const void *src, size_t srcSize); - -/** - * struct ZSTD_frameParams - zstd frame parameters stored in the frame header - * @frameContentSize: The frame content size, or 0 if not present. - * @windowSize: The window size, or 0 if the frame is a skippable frame. - * @dictID: The dictionary id, or 0 if not present. - * @checksumFlag: Whether a checksum was used. - */ -typedef struct { - unsigned long long frameContentSize; - unsigned int windowSize; - unsigned int dictID; - unsigned int checksumFlag; -} ZSTD_frameParams; - -/** - * ZSTD_getFrameParams() - extracts parameters from a zstd or skippable frame - * @fparamsPtr: On success the frame parameters are written here. - * @src: The source buffer. It must point to a zstd or skippable frame. - * @srcSize: The size of the source buffer. `ZSTD_frameHeaderSize_max` is - * always large enough to succeed. - * - * Return: 0 on success. If more data is required it returns how many bytes - * must be provided to make forward progress. Otherwise it returns - * an error, which can be checked using ZSTD_isError(). - */ -size_t ZSTD_getFrameParams(ZSTD_frameParams *fparamsPtr, const void *src, - size_t srcSize); - -/*-***************************************************************************** - * Buffer-less and synchronous inner streaming functions - * - * This is an advanced API, giving full control over buffer management, for - * users which need direct control over memory. - * But it's also a complex one, with many restrictions (documented below). - * Prefer using normal streaming API for an easier experience - ******************************************************************************/ - -/*-***************************************************************************** - * Buffer-less streaming compression (synchronous mode) - * - * A ZSTD_CCtx object is required to track streaming operations. - * Use ZSTD_initCCtx() to initialize a context. - * ZSTD_CCtx object can be re-used multiple times within successive compression - * operations. - * - * Start by initializing a context. - * Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary - * compression, - * or ZSTD_compressBegin_advanced(), for finer parameter control. - * It's also possible to duplicate a reference context which has already been - * initialized, using ZSTD_copyCCtx() - * - * Then, consume your input using ZSTD_compressContinue(). - * There are some important considerations to keep in mind when using this - * advanced function : - * - ZSTD_compressContinue() has no internal buffer. It uses externally provided - * buffer only. - * - Interface is synchronous : input is consumed entirely and produce 1+ - * (or more) compressed blocks. - * - Caller must ensure there is enough space in `dst` to store compressed data - * under worst case scenario. Worst case evaluation is provided by - * ZSTD_compressBound(). - * ZSTD_compressContinue() doesn't guarantee recover after a failed - * compression. - * - ZSTD_compressContinue() presumes prior input ***is still accessible and - * unmodified*** (up to maximum distance size, see WindowLog). - * It remembers all previous contiguous blocks, plus one separated memory - * segment (which can itself consists of multiple contiguous blocks) - * - ZSTD_compressContinue() detects that prior input has been overwritten when - * `src` buffer overlaps. In which case, it will "discard" the relevant memory - * section from its history. - * - * Finish a frame with ZSTD_compressEnd(), which will write the last block(s) - * and optional checksum. It's possible to use srcSize==0, in which case, it - * will write a final empty block to end the frame. Without last block mark, - * frames will be considered unfinished (corrupted) by decoders. - * - * `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new - * frame. - ******************************************************************************/ - -/*===== Buffer-less streaming compression functions =====*/ -size_t ZSTD_compressBegin(ZSTD_CCtx *cctx, int compressionLevel); -size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict, - size_t dictSize, int compressionLevel); -size_t ZSTD_compressBegin_advanced(ZSTD_CCtx *cctx, const void *dict, - size_t dictSize, ZSTD_parameters params, - unsigned long long pledgedSrcSize); -size_t ZSTD_copyCCtx(ZSTD_CCtx *cctx, const ZSTD_CCtx *preparedCCtx, - unsigned long long pledgedSrcSize); -size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, - unsigned long long pledgedSrcSize); -size_t ZSTD_compressContinue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize); -size_t ZSTD_compressEnd(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize); - - - -/*-***************************************************************************** - * Buffer-less streaming decompression (synchronous mode) - * - * A ZSTD_DCtx object is required to track streaming operations. - * Use ZSTD_initDCtx() to initialize a context. - * A ZSTD_DCtx object can be re-used multiple times. - * - * First typical operation is to retrieve frame parameters, using - * ZSTD_getFrameParams(). It fills a ZSTD_frameParams structure which provide - * important information to correctly decode the frame, such as the minimum - * rolling buffer size to allocate to decompress data (`windowSize`), and the - * dictionary ID used. - * Note: content size is optional, it may not be present. 0 means unknown. - * Note that these values could be wrong, either because of data malformation, - * or because an attacker is spoofing deliberate false information. As a - * consequence, check that values remain within valid application range, - * especially `windowSize`, before allocation. Each application can set its own - * limit, depending on local restrictions. For extended interoperability, it is - * recommended to support at least 8 MB. - * Frame parameters are extracted from the beginning of the compressed frame. - * Data fragment must be large enough to ensure successful decoding, typically - * `ZSTD_frameHeaderSize_max` bytes. - * Result: 0: successful decoding, the `ZSTD_frameParams` structure is filled. - * >0: `srcSize` is too small, provide at least this many bytes. - * errorCode, which can be tested using ZSTD_isError(). - * - * Start decompression, with ZSTD_decompressBegin() or - * ZSTD_decompressBegin_usingDict(). Alternatively, you can copy a prepared - * context, using ZSTD_copyDCtx(). - * - * Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() - * alternatively. - * ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' - * to ZSTD_decompressContinue(). - * ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will - * fail. - * - * The result of ZSTD_decompressContinue() is the number of bytes regenerated - * within 'dst' (necessarily <= dstCapacity). It can be zero, which is not an - * error; it just means ZSTD_decompressContinue() has decoded some metadata - * item. It can also be an error code, which can be tested with ZSTD_isError(). - * - * ZSTD_decompressContinue() needs previous data blocks during decompression, up - * to `windowSize`. They should preferably be located contiguously, prior to - * current block. Alternatively, a round buffer of sufficient size is also - * possible. Sufficient size is determined by frame parameters. - * ZSTD_decompressContinue() is very sensitive to contiguity, if 2 blocks don't - * follow each other, make sure that either the compressor breaks contiguity at - * the same place, or that previous contiguous segment is large enough to - * properly handle maximum back-reference. - * - * A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. - * Context can then be reset to start a new decompression. - * - * Note: it's possible to know if next input to present is a header or a block, - * using ZSTD_nextInputType(). This information is not required to properly - * decode a frame. - * - * == Special case: skippable frames == - * - * Skippable frames allow integration of user-defined data into a flow of - * concatenated frames. Skippable frames will be ignored (skipped) by a - * decompressor. The format of skippable frames is as follows: - * a) Skippable frame ID - 4 Bytes, Little endian format, any value from - * 0x184D2A50 to 0x184D2A5F - * b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits - * c) Frame Content - any content (User Data) of length equal to Frame Size - * For skippable frames ZSTD_decompressContinue() always returns 0. - * For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 - * what means that a frame is skippable. - * Note: If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might - * actually be a zstd encoded frame with no content. For purposes of - * decompression, it is valid in both cases to skip the frame using - * ZSTD_findFrameCompressedSize() to find its size in bytes. - * It also returns frame size as fparamsPtr->frameContentSize. - ******************************************************************************/ - -/*===== Buffer-less streaming decompression functions =====*/ -size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx); -size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx *dctx, const void *dict, - size_t dictSize); -void ZSTD_copyDCtx(ZSTD_DCtx *dctx, const ZSTD_DCtx *preparedDCtx); -size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx *dctx); -size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize); -typedef enum { - ZSTDnit_frameHeader, - ZSTDnit_blockHeader, - ZSTDnit_block, - ZSTDnit_lastBlock, - ZSTDnit_checksum, - ZSTDnit_skippableFrame -} ZSTD_nextInputType_e; -ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx *dctx); - -/*-***************************************************************************** - * Block functions - * - * Block functions produce and decode raw zstd blocks, without frame metadata. - * Frame metadata cost is typically ~18 bytes, which can be non-negligible for - * very small blocks (< 100 bytes). User will have to take in charge required - * information to regenerate data, such as compressed and content sizes. - * - * A few rules to respect: - * - Compressing and decompressing require a context structure - * + Use ZSTD_initCCtx() and ZSTD_initDCtx() - * - It is necessary to init context before starting - * + compression : ZSTD_compressBegin() - * + decompression : ZSTD_decompressBegin() - * + variants _usingDict() are also allowed - * + copyCCtx() and copyDCtx() work too - * - Block size is limited, it must be <= ZSTD_getBlockSizeMax() - * + If you need to compress more, cut data into multiple blocks - * + Consider using the regular ZSTD_compress() instead, as frame metadata - * costs become negligible when source size is large. - * - When a block is considered not compressible enough, ZSTD_compressBlock() - * result will be zero. In which case, nothing is produced into `dst`. - * + User must test for such outcome and deal directly with uncompressed data - * + ZSTD_decompressBlock() doesn't accept uncompressed data as input!!! - * + In case of multiple successive blocks, decoder must be informed of - * uncompressed block existence to follow proper history. Use - * ZSTD_insertBlock() in such a case. - ******************************************************************************/ - -/* Define for static allocation */ -#define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) -/*===== Raw zstd block functions =====*/ -size_t ZSTD_getBlockSizeMax(ZSTD_CCtx *cctx); -size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize); -size_t ZSTD_decompressBlock(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, - const void *src, size_t srcSize); -size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void *blockStart, - size_t blockSize); - -#endif /* ZSTD_H */ diff --git a/contrib/linux-kernel/kernelize.sh b/contrib/linux-kernel/kernelize.sh deleted file mode 100755 index 21aa2ecd..00000000 --- a/contrib/linux-kernel/kernelize.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/sh -set -e - -# Constants -SED_COMMANDS="commands.tmp" -CLANG_FORMAT="clang-format-3.9" -INCLUDE='include/linux/' -LIB='lib/zstd/' -SPACES=' ' -TAB=$'\t' -TMP="replacements.tmp" - -function prompt() { - while true; do - read -p "$1 [Y/n]" yn - case $yn in - '' ) yes='yes'; break;; - [Yy]* ) yes='yes'; break;; - [Nn]* ) yes=''; break;; - * ) echo "Please answer yes or no.";; - esac -done -} - -function check_not_present() { - grep "$1" $INCLUDE*.h ${LIB}*.{h,c} && exit 1 || true -} - -function check_not_present_in_file() { - grep "$1" "$2" && exit 1 || true -} - -function check_present_in_file() { - grep "$1" "$2" > /dev/null 2> /dev/null || exit 1 -} - -echo "Files: " $INCLUDE*.h $LIB*.{h,c} - -prompt "Do you wish to replace 4 spaces with a tab?" -if [ ! -z "$yes" ] -then - # Check files for existing tabs - grep "$TAB" $INCLUDE*.h $LIB*.{h,c} && exit 1 || true - # Replace the first tab on every line - sed -i '' "s/^$SPACES/$TAB/" $INCLUDE*.h $LIB*.{h,c} - - # Execute once and then execute as long as replacements are happening - more_work="yes" - while [ ! -z "$more_work" ] - do - rm -f $TMP - # Replaces $SPACES that directly follow a $TAB with a $TAB. - # $TMP will be non-empty if any replacements took place. - sed -i '' "s/$TAB$SPACES/$TAB$TAB/w $TMP" $INCLUDE*.h $LIB*.{h,c} - more_work=$(cat "$TMP") - done - rm -f $TMP -fi - -prompt "Do you wish to replace '{ ' with a tab?" -if [ ! -z "$yes" ] -then - sed -i '' "s/$TAB{ /$TAB{$TAB/g" $INCLUDE*.h $LIB*.{h,c} -fi - -rm -f $SED_COMMANDS -cat > $SED_COMMANDS <<EOF -s/current/curr/g -s/MEM_STATIC/ZSTD_STATIC/g -s/MEM_check/ZSTD_check/g -s/MEM_32bits/ZSTD_32bits/g -s/MEM_64bits/ZSTD_64bits/g -s/MEM_LITTLE_ENDIAN/ZSTD_LITTLE_ENDIAN/g -s/MEM_isLittleEndian/ZSTD_isLittleEndian/g -s/MEM_read/ZSTD_read/g -s/MEM_write/ZSTD_write/g -EOF - -prompt "Do you wish to run these sed commands $(cat $SED_COMMANDS)?" -if [ ! -z "$yes" ] -then - sed -i '' -f $SED_COMMANDS $LIB*.{h,c} -fi -rm -f $SED_COMMANDS - -prompt "Do you wish to clang-format $LIB*.{h,c}?" -if [ ! -z "$yes" ] -then - $CLANG_FORMAT -i ${LIB}*.{h,c} -fi - -prompt "Do you wish to run some checks?" -if [ ! -z "$yes" ] -then - check_present_in_file ZSTD_STATIC_ASSERT ${LIB}zstd_internal.h - check_not_present_in_file STATIC_ASSERT ${LIB}mem.h - check_not_present_in_file "#define ZSTD_STATIC_ASSERT" ${LIB}compress.c - check_not_present MEM_STATIC - check_not_present FSE_COMMONDEFS_ONLY - check_not_present "#if 0" - check_not_present "#if 1" - check_not_present _MSC_VER - check_not_present __cplusplus - check_not_present __STDC_VERSION__ - check_not_present __VMS - check_not_present __GNUC__ - check_not_present __INTEL_COMPILER - check_not_present FORCE_MEMORY_ACCESS - check_not_present STATIC_LINKING_ONLY -fi diff --git a/contrib/linux-kernel/lib/Kconfig.diff b/contrib/linux-kernel/lib/Kconfig.diff deleted file mode 100644 index 227c6e2c..00000000 --- a/contrib/linux-kernel/lib/Kconfig.diff +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/lib/Kconfig b/lib/Kconfig -index b6009d7..f00ddab 100644 ---- a/lib/Kconfig -+++ b/lib/Kconfig -@@ -241,6 +241,14 @@ config LZ4HC_COMPRESS - config LZ4_DECOMPRESS - tristate - -+config ZSTD_COMPRESS -+ select XXHASH -+ tristate -+ -+config ZSTD_DECOMPRESS -+ select XXHASH -+ tristate -+ - source "lib/xz/Kconfig" - - # diff --git a/contrib/linux-kernel/lib/Makefile.diff b/contrib/linux-kernel/lib/Makefile.diff deleted file mode 100644 index f92efe8e..00000000 --- a/contrib/linux-kernel/lib/Makefile.diff +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/lib/Makefile b/lib/Makefile -index e16f94a..0cfd529 100644 ---- a/lib/Makefile -+++ b/lib/Makefile -@@ -115,6 +115,8 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/ - obj-$(CONFIG_LZ4_COMPRESS) += lz4/ - obj-$(CONFIG_LZ4HC_COMPRESS) += lz4/ - obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/ -+obj-$(CONFIG_ZSTD_COMPRESS) += zstd/ -+obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd/ - obj-$(CONFIG_XZ_DEC) += xz/ - obj-$(CONFIG_RAID6_PQ) += raid6/ - diff --git a/contrib/linux-kernel/lib/zstd/.clang-format b/contrib/linux-kernel/lib/zstd/.clang-format deleted file mode 100644 index 0c6cf3ba..00000000 --- a/contrib/linux-kernel/lib/zstd/.clang-format +++ /dev/null @@ -1,11 +0,0 @@ -BasedOnStyle: LLVM -IndentWidth: 8 -UseTab: Always -BreakBeforeBraces: Linux -AllowShortIfStatementsOnASingleLine: false -IndentCaseLabels: false - -ColumnLimit: 160 -AlignEscapedNewlinesLeft: true -ReflowComments: true -AllowShortCaseLabelsOnASingleLine: true diff --git a/contrib/linux-kernel/lib/zstd/Makefile b/contrib/linux-kernel/lib/zstd/Makefile deleted file mode 100644 index dd0a359c..00000000 --- a/contrib/linux-kernel/lib/zstd/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o -obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o - -ccflags-y += -O3 - -# Object files unique to zstd_compress and zstd_decompress -zstd_compress-y := fse_compress.o huf_compress.o compress.o -zstd_decompress-y := huf_decompress.o decompress.o - -# These object files are shared between the modules. -# Always add them to zstd_compress. -# Unless both zstd_compress and zstd_decompress are built in -# then also add them to zstd_decompress. -zstd_compress-y += entropy_common.o fse_decompress.o zstd_common.o - -ifneq ($(CONFIG_ZSTD_COMPRESS)$(CONFIG_ZSTD_DECOMPRESS),yy) - zstd_decompress-y += entropy_common.o fse_decompress.o zstd_common.o -endif diff --git a/contrib/linux-kernel/lib/zstd/bitstream.h b/contrib/linux-kernel/lib/zstd/bitstream.h deleted file mode 100644 index a826b99e..00000000 --- a/contrib/linux-kernel/lib/zstd/bitstream.h +++ /dev/null @@ -1,374 +0,0 @@ -/* - * bitstream - * Part of FSE library - * header file (to include) - * Copyright (C) 2013-2016, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ -#ifndef BITSTREAM_H_MODULE -#define BITSTREAM_H_MODULE - -/* -* This API consists of small unitary functions, which must be inlined for best performance. -* Since link-time-optimization is not available for all compilers, -* these functions are defined into a .h to be included. -*/ - -/*-**************************************** -* Dependencies -******************************************/ -#include "error_private.h" /* error codes and messages */ -#include "mem.h" /* unaligned access routines */ - -/*========================================= -* Target specific -=========================================*/ -#define STREAM_ACCUMULATOR_MIN_32 25 -#define STREAM_ACCUMULATOR_MIN_64 57 -#define STREAM_ACCUMULATOR_MIN ((U32)(ZSTD_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) - -/*-****************************************** -* bitStream encoding API (write forward) -********************************************/ -/* bitStream can mix input from multiple sources. -* A critical property of these streams is that they encode and decode in **reverse** direction. -* So the first bit sequence you add will be the last to be read, like a LIFO stack. -*/ -typedef struct { - size_t bitContainer; - int bitPos; - char *startPtr; - char *ptr; - char *endPtr; -} BIT_CStream_t; - -ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *dstBuffer, size_t dstCapacity); -ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits); -ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC); -ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC); - -/* Start with initCStream, providing the size of buffer to write into. -* bitStream will never write outside of this buffer. -* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. -* -* bits are first added to a local register. -* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. -* Writing data into memory is an explicit operation, performed by the flushBits function. -* Hence keep track how many bits are potentially stored into local register to avoid register overflow. -* After a flushBits, a maximum of 7 bits might still be stored into local register. -* -* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. -* -* Last operation is to close the bitStream. -* The function returns the final size of CStream in bytes. -* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) -*/ - -/*-******************************************** -* bitStream decoding API (read backward) -**********************************************/ -typedef struct { - size_t bitContainer; - unsigned bitsConsumed; - const char *ptr; - const char *start; -} BIT_DStream_t; - -typedef enum { - BIT_DStream_unfinished = 0, - BIT_DStream_endOfBuffer = 1, - BIT_DStream_completed = 2, - BIT_DStream_overflow = 3 -} BIT_DStream_status; /* result of BIT_reloadDStream() */ -/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ - -ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize); -ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, unsigned nbBits); -ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD); -ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *bitD); - -/* Start by invoking BIT_initDStream(). -* A chunk of the bitStream is then stored into a local register. -* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). -* You can then retrieve bitFields stored into the local register, **in reverse order**. -* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. -* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. -* Otherwise, it can be less than that, so proceed accordingly. -* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). -*/ - -/*-**************************************** -* unsafe API -******************************************/ -ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits); -/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ - -ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC); -/* unsafe version; does not check buffer overflow */ - -ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, unsigned nbBits); -/* faster, but works only if nbBits >= 1 */ - -/*-************************************************************** -* Internal functions -****************************************************************/ -ZSTD_STATIC unsigned BIT_highbit32(register U32 val) { return 31 - __builtin_clz(val); } - -/*===== Local Constants =====*/ -static const unsigned BIT_mask[] = {0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, - 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, - 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF}; /* up to 26 bits */ - -/*-************************************************************** -* bitStream encoding -****************************************************************/ -/*! BIT_initCStream() : - * `dstCapacity` must be > sizeof(void*) - * @return : 0 if success, - otherwise an error code (can be tested using ERR_isError() ) */ -ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *startPtr, size_t dstCapacity) -{ - bitC->bitContainer = 0; - bitC->bitPos = 0; - bitC->startPtr = (char *)startPtr; - bitC->ptr = bitC->startPtr; - bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr); - if (dstCapacity <= sizeof(bitC->ptr)) - return ERROR(dstSize_tooSmall); - return 0; -} - -/*! BIT_addBits() : - can add up to 26 bits into `bitC`. - Does not check for register overflow ! */ -ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits) -{ - bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; - bitC->bitPos += nbBits; -} - -/*! BIT_addBitsFast() : - * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ -ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits) -{ - bitC->bitContainer |= value << bitC->bitPos; - bitC->bitPos += nbBits; -} - -/*! BIT_flushBitsFast() : - * unsafe version; does not check buffer overflow */ -ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC) -{ - size_t const nbBytes = bitC->bitPos >> 3; - ZSTD_writeLEST(bitC->ptr, bitC->bitContainer); - bitC->ptr += nbBytes; - bitC->bitPos &= 7; - bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ -} - -/*! BIT_flushBits() : - * safe version; check for buffer overflow, and prevents it. - * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */ -ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC) -{ - size_t const nbBytes = bitC->bitPos >> 3; - ZSTD_writeLEST(bitC->ptr, bitC->bitContainer); - bitC->ptr += nbBytes; - if (bitC->ptr > bitC->endPtr) - bitC->ptr = bitC->endPtr; - bitC->bitPos &= 7; - bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ -} - -/*! BIT_closeCStream() : - * @return : size of CStream, in bytes, - or 0 if it could not fit into dstBuffer */ -ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC) -{ - BIT_addBitsFast(bitC, 1, 1); /* endMark */ - BIT_flushBits(bitC); - - if (bitC->ptr >= bitC->endPtr) - return 0; /* doesn't fit within authorized budget : cancel */ - - return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); -} - -/*-******************************************************** -* bitStream decoding -**********************************************************/ -/*! BIT_initDStream() : -* Initialize a BIT_DStream_t. -* `bitD` : a pointer to an already allocated BIT_DStream_t structure. -* `srcSize` must be the *exact* size of the bitStream, in bytes. -* @return : size of stream (== srcSize) or an errorCode if a problem is detected -*/ -ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize) -{ - if (srcSize < 1) { - memset(bitD, 0, sizeof(*bitD)); - return ERROR(srcSize_wrong); - } - - if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ - bitD->start = (const char *)srcBuffer; - bitD->ptr = (const char *)srcBuffer + srcSize - sizeof(bitD->bitContainer); - bitD->bitContainer = ZSTD_readLEST(bitD->ptr); - { - BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ - if (lastByte == 0) - return ERROR(GENERIC); /* endMark not present */ - } - } else { - bitD->start = (const char *)srcBuffer; - bitD->ptr = bitD->start; - bitD->bitContainer = *(const BYTE *)(bitD->start); - switch (srcSize) { - case 7: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[6]) << (sizeof(bitD->bitContainer) * 8 - 16); - case 6: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[5]) << (sizeof(bitD->bitContainer) * 8 - 24); - case 5: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[4]) << (sizeof(bitD->bitContainer) * 8 - 32); - case 4: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[3]) << 24; - case 3: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[2]) << 16; - case 2: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[1]) << 8; - default:; - } - { - BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; - if (lastByte == 0) - return ERROR(GENERIC); /* endMark not present */ - } - bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize) * 8; - } - - return srcSize; -} - -ZSTD_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) { return bitContainer >> start; } - -ZSTD_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) { return (bitContainer >> start) & BIT_mask[nbBits]; } - -ZSTD_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) { return bitContainer & BIT_mask[nbBits]; } - -/*! BIT_lookBits() : - * Provides next n bits from local register. - * local register is not modified. - * On 32-bits, maxNbBits==24. - * On 64-bits, maxNbBits==56. - * @return : value extracted - */ -ZSTD_STATIC size_t BIT_lookBits(const BIT_DStream_t *bitD, U32 nbBits) -{ - U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1; - return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask - nbBits) & bitMask); -} - -/*! BIT_lookBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ -ZSTD_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t *bitD, U32 nbBits) -{ - U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1; - return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask + 1) - nbBits) & bitMask); -} - -ZSTD_STATIC void BIT_skipBits(BIT_DStream_t *bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; } - -/*! BIT_readBits() : - * Read (consume) next n bits from local register and update. - * Pay attention to not read more than nbBits contained into local register. - * @return : extracted value. - */ -ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, U32 nbBits) -{ - size_t const value = BIT_lookBits(bitD, nbBits); - BIT_skipBits(bitD, nbBits); - return value; -} - -/*! BIT_readBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ -ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, U32 nbBits) -{ - size_t const value = BIT_lookBitsFast(bitD, nbBits); - BIT_skipBits(bitD, nbBits); - return value; -} - -/*! BIT_reloadDStream() : -* Refill `bitD` from buffer previously set in BIT_initDStream() . -* This function is safe, it guarantees it will not read beyond src buffer. -* @return : status of `BIT_DStream_t` internal register. - if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ -ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD) -{ - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer) * 8)) /* should not happen => corruption detected */ - return BIT_DStream_overflow; - - if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { - bitD->ptr -= bitD->bitsConsumed >> 3; - bitD->bitsConsumed &= 7; - bitD->bitContainer = ZSTD_readLEST(bitD->ptr); - return BIT_DStream_unfinished; - } - if (bitD->ptr == bitD->start) { - if (bitD->bitsConsumed < sizeof(bitD->bitContainer) * 8) - return BIT_DStream_endOfBuffer; - return BIT_DStream_completed; - } - { - U32 nbBytes = bitD->bitsConsumed >> 3; - BIT_DStream_status result = BIT_DStream_unfinished; - if (bitD->ptr - nbBytes < bitD->start) { - nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ - result = BIT_DStream_endOfBuffer; - } - bitD->ptr -= nbBytes; - bitD->bitsConsumed -= nbBytes * 8; - bitD->bitContainer = ZSTD_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ - return result; - } -} - -/*! BIT_endOfDStream() : -* @return Tells if DStream has exactly reached its end (all bits consumed). -*/ -ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *DStream) -{ - return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer) * 8)); -} - -#endif /* BITSTREAM_H_MODULE */ diff --git a/contrib/linux-kernel/lib/zstd/compress.c b/contrib/linux-kernel/lib/zstd/compress.c deleted file mode 100644 index 43535b8d..00000000 --- a/contrib/linux-kernel/lib/zstd/compress.c +++ /dev/null @@ -1,3482 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -/*-************************************* -* Dependencies -***************************************/ -#include "fse.h" -#include "huf.h" -#include "mem.h" -#include "zstd_internal.h" /* includes zstd.h */ -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/string.h> /* memset */ - -/*-************************************* -* Constants -***************************************/ -static const U32 g_searchStrength = 8; /* control skip over incompressible data */ -#define HASH_READ_SIZE 8 -typedef enum { ZSTDcs_created = 0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; - -/*-************************************* -* Helper functions -***************************************/ -size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; } - -/*-************************************* -* Sequence storage -***************************************/ -static void ZSTD_resetSeqStore(seqStore_t *ssPtr) -{ - ssPtr->lit = ssPtr->litStart; - ssPtr->sequences = ssPtr->sequencesStart; - ssPtr->longLengthID = 0; -} - -/*-************************************* -* Context memory management -***************************************/ -struct ZSTD_CCtx_s { - const BYTE *nextSrc; /* next block here to continue on curr prefix */ - const BYTE *base; /* All regular indexes relative to this position */ - const BYTE *dictBase; /* extDict indexes relative to this position */ - U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more data */ - U32 nextToUpdate; /* index from which to continue dictionary update */ - U32 nextToUpdate3; /* index from which to continue dictionary update */ - U32 hashLog3; /* dispatch table : larger == faster, more memory */ - U32 loadedDictEnd; /* index of end of dictionary */ - U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */ - U32 forceRawDict; /* Force loading dictionary in "content-only" mode (no header analysis) */ - ZSTD_compressionStage_e stage; - U32 rep[ZSTD_REP_NUM]; - U32 repToConfirm[ZSTD_REP_NUM]; - U32 dictID; - ZSTD_parameters params; - void *workSpace; - size_t workSpaceSize; - size_t blockSize; - U64 frameContentSize; - struct xxh64_state xxhState; - ZSTD_customMem customMem; - - seqStore_t seqStore; /* sequences storage ptrs */ - U32 *hashTable; - U32 *hashTable3; - U32 *chainTable; - HUF_CElt *hufTable; - U32 flagStaticTables; - HUF_repeat flagStaticHufTable; - FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; - FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; - FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; - unsigned tmpCounters[HUF_COMPRESS_WORKSPACE_SIZE_U32]; -}; - -size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams) -{ - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog); - U32 const divider = (cParams.searchLength == 3) ? 3 : 4; - size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = blockSize + 11 * maxNbSeq; - size_t const chainSize = (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog); - size_t const hSize = ((size_t)1) << cParams.hashLog; - U32 const hashLog3 = (cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog); - size_t const h3Size = ((size_t)1) << hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - size_t const optSpace = - ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) + (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); - size_t const workspaceSize = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace + - (((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); - - return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_CCtx)) + ZSTD_ALIGN(workspaceSize); -} - -static ZSTD_CCtx *ZSTD_createCCtx_advanced(ZSTD_customMem customMem) -{ - ZSTD_CCtx *cctx; - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - cctx = (ZSTD_CCtx *)ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); - if (!cctx) - return NULL; - memset(cctx, 0, sizeof(ZSTD_CCtx)); - cctx->customMem = customMem; - return cctx; -} - -ZSTD_CCtx *ZSTD_initCCtx(void *workspace, size_t workspaceSize) -{ - ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); - ZSTD_CCtx *cctx = ZSTD_createCCtx_advanced(stackMem); - if (cctx) { - cctx->workSpace = ZSTD_stackAllocAll(cctx->customMem.opaque, &cctx->workSpaceSize); - } - return cctx; -} - -size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx) -{ - if (cctx == NULL) - return 0; /* support free on NULL */ - ZSTD_free(cctx->workSpace, cctx->customMem); - ZSTD_free(cctx, cctx->customMem); - return 0; /* reserved as a potential error code in the future */ -} - -const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx) /* hidden interface */ { return &(ctx->seqStore); } - -static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx *cctx) { return cctx->params; } - -/** ZSTD_checkParams() : - ensure param values remain within authorized range. - @return : 0, or an error code if one value is beyond authorized range */ -size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) -{ -#define CLAMPCHECK(val, min, max) \ - { \ - if ((val < min) | (val > max)) \ - return ERROR(compressionParameter_unsupported); \ - } - CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); - CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); - CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); - CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); - CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); - CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); - if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2) - return ERROR(compressionParameter_unsupported); - return 0; -} - -/** ZSTD_cycleLog() : - * condition for correct operation : hashLog > 1 */ -static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) -{ - U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); - return hashLog - btScale; -} - -/** ZSTD_adjustCParams() : - optimize `cPar` for a given input (`srcSize` and `dictSize`). - mostly downsizing to reduce memory consumption and initialization. - Both `srcSize` and `dictSize` are optional (use 0 if unknown), - but if both are 0, no optimization can be done. - Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ -ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) -{ - if (srcSize + dictSize == 0) - return cPar; /* no size information available : no adjustment */ - - /* resize params, to use less memory when necessary */ - { - U32 const minSrcSize = (srcSize == 0) ? 500 : 0; - U64 const rSize = srcSize + dictSize + minSrcSize; - if (rSize < ((U64)1 << ZSTD_WINDOWLOG_MAX)) { - U32 const srcLog = MAX(ZSTD_HASHLOG_MIN, ZSTD_highbit32((U32)(rSize)-1) + 1); - if (cPar.windowLog > srcLog) - cPar.windowLog = srcLog; - } - } - if (cPar.hashLog > cPar.windowLog) - cPar.hashLog = cPar.windowLog; - { - U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); - if (cycleLog > cPar.windowLog) - cPar.chainLog -= (cycleLog - cPar.windowLog); - } - - if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) - cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ - - return cPar; -} - -static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2) -{ - return (param1.cParams.hashLog == param2.cParams.hashLog) & (param1.cParams.chainLog == param2.cParams.chainLog) & - (param1.cParams.strategy == param2.cParams.strategy) & ((param1.cParams.searchLength == 3) == (param2.cParams.searchLength == 3)); -} - -/*! ZSTD_continueCCtx() : - reuse CCtx without reset (note : requires no dictionary) */ -static size_t ZSTD_continueCCtx(ZSTD_CCtx *cctx, ZSTD_parameters params, U64 frameContentSize) -{ - U32 const end = (U32)(cctx->nextSrc - cctx->base); - cctx->params = params; - cctx->frameContentSize = frameContentSize; - cctx->lowLimit = end; - cctx->dictLimit = end; - cctx->nextToUpdate = end + 1; - cctx->stage = ZSTDcs_init; - cctx->dictID = 0; - cctx->loadedDictEnd = 0; - { - int i; - for (i = 0; i < ZSTD_REP_NUM; i++) - cctx->rep[i] = repStartValue[i]; - } - cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */ - xxh64_reset(&cctx->xxhState, 0); - return 0; -} - -typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e; - -/*! ZSTD_resetCCtx_advanced() : - note : `params` must be validated */ -static size_t ZSTD_resetCCtx_advanced(ZSTD_CCtx *zc, ZSTD_parameters params, U64 frameContentSize, ZSTD_compResetPolicy_e const crp) -{ - if (crp == ZSTDcrp_continue) - if (ZSTD_equivalentParams(params, zc->params)) { - zc->flagStaticTables = 0; - zc->flagStaticHufTable = HUF_repeat_none; - return ZSTD_continueCCtx(zc, params, frameContentSize); - } - - { - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog); - U32 const divider = (params.cParams.searchLength == 3) ? 3 : 4; - size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = blockSize + 11 * maxNbSeq; - size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); - size_t const hSize = ((size_t)1) << params.cParams.hashLog; - U32 const hashLog3 = (params.cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog); - size_t const h3Size = ((size_t)1) << hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - void *ptr; - - /* Check if workSpace is large enough, alloc a new one if needed */ - { - size_t const optSpace = ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) + - (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); - size_t const neededSpace = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace + - (((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); - if (zc->workSpaceSize < neededSpace) { - ZSTD_free(zc->workSpace, zc->customMem); - zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); - if (zc->workSpace == NULL) - return ERROR(memory_allocation); - zc->workSpaceSize = neededSpace; - } - } - - if (crp != ZSTDcrp_noMemset) - memset(zc->workSpace, 0, tableSpace); /* reset tables only */ - xxh64_reset(&zc->xxhState, 0); - zc->hashLog3 = hashLog3; - zc->hashTable = (U32 *)(zc->workSpace); - zc->chainTable = zc->hashTable + hSize; - zc->hashTable3 = zc->chainTable + chainSize; - ptr = zc->hashTable3 + h3Size; - zc->hufTable = (HUF_CElt *)ptr; - zc->flagStaticTables = 0; - zc->flagStaticHufTable = HUF_repeat_none; - ptr = ((U32 *)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */ - - zc->nextToUpdate = 1; - zc->nextSrc = NULL; - zc->base = NULL; - zc->dictBase = NULL; - zc->dictLimit = 0; - zc->lowLimit = 0; - zc->params = params; - zc->blockSize = blockSize; - zc->frameContentSize = frameContentSize; - { - int i; - for (i = 0; i < ZSTD_REP_NUM; i++) - zc->rep[i] = repStartValue[i]; - } - - if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) { - zc->seqStore.litFreq = (U32 *)ptr; - zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1 << Litbits); - zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL + 1); - zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML + 1); - ptr = zc->seqStore.offCodeFreq + (MaxOff + 1); - zc->seqStore.matchTable = (ZSTD_match_t *)ptr; - ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM + 1; - zc->seqStore.priceTable = (ZSTD_optimal_t *)ptr; - ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM + 1; - zc->seqStore.litLengthSum = 0; - } - zc->seqStore.sequencesStart = (seqDef *)ptr; - ptr = zc->seqStore.sequencesStart + maxNbSeq; - zc->seqStore.llCode = (BYTE *)ptr; - zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; - zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; - zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; - - zc->stage = ZSTDcs_init; - zc->dictID = 0; - zc->loadedDictEnd = 0; - - return 0; - } -} - -/* ZSTD_invalidateRepCodes() : - * ensures next compression will not use repcodes from previous block. - * Note : only works with regular variant; - * do not use with extDict variant ! */ -void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx) -{ - int i; - for (i = 0; i < ZSTD_REP_NUM; i++) - cctx->rep[i] = 0; -} - -/*! ZSTD_copyCCtx() : -* Duplicate an existing context `srcCCtx` into another one `dstCCtx`. -* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). -* @return : 0, or an error code */ -size_t ZSTD_copyCCtx(ZSTD_CCtx *dstCCtx, const ZSTD_CCtx *srcCCtx, unsigned long long pledgedSrcSize) -{ - if (srcCCtx->stage != ZSTDcs_init) - return ERROR(stage_wrong); - - memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); - { - ZSTD_parameters params = srcCCtx->params; - params.fParams.contentSizeFlag = (pledgedSrcSize > 0); - ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset); - } - - /* copy tables */ - { - size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); - size_t const hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog; - size_t const h3Size = (size_t)1 << srcCCtx->hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); - } - - /* copy dictionary offsets */ - dstCCtx->nextToUpdate = srcCCtx->nextToUpdate; - dstCCtx->nextToUpdate3 = srcCCtx->nextToUpdate3; - dstCCtx->nextSrc = srcCCtx->nextSrc; - dstCCtx->base = srcCCtx->base; - dstCCtx->dictBase = srcCCtx->dictBase; - dstCCtx->dictLimit = srcCCtx->dictLimit; - dstCCtx->lowLimit = srcCCtx->lowLimit; - dstCCtx->loadedDictEnd = srcCCtx->loadedDictEnd; - dstCCtx->dictID = srcCCtx->dictID; - - /* copy entropy tables */ - dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; - dstCCtx->flagStaticHufTable = srcCCtx->flagStaticHufTable; - if (srcCCtx->flagStaticTables) { - memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable)); - memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable)); - memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable)); - } - if (srcCCtx->flagStaticHufTable) { - memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256 * 4); - } - - return 0; -} - -/*! ZSTD_reduceTable() : -* reduce table indexes by `reducerValue` */ -static void ZSTD_reduceTable(U32 *const table, U32 const size, U32 const reducerValue) -{ - U32 u; - for (u = 0; u < size; u++) { - if (table[u] < reducerValue) - table[u] = 0; - else - table[u] -= reducerValue; - } -} - -/*! ZSTD_reduceIndex() : -* rescale all indexes to avoid future overflow (indexes are U32) */ -static void ZSTD_reduceIndex(ZSTD_CCtx *zc, const U32 reducerValue) -{ - { - U32 const hSize = 1 << zc->params.cParams.hashLog; - ZSTD_reduceTable(zc->hashTable, hSize, reducerValue); - } - - { - U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog); - ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); - } - - { - U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; - ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); - } -} - -/*-******************************************************* -* Block entropic compression -*********************************************************/ - -/* See doc/zstd_compression_format.md for detailed format description */ - -size_t ZSTD_noCompressBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - if (srcSize + ZSTD_blockHeaderSize > dstCapacity) - return ERROR(dstSize_tooSmall); - memcpy((BYTE *)dst + ZSTD_blockHeaderSize, src, srcSize); - ZSTD_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw); - return ZSTD_blockHeaderSize + srcSize; -} - -static size_t ZSTD_noCompressLiterals(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - BYTE *const ostart = (BYTE * const)dst; - U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095); - - if (srcSize + flSize > dstCapacity) - return ERROR(dstSize_tooSmall); - - switch (flSize) { - case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_basic + (srcSize << 3)); break; - case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_basic + (1 << 2) + (srcSize << 4))); break; - default: /*note : should not be necessary : flSize is within {1,2,3} */ - case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_basic + (3 << 2) + (srcSize << 4))); break; - } - - memcpy(ostart + flSize, src, srcSize); - return srcSize + flSize; -} - -static size_t ZSTD_compressRleLiteralsBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - BYTE *const ostart = (BYTE * const)dst; - U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095); - - (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ - - switch (flSize) { - case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_rle + (srcSize << 3)); break; - case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_rle + (1 << 2) + (srcSize << 4))); break; - default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */ - case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_rle + (3 << 2) + (srcSize << 4))); break; - } - - ostart[flSize] = *(const BYTE *)src; - return flSize + 1; -} - -static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; } - -static size_t ZSTD_compressLiterals(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - size_t const minGain = ZSTD_minGain(srcSize); - size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); - BYTE *const ostart = (BYTE *)dst; - U32 singleStream = srcSize < 256; - symbolEncodingType_e hType = set_compressed; - size_t cLitSize; - -/* small ? don't even attempt compression (speed opt) */ -#define LITERAL_NOENTROPY 63 - { - size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY; - if (srcSize <= minLitSize) - return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } - - if (dstCapacity < lhSize + 1) - return ERROR(dstSize_tooSmall); /* not enough space for compression */ - { - HUF_repeat repeat = zc->flagStaticHufTable; - int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0; - if (repeat == HUF_repeat_valid && lhSize == 3) - singleStream = 1; - cLitSize = singleStream ? HUF_compress1X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters, - sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat) - : HUF_compress4X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters, - sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat); - if (repeat != HUF_repeat_none) { - hType = set_repeat; - } /* reused the existing table */ - else { - zc->flagStaticHufTable = HUF_repeat_check; - } /* now have a table to reuse */ - } - - if ((cLitSize == 0) | (cLitSize >= srcSize - minGain)) { - zc->flagStaticHufTable = HUF_repeat_none; - return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } - if (cLitSize == 1) { - zc->flagStaticHufTable = HUF_repeat_none; - return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); - } - - /* Build header */ - switch (lhSize) { - case 3: /* 2 - 2 - 10 - 10 */ - { - U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 14); - ZSTD_writeLE24(ostart, lhc); - break; - } - case 4: /* 2 - 2 - 14 - 14 */ - { - U32 const lhc = hType + (2 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 18); - ZSTD_writeLE32(ostart, lhc); - break; - } - default: /* should not be necessary, lhSize is only {3,4,5} */ - case 5: /* 2 - 2 - 18 - 18 */ - { - U32 const lhc = hType + (3 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 22); - ZSTD_writeLE32(ostart, lhc); - ostart[4] = (BYTE)(cLitSize >> 10); - break; - } - } - return lhSize + cLitSize; -} - -static const BYTE LL_Code[64] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, - 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, - 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24}; - -static const BYTE ML_Code[128] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, - 26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, - 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 40, 40, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42}; - -void ZSTD_seqToCodes(const seqStore_t *seqStorePtr) -{ - BYTE const LL_deltaCode = 19; - BYTE const ML_deltaCode = 36; - const seqDef *const sequences = seqStorePtr->sequencesStart; - BYTE *const llCodeTable = seqStorePtr->llCode; - BYTE *const ofCodeTable = seqStorePtr->ofCode; - BYTE *const mlCodeTable = seqStorePtr->mlCode; - U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - U32 u; - for (u = 0; u < nbSeq; u++) { - U32 const llv = sequences[u].litLength; - U32 const mlv = sequences[u].matchLength; - llCodeTable[u] = (llv > 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv]; - ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset); - mlCodeTable[u] = (mlv > 127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv]; - } - if (seqStorePtr->longLengthID == 1) - llCodeTable[seqStorePtr->longLengthPos] = MaxLL; - if (seqStorePtr->longLengthID == 2) - mlCodeTable[seqStorePtr->longLengthPos] = MaxML; -} - -ZSTD_STATIC size_t ZSTD_compressSequences_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity) -{ - const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN; - const seqStore_t *seqStorePtr = &(zc->seqStore); - FSE_CTable *CTable_LitLength = zc->litlengthCTable; - FSE_CTable *CTable_OffsetBits = zc->offcodeCTable; - FSE_CTable *CTable_MatchLength = zc->matchlengthCTable; - U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ - const seqDef *const sequences = seqStorePtr->sequencesStart; - const BYTE *const ofCodeTable = seqStorePtr->ofCode; - const BYTE *const llCodeTable = seqStorePtr->llCode; - const BYTE *const mlCodeTable = seqStorePtr->mlCode; - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstCapacity; - BYTE *op = ostart; - size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; - BYTE *seqHead; - - U32 *count; - S16 *norm; - U32 *workspace; - size_t workspaceSize = sizeof(zc->tmpCounters); - { - size_t spaceUsed32 = 0; - count = (U32 *)zc->tmpCounters + spaceUsed32; - spaceUsed32 += MaxSeq + 1; - norm = (S16 *)((U32 *)zc->tmpCounters + spaceUsed32); - spaceUsed32 += ALIGN(sizeof(S16) * (MaxSeq + 1), sizeof(U32)) >> 2; - - workspace = (U32 *)zc->tmpCounters + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - } - - /* Compress literals */ - { - const BYTE *const literals = seqStorePtr->litStart; - size_t const litSize = seqStorePtr->lit - literals; - size_t const cSize = ZSTD_compressLiterals(zc, op, dstCapacity, literals, litSize); - if (ZSTD_isError(cSize)) - return cSize; - op += cSize; - } - - /* Sequences Header */ - if ((oend - op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */) - return ERROR(dstSize_tooSmall); - if (nbSeq < 0x7F) - *op++ = (BYTE)nbSeq; - else if (nbSeq < LONGNBSEQ) - op[0] = (BYTE)((nbSeq >> 8) + 0x80), op[1] = (BYTE)nbSeq, op += 2; - else - op[0] = 0xFF, ZSTD_writeLE16(op + 1, (U16)(nbSeq - LONGNBSEQ)), op += 3; - if (nbSeq == 0) - return op - ostart; - - /* seqHead : flags for FSE encoding type */ - seqHead = op++; - -#define MIN_SEQ_FOR_DYNAMIC_FSE 64 -#define MAX_SEQ_FOR_STATIC_FSE 1000 - - /* convert length/distances into codes */ - ZSTD_seqToCodes(seqStorePtr); - - /* CTable for Literal Lengths */ - { - U32 max = MaxLL; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace); - if ((mostFrequent == nbSeq) && (nbSeq > 2)) { - *op++ = llCodeTable[0]; - FSE_buildCTable_rle(CTable_LitLength, (BYTE)max); - LLtype = set_rle; - } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { - LLtype = set_repeat; - } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog - 1)))) { - FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, workspace, workspaceSize); - LLtype = set_basic; - } else { - size_t nbSeq_1 = nbSeq; - const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max); - if (count[llCodeTable[nbSeq - 1]] > 1) { - count[llCodeTable[nbSeq - 1]]--; - nbSeq_1--; - } - FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); - { - size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) - return NCountSize; - op += NCountSize; - } - FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, workspace, workspaceSize); - LLtype = set_compressed; - } - } - - /* CTable for Offsets */ - { - U32 max = MaxOff; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace); - if ((mostFrequent == nbSeq) && (nbSeq > 2)) { - *op++ = ofCodeTable[0]; - FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max); - Offtype = set_rle; - } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { - Offtype = set_repeat; - } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog - 1)))) { - FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, workspace, workspaceSize); - Offtype = set_basic; - } else { - size_t nbSeq_1 = nbSeq; - const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max); - if (count[ofCodeTable[nbSeq - 1]] > 1) { - count[ofCodeTable[nbSeq - 1]]--; - nbSeq_1--; - } - FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); - { - size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) - return NCountSize; - op += NCountSize; - } - FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, workspace, workspaceSize); - Offtype = set_compressed; - } - } - - /* CTable for MatchLengths */ - { - U32 max = MaxML; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace); - if ((mostFrequent == nbSeq) && (nbSeq > 2)) { - *op++ = *mlCodeTable; - FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max); - MLtype = set_rle; - } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { - MLtype = set_repeat; - } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog - 1)))) { - FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, workspace, workspaceSize); - MLtype = set_basic; - } else { - size_t nbSeq_1 = nbSeq; - const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max); - if (count[mlCodeTable[nbSeq - 1]] > 1) { - count[mlCodeTable[nbSeq - 1]]--; - nbSeq_1--; - } - FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); - { - size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) - return NCountSize; - op += NCountSize; - } - FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, workspace, workspaceSize); - MLtype = set_compressed; - } - } - - *seqHead = (BYTE)((LLtype << 6) + (Offtype << 4) + (MLtype << 2)); - zc->flagStaticTables = 0; - - /* Encoding Sequences */ - { - BIT_CStream_t blockStream; - FSE_CState_t stateMatchLength; - FSE_CState_t stateOffsetBits; - FSE_CState_t stateLitLength; - - CHECK_E(BIT_initCStream(&blockStream, op, oend - op), dstSize_tooSmall); /* not enough space remaining */ - - /* first symbols */ - FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq - 1]); - FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq - 1]); - FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq - 1]); - BIT_addBits(&blockStream, sequences[nbSeq - 1].litLength, LL_bits[llCodeTable[nbSeq - 1]]); - if (ZSTD_32bits()) - BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[nbSeq - 1].matchLength, ML_bits[mlCodeTable[nbSeq - 1]]); - if (ZSTD_32bits()) - BIT_flushBits(&blockStream); - if (longOffsets) { - U32 const ofBits = ofCodeTable[nbSeq - 1]; - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, extraBits); - BIT_flushBits(&blockStream); - } - BIT_addBits(&blockStream, sequences[nbSeq - 1].offset >> extraBits, ofBits - extraBits); - } else { - BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, ofCodeTable[nbSeq - 1]); - } - BIT_flushBits(&blockStream); - - { - size_t n; - for (n = nbSeq - 2; n < nbSeq; n--) { /* intentional underflow */ - BYTE const llCode = llCodeTable[n]; - BYTE const ofCode = ofCodeTable[n]; - BYTE const mlCode = mlCodeTable[n]; - U32 const llBits = LL_bits[llCode]; - U32 const ofBits = ofCode; /* 32b*/ /* 64b*/ - U32 const mlBits = ML_bits[mlCode]; - /* (7)*/ /* (7)*/ - FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ - FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */ - if (ZSTD_32bits()) - BIT_flushBits(&blockStream); /* (7)*/ - FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */ - if (ZSTD_32bits() || (ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + OffFSELog))) - BIT_flushBits(&blockStream); /* (7)*/ - BIT_addBits(&blockStream, sequences[n].litLength, llBits); - if (ZSTD_32bits() && ((llBits + mlBits) > 24)) - BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); - if (ZSTD_32bits()) - BIT_flushBits(&blockStream); /* (7)*/ - if (longOffsets) { - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[n].offset, extraBits); - BIT_flushBits(&blockStream); /* (7)*/ - } - BIT_addBits(&blockStream, sequences[n].offset >> extraBits, ofBits - extraBits); /* 31 */ - } else { - BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ - } - BIT_flushBits(&blockStream); /* (7)*/ - } - } - - FSE_flushCState(&blockStream, &stateMatchLength); - FSE_flushCState(&blockStream, &stateOffsetBits); - FSE_flushCState(&blockStream, &stateLitLength); - - { - size_t const streamSize = BIT_closeCStream(&blockStream); - if (streamSize == 0) - return ERROR(dstSize_tooSmall); /* not enough space */ - op += streamSize; - } - } - return op - ostart; -} - -ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, size_t srcSize) -{ - size_t const cSize = ZSTD_compressSequences_internal(zc, dst, dstCapacity); - size_t const minGain = ZSTD_minGain(srcSize); - size_t const maxCSize = srcSize - minGain; - /* If the srcSize <= dstCapacity, then there is enough space to write a - * raw uncompressed block. Since we ran out of space, the block must not - * be compressible, so fall back to a raw uncompressed block. - */ - int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity; - int i; - - if (ZSTD_isError(cSize) && !uncompressibleError) - return cSize; - if (cSize >= maxCSize || uncompressibleError) { - zc->flagStaticHufTable = HUF_repeat_none; - return 0; - } - /* confirm repcodes */ - for (i = 0; i < ZSTD_REP_NUM; i++) - zc->rep[i] = zc->repToConfirm[i]; - return cSize; -} - -/*! ZSTD_storeSeq() : - Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. - `offsetCode` : distance to match, or 0 == repCode. - `matchCode` : matchLength - MINMATCH -*/ -ZSTD_STATIC void ZSTD_storeSeq(seqStore_t *seqStorePtr, size_t litLength, const void *literals, U32 offsetCode, size_t matchCode) -{ - /* copy Literals */ - ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); - seqStorePtr->lit += litLength; - - /* literal Length */ - if (litLength > 0xFFFF) { - seqStorePtr->longLengthID = 1; - seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - } - seqStorePtr->sequences[0].litLength = (U16)litLength; - - /* match offset */ - seqStorePtr->sequences[0].offset = offsetCode + 1; - - /* match Length */ - if (matchCode > 0xFFFF) { - seqStorePtr->longLengthID = 2; - seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - } - seqStorePtr->sequences[0].matchLength = (U16)matchCode; - - seqStorePtr->sequences++; -} - -/*-************************************* -* Match length counter -***************************************/ -static unsigned ZSTD_NbCommonBytes(register size_t val) -{ - if (ZSTD_isLittleEndian()) { - if (ZSTD_64bits()) { - return (__builtin_ctzll((U64)val) >> 3); - } else { /* 32 bits */ - return (__builtin_ctz((U32)val) >> 3); - } - } else { /* Big Endian CPU */ - if (ZSTD_64bits()) { - return (__builtin_clzll(val) >> 3); - } else { /* 32 bits */ - return (__builtin_clz((U32)val) >> 3); - } - } -} - -static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch, const BYTE *const pInLimit) -{ - const BYTE *const pStart = pIn; - const BYTE *const pInLoopLimit = pInLimit - (sizeof(size_t) - 1); - - while (pIn < pInLoopLimit) { - size_t const diff = ZSTD_readST(pMatch) ^ ZSTD_readST(pIn); - if (!diff) { - pIn += sizeof(size_t); - pMatch += sizeof(size_t); - continue; - } - pIn += ZSTD_NbCommonBytes(diff); - return (size_t)(pIn - pStart); - } - if (ZSTD_64bits()) - if ((pIn < (pInLimit - 3)) && (ZSTD_read32(pMatch) == ZSTD_read32(pIn))) { - pIn += 4; - pMatch += 4; - } - if ((pIn < (pInLimit - 1)) && (ZSTD_read16(pMatch) == ZSTD_read16(pIn))) { - pIn += 2; - pMatch += 2; - } - if ((pIn < pInLimit) && (*pMatch == *pIn)) - pIn++; - return (size_t)(pIn - pStart); -} - -/** ZSTD_count_2segments() : -* can count match length with `ip` & `match` in 2 different segments. -* convention : on reaching mEnd, match count continue starting from iStart -*/ -static size_t ZSTD_count_2segments(const BYTE *ip, const BYTE *match, const BYTE *iEnd, const BYTE *mEnd, const BYTE *iStart) -{ - const BYTE *const vEnd = MIN(ip + (mEnd - match), iEnd); - size_t const matchLength = ZSTD_count(ip, match, vEnd); - if (match + matchLength != mEnd) - return matchLength; - return matchLength + ZSTD_count(ip + matchLength, iStart, iEnd); -} - -/*-************************************* -* Hashes -***************************************/ -static const U32 prime3bytes = 506832829U; -static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32 - 24)) * prime3bytes) >> (32 - h); } -ZSTD_STATIC size_t ZSTD_hash3Ptr(const void *ptr, U32 h) { return ZSTD_hash3(ZSTD_readLE32(ptr), h); } /* only in zstd_opt.h */ - -static const U32 prime4bytes = 2654435761U; -static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32 - h); } -static size_t ZSTD_hash4Ptr(const void *ptr, U32 h) { return ZSTD_hash4(ZSTD_read32(ptr), h); } - -static const U64 prime5bytes = 889523592379ULL; -static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64 - 40)) * prime5bytes) >> (64 - h)); } -static size_t ZSTD_hash5Ptr(const void *p, U32 h) { return ZSTD_hash5(ZSTD_readLE64(p), h); } - -static const U64 prime6bytes = 227718039650203ULL; -static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64 - 48)) * prime6bytes) >> (64 - h)); } -static size_t ZSTD_hash6Ptr(const void *p, U32 h) { return ZSTD_hash6(ZSTD_readLE64(p), h); } - -static const U64 prime7bytes = 58295818150454627ULL; -static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64 - 56)) * prime7bytes) >> (64 - h)); } -static size_t ZSTD_hash7Ptr(const void *p, U32 h) { return ZSTD_hash7(ZSTD_readLE64(p), h); } - -static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; -static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u)*prime8bytes) >> (64 - h)); } -static size_t ZSTD_hash8Ptr(const void *p, U32 h) { return ZSTD_hash8(ZSTD_readLE64(p), h); } - -static size_t ZSTD_hashPtr(const void *p, U32 hBits, U32 mls) -{ - switch (mls) { - // case 3: return ZSTD_hash3Ptr(p, hBits); - default: - case 4: return ZSTD_hash4Ptr(p, hBits); - case 5: return ZSTD_hash5Ptr(p, hBits); - case 6: return ZSTD_hash6Ptr(p, hBits); - case 7: return ZSTD_hash7Ptr(p, hBits); - case 8: return ZSTD_hash8Ptr(p, hBits); - } -} - -/*-************************************* -* Fast Scan -***************************************/ -static void ZSTD_fillHashTable(ZSTD_CCtx *zc, const void *end, const U32 mls) -{ - U32 *const hashTable = zc->hashTable; - U32 const hBits = zc->params.cParams.hashLog; - const BYTE *const base = zc->base; - const BYTE *ip = base + zc->nextToUpdate; - const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE; - const size_t fastHashFillStep = 3; - - while (ip <= iend) { - hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base); - ip += fastHashFillStep; - } -} - -FORCE_INLINE -void ZSTD_compressBlock_fast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls) -{ - U32 *const hashTable = cctx->hashTable; - U32 const hBits = cctx->params.cParams.hashLog; - seqStore_t *seqStorePtr = &(cctx->seqStore); - const BYTE *const base = cctx->base; - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const U32 lowestIndex = cctx->dictLimit; - const BYTE *const lowest = base + lowestIndex; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - HASH_READ_SIZE; - U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1]; - U32 offsetSaved = 0; - - /* init */ - ip += (ip == lowest); - { - U32 const maxRep = (U32)(ip - lowest); - if (offset_2 > maxRep) - offsetSaved = offset_2, offset_2 = 0; - if (offset_1 > maxRep) - offsetSaved = offset_1, offset_1 = 0; - } - - /* Main Search Loop */ - while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ - size_t mLength; - size_t const h = ZSTD_hashPtr(ip, hBits, mls); - U32 const curr = (U32)(ip - base); - U32 const matchIndex = hashTable[h]; - const BYTE *match = base + matchIndex; - hashTable[h] = curr; /* update hash table */ - - if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) { - mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4; - ip++; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH); - } else { - U32 offset; - if ((matchIndex <= lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) { - ip += ((ip - anchor) >> g_searchStrength) + 1; - continue; - } - mLength = ZSTD_count(ip + 4, match + 4, iend) + 4; - offset = (U32)(ip - match); - while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) { - ip--; - match--; - mLength++; - } /* catch up */ - offset_2 = offset_1; - offset_1 = offset; - - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); - } - - /* match found */ - ip += mLength; - anchor = ip; - - if (ip <= ilimit) { - /* Fill Table */ - hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2; /* here because curr+2 could be > iend-8 */ - hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base); - /* check immediate repcode */ - while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) { - /* store sequence */ - size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4; - { - U32 const tmpOff = offset_2; - offset_2 = offset_1; - offset_1 = tmpOff; - } /* swap offset_2 <=> offset_1 */ - hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH); - ip += rLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } - } - } - - /* save reps for next block */ - cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; - cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -static void ZSTD_compressBlock_fast(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - const U32 mls = ctx->params.cParams.searchLength; - switch (mls) { - default: /* includes case 3 */ - case 4: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); return; - case 5: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); return; - case 6: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); return; - case 7: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); return; - } -} - -static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls) -{ - U32 *hashTable = ctx->hashTable; - const U32 hBits = ctx->params.cParams.hashLog; - seqStore_t *seqStorePtr = &(ctx->seqStore); - const BYTE *const base = ctx->base; - const BYTE *const dictBase = ctx->dictBase; - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const U32 lowestIndex = ctx->lowLimit; - const BYTE *const dictStart = dictBase + lowestIndex; - const U32 dictLimit = ctx->dictLimit; - const BYTE *const lowPrefixPtr = base + dictLimit; - const BYTE *const dictEnd = dictBase + dictLimit; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - 8; - U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1]; - - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ - const size_t h = ZSTD_hashPtr(ip, hBits, mls); - const U32 matchIndex = hashTable[h]; - const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base; - const BYTE *match = matchBase + matchIndex; - const U32 curr = (U32)(ip - base); - const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ - const BYTE *repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *repMatch = repBase + repIndex; - size_t mLength; - hashTable[h] = curr; /* update hash table */ - - if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) && - (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) { - const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32; - ip++; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH); - } else { - if ((matchIndex < lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) { - ip += ((ip - anchor) >> g_searchStrength) + 1; - continue; - } - { - const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend; - const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; - U32 offset; - mLength = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32; - while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) { - ip--; - match--; - mLength++; - } /* catch up */ - offset = curr - matchIndex; - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); - } - } - - /* found a match : store it */ - ip += mLength; - anchor = ip; - - if (ip <= ilimit) { - /* Fill Table */ - hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2; - hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base); - /* check immediate repcode */ - while (ip <= ilimit) { - U32 const curr2 = (U32)(ip - base); - U32 const repIndex2 = curr2 - offset_2; - const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; - if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ - && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) { - const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; - size_t repLength2 = - ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; - U32 tmpOffset = offset_2; - offset_2 = offset_1; - offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH); - hashTable[ZSTD_hashPtr(ip, hBits, mls)] = curr2; - ip += repLength2; - anchor = ip; - continue; - } - break; - } - } - } - - /* save reps for next block */ - ctx->repToConfirm[0] = offset_1; - ctx->repToConfirm[1] = offset_2; - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - U32 const mls = ctx->params.cParams.searchLength; - switch (mls) { - default: /* includes case 3 */ - case 4: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); return; - case 5: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); return; - case 6: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); return; - case 7: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); return; - } -} - -/*-************************************* -* Double Fast -***************************************/ -static void ZSTD_fillDoubleHashTable(ZSTD_CCtx *cctx, const void *end, const U32 mls) -{ - U32 *const hashLarge = cctx->hashTable; - U32 const hBitsL = cctx->params.cParams.hashLog; - U32 *const hashSmall = cctx->chainTable; - U32 const hBitsS = cctx->params.cParams.chainLog; - const BYTE *const base = cctx->base; - const BYTE *ip = base + cctx->nextToUpdate; - const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE; - const size_t fastHashFillStep = 3; - - while (ip <= iend) { - hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base); - hashLarge[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base); - ip += fastHashFillStep; - } -} - -FORCE_INLINE -void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls) -{ - U32 *const hashLong = cctx->hashTable; - const U32 hBitsL = cctx->params.cParams.hashLog; - U32 *const hashSmall = cctx->chainTable; - const U32 hBitsS = cctx->params.cParams.chainLog; - seqStore_t *seqStorePtr = &(cctx->seqStore); - const BYTE *const base = cctx->base; - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const U32 lowestIndex = cctx->dictLimit; - const BYTE *const lowest = base + lowestIndex; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - HASH_READ_SIZE; - U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1]; - U32 offsetSaved = 0; - - /* init */ - ip += (ip == lowest); - { - U32 const maxRep = (U32)(ip - lowest); - if (offset_2 > maxRep) - offsetSaved = offset_2, offset_2 = 0; - if (offset_1 > maxRep) - offsetSaved = offset_1, offset_1 = 0; - } - - /* Main Search Loop */ - while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ - size_t mLength; - size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); - size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); - U32 const curr = (U32)(ip - base); - U32 const matchIndexL = hashLong[h2]; - U32 const matchIndexS = hashSmall[h]; - const BYTE *matchLong = base + matchIndexL; - const BYTE *match = base + matchIndexS; - hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ - - if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) { /* note : by construction, offset_1 <= curr */ - mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4; - ip++; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH); - } else { - U32 offset; - if ((matchIndexL > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) { - mLength = ZSTD_count(ip + 8, matchLong + 8, iend) + 8; - offset = (U32)(ip - matchLong); - while (((ip > anchor) & (matchLong > lowest)) && (ip[-1] == matchLong[-1])) { - ip--; - matchLong--; - mLength++; - } /* catch up */ - } else if ((matchIndexS > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) { - size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8); - U32 const matchIndex3 = hashLong[h3]; - const BYTE *match3 = base + matchIndex3; - hashLong[h3] = curr + 1; - if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) { - mLength = ZSTD_count(ip + 9, match3 + 8, iend) + 8; - ip++; - offset = (U32)(ip - match3); - while (((ip > anchor) & (match3 > lowest)) && (ip[-1] == match3[-1])) { - ip--; - match3--; - mLength++; - } /* catch up */ - } else { - mLength = ZSTD_count(ip + 4, match + 4, iend) + 4; - offset = (U32)(ip - match); - while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) { - ip--; - match--; - mLength++; - } /* catch up */ - } - } else { - ip += ((ip - anchor) >> g_searchStrength) + 1; - continue; - } - - offset_2 = offset_1; - offset_1 = offset; - - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); - } - - /* match found */ - ip += mLength; - anchor = ip; - - if (ip <= ilimit) { - /* Fill Table */ - hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] = - curr + 2; /* here because curr+2 could be > iend-8 */ - hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base); - - /* check immediate repcode */ - while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) { - /* store sequence */ - size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4; - { - U32 const tmpOff = offset_2; - offset_2 = offset_1; - offset_1 = tmpOff; - } /* swap offset_2 <=> offset_1 */ - hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base); - hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH); - ip += rLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } - } - } - - /* save reps for next block */ - cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; - cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - const U32 mls = ctx->params.cParams.searchLength; - switch (mls) { - default: /* includes case 3 */ - case 4: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return; - case 5: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); return; - case 6: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); return; - case 7: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); return; - } -} - -static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls) -{ - U32 *const hashLong = ctx->hashTable; - U32 const hBitsL = ctx->params.cParams.hashLog; - U32 *const hashSmall = ctx->chainTable; - U32 const hBitsS = ctx->params.cParams.chainLog; - seqStore_t *seqStorePtr = &(ctx->seqStore); - const BYTE *const base = ctx->base; - const BYTE *const dictBase = ctx->dictBase; - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const U32 lowestIndex = ctx->lowLimit; - const BYTE *const dictStart = dictBase + lowestIndex; - const U32 dictLimit = ctx->dictLimit; - const BYTE *const lowPrefixPtr = base + dictLimit; - const BYTE *const dictEnd = dictBase + dictLimit; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - 8; - U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1]; - - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ - const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); - const U32 matchIndex = hashSmall[hSmall]; - const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base; - const BYTE *match = matchBase + matchIndex; - - const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); - const U32 matchLongIndex = hashLong[hLong]; - const BYTE *matchLongBase = matchLongIndex < dictLimit ? dictBase : base; - const BYTE *matchLong = matchLongBase + matchLongIndex; - - const U32 curr = (U32)(ip - base); - const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ - const BYTE *repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *repMatch = repBase + repIndex; - size_t mLength; - hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ - - if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) && - (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) { - const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, lowPrefixPtr) + 4; - ip++; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH); - } else { - if ((matchLongIndex > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) { - const BYTE *matchEnd = matchLongIndex < dictLimit ? dictEnd : iend; - const BYTE *lowMatchPtr = matchLongIndex < dictLimit ? dictStart : lowPrefixPtr; - U32 offset; - mLength = ZSTD_count_2segments(ip + 8, matchLong + 8, iend, matchEnd, lowPrefixPtr) + 8; - offset = curr - matchLongIndex; - while (((ip > anchor) & (matchLong > lowMatchPtr)) && (ip[-1] == matchLong[-1])) { - ip--; - matchLong--; - mLength++; - } /* catch up */ - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); - - } else if ((matchIndex > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) { - size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8); - U32 const matchIndex3 = hashLong[h3]; - const BYTE *const match3Base = matchIndex3 < dictLimit ? dictBase : base; - const BYTE *match3 = match3Base + matchIndex3; - U32 offset; - hashLong[h3] = curr + 1; - if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) { - const BYTE *matchEnd = matchIndex3 < dictLimit ? dictEnd : iend; - const BYTE *lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr; - mLength = ZSTD_count_2segments(ip + 9, match3 + 8, iend, matchEnd, lowPrefixPtr) + 8; - ip++; - offset = curr + 1 - matchIndex3; - while (((ip > anchor) & (match3 > lowMatchPtr)) && (ip[-1] == match3[-1])) { - ip--; - match3--; - mLength++; - } /* catch up */ - } else { - const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend; - const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; - mLength = ZSTD_count_2segments(ip + 4, match + 4, iend, matchEnd, lowPrefixPtr) + 4; - offset = curr - matchIndex; - while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) { - ip--; - match--; - mLength++; - } /* catch up */ - } - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); - - } else { - ip += ((ip - anchor) >> g_searchStrength) + 1; - continue; - } - } - - /* found a match : store it */ - ip += mLength; - anchor = ip; - - if (ip <= ilimit) { - /* Fill Table */ - hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] = curr + 2; - hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = curr + 2; - hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base); - hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = (U32)(ip - 2 - base); - /* check immediate repcode */ - while (ip <= ilimit) { - U32 const curr2 = (U32)(ip - base); - U32 const repIndex2 = curr2 - offset_2; - const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; - if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ - && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) { - const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; - size_t const repLength2 = - ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; - U32 tmpOffset = offset_2; - offset_2 = offset_1; - offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH); - hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = curr2; - hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = curr2; - ip += repLength2; - anchor = ip; - continue; - } - break; - } - } - } - - /* save reps for next block */ - ctx->repToConfirm[0] = offset_1; - ctx->repToConfirm[1] = offset_2; - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - U32 const mls = ctx->params.cParams.searchLength; - switch (mls) { - default: /* includes case 3 */ - case 4: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return; - case 5: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); return; - case 6: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); return; - case 7: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); return; - } -} - -/*-************************************* -* Binary Tree search -***************************************/ -/** ZSTD_insertBt1() : add one or multiple positions to tree. -* ip : assumed <= iend-8 . -* @return : nb of positions added */ -static U32 ZSTD_insertBt1(ZSTD_CCtx *zc, const BYTE *const ip, const U32 mls, const BYTE *const iend, U32 nbCompares, U32 extDict) -{ - U32 *const hashTable = zc->hashTable; - U32 const hashLog = zc->params.cParams.hashLog; - size_t const h = ZSTD_hashPtr(ip, hashLog, mls); - U32 *const bt = zc->chainTable; - U32 const btLog = zc->params.cParams.chainLog - 1; - U32 const btMask = (1 << btLog) - 1; - U32 matchIndex = hashTable[h]; - size_t commonLengthSmaller = 0, commonLengthLarger = 0; - const BYTE *const base = zc->base; - const BYTE *const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; - const BYTE *const dictEnd = dictBase + dictLimit; - const BYTE *const prefixStart = base + dictLimit; - const BYTE *match; - const U32 curr = (U32)(ip - base); - const U32 btLow = btMask >= curr ? 0 : curr - btMask; - U32 *smallerPtr = bt + 2 * (curr & btMask); - U32 *largerPtr = smallerPtr + 1; - U32 dummy32; /* to be nullified at the end */ - U32 const windowLow = zc->lowLimit; - U32 matchEndIdx = curr + 8; - size_t bestLength = 8; - - hashTable[h] = curr; /* Update Hash Table */ - - while (nbCompares-- && (matchIndex > windowLow)) { - U32 *const nextPtr = bt + 2 * (matchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - - if ((!extDict) || (matchIndex + matchLength >= dictLimit)) { - match = base + matchIndex; - if (match[matchLength] == ip[matchLength]) - matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1; - } else { - match = dictBase + matchIndex; - matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart); - if (matchIndex + matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ - } - - if (matchLength > bestLength) { - bestLength = matchLength; - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - } - - if (ip + matchLength == iend) /* equal : no way to know if inf or sup */ - break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ - - if (match[matchLength] < ip[matchLength]) { /* necessarily within correct buffer */ - /* match is smaller than curr */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { - smallerPtr = &dummy32; - break; - } /* beyond tree size, stop the search */ - smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ - matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ - } else { - /* match is larger than curr */ - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { - largerPtr = &dummy32; - break; - } /* beyond tree size, stop the search */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - } - } - - *smallerPtr = *largerPtr = 0; - if (bestLength > 384) - return MIN(192, (U32)(bestLength - 384)); /* speed optimization */ - if (matchEndIdx > curr + 8) - return matchEndIdx - curr - 8; - return 1; -} - -static size_t ZSTD_insertBtAndFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, size_t *offsetPtr, U32 nbCompares, const U32 mls, - U32 extDict) -{ - U32 *const hashTable = zc->hashTable; - U32 const hashLog = zc->params.cParams.hashLog; - size_t const h = ZSTD_hashPtr(ip, hashLog, mls); - U32 *const bt = zc->chainTable; - U32 const btLog = zc->params.cParams.chainLog - 1; - U32 const btMask = (1 << btLog) - 1; - U32 matchIndex = hashTable[h]; - size_t commonLengthSmaller = 0, commonLengthLarger = 0; - const BYTE *const base = zc->base; - const BYTE *const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; - const BYTE *const dictEnd = dictBase + dictLimit; - const BYTE *const prefixStart = base + dictLimit; - const U32 curr = (U32)(ip - base); - const U32 btLow = btMask >= curr ? 0 : curr - btMask; - const U32 windowLow = zc->lowLimit; - U32 *smallerPtr = bt + 2 * (curr & btMask); - U32 *largerPtr = bt + 2 * (curr & btMask) + 1; - U32 matchEndIdx = curr + 8; - U32 dummy32; /* to be nullified at the end */ - size_t bestLength = 0; - - hashTable[h] = curr; /* Update Hash Table */ - - while (nbCompares-- && (matchIndex > windowLow)) { - U32 *const nextPtr = bt + 2 * (matchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - const BYTE *match; - - if ((!extDict) || (matchIndex + matchLength >= dictLimit)) { - match = base + matchIndex; - if (match[matchLength] == ip[matchLength]) - matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1; - } else { - match = dictBase + matchIndex; - matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart); - if (matchIndex + matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ - } - - if (matchLength > bestLength) { - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - if ((4 * (int)(matchLength - bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)offsetPtr[0] + 1))) - bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; - if (ip + matchLength == iend) /* equal : no way to know if inf or sup */ - break; /* drop, to guarantee consistency (miss a little bit of compression) */ - } - - if (match[matchLength] < ip[matchLength]) { - /* match is smaller than curr */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { - smallerPtr = &dummy32; - break; - } /* beyond tree size, stop the search */ - smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ - matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ - } else { - /* match is larger than curr */ - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { - largerPtr = &dummy32; - break; - } /* beyond tree size, stop the search */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - } - } - - *smallerPtr = *largerPtr = 0; - - zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1; - return bestLength; -} - -static void ZSTD_updateTree(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls) -{ - const BYTE *const base = zc->base; - const U32 target = (U32)(ip - base); - U32 idx = zc->nextToUpdate; - - while (idx < target) - idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 0); -} - -/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ -static size_t ZSTD_BtFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls) -{ - if (ip < zc->base + zc->nextToUpdate) - return 0; /* skipped area */ - ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0); -} - -static size_t ZSTD_BtFindBestMatch_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */ - const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 matchLengthSearch) -{ - switch (matchLengthSearch) { - default: /* includes case 3 */ - case 4: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); - case 5: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); - case 7: - case 6: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); - } -} - -static void ZSTD_updateTree_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls) -{ - const BYTE *const base = zc->base; - const U32 target = (U32)(ip - base); - U32 idx = zc->nextToUpdate; - - while (idx < target) - idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 1); -} - -/** Tree updater, providing best match */ -static size_t ZSTD_BtFindBestMatch_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, - const U32 mls) -{ - if (ip < zc->base + zc->nextToUpdate) - return 0; /* skipped area */ - ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1); -} - -static size_t ZSTD_BtFindBestMatch_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */ - const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, - const U32 matchLengthSearch) -{ - switch (matchLengthSearch) { - default: /* includes case 3 */ - case 4: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); - case 5: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); - case 7: - case 6: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); - } -} - -/* ********************************* -* Hash Chain -***********************************/ -#define NEXT_IN_CHAIN(d, mask) chainTable[(d)&mask] - -/* Update chains up to ip (excluded) - Assumption : always within prefix (i.e. not within extDict) */ -FORCE_INLINE -U32 ZSTD_insertAndFindFirstIndex(ZSTD_CCtx *zc, const BYTE *ip, U32 mls) -{ - U32 *const hashTable = zc->hashTable; - const U32 hashLog = zc->params.cParams.hashLog; - U32 *const chainTable = zc->chainTable; - const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1; - const BYTE *const base = zc->base; - const U32 target = (U32)(ip - base); - U32 idx = zc->nextToUpdate; - - while (idx < target) { /* catch up */ - size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); - NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; - hashTable[h] = idx; - idx++; - } - - zc->nextToUpdate = target; - return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; -} - -/* inlining is important to hardwire a hot branch (template emulation) */ -FORCE_INLINE -size_t ZSTD_HcFindBestMatch_generic(ZSTD_CCtx *zc, /* Index table will be updated */ - const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls, - const U32 extDict) -{ - U32 *const chainTable = zc->chainTable; - const U32 chainSize = (1 << zc->params.cParams.chainLog); - const U32 chainMask = chainSize - 1; - const BYTE *const base = zc->base; - const BYTE *const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; - const BYTE *const prefixStart = base + dictLimit; - const BYTE *const dictEnd = dictBase + dictLimit; - const U32 lowLimit = zc->lowLimit; - const U32 curr = (U32)(ip - base); - const U32 minChain = curr > chainSize ? curr - chainSize : 0; - int nbAttempts = maxNbAttempts; - size_t ml = EQUAL_READ32 - 1; - - /* HC4 match finder */ - U32 matchIndex = ZSTD_insertAndFindFirstIndex(zc, ip, mls); - - for (; (matchIndex > lowLimit) & (nbAttempts > 0); nbAttempts--) { - const BYTE *match; - size_t currMl = 0; - if ((!extDict) || matchIndex >= dictLimit) { - match = base + matchIndex; - if (match[ml] == ip[ml]) /* potentially better */ - currMl = ZSTD_count(ip, match, iLimit); - } else { - match = dictBase + matchIndex; - if (ZSTD_read32(match) == ZSTD_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ - currMl = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iLimit, dictEnd, prefixStart) + EQUAL_READ32; - } - - /* save best solution */ - if (currMl > ml) { - ml = currMl; - *offsetPtr = curr - matchIndex + ZSTD_REP_MOVE; - if (ip + currMl == iLimit) - break; /* best possible, and avoid read overflow*/ - } - - if (matchIndex <= minChain) - break; - matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); - } - - return ml; -} - -FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, - const U32 matchLengthSearch) -{ - switch (matchLengthSearch) { - default: /* includes case 3 */ - case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0); - case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0); - case 7: - case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0); - } -} - -FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, - const U32 matchLengthSearch) -{ - switch (matchLengthSearch) { - default: /* includes case 3 */ - case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1); - case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1); - case 7: - case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1); - } -} - -/* ******************************* -* Common parser - lazy strategy -*********************************/ -FORCE_INLINE -void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth) -{ - seqStore_t *seqStorePtr = &(ctx->seqStore); - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - 8; - const BYTE *const base = ctx->base + ctx->dictLimit; - - U32 const maxSearches = 1 << ctx->params.cParams.searchLog; - U32 const mls = ctx->params.cParams.searchLength; - - typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch); - searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; - U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset = 0; - - /* init */ - ip += (ip == base); - ctx->nextToUpdate3 = ctx->nextToUpdate; - { - U32 const maxRep = (U32)(ip - base); - if (offset_2 > maxRep) - savedOffset = offset_2, offset_2 = 0; - if (offset_1 > maxRep) - savedOffset = offset_1, offset_1 = 0; - } - - /* Match Loop */ - while (ip < ilimit) { - size_t matchLength = 0; - size_t offset = 0; - const BYTE *start = ip + 1; - - /* check repCode */ - if ((offset_1 > 0) & (ZSTD_read32(ip + 1) == ZSTD_read32(ip + 1 - offset_1))) { - /* repcode : we take it */ - matchLength = ZSTD_count(ip + 1 + EQUAL_READ32, ip + 1 + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32; - if (depth == 0) - goto _storeSequence; - } - - /* first search (depth 0) */ - { - size_t offsetFound = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); - if (ml2 > matchLength) - matchLength = ml2, start = ip, offset = offsetFound; - } - - if (matchLength < EQUAL_READ32) { - ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ - continue; - } - - /* let's try to find a better solution */ - if (depth >= 1) - while (ip < ilimit) { - ip++; - if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) { - size_t const mlRep = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32; - int const gain2 = (int)(mlRep * 3); - int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1); - if ((mlRep >= EQUAL_READ32) && (gain2 > gain1)) - matchLength = mlRep, offset = 0, start = ip; - } - { - size_t offset2 = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ - int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; - continue; /* search a better one */ - } - } - - /* let's find an even better one */ - if ((depth == 2) && (ip < ilimit)) { - ip++; - if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) { - size_t const ml2 = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32; - int const gain2 = (int)(ml2 * 4); - int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) - matchLength = ml2, offset = 0, start = ip; - } - { - size_t offset2 = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ - int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; - continue; - } - } - } - break; /* nothing found : store previous solution */ - } - - /* NOTE: - * start[-offset+ZSTD_REP_MOVE-1] is undefined behavior. - * (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which - * overflows the pointer, which is undefined behavior. - */ - /* catch up */ - if (offset) { - while ((start > anchor) && (start > base + offset - ZSTD_REP_MOVE) && - (start[-1] == (start-offset+ZSTD_REP_MOVE)[-1])) /* only search for offset within prefix */ - { - start--; - matchLength++; - } - offset_2 = offset_1; - offset_1 = (U32)(offset - ZSTD_REP_MOVE); - } - - /* store sequence */ -_storeSequence: - { - size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH); - anchor = ip = start + matchLength; - } - - /* check immediate repcode */ - while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) { - /* store sequence */ - matchLength = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_2, iend) + EQUAL_READ32; - offset = offset_2; - offset_2 = offset_1; - offset_1 = (U32)offset; /* swap repcodes */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH); - ip += matchLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } - } - - /* Save reps for next block */ - ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset; - ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset; - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2); } - -static void ZSTD_compressBlock_lazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2); } - -static void ZSTD_compressBlock_lazy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1); } - -static void ZSTD_compressBlock_greedy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0); } - -FORCE_INLINE -void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth) -{ - seqStore_t *seqStorePtr = &(ctx->seqStore); - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - 8; - const BYTE *const base = ctx->base; - const U32 dictLimit = ctx->dictLimit; - const U32 lowestIndex = ctx->lowLimit; - const BYTE *const prefixStart = base + dictLimit; - const BYTE *const dictBase = ctx->dictBase; - const BYTE *const dictEnd = dictBase + dictLimit; - const BYTE *const dictStart = dictBase + ctx->lowLimit; - - const U32 maxSearches = 1 << ctx->params.cParams.searchLog; - const U32 mls = ctx->params.cParams.searchLength; - - typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch); - searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS; - - U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1]; - - /* init */ - ctx->nextToUpdate3 = ctx->nextToUpdate; - ip += (ip == prefixStart); - - /* Match Loop */ - while (ip < ilimit) { - size_t matchLength = 0; - size_t offset = 0; - const BYTE *start = ip + 1; - U32 curr = (U32)(ip - base); - - /* check repCode */ - { - const U32 repIndex = (U32)(curr + 1 - offset_1); - const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *const repMatch = repBase + repIndex; - if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ - if (ZSTD_read32(ip + 1) == ZSTD_read32(repMatch)) { - /* repcode detected we should take it */ - const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; - matchLength = - ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; - if (depth == 0) - goto _storeSequence; - } - } - - /* first search (depth 0) */ - { - size_t offsetFound = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); - if (ml2 > matchLength) - matchLength = ml2, start = ip, offset = offsetFound; - } - - if (matchLength < EQUAL_READ32) { - ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ - continue; - } - - /* let's try to find a better solution */ - if (depth >= 1) - while (ip < ilimit) { - ip++; - curr++; - /* check repCode */ - if (offset) { - const U32 repIndex = (U32)(curr - offset_1); - const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *const repMatch = repBase + repIndex; - if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ - if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { - /* repcode detected */ - const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; - size_t const repLength = - ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + - EQUAL_READ32; - int const gain2 = (int)(repLength * 3); - int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1); - if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) - matchLength = repLength, offset = 0, start = ip; - } - } - - /* search match, depth 1 */ - { - size_t offset2 = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ - int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; - continue; /* search a better one */ - } - } - - /* let's find an even better one */ - if ((depth == 2) && (ip < ilimit)) { - ip++; - curr++; - /* check repCode */ - if (offset) { - const U32 repIndex = (U32)(curr - offset_1); - const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *const repMatch = repBase + repIndex; - if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ - if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { - /* repcode detected */ - const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; - size_t repLength = ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, - repEnd, prefixStart) + - EQUAL_READ32; - int gain2 = (int)(repLength * 4); - int gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1); - if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) - matchLength = repLength, offset = 0, start = ip; - } - } - - /* search match, depth 2 */ - { - size_t offset2 = 99999999; - size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ - int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; - continue; - } - } - } - break; /* nothing found : store previous solution */ - } - - /* catch up */ - if (offset) { - U32 const matchIndex = (U32)((start - base) - (offset - ZSTD_REP_MOVE)); - const BYTE *match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; - const BYTE *const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; - while ((start > anchor) && (match > mStart) && (start[-1] == match[-1])) { - start--; - match--; - matchLength++; - } /* catch up */ - offset_2 = offset_1; - offset_1 = (U32)(offset - ZSTD_REP_MOVE); - } - - /* store sequence */ - _storeSequence : { - size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH); - anchor = ip = start + matchLength; - } - - /* check immediate repcode */ - while (ip <= ilimit) { - const U32 repIndex = (U32)((ip - base) - offset_2); - const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *const repMatch = repBase + repIndex; - if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ - if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) { - /* repcode detected we should take it */ - const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; - matchLength = - ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; - offset = offset_2; - offset_2 = offset_1; - offset_1 = (U32)offset; /* swap offset history */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH); - ip += matchLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } - break; - } - } - - /* Save reps for next block */ - ctx->repToConfirm[0] = offset_1; - ctx->repToConfirm[1] = offset_2; - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -void ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0); } - -static void ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 1); -} - -static void ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 2); -} - -static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ - ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2); -} - -/* The optimal parser */ -#include "zstd_opt.h" - -static void ZSTD_compressBlock_btopt(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ -#ifdef ZSTD_OPT_H_91842398743 - ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0); -#else - (void)ctx; - (void)src; - (void)srcSize; - return; -#endif -} - -static void ZSTD_compressBlock_btopt2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ -#ifdef ZSTD_OPT_H_91842398743 - ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1); -#else - (void)ctx; - (void)src; - (void)srcSize; - return; -#endif -} - -static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ -#ifdef ZSTD_OPT_H_91842398743 - ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0); -#else - (void)ctx; - (void)src; - (void)srcSize; - return; -#endif -} - -static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) -{ -#ifdef ZSTD_OPT_H_91842398743 - ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1); -#else - (void)ctx; - (void)src; - (void)srcSize; - return; -#endif -} - -typedef void (*ZSTD_blockCompressor)(ZSTD_CCtx *ctx, const void *src, size_t srcSize); - -static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) -{ - static const ZSTD_blockCompressor blockCompressor[2][8] = { - {ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, - ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2}, - {ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict, - ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict}}; - - return blockCompressor[extDict][(U32)strat]; -} - -static size_t ZSTD_compressBlock_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit); - const BYTE *const base = zc->base; - const BYTE *const istart = (const BYTE *)src; - const U32 curr = (U32)(istart - base); - if (srcSize < MIN_CBLOCK_SIZE + ZSTD_blockHeaderSize + 1) - return 0; /* don't even attempt compression below a certain srcSize */ - ZSTD_resetSeqStore(&(zc->seqStore)); - if (curr > zc->nextToUpdate + 384) - zc->nextToUpdate = curr - MIN(192, (U32)(curr - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */ - blockCompressor(zc, src, srcSize); - return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize); -} - -/*! ZSTD_compress_generic() : -* Compress a chunk of data into one or multiple blocks. -* All blocks will be terminated, all input will be consumed. -* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. -* Frame is supposed already started (header already produced) -* @return : compressed size, or an error code -*/ -static size_t ZSTD_compress_generic(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 lastFrameChunk) -{ - size_t blockSize = cctx->blockSize; - size_t remaining = srcSize; - const BYTE *ip = (const BYTE *)src; - BYTE *const ostart = (BYTE *)dst; - BYTE *op = ostart; - U32 const maxDist = 1 << cctx->params.cParams.windowLog; - - if (cctx->params.fParams.checksumFlag && srcSize) - xxh64_update(&cctx->xxhState, src, srcSize); - - while (remaining) { - U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); - size_t cSize; - - if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) - return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ - if (remaining < blockSize) - blockSize = remaining; - - /* preemptive overflow correction */ - if (cctx->lowLimit > (3U << 29)) { - U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1; - U32 const curr = (U32)(ip - cctx->base); - U32 const newCurr = (curr & cycleMask) + (1 << cctx->params.cParams.windowLog); - U32 const correction = curr - newCurr; - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30); - ZSTD_reduceIndex(cctx, correction); - cctx->base += correction; - cctx->dictBase += correction; - cctx->lowLimit -= correction; - cctx->dictLimit -= correction; - if (cctx->nextToUpdate < correction) - cctx->nextToUpdate = 0; - else - cctx->nextToUpdate -= correction; - } - - if ((U32)(ip + blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { - /* enforce maxDist */ - U32 const newLowLimit = (U32)(ip + blockSize - cctx->base) - maxDist; - if (cctx->lowLimit < newLowLimit) - cctx->lowLimit = newLowLimit; - if (cctx->dictLimit < cctx->lowLimit) - cctx->dictLimit = cctx->lowLimit; - } - - cSize = ZSTD_compressBlock_internal(cctx, op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, ip, blockSize); - if (ZSTD_isError(cSize)) - return cSize; - - if (cSize == 0) { /* block is not compressible */ - U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw) << 1) + (U32)(blockSize << 3); - if (blockSize + ZSTD_blockHeaderSize > dstCapacity) - return ERROR(dstSize_tooSmall); - ZSTD_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */ - memcpy(op + ZSTD_blockHeaderSize, ip, blockSize); - cSize = ZSTD_blockHeaderSize + blockSize; - } else { - U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed) << 1) + (U32)(cSize << 3); - ZSTD_writeLE24(op, cBlockHeader24); - cSize += ZSTD_blockHeaderSize; - } - - remaining -= blockSize; - dstCapacity -= cSize; - ip += blockSize; - op += cSize; - } - - if (lastFrameChunk && (op > ostart)) - cctx->stage = ZSTDcs_ending; - return op - ostart; -} - -static size_t ZSTD_writeFrameHeader(void *dst, size_t dstCapacity, ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID) -{ - BYTE *const op = (BYTE *)dst; - U32 const dictIDSizeCode = (dictID > 0) + (dictID >= 256) + (dictID >= 65536); /* 0-3 */ - U32 const checksumFlag = params.fParams.checksumFlag > 0; - U32 const windowSize = 1U << params.cParams.windowLog; - U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); - BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); - U32 const fcsCode = - params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */ - BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6)); - size_t pos; - - if (dstCapacity < ZSTD_frameHeaderSize_max) - return ERROR(dstSize_tooSmall); - - ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER); - op[4] = frameHeaderDescriptionByte; - pos = 5; - if (!singleSegment) - op[pos++] = windowLogByte; - switch (dictIDSizeCode) { - default: /* impossible */ - case 0: break; - case 1: - op[pos] = (BYTE)(dictID); - pos++; - break; - case 2: - ZSTD_writeLE16(op + pos, (U16)dictID); - pos += 2; - break; - case 3: - ZSTD_writeLE32(op + pos, dictID); - pos += 4; - break; - } - switch (fcsCode) { - default: /* impossible */ - case 0: - if (singleSegment) - op[pos++] = (BYTE)(pledgedSrcSize); - break; - case 1: - ZSTD_writeLE16(op + pos, (U16)(pledgedSrcSize - 256)); - pos += 2; - break; - case 2: - ZSTD_writeLE32(op + pos, (U32)(pledgedSrcSize)); - pos += 4; - break; - case 3: - ZSTD_writeLE64(op + pos, (U64)(pledgedSrcSize)); - pos += 8; - break; - } - return pos; -} - -static size_t ZSTD_compressContinue_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 frame, U32 lastFrameChunk) -{ - const BYTE *const ip = (const BYTE *)src; - size_t fhSize = 0; - - if (cctx->stage == ZSTDcs_created) - return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ - - if (frame && (cctx->stage == ZSTDcs_init)) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID); - if (ZSTD_isError(fhSize)) - return fhSize; - dstCapacity -= fhSize; - dst = (char *)dst + fhSize; - cctx->stage = ZSTDcs_ongoing; - } - - /* Check if blocks follow each other */ - if (src != cctx->nextSrc) { - /* not contiguous */ - ptrdiff_t const delta = cctx->nextSrc - ip; - cctx->lowLimit = cctx->dictLimit; - cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base); - cctx->dictBase = cctx->base; - cctx->base -= delta; - cctx->nextToUpdate = cctx->dictLimit; - if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) - cctx->lowLimit = cctx->dictLimit; /* too small extDict */ - } - - /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ - if ((ip + srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) { - ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase; - U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx; - cctx->lowLimit = lowLimitMax; - } - - cctx->nextSrc = ip + srcSize; - - if (srcSize) { - size_t const cSize = frame ? ZSTD_compress_generic(cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) - : ZSTD_compressBlock_internal(cctx, dst, dstCapacity, src, srcSize); - if (ZSTD_isError(cSize)) - return cSize; - return cSize + fhSize; - } else - return fhSize; -} - -size_t ZSTD_compressContinue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0); -} - -size_t ZSTD_getBlockSizeMax(ZSTD_CCtx *cctx) { return MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog); } - -size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx); - if (srcSize > blockSizeMax) - return ERROR(srcSize_wrong); - return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0); -} - -/*! ZSTD_loadDictionaryContent() : - * @return : 0, or an error code - */ -static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx *zc, const void *src, size_t srcSize) -{ - const BYTE *const ip = (const BYTE *)src; - const BYTE *const iend = ip + srcSize; - - /* input becomes curr prefix */ - zc->lowLimit = zc->dictLimit; - zc->dictLimit = (U32)(zc->nextSrc - zc->base); - zc->dictBase = zc->base; - zc->base += ip - zc->nextSrc; - zc->nextToUpdate = zc->dictLimit; - zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base); - - zc->nextSrc = iend; - if (srcSize <= HASH_READ_SIZE) - return 0; - - switch (zc->params.cParams.strategy) { - case ZSTD_fast: ZSTD_fillHashTable(zc, iend, zc->params.cParams.searchLength); break; - - case ZSTD_dfast: ZSTD_fillDoubleHashTable(zc, iend, zc->params.cParams.searchLength); break; - - case ZSTD_greedy: - case ZSTD_lazy: - case ZSTD_lazy2: - if (srcSize >= HASH_READ_SIZE) - ZSTD_insertAndFindFirstIndex(zc, iend - HASH_READ_SIZE, zc->params.cParams.searchLength); - break; - - case ZSTD_btlazy2: - case ZSTD_btopt: - case ZSTD_btopt2: - if (srcSize >= HASH_READ_SIZE) - ZSTD_updateTree(zc, iend - HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength); - break; - - default: - return ERROR(GENERIC); /* strategy doesn't exist; impossible */ - } - - zc->nextToUpdate = (U32)(iend - zc->base); - return 0; -} - -/* Dictionaries that assign zero probability to symbols that show up causes problems - when FSE encoding. Refuse dictionaries that assign zero probability to symbols - that we may encounter during compression. - NOTE: This behavior is not standard and could be improved in the future. */ -static size_t ZSTD_checkDictNCount(short *normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) -{ - U32 s; - if (dictMaxSymbolValue < maxSymbolValue) - return ERROR(dictionary_corrupted); - for (s = 0; s <= maxSymbolValue; ++s) { - if (normalizedCounter[s] == 0) - return ERROR(dictionary_corrupted); - } - return 0; -} - -/* Dictionary format : - * See : - * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format - */ -/*! ZSTD_loadZstdDictionary() : - * @return : 0, or an error code - * assumptions : magic number supposed already checked - * dictSize supposed > 8 - */ -static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize) -{ - const BYTE *dictPtr = (const BYTE *)dict; - const BYTE *const dictEnd = dictPtr + dictSize; - short offcodeNCount[MaxOff + 1]; - unsigned offcodeMaxValue = MaxOff; - - dictPtr += 4; /* skip magic number */ - cctx->dictID = cctx->params.fParams.noDictIDFlag ? 0 : ZSTD_readLE32(dictPtr); - dictPtr += 4; - - { - size_t const hufHeaderSize = HUF_readCTable_wksp(cctx->hufTable, 255, dictPtr, dictEnd - dictPtr, cctx->tmpCounters, sizeof(cctx->tmpCounters)); - if (HUF_isError(hufHeaderSize)) - return ERROR(dictionary_corrupted); - dictPtr += hufHeaderSize; - } - - { - unsigned offcodeLog; - size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr); - if (FSE_isError(offcodeHeaderSize)) - return ERROR(dictionary_corrupted); - if (offcodeLog > OffFSELog) - return ERROR(dictionary_corrupted); - /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ - CHECK_E(FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)), - dictionary_corrupted); - dictPtr += offcodeHeaderSize; - } - - { - short matchlengthNCount[MaxML + 1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog; - size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr); - if (FSE_isError(matchlengthHeaderSize)) - return ERROR(dictionary_corrupted); - if (matchlengthLog > MLFSELog) - return ERROR(dictionary_corrupted); - /* Every match length code must have non-zero probability */ - CHECK_F(ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); - CHECK_E( - FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)), - dictionary_corrupted); - dictPtr += matchlengthHeaderSize; - } - - { - short litlengthNCount[MaxLL + 1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog; - size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr); - if (FSE_isError(litlengthHeaderSize)) - return ERROR(dictionary_corrupted); - if (litlengthLog > LLFSELog) - return ERROR(dictionary_corrupted); - /* Every literal length code must have non-zero probability */ - CHECK_F(ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); - CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)), - dictionary_corrupted); - dictPtr += litlengthHeaderSize; - } - - if (dictPtr + 12 > dictEnd) - return ERROR(dictionary_corrupted); - cctx->rep[0] = ZSTD_readLE32(dictPtr + 0); - cctx->rep[1] = ZSTD_readLE32(dictPtr + 4); - cctx->rep[2] = ZSTD_readLE32(dictPtr + 8); - dictPtr += 12; - - { - size_t const dictContentSize = (size_t)(dictEnd - dictPtr); - U32 offcodeMax = MaxOff; - if (dictContentSize <= ((U32)-1) - 128 KB) { - U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ - offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ - } - /* All offset values <= dictContentSize + 128 KB must be representable */ - CHECK_F(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); - /* All repCodes must be <= dictContentSize and != 0*/ - { - U32 u; - for (u = 0; u < 3; u++) { - if (cctx->rep[u] == 0) - return ERROR(dictionary_corrupted); - if (cctx->rep[u] > dictContentSize) - return ERROR(dictionary_corrupted); - } - } - - cctx->flagStaticTables = 1; - cctx->flagStaticHufTable = HUF_repeat_valid; - return ZSTD_loadDictionaryContent(cctx, dictPtr, dictContentSize); - } -} - -/** ZSTD_compress_insertDictionary() : -* @return : 0, or an error code */ -static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize) -{ - if ((dict == NULL) || (dictSize <= 8)) - return 0; - - /* dict as pure content */ - if ((ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) || (cctx->forceRawDict)) - return ZSTD_loadDictionaryContent(cctx, dict, dictSize); - - /* dict as zstd dictionary */ - return ZSTD_loadZstdDictionary(cctx, dict, dictSize); -} - -/*! ZSTD_compressBegin_internal() : -* @return : 0, or an error code */ -static size_t ZSTD_compressBegin_internal(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) -{ - ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue; - CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp)); - return ZSTD_compress_insertDictionary(cctx, dict, dictSize); -} - -/*! ZSTD_compressBegin_advanced() : -* @return : 0, or an error code */ -size_t ZSTD_compressBegin_advanced(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) -{ - /* compression parameters verification and optimization */ - CHECK_F(ZSTD_checkCParams(params.cParams)); - return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); -} - -size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, int compressionLevel) -{ - ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); - return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0); -} - -size_t ZSTD_compressBegin(ZSTD_CCtx *cctx, int compressionLevel) { return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); } - -/*! ZSTD_writeEpilogue() : -* Ends a frame. -* @return : nb of bytes written into dst (or an error code) */ -static size_t ZSTD_writeEpilogue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity) -{ - BYTE *const ostart = (BYTE *)dst; - BYTE *op = ostart; - size_t fhSize = 0; - - if (cctx->stage == ZSTDcs_created) - return ERROR(stage_wrong); /* init missing */ - - /* special case : empty frame */ - if (cctx->stage == ZSTDcs_init) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0); - if (ZSTD_isError(fhSize)) - return fhSize; - dstCapacity -= fhSize; - op += fhSize; - cctx->stage = ZSTDcs_ongoing; - } - - if (cctx->stage != ZSTDcs_ending) { - /* write one last empty block, make it the "last" block */ - U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw) << 1) + 0; - if (dstCapacity < 4) - return ERROR(dstSize_tooSmall); - ZSTD_writeLE32(op, cBlockHeader24); - op += ZSTD_blockHeaderSize; - dstCapacity -= ZSTD_blockHeaderSize; - } - - if (cctx->params.fParams.checksumFlag) { - U32 const checksum = (U32)xxh64_digest(&cctx->xxhState); - if (dstCapacity < 4) - return ERROR(dstSize_tooSmall); - ZSTD_writeLE32(op, checksum); - op += 4; - } - - cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ - return op - ostart; -} - -size_t ZSTD_compressEnd(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - size_t endResult; - size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1); - if (ZSTD_isError(cSize)) - return cSize; - endResult = ZSTD_writeEpilogue(cctx, (char *)dst + cSize, dstCapacity - cSize); - if (ZSTD_isError(endResult)) - return endResult; - return cSize + endResult; -} - -static size_t ZSTD_compress_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, - ZSTD_parameters params) -{ - CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize)); - return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); -} - -size_t ZSTD_compress_usingDict(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, - ZSTD_parameters params) -{ - return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); -} - -size_t ZSTD_compressCCtx(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, ZSTD_parameters params) -{ - return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, NULL, 0, params); -} - -/* ===== Dictionary API ===== */ - -struct ZSTD_CDict_s { - void *dictBuffer; - const void *dictContent; - size_t dictContentSize; - ZSTD_CCtx *refContext; -}; /* typedef'd tp ZSTD_CDict within "zstd.h" */ - -size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams) { return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CDict)); } - -static ZSTD_CDict *ZSTD_createCDict_advanced(const void *dictBuffer, size_t dictSize, unsigned byReference, ZSTD_parameters params, ZSTD_customMem customMem) -{ - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - { - ZSTD_CDict *const cdict = (ZSTD_CDict *)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); - ZSTD_CCtx *const cctx = ZSTD_createCCtx_advanced(customMem); - - if (!cdict || !cctx) { - ZSTD_free(cdict, customMem); - ZSTD_freeCCtx(cctx); - return NULL; - } - - if ((byReference) || (!dictBuffer) || (!dictSize)) { - cdict->dictBuffer = NULL; - cdict->dictContent = dictBuffer; - } else { - void *const internalBuffer = ZSTD_malloc(dictSize, customMem); - if (!internalBuffer) { - ZSTD_free(cctx, customMem); - ZSTD_free(cdict, customMem); - return NULL; - } - memcpy(internalBuffer, dictBuffer, dictSize); - cdict->dictBuffer = internalBuffer; - cdict->dictContent = internalBuffer; - } - - { - size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0); - if (ZSTD_isError(errorCode)) { - ZSTD_free(cdict->dictBuffer, customMem); - ZSTD_free(cdict, customMem); - ZSTD_freeCCtx(cctx); - return NULL; - } - } - - cdict->refContext = cctx; - cdict->dictContentSize = dictSize; - return cdict; - } -} - -ZSTD_CDict *ZSTD_initCDict(const void *dict, size_t dictSize, ZSTD_parameters params, void *workspace, size_t workspaceSize) -{ - ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); - return ZSTD_createCDict_advanced(dict, dictSize, 1, params, stackMem); -} - -size_t ZSTD_freeCDict(ZSTD_CDict *cdict) -{ - if (cdict == NULL) - return 0; /* support free on NULL */ - { - ZSTD_customMem const cMem = cdict->refContext->customMem; - ZSTD_freeCCtx(cdict->refContext); - ZSTD_free(cdict->dictBuffer, cMem); - ZSTD_free(cdict, cMem); - return 0; - } -} - -static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict *cdict) { return ZSTD_getParamsFromCCtx(cdict->refContext); } - -size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize) -{ - if (cdict->dictContentSize) - CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) - else { - ZSTD_parameters params = cdict->refContext->params; - params.fParams.contentSizeFlag = (pledgedSrcSize > 0); - CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, params, pledgedSrcSize)); - } - return 0; -} - -/*! ZSTD_compress_usingCDict() : -* Compression using a digested Dictionary. -* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. -* Note that compression level is decided during dictionary creation */ -size_t ZSTD_compress_usingCDict(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_CDict *cdict) -{ - CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize)); - - if (cdict->refContext->params.fParams.contentSizeFlag == 1) { - cctx->params.fParams.contentSizeFlag = 1; - cctx->frameContentSize = srcSize; - } else { - cctx->params.fParams.contentSizeFlag = 0; - } - - return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); -} - -/* ****************************************************************** -* Streaming -********************************************************************/ - -typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; - -struct ZSTD_CStream_s { - ZSTD_CCtx *cctx; - ZSTD_CDict *cdictLocal; - const ZSTD_CDict *cdict; - char *inBuff; - size_t inBuffSize; - size_t inToCompress; - size_t inBuffPos; - size_t inBuffTarget; - size_t blockSize; - char *outBuff; - size_t outBuffSize; - size_t outBuffContentSize; - size_t outBuffFlushedSize; - ZSTD_cStreamStage stage; - U32 checksum; - U32 frameEnded; - U64 pledgedSrcSize; - U64 inputProcessed; - ZSTD_parameters params; - ZSTD_customMem customMem; -}; /* typedef'd to ZSTD_CStream within "zstd.h" */ - -size_t ZSTD_CStreamWorkspaceBound(ZSTD_compressionParameters cParams) -{ - size_t const inBuffSize = (size_t)1 << cParams.windowLog; - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, inBuffSize); - size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; - - return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize); -} - -ZSTD_CStream *ZSTD_createCStream_advanced(ZSTD_customMem customMem) -{ - ZSTD_CStream *zcs; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - zcs = (ZSTD_CStream *)ZSTD_malloc(sizeof(ZSTD_CStream), customMem); - if (zcs == NULL) - return NULL; - memset(zcs, 0, sizeof(ZSTD_CStream)); - memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); - zcs->cctx = ZSTD_createCCtx_advanced(customMem); - if (zcs->cctx == NULL) { - ZSTD_freeCStream(zcs); - return NULL; - } - return zcs; -} - -size_t ZSTD_freeCStream(ZSTD_CStream *zcs) -{ - if (zcs == NULL) - return 0; /* support free on NULL */ - { - ZSTD_customMem const cMem = zcs->customMem; - ZSTD_freeCCtx(zcs->cctx); - zcs->cctx = NULL; - ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = NULL; - ZSTD_free(zcs->inBuff, cMem); - zcs->inBuff = NULL; - ZSTD_free(zcs->outBuff, cMem); - zcs->outBuff = NULL; - ZSTD_free(zcs, cMem); - return 0; - } -} - -/*====== Initialization ======*/ - -size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } -size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */; } - -static size_t ZSTD_resetCStream_internal(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize) -{ - if (zcs->inBuffSize == 0) - return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */ - - if (zcs->cdict) - CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)) - else - CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize)); - - zcs->inToCompress = 0; - zcs->inBuffPos = 0; - zcs->inBuffTarget = zcs->blockSize; - zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; - zcs->stage = zcss_load; - zcs->frameEnded = 0; - zcs->pledgedSrcSize = pledgedSrcSize; - zcs->inputProcessed = 0; - return 0; /* ready to go */ -} - -size_t ZSTD_resetCStream(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize) -{ - - zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0); - - return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); -} - -static size_t ZSTD_initCStream_advanced(ZSTD_CStream *zcs, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) -{ - /* allocate buffers */ - { - size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; - if (zcs->inBuffSize < neededInBuffSize) { - zcs->inBuffSize = neededInBuffSize; - ZSTD_free(zcs->inBuff, zcs->customMem); - zcs->inBuff = (char *)ZSTD_malloc(neededInBuffSize, zcs->customMem); - if (zcs->inBuff == NULL) - return ERROR(memory_allocation); - } - zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); - } - if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize) + 1) { - zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize) + 1; - ZSTD_free(zcs->outBuff, zcs->customMem); - zcs->outBuff = (char *)ZSTD_malloc(zcs->outBuffSize, zcs->customMem); - if (zcs->outBuff == NULL) - return ERROR(memory_allocation); - } - - if (dict && dictSize >= 8) { - ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem); - if (zcs->cdictLocal == NULL) - return ERROR(memory_allocation); - zcs->cdict = zcs->cdictLocal; - } else - zcs->cdict = NULL; - - zcs->checksum = params.fParams.checksumFlag > 0; - zcs->params = params; - - return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); -} - -ZSTD_CStream *ZSTD_initCStream(ZSTD_parameters params, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize) -{ - ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); - ZSTD_CStream *const zcs = ZSTD_createCStream_advanced(stackMem); - if (zcs) { - size_t const code = ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize); - if (ZSTD_isError(code)) { - return NULL; - } - } - return zcs; -} - -ZSTD_CStream *ZSTD_initCStream_usingCDict(const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize) -{ - ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict); - ZSTD_CStream *const zcs = ZSTD_initCStream(params, pledgedSrcSize, workspace, workspaceSize); - if (zcs) { - zcs->cdict = cdict; - if (ZSTD_isError(ZSTD_resetCStream_internal(zcs, pledgedSrcSize))) { - return NULL; - } - } - return zcs; -} - -/*====== Compression ======*/ - -typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e; - -ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - size_t const length = MIN(dstCapacity, srcSize); - memcpy(dst, src, length); - return length; -} - -static size_t ZSTD_compressStream_generic(ZSTD_CStream *zcs, void *dst, size_t *dstCapacityPtr, const void *src, size_t *srcSizePtr, ZSTD_flush_e const flush) -{ - U32 someMoreWork = 1; - const char *const istart = (const char *)src; - const char *const iend = istart + *srcSizePtr; - const char *ip = istart; - char *const ostart = (char *)dst; - char *const oend = ostart + *dstCapacityPtr; - char *op = ostart; - - while (someMoreWork) { - switch (zcs->stage) { - case zcss_init: - return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ - - case zcss_load: - /* complete inBuffer */ - { - size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; - size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend - ip); - zcs->inBuffPos += loaded; - ip += loaded; - if ((zcs->inBuffPos == zcs->inToCompress) || (!flush && (toLoad != loaded))) { - someMoreWork = 0; - break; /* not enough input to get a full block : stop there, wait for more */ - } - } - /* compress curr block (note : this stage cannot be stopped in the middle) */ - { - void *cDst; - size_t cSize; - size_t const iSize = zcs->inBuffPos - zcs->inToCompress; - size_t oSize = oend - op; - if (oSize >= ZSTD_compressBound(iSize)) - cDst = op; /* compress directly into output buffer (avoid flush stage) */ - else - cDst = zcs->outBuff, oSize = zcs->outBuffSize; - cSize = (flush == zsf_end) ? ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) - : ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); - if (ZSTD_isError(cSize)) - return cSize; - if (flush == zsf_end) - zcs->frameEnded = 1; - /* prepare next block */ - zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; - if (zcs->inBuffTarget > zcs->inBuffSize) - zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */ - zcs->inToCompress = zcs->inBuffPos; - if (cDst == op) { - op += cSize; - break; - } /* no need to flush */ - zcs->outBuffContentSize = cSize; - zcs->outBuffFlushedSize = 0; - zcs->stage = zcss_flush; /* pass-through to flush stage */ - } - - case zcss_flush: { - size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; - size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); - op += flushed; - zcs->outBuffFlushedSize += flushed; - if (toFlush != flushed) { - someMoreWork = 0; - break; - } /* dst too small to store flushed data : stop there */ - zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; - zcs->stage = zcss_load; - break; - } - - case zcss_final: - someMoreWork = 0; /* do nothing */ - break; - - default: - return ERROR(GENERIC); /* impossible */ - } - } - - *srcSizePtr = ip - istart; - *dstCapacityPtr = op - ostart; - zcs->inputProcessed += *srcSizePtr; - if (zcs->frameEnded) - return 0; - { - size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; - if (hintInSize == 0) - hintInSize = zcs->blockSize; - return hintInSize; - } -} - -size_t ZSTD_compressStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output, ZSTD_inBuffer *input) -{ - size_t sizeRead = input->size - input->pos; - size_t sizeWritten = output->size - output->pos; - size_t const result = - ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, (const char *)(input->src) + input->pos, &sizeRead, zsf_gather); - input->pos += sizeRead; - output->pos += sizeWritten; - return result; -} - -/*====== Finalize ======*/ - -/*! ZSTD_flushStream() : -* @return : amount of data remaining to flush */ -size_t ZSTD_flushStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output) -{ - size_t srcSize = 0; - size_t sizeWritten = output->size - output->pos; - size_t const result = ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, &srcSize, - &srcSize, /* use a valid src address instead of NULL */ - zsf_flush); - output->pos += sizeWritten; - if (ZSTD_isError(result)) - return result; - return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ -} - -size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output) -{ - BYTE *const ostart = (BYTE *)(output->dst) + output->pos; - BYTE *const oend = (BYTE *)(output->dst) + output->size; - BYTE *op = ostart; - - if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize)) - return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */ - - if (zcs->stage != zcss_final) { - /* flush whatever remains */ - size_t srcSize = 0; - size_t sizeWritten = output->size - output->pos; - size_t const notEnded = - ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */ - size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; - op += sizeWritten; - if (remainingToFlush) { - output->pos += sizeWritten; - return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4); - } - /* create epilogue */ - zcs->stage = zcss_final; - zcs->outBuffContentSize = !notEnded ? 0 : ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, - 0); /* write epilogue, including final empty block, into outBuff */ - } - - /* flush epilogue */ - { - size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; - size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); - op += flushed; - zcs->outBuffFlushedSize += flushed; - output->pos += op - ostart; - if (toFlush == flushed) - zcs->stage = zcss_init; /* end reached */ - return toFlush - flushed; - } -} - -/*-===== Pre-defined compression levels =====-*/ - -#define ZSTD_DEFAULT_CLEVEL 1 -#define ZSTD_MAX_CLEVEL 22 -int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } - -static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL + 1] = { - { - /* "default" */ - /* W, C, H, S, L, TL, strat */ - {18, 12, 12, 1, 7, 16, ZSTD_fast}, /* level 0 - never used */ - {19, 13, 14, 1, 7, 16, ZSTD_fast}, /* level 1 */ - {19, 15, 16, 1, 6, 16, ZSTD_fast}, /* level 2 */ - {20, 16, 17, 1, 5, 16, ZSTD_dfast}, /* level 3.*/ - {20, 18, 18, 1, 5, 16, ZSTD_dfast}, /* level 4.*/ - {20, 15, 18, 3, 5, 16, ZSTD_greedy}, /* level 5 */ - {21, 16, 19, 2, 5, 16, ZSTD_lazy}, /* level 6 */ - {21, 17, 20, 3, 5, 16, ZSTD_lazy}, /* level 7 */ - {21, 18, 20, 3, 5, 16, ZSTD_lazy2}, /* level 8 */ - {21, 20, 20, 3, 5, 16, ZSTD_lazy2}, /* level 9 */ - {21, 19, 21, 4, 5, 16, ZSTD_lazy2}, /* level 10 */ - {22, 20, 22, 4, 5, 16, ZSTD_lazy2}, /* level 11 */ - {22, 20, 22, 5, 5, 16, ZSTD_lazy2}, /* level 12 */ - {22, 21, 22, 5, 5, 16, ZSTD_lazy2}, /* level 13 */ - {22, 21, 22, 6, 5, 16, ZSTD_lazy2}, /* level 14 */ - {22, 21, 21, 5, 5, 16, ZSTD_btlazy2}, /* level 15 */ - {23, 22, 22, 5, 5, 16, ZSTD_btlazy2}, /* level 16 */ - {23, 21, 22, 4, 5, 24, ZSTD_btopt}, /* level 17 */ - {23, 23, 22, 6, 5, 32, ZSTD_btopt}, /* level 18 */ - {23, 23, 22, 6, 3, 48, ZSTD_btopt}, /* level 19 */ - {25, 25, 23, 7, 3, 64, ZSTD_btopt2}, /* level 20 */ - {26, 26, 23, 7, 3, 256, ZSTD_btopt2}, /* level 21 */ - {27, 27, 25, 9, 3, 512, ZSTD_btopt2}, /* level 22 */ - }, - { - /* for srcSize <= 256 KB */ - /* W, C, H, S, L, T, strat */ - {0, 0, 0, 0, 0, 0, ZSTD_fast}, /* level 0 - not used */ - {18, 13, 14, 1, 6, 8, ZSTD_fast}, /* level 1 */ - {18, 14, 13, 1, 5, 8, ZSTD_dfast}, /* level 2 */ - {18, 16, 15, 1, 5, 8, ZSTD_dfast}, /* level 3 */ - {18, 15, 17, 1, 5, 8, ZSTD_greedy}, /* level 4.*/ - {18, 16, 17, 4, 5, 8, ZSTD_greedy}, /* level 5.*/ - {18, 16, 17, 3, 5, 8, ZSTD_lazy}, /* level 6.*/ - {18, 17, 17, 4, 4, 8, ZSTD_lazy}, /* level 7 */ - {18, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */ - {18, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */ - {18, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */ - {18, 18, 17, 6, 4, 8, ZSTD_lazy2}, /* level 11.*/ - {18, 18, 17, 7, 4, 8, ZSTD_lazy2}, /* level 12.*/ - {18, 19, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13 */ - {18, 18, 18, 4, 4, 16, ZSTD_btopt}, /* level 14.*/ - {18, 18, 18, 4, 3, 16, ZSTD_btopt}, /* level 15.*/ - {18, 19, 18, 6, 3, 32, ZSTD_btopt}, /* level 16.*/ - {18, 19, 18, 8, 3, 64, ZSTD_btopt}, /* level 17.*/ - {18, 19, 18, 9, 3, 128, ZSTD_btopt}, /* level 18.*/ - {18, 19, 18, 10, 3, 256, ZSTD_btopt}, /* level 19.*/ - {18, 19, 18, 11, 3, 512, ZSTD_btopt2}, /* level 20.*/ - {18, 19, 18, 12, 3, 512, ZSTD_btopt2}, /* level 21.*/ - {18, 19, 18, 13, 3, 512, ZSTD_btopt2}, /* level 22.*/ - }, - { - /* for srcSize <= 128 KB */ - /* W, C, H, S, L, T, strat */ - {17, 12, 12, 1, 7, 8, ZSTD_fast}, /* level 0 - not used */ - {17, 12, 13, 1, 6, 8, ZSTD_fast}, /* level 1 */ - {17, 13, 16, 1, 5, 8, ZSTD_fast}, /* level 2 */ - {17, 16, 16, 2, 5, 8, ZSTD_dfast}, /* level 3 */ - {17, 13, 15, 3, 4, 8, ZSTD_greedy}, /* level 4 */ - {17, 15, 17, 4, 4, 8, ZSTD_greedy}, /* level 5 */ - {17, 16, 17, 3, 4, 8, ZSTD_lazy}, /* level 6 */ - {17, 15, 17, 4, 4, 8, ZSTD_lazy2}, /* level 7 */ - {17, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */ - {17, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */ - {17, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */ - {17, 17, 17, 7, 4, 8, ZSTD_lazy2}, /* level 11 */ - {17, 17, 17, 8, 4, 8, ZSTD_lazy2}, /* level 12 */ - {17, 18, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13.*/ - {17, 17, 17, 7, 3, 8, ZSTD_btopt}, /* level 14.*/ - {17, 17, 17, 7, 3, 16, ZSTD_btopt}, /* level 15.*/ - {17, 18, 17, 7, 3, 32, ZSTD_btopt}, /* level 16.*/ - {17, 18, 17, 7, 3, 64, ZSTD_btopt}, /* level 17.*/ - {17, 18, 17, 7, 3, 256, ZSTD_btopt}, /* level 18.*/ - {17, 18, 17, 8, 3, 256, ZSTD_btopt}, /* level 19.*/ - {17, 18, 17, 9, 3, 256, ZSTD_btopt2}, /* level 20.*/ - {17, 18, 17, 10, 3, 256, ZSTD_btopt2}, /* level 21.*/ - {17, 18, 17, 11, 3, 512, ZSTD_btopt2}, /* level 22.*/ - }, - { - /* for srcSize <= 16 KB */ - /* W, C, H, S, L, T, strat */ - {14, 12, 12, 1, 7, 6, ZSTD_fast}, /* level 0 - not used */ - {14, 14, 14, 1, 6, 6, ZSTD_fast}, /* level 1 */ - {14, 14, 14, 1, 4, 6, ZSTD_fast}, /* level 2 */ - {14, 14, 14, 1, 4, 6, ZSTD_dfast}, /* level 3.*/ - {14, 14, 14, 4, 4, 6, ZSTD_greedy}, /* level 4.*/ - {14, 14, 14, 3, 4, 6, ZSTD_lazy}, /* level 5.*/ - {14, 14, 14, 4, 4, 6, ZSTD_lazy2}, /* level 6 */ - {14, 14, 14, 5, 4, 6, ZSTD_lazy2}, /* level 7 */ - {14, 14, 14, 6, 4, 6, ZSTD_lazy2}, /* level 8.*/ - {14, 15, 14, 6, 4, 6, ZSTD_btlazy2}, /* level 9.*/ - {14, 15, 14, 3, 3, 6, ZSTD_btopt}, /* level 10.*/ - {14, 15, 14, 6, 3, 8, ZSTD_btopt}, /* level 11.*/ - {14, 15, 14, 6, 3, 16, ZSTD_btopt}, /* level 12.*/ - {14, 15, 14, 6, 3, 24, ZSTD_btopt}, /* level 13.*/ - {14, 15, 15, 6, 3, 48, ZSTD_btopt}, /* level 14.*/ - {14, 15, 15, 6, 3, 64, ZSTD_btopt}, /* level 15.*/ - {14, 15, 15, 6, 3, 96, ZSTD_btopt}, /* level 16.*/ - {14, 15, 15, 6, 3, 128, ZSTD_btopt}, /* level 17.*/ - {14, 15, 15, 6, 3, 256, ZSTD_btopt}, /* level 18.*/ - {14, 15, 15, 7, 3, 256, ZSTD_btopt}, /* level 19.*/ - {14, 15, 15, 8, 3, 256, ZSTD_btopt2}, /* level 20.*/ - {14, 15, 15, 9, 3, 256, ZSTD_btopt2}, /* level 21.*/ - {14, 15, 15, 10, 3, 256, ZSTD_btopt2}, /* level 22.*/ - }, -}; - -/*! ZSTD_getCParams() : -* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`. -* Size values are optional, provide 0 if not known or unused */ -ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) -{ - ZSTD_compressionParameters cp; - size_t const addedSize = srcSize ? 0 : 500; - U64 const rSize = srcSize + dictSize ? srcSize + dictSize + addedSize : (U64)-1; - U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ - if (compressionLevel <= 0) - compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */ - if (compressionLevel > ZSTD_MAX_CLEVEL) - compressionLevel = ZSTD_MAX_CLEVEL; - cp = ZSTD_defaultCParameters[tableID][compressionLevel]; - if (ZSTD_32bits()) { /* auto-correction, for 32-bits mode */ - if (cp.windowLog > ZSTD_WINDOWLOG_MAX) - cp.windowLog = ZSTD_WINDOWLOG_MAX; - if (cp.chainLog > ZSTD_CHAINLOG_MAX) - cp.chainLog = ZSTD_CHAINLOG_MAX; - if (cp.hashLog > ZSTD_HASHLOG_MAX) - cp.hashLog = ZSTD_HASHLOG_MAX; - } - cp = ZSTD_adjustCParams(cp, srcSize, dictSize); - return cp; -} - -/*! ZSTD_getParams() : -* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). -* All fields of `ZSTD_frameParameters` are set to default (0) */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) -{ - ZSTD_parameters params; - ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); - memset(¶ms, 0, sizeof(params)); - params.cParams = cParams; - return params; -} - -EXPORT_SYMBOL(ZSTD_maxCLevel); -EXPORT_SYMBOL(ZSTD_compressBound); - -EXPORT_SYMBOL(ZSTD_CCtxWorkspaceBound); -EXPORT_SYMBOL(ZSTD_initCCtx); -EXPORT_SYMBOL(ZSTD_compressCCtx); -EXPORT_SYMBOL(ZSTD_compress_usingDict); - -EXPORT_SYMBOL(ZSTD_CDictWorkspaceBound); -EXPORT_SYMBOL(ZSTD_initCDict); -EXPORT_SYMBOL(ZSTD_compress_usingCDict); - -EXPORT_SYMBOL(ZSTD_CStreamWorkspaceBound); -EXPORT_SYMBOL(ZSTD_initCStream); -EXPORT_SYMBOL(ZSTD_initCStream_usingCDict); -EXPORT_SYMBOL(ZSTD_resetCStream); -EXPORT_SYMBOL(ZSTD_compressStream); -EXPORT_SYMBOL(ZSTD_flushStream); -EXPORT_SYMBOL(ZSTD_endStream); -EXPORT_SYMBOL(ZSTD_CStreamInSize); -EXPORT_SYMBOL(ZSTD_CStreamOutSize); - -EXPORT_SYMBOL(ZSTD_getCParams); -EXPORT_SYMBOL(ZSTD_getParams); -EXPORT_SYMBOL(ZSTD_checkCParams); -EXPORT_SYMBOL(ZSTD_adjustCParams); - -EXPORT_SYMBOL(ZSTD_compressBegin); -EXPORT_SYMBOL(ZSTD_compressBegin_usingDict); -EXPORT_SYMBOL(ZSTD_compressBegin_advanced); -EXPORT_SYMBOL(ZSTD_copyCCtx); -EXPORT_SYMBOL(ZSTD_compressBegin_usingCDict); -EXPORT_SYMBOL(ZSTD_compressContinue); -EXPORT_SYMBOL(ZSTD_compressEnd); - -EXPORT_SYMBOL(ZSTD_getBlockSizeMax); -EXPORT_SYMBOL(ZSTD_compressBlock); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_DESCRIPTION("Zstd Compressor"); diff --git a/contrib/linux-kernel/lib/zstd/decompress.c b/contrib/linux-kernel/lib/zstd/decompress.c deleted file mode 100644 index 72df4828..00000000 --- a/contrib/linux-kernel/lib/zstd/decompress.c +++ /dev/null @@ -1,2526 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -/* *************************************************************** -* Tuning parameters -*****************************************************************/ -/*! -* MAXWINDOWSIZE_DEFAULT : -* maximum window size accepted by DStream, by default. -* Frames requiring more memory will be rejected. -*/ -#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT -#define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */ -#endif - -/*-******************************************************* -* Dependencies -*********************************************************/ -#include "fse.h" -#include "huf.h" -#include "mem.h" /* low level memory routines */ -#include "zstd_internal.h" -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/string.h> /* memcpy, memmove, memset */ - -#define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0) - -/*-************************************* -* Macros -***************************************/ -#define ZSTD_isError ERR_isError /* for inlining */ -#define FSE_isError ERR_isError -#define HUF_isError ERR_isError - -/*_******************************************************* -* Memory operations -**********************************************************/ -static void ZSTD_copy4(void *dst, const void *src) { memcpy(dst, src, 4); } - -/*-************************************************************* -* Context management -***************************************************************/ -typedef enum { - ZSTDds_getFrameHeaderSize, - ZSTDds_decodeFrameHeader, - ZSTDds_decodeBlockHeader, - ZSTDds_decompressBlock, - ZSTDds_decompressLastBlock, - ZSTDds_checkChecksum, - ZSTDds_decodeSkippableHeader, - ZSTDds_skipFrame -} ZSTD_dStage; - -typedef struct { - FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; - FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; - FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; - HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ - U64 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32 / 2]; - U32 rep[ZSTD_REP_NUM]; -} ZSTD_entropyTables_t; - -struct ZSTD_DCtx_s { - const FSE_DTable *LLTptr; - const FSE_DTable *MLTptr; - const FSE_DTable *OFTptr; - const HUF_DTable *HUFptr; - ZSTD_entropyTables_t entropy; - const void *previousDstEnd; /* detect continuity */ - const void *base; /* start of curr segment */ - const void *vBase; /* virtual start of previous segment if it was just before curr one */ - const void *dictEnd; /* end of previous segment */ - size_t expected; - ZSTD_frameParams fParams; - blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ - ZSTD_dStage stage; - U32 litEntropy; - U32 fseEntropy; - struct xxh64_state xxhState; - size_t headerSize; - U32 dictID; - const BYTE *litPtr; - ZSTD_customMem customMem; - size_t litSize; - size_t rleSize; - BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; -}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ - -size_t ZSTD_DCtxWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DCtx)); } - -size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx) -{ - dctx->expected = ZSTD_frameHeaderSize_prefix; - dctx->stage = ZSTDds_getFrameHeaderSize; - dctx->previousDstEnd = NULL; - dctx->base = NULL; - dctx->vBase = NULL; - dctx->dictEnd = NULL; - dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ - dctx->litEntropy = dctx->fseEntropy = 0; - dctx->dictID = 0; - ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); - memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ - dctx->LLTptr = dctx->entropy.LLTable; - dctx->MLTptr = dctx->entropy.MLTable; - dctx->OFTptr = dctx->entropy.OFTable; - dctx->HUFptr = dctx->entropy.hufTable; - return 0; -} - -ZSTD_DCtx *ZSTD_createDCtx_advanced(ZSTD_customMem customMem) -{ - ZSTD_DCtx *dctx; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - dctx = (ZSTD_DCtx *)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem); - if (!dctx) - return NULL; - memcpy(&dctx->customMem, &customMem, sizeof(customMem)); - ZSTD_decompressBegin(dctx); - return dctx; -} - -ZSTD_DCtx *ZSTD_initDCtx(void *workspace, size_t workspaceSize) -{ - ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); - return ZSTD_createDCtx_advanced(stackMem); -} - -size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx) -{ - if (dctx == NULL) - return 0; /* support free on NULL */ - ZSTD_free(dctx, dctx->customMem); - return 0; /* reserved as a potential error code in the future */ -} - -void ZSTD_copyDCtx(ZSTD_DCtx *dstDCtx, const ZSTD_DCtx *srcDCtx) -{ - size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max; - memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ -} - -static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict); - -/*-************************************************************* -* Decompression section -***************************************************************/ - -/*! ZSTD_isFrame() : - * Tells if the content of `buffer` starts with a valid Frame Identifier. - * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. - * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. - * Note 3 : Skippable Frame Identifiers are considered valid. */ -unsigned ZSTD_isFrame(const void *buffer, size_t size) -{ - if (size < 4) - return 0; - { - U32 const magic = ZSTD_readLE32(buffer); - if (magic == ZSTD_MAGICNUMBER) - return 1; - if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) - return 1; - } - return 0; -} - -/** ZSTD_frameHeaderSize() : -* srcSize must be >= ZSTD_frameHeaderSize_prefix. -* @return : size of the Frame Header */ -static size_t ZSTD_frameHeaderSize(const void *src, size_t srcSize) -{ - if (srcSize < ZSTD_frameHeaderSize_prefix) - return ERROR(srcSize_wrong); - { - BYTE const fhd = ((const BYTE *)src)[4]; - U32 const dictID = fhd & 3; - U32 const singleSegment = (fhd >> 5) & 1; - U32 const fcsId = fhd >> 6; - return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + (singleSegment && !fcsId); - } -} - -/** ZSTD_getFrameParams() : -* decode Frame Header, or require larger `srcSize`. -* @return : 0, `fparamsPtr` is correctly filled, -* >0, `srcSize` is too small, result is expected `srcSize`, -* or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_getFrameParams(ZSTD_frameParams *fparamsPtr, const void *src, size_t srcSize) -{ - const BYTE *ip = (const BYTE *)src; - - if (srcSize < ZSTD_frameHeaderSize_prefix) - return ZSTD_frameHeaderSize_prefix; - if (ZSTD_readLE32(src) != ZSTD_MAGICNUMBER) { - if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { - if (srcSize < ZSTD_skippableHeaderSize) - return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ - memset(fparamsPtr, 0, sizeof(*fparamsPtr)); - fparamsPtr->frameContentSize = ZSTD_readLE32((const char *)src + 4); - fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ - return 0; - } - return ERROR(prefix_unknown); - } - - /* ensure there is enough `srcSize` to fully read/decode frame header */ - { - size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); - if (srcSize < fhsize) - return fhsize; - } - - { - BYTE const fhdByte = ip[4]; - size_t pos = 5; - U32 const dictIDSizeCode = fhdByte & 3; - U32 const checksumFlag = (fhdByte >> 2) & 1; - U32 const singleSegment = (fhdByte >> 5) & 1; - U32 const fcsID = fhdByte >> 6; - U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; - U32 windowSize = 0; - U32 dictID = 0; - U64 frameContentSize = 0; - if ((fhdByte & 0x08) != 0) - return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */ - if (!singleSegment) { - BYTE const wlByte = ip[pos++]; - U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - if (windowLog > ZSTD_WINDOWLOG_MAX) - return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */ - windowSize = (1U << windowLog); - windowSize += (windowSize >> 3) * (wlByte & 7); - } - - switch (dictIDSizeCode) { - default: /* impossible */ - case 0: break; - case 1: - dictID = ip[pos]; - pos++; - break; - case 2: - dictID = ZSTD_readLE16(ip + pos); - pos += 2; - break; - case 3: - dictID = ZSTD_readLE32(ip + pos); - pos += 4; - break; - } - switch (fcsID) { - default: /* impossible */ - case 0: - if (singleSegment) - frameContentSize = ip[pos]; - break; - case 1: frameContentSize = ZSTD_readLE16(ip + pos) + 256; break; - case 2: frameContentSize = ZSTD_readLE32(ip + pos); break; - case 3: frameContentSize = ZSTD_readLE64(ip + pos); break; - } - if (!windowSize) - windowSize = (U32)frameContentSize; - if (windowSize > windowSizeMax) - return ERROR(frameParameter_windowTooLarge); - fparamsPtr->frameContentSize = frameContentSize; - fparamsPtr->windowSize = windowSize; - fparamsPtr->dictID = dictID; - fparamsPtr->checksumFlag = checksumFlag; - } - return 0; -} - -/** ZSTD_getFrameContentSize() : -* compatible with legacy mode -* @return : decompressed size of the single frame pointed to be `src` if known, otherwise -* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined -* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ -unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) -{ - { - ZSTD_frameParams fParams; - if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0) - return ZSTD_CONTENTSIZE_ERROR; - if (fParams.windowSize == 0) { - /* Either skippable or empty frame, size == 0 either way */ - return 0; - } else if (fParams.frameContentSize != 0) { - return fParams.frameContentSize; - } else { - return ZSTD_CONTENTSIZE_UNKNOWN; - } - } -} - -/** ZSTD_findDecompressedSize() : - * compatible with legacy mode - * `srcSize` must be the exact length of some number of ZSTD compressed and/or - * skippable frames - * @return : decompressed size of the frames contained */ -unsigned long long ZSTD_findDecompressedSize(const void *src, size_t srcSize) -{ - { - unsigned long long totalDstSize = 0; - while (srcSize >= ZSTD_frameHeaderSize_prefix) { - const U32 magicNumber = ZSTD_readLE32(src); - - if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { - size_t skippableSize; - if (srcSize < ZSTD_skippableHeaderSize) - return ERROR(srcSize_wrong); - skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize; - if (srcSize < skippableSize) { - return ZSTD_CONTENTSIZE_ERROR; - } - - src = (const BYTE *)src + skippableSize; - srcSize -= skippableSize; - continue; - } - - { - unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); - if (ret >= ZSTD_CONTENTSIZE_ERROR) - return ret; - - /* check for overflow */ - if (totalDstSize + ret < totalDstSize) - return ZSTD_CONTENTSIZE_ERROR; - totalDstSize += ret; - } - { - size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); - if (ZSTD_isError(frameSrcSize)) { - return ZSTD_CONTENTSIZE_ERROR; - } - - src = (const BYTE *)src + frameSrcSize; - srcSize -= frameSrcSize; - } - } - - if (srcSize) { - return ZSTD_CONTENTSIZE_ERROR; - } - - return totalDstSize; - } -} - -/** ZSTD_decodeFrameHeader() : -* `headerSize` must be the size provided by ZSTD_frameHeaderSize(). -* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ -static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx *dctx, const void *src, size_t headerSize) -{ - size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize); - if (ZSTD_isError(result)) - return result; /* invalid header */ - if (result > 0) - return ERROR(srcSize_wrong); /* headerSize too small */ - if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) - return ERROR(dictionary_wrong); - if (dctx->fParams.checksumFlag) - xxh64_reset(&dctx->xxhState, 0); - return 0; -} - -typedef struct { - blockType_e blockType; - U32 lastBlock; - U32 origSize; -} blockProperties_t; - -/*! ZSTD_getcBlockSize() : -* Provides the size of compressed block from block header `src` */ -size_t ZSTD_getcBlockSize(const void *src, size_t srcSize, blockProperties_t *bpPtr) -{ - if (srcSize < ZSTD_blockHeaderSize) - return ERROR(srcSize_wrong); - { - U32 const cBlockHeader = ZSTD_readLE24(src); - U32 const cSize = cBlockHeader >> 3; - bpPtr->lastBlock = cBlockHeader & 1; - bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); - bpPtr->origSize = cSize; /* only useful for RLE */ - if (bpPtr->blockType == bt_rle) - return 1; - if (bpPtr->blockType == bt_reserved) - return ERROR(corruption_detected); - return cSize; - } -} - -static size_t ZSTD_copyRawBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - if (srcSize > dstCapacity) - return ERROR(dstSize_tooSmall); - memcpy(dst, src, srcSize); - return srcSize; -} - -static size_t ZSTD_setRleBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize, size_t regenSize) -{ - if (srcSize != 1) - return ERROR(srcSize_wrong); - if (regenSize > dstCapacity) - return ERROR(dstSize_tooSmall); - memset(dst, *(const BYTE *)src, regenSize); - return regenSize; -} - -/*! ZSTD_decodeLiteralsBlock() : - @return : nb of bytes read from src (< srcSize ) */ -size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx *dctx, const void *src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ -{ - if (srcSize < MIN_CBLOCK_SIZE) - return ERROR(corruption_detected); - - { - const BYTE *const istart = (const BYTE *)src; - symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); - - switch (litEncType) { - case set_repeat: - if (dctx->litEntropy == 0) - return ERROR(dictionary_corrupted); - /* fall-through */ - case set_compressed: - if (srcSize < 5) - return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ - { - size_t lhSize, litSize, litCSize; - U32 singleStream = 0; - U32 const lhlCode = (istart[0] >> 2) & 3; - U32 const lhc = ZSTD_readLE32(istart); - switch (lhlCode) { - case 0: - case 1: - default: /* note : default is impossible, since lhlCode into [0..3] */ - /* 2 - 2 - 10 - 10 */ - singleStream = !lhlCode; - lhSize = 3; - litSize = (lhc >> 4) & 0x3FF; - litCSize = (lhc >> 14) & 0x3FF; - break; - case 2: - /* 2 - 2 - 14 - 14 */ - lhSize = 4; - litSize = (lhc >> 4) & 0x3FFF; - litCSize = lhc >> 18; - break; - case 3: - /* 2 - 2 - 18 - 18 */ - lhSize = 5; - litSize = (lhc >> 4) & 0x3FFFF; - litCSize = (lhc >> 22) + (istart[4] << 10); - break; - } - if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) - return ERROR(corruption_detected); - if (litCSize + lhSize > srcSize) - return ERROR(corruption_detected); - - if (HUF_isError( - (litEncType == set_repeat) - ? (singleStream ? HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr) - : HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr)) - : (singleStream - ? HUF_decompress1X2_DCtx_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize, - dctx->entropy.workspace, sizeof(dctx->entropy.workspace)) - : HUF_decompress4X_hufOnly_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize, - dctx->entropy.workspace, sizeof(dctx->entropy.workspace))))) - return ERROR(corruption_detected); - - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - dctx->litEntropy = 1; - if (litEncType == set_compressed) - dctx->HUFptr = dctx->entropy.hufTable; - memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); - return litCSize + lhSize; - } - - case set_basic: { - size_t litSize, lhSize; - U32 const lhlCode = ((istart[0]) >> 2) & 3; - switch (lhlCode) { - case 0: - case 2: - default: /* note : default is impossible, since lhlCode into [0..3] */ - lhSize = 1; - litSize = istart[0] >> 3; - break; - case 1: - lhSize = 2; - litSize = ZSTD_readLE16(istart) >> 4; - break; - case 3: - lhSize = 3; - litSize = ZSTD_readLE24(istart) >> 4; - break; - } - - if (lhSize + litSize + WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ - if (litSize + lhSize > srcSize) - return ERROR(corruption_detected); - memcpy(dctx->litBuffer, istart + lhSize, litSize); - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); - return lhSize + litSize; - } - /* direct reference into compressed stream */ - dctx->litPtr = istart + lhSize; - dctx->litSize = litSize; - return lhSize + litSize; - } - - case set_rle: { - U32 const lhlCode = ((istart[0]) >> 2) & 3; - size_t litSize, lhSize; - switch (lhlCode) { - case 0: - case 2: - default: /* note : default is impossible, since lhlCode into [0..3] */ - lhSize = 1; - litSize = istart[0] >> 3; - break; - case 1: - lhSize = 2; - litSize = ZSTD_readLE16(istart) >> 4; - break; - case 3: - lhSize = 3; - litSize = ZSTD_readLE24(istart) >> 4; - if (srcSize < 4) - return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ - break; - } - if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) - return ERROR(corruption_detected); - memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - return lhSize + 1; - } - default: - return ERROR(corruption_detected); /* impossible */ - } - } -} - -typedef union { - FSE_decode_t realData; - U32 alignedBy4; -} FSE_decode_t4; - -static const FSE_decode_t4 LL_defaultDTable[(1 << LL_DEFAULTNORMLOG) + 1] = { - {{LL_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */ - {{0, 0, 4}}, /* 0 : base, symbol, bits */ - {{16, 0, 4}}, - {{32, 1, 5}}, - {{0, 3, 5}}, - {{0, 4, 5}}, - {{0, 6, 5}}, - {{0, 7, 5}}, - {{0, 9, 5}}, - {{0, 10, 5}}, - {{0, 12, 5}}, - {{0, 14, 6}}, - {{0, 16, 5}}, - {{0, 18, 5}}, - {{0, 19, 5}}, - {{0, 21, 5}}, - {{0, 22, 5}}, - {{0, 24, 5}}, - {{32, 25, 5}}, - {{0, 26, 5}}, - {{0, 27, 6}}, - {{0, 29, 6}}, - {{0, 31, 6}}, - {{32, 0, 4}}, - {{0, 1, 4}}, - {{0, 2, 5}}, - {{32, 4, 5}}, - {{0, 5, 5}}, - {{32, 7, 5}}, - {{0, 8, 5}}, - {{32, 10, 5}}, - {{0, 11, 5}}, - {{0, 13, 6}}, - {{32, 16, 5}}, - {{0, 17, 5}}, - {{32, 19, 5}}, - {{0, 20, 5}}, - {{32, 22, 5}}, - {{0, 23, 5}}, - {{0, 25, 4}}, - {{16, 25, 4}}, - {{32, 26, 5}}, - {{0, 28, 6}}, - {{0, 30, 6}}, - {{48, 0, 4}}, - {{16, 1, 4}}, - {{32, 2, 5}}, - {{32, 3, 5}}, - {{32, 5, 5}}, - {{32, 6, 5}}, - {{32, 8, 5}}, - {{32, 9, 5}}, - {{32, 11, 5}}, - {{32, 12, 5}}, - {{0, 15, 6}}, - {{32, 17, 5}}, - {{32, 18, 5}}, - {{32, 20, 5}}, - {{32, 21, 5}}, - {{32, 23, 5}}, - {{32, 24, 5}}, - {{0, 35, 6}}, - {{0, 34, 6}}, - {{0, 33, 6}}, - {{0, 32, 6}}, -}; /* LL_defaultDTable */ - -static const FSE_decode_t4 ML_defaultDTable[(1 << ML_DEFAULTNORMLOG) + 1] = { - {{ML_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */ - {{0, 0, 6}}, /* 0 : base, symbol, bits */ - {{0, 1, 4}}, - {{32, 2, 5}}, - {{0, 3, 5}}, - {{0, 5, 5}}, - {{0, 6, 5}}, - {{0, 8, 5}}, - {{0, 10, 6}}, - {{0, 13, 6}}, - {{0, 16, 6}}, - {{0, 19, 6}}, - {{0, 22, 6}}, - {{0, 25, 6}}, - {{0, 28, 6}}, - {{0, 31, 6}}, - {{0, 33, 6}}, - {{0, 35, 6}}, - {{0, 37, 6}}, - {{0, 39, 6}}, - {{0, 41, 6}}, - {{0, 43, 6}}, - {{0, 45, 6}}, - {{16, 1, 4}}, - {{0, 2, 4}}, - {{32, 3, 5}}, - {{0, 4, 5}}, - {{32, 6, 5}}, - {{0, 7, 5}}, - {{0, 9, 6}}, - {{0, 12, 6}}, - {{0, 15, 6}}, - {{0, 18, 6}}, - {{0, 21, 6}}, - {{0, 24, 6}}, - {{0, 27, 6}}, - {{0, 30, 6}}, - {{0, 32, 6}}, - {{0, 34, 6}}, - {{0, 36, 6}}, - {{0, 38, 6}}, - {{0, 40, 6}}, - {{0, 42, 6}}, - {{0, 44, 6}}, - {{32, 1, 4}}, - {{48, 1, 4}}, - {{16, 2, 4}}, - {{32, 4, 5}}, - {{32, 5, 5}}, - {{32, 7, 5}}, - {{32, 8, 5}}, - {{0, 11, 6}}, - {{0, 14, 6}}, - {{0, 17, 6}}, - {{0, 20, 6}}, - {{0, 23, 6}}, - {{0, 26, 6}}, - {{0, 29, 6}}, - {{0, 52, 6}}, - {{0, 51, 6}}, - {{0, 50, 6}}, - {{0, 49, 6}}, - {{0, 48, 6}}, - {{0, 47, 6}}, - {{0, 46, 6}}, -}; /* ML_defaultDTable */ - -static const FSE_decode_t4 OF_defaultDTable[(1 << OF_DEFAULTNORMLOG) + 1] = { - {{OF_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */ - {{0, 0, 5}}, /* 0 : base, symbol, bits */ - {{0, 6, 4}}, - {{0, 9, 5}}, - {{0, 15, 5}}, - {{0, 21, 5}}, - {{0, 3, 5}}, - {{0, 7, 4}}, - {{0, 12, 5}}, - {{0, 18, 5}}, - {{0, 23, 5}}, - {{0, 5, 5}}, - {{0, 8, 4}}, - {{0, 14, 5}}, - {{0, 20, 5}}, - {{0, 2, 5}}, - {{16, 7, 4}}, - {{0, 11, 5}}, - {{0, 17, 5}}, - {{0, 22, 5}}, - {{0, 4, 5}}, - {{16, 8, 4}}, - {{0, 13, 5}}, - {{0, 19, 5}}, - {{0, 1, 5}}, - {{16, 6, 4}}, - {{0, 10, 5}}, - {{0, 16, 5}}, - {{0, 28, 5}}, - {{0, 27, 5}}, - {{0, 26, 5}}, - {{0, 25, 5}}, - {{0, 24, 5}}, -}; /* OF_defaultDTable */ - -/*! ZSTD_buildSeqTable() : - @return : nb bytes read from src, - or an error code if it fails, testable with ZSTD_isError() -*/ -static size_t ZSTD_buildSeqTable(FSE_DTable *DTableSpace, const FSE_DTable **DTablePtr, symbolEncodingType_e type, U32 max, U32 maxLog, const void *src, - size_t srcSize, const FSE_decode_t4 *defaultTable, U32 flagRepeatTable, void *workspace, size_t workspaceSize) -{ - const void *const tmpPtr = defaultTable; /* bypass strict aliasing */ - switch (type) { - case set_rle: - if (!srcSize) - return ERROR(srcSize_wrong); - if ((*(const BYTE *)src) > max) - return ERROR(corruption_detected); - FSE_buildDTable_rle(DTableSpace, *(const BYTE *)src); - *DTablePtr = DTableSpace; - return 1; - case set_basic: *DTablePtr = (const FSE_DTable *)tmpPtr; return 0; - case set_repeat: - if (!flagRepeatTable) - return ERROR(corruption_detected); - return 0; - default: /* impossible */ - case set_compressed: { - U32 tableLog; - S16 *norm = (S16 *)workspace; - size_t const spaceUsed32 = ALIGN(sizeof(S16) * (MaxSeq + 1), sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(GENERIC); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - { - size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); - if (FSE_isError(headerSize)) - return ERROR(corruption_detected); - if (tableLog > maxLog) - return ERROR(corruption_detected); - FSE_buildDTable_wksp(DTableSpace, norm, max, tableLog, workspace, workspaceSize); - *DTablePtr = DTableSpace; - return headerSize; - } - } - } -} - -size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx *dctx, int *nbSeqPtr, const void *src, size_t srcSize) -{ - const BYTE *const istart = (const BYTE *const)src; - const BYTE *const iend = istart + srcSize; - const BYTE *ip = istart; - - /* check */ - if (srcSize < MIN_SEQUENCES_SIZE) - return ERROR(srcSize_wrong); - - /* SeqHead */ - { - int nbSeq = *ip++; - if (!nbSeq) { - *nbSeqPtr = 0; - return 1; - } - if (nbSeq > 0x7F) { - if (nbSeq == 0xFF) { - if (ip + 2 > iend) - return ERROR(srcSize_wrong); - nbSeq = ZSTD_readLE16(ip) + LONGNBSEQ, ip += 2; - } else { - if (ip >= iend) - return ERROR(srcSize_wrong); - nbSeq = ((nbSeq - 0x80) << 8) + *ip++; - } - } - *nbSeqPtr = nbSeq; - } - - /* FSE table descriptors */ - if (ip + 4 > iend) - return ERROR(srcSize_wrong); /* minimum possible size */ - { - symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); - symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); - symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); - ip++; - - /* Build DTables */ - { - size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, LLtype, MaxLL, LLFSELog, ip, iend - ip, - LL_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace)); - if (ZSTD_isError(llhSize)) - return ERROR(corruption_detected); - ip += llhSize; - } - { - size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, OFtype, MaxOff, OffFSELog, ip, iend - ip, - OF_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace)); - if (ZSTD_isError(ofhSize)) - return ERROR(corruption_detected); - ip += ofhSize; - } - { - size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, MLtype, MaxML, MLFSELog, ip, iend - ip, - ML_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace)); - if (ZSTD_isError(mlhSize)) - return ERROR(corruption_detected); - ip += mlhSize; - } - } - - return ip - istart; -} - -typedef struct { - size_t litLength; - size_t matchLength; - size_t offset; - const BYTE *match; -} seq_t; - -typedef struct { - BIT_DStream_t DStream; - FSE_DState_t stateLL; - FSE_DState_t stateOffb; - FSE_DState_t stateML; - size_t prevOffset[ZSTD_REP_NUM]; - const BYTE *base; - size_t pos; - uPtrDiff gotoDict; -} seqState_t; - -FORCE_NOINLINE -size_t ZSTD_execSequenceLast7(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base, - const BYTE *const vBase, const BYTE *const dictEnd) -{ - BYTE *const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH; - const BYTE *const iLitEnd = *litPtr + sequence.litLength; - const BYTE *match = oLitEnd - sequence.offset; - - /* check */ - if (oMatchEnd > oend) - return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit) - return ERROR(corruption_detected); /* over-read beyond lit buffer */ - if (oLitEnd <= oend_w) - return ERROR(GENERIC); /* Precondition */ - - /* copy literals */ - if (op < oend_w) { - ZSTD_wildcopy(op, *litPtr, oend_w - op); - *litPtr += oend_w - op; - op = oend_w; - } - while (op < oLitEnd) - *op++ = *(*litPtr)++; - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - base)) { - /* offset beyond prefix */ - if (sequence.offset > (size_t)(oLitEnd - vBase)) - return ERROR(corruption_detected); - match = dictEnd - (base - match); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currPrefixSegment */ - { - size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = base; - } - } - while (op < oMatchEnd) - *op++ = *match++; - return sequenceLength; -} - -static seq_t ZSTD_decodeSequence(seqState_t *seqState) -{ - seq_t seq; - - U32 const llCode = FSE_peekSymbol(&seqState->stateLL); - U32 const mlCode = FSE_peekSymbol(&seqState->stateML); - U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */ - - U32 const llBits = LL_bits[llCode]; - U32 const mlBits = ML_bits[mlCode]; - U32 const ofBits = ofCode; - U32 const totalBits = llBits + mlBits + ofBits; - - static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, - 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000}; - - static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41, - 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003}; - - static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD, - 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, - 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD}; - - /* sequence */ - { - size_t offset; - if (!ofCode) - offset = 0; - else { - offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (ZSTD_32bits()) - BIT_reloadDStream(&seqState->DStream); - } - - if (ofCode <= 1) { - offset += (llCode == 0); - if (offset) { - size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ - if (offset != 1) - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } else { - offset = seqState->prevOffset[0]; - } - } else { - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } - seq.offset = offset; - } - - seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ - if (ZSTD_32bits() && (mlBits + llBits > 24)) - BIT_reloadDStream(&seqState->DStream); - - seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ - if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - - /* ANS state update */ - FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ - if (ZSTD_32bits()) - BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ - - seq.match = NULL; - - return seq; -} - -FORCE_INLINE -size_t ZSTD_execSequence(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base, - const BYTE *const vBase, const BYTE *const dictEnd) -{ - BYTE *const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH; - const BYTE *const iLitEnd = *litPtr + sequence.litLength; - const BYTE *match = oLitEnd - sequence.offset; - - /* check */ - if (oMatchEnd > oend) - return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit) - return ERROR(corruption_detected); /* over-read beyond lit buffer */ - if (oLitEnd > oend_w) - return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); - - /* copy Literals */ - ZSTD_copy8(op, *litPtr); - if (sequence.litLength > 8) - ZSTD_wildcopy(op + 8, (*litPtr) + 8, - sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ - op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - base)) { - /* offset beyond prefix */ - if (sequence.offset > (size_t)(oLitEnd - vBase)) - return ERROR(corruption_detected); - match = dictEnd + (match - base); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currPrefixSegment */ - { - size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = base; - if (op > oend_w || sequence.matchLength < MINMATCH) { - U32 i; - for (i = 0; i < sequence.matchLength; ++i) - op[i] = match[i]; - return sequenceLength; - } - } - } - /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ - - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */ - static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTD_copy4(op + 4, match); - match -= sub2; - } else { - ZSTD_copy8(op, match); - } - op += 8; - match += 8; - - if (oMatchEnd > oend - (16 - MINMATCH)) { - if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) - *op++ = *match++; - } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */ - } - return sequenceLength; -} - -static size_t ZSTD_decompressSequences(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize) -{ - const BYTE *ip = (const BYTE *)seqStart; - const BYTE *const iend = ip + seqSize; - BYTE *const ostart = (BYTE * const)dst; - BYTE *const oend = ostart + maxDstSize; - BYTE *op = ostart; - const BYTE *litPtr = dctx->litPtr; - const BYTE *const litEnd = litPtr + dctx->litSize; - const BYTE *const base = (const BYTE *)(dctx->base); - const BYTE *const vBase = (const BYTE *)(dctx->vBase); - const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd); - int nbSeq; - - /* Build Decoding Tables */ - { - size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); - if (ZSTD_isError(seqHSize)) - return seqHSize; - ip += seqHSize; - } - - /* Regen sequences */ - if (nbSeq) { - seqState_t seqState; - dctx->fseEntropy = 1; - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - seqState.prevOffset[i] = dctx->entropy.rep[i]; - } - CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected); - FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); - FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); - FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); - - for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq;) { - nbSeq--; - { - seq_t const sequence = ZSTD_decodeSequence(&seqState); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd); - if (ZSTD_isError(oneSeqSize)) - return oneSeqSize; - op += oneSeqSize; - } - } - - /* check if reached exact end */ - if (nbSeq) - return ERROR(corruption_detected); - /* save reps for next block */ - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); - } - } - - /* last literal segment */ - { - size_t const lastLLSize = litEnd - litPtr; - if (lastLLSize > (size_t)(oend - op)) - return ERROR(dstSize_tooSmall); - memcpy(op, litPtr, lastLLSize); - op += lastLLSize; - } - - return op - ostart; -} - -FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t *seqState, int const longOffsets) -{ - seq_t seq; - - U32 const llCode = FSE_peekSymbol(&seqState->stateLL); - U32 const mlCode = FSE_peekSymbol(&seqState->stateML); - U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */ - - U32 const llBits = LL_bits[llCode]; - U32 const mlBits = ML_bits[mlCode]; - U32 const ofBits = ofCode; - U32 const totalBits = llBits + mlBits + ofBits; - - static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, - 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000}; - - static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41, - 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003}; - - static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD, - 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, - 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD}; - - /* sequence */ - { - size_t offset; - if (!ofCode) - offset = 0; - else { - if (longOffsets) { - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN); - offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); - if (ZSTD_32bits() || extraBits) - BIT_reloadDStream(&seqState->DStream); - if (extraBits) - offset += BIT_readBitsFast(&seqState->DStream, extraBits); - } else { - offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (ZSTD_32bits()) - BIT_reloadDStream(&seqState->DStream); - } - } - - if (ofCode <= 1) { - offset += (llCode == 0); - if (offset) { - size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ - if (offset != 1) - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } else { - offset = seqState->prevOffset[0]; - } - } else { - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } - seq.offset = offset; - } - - seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ - if (ZSTD_32bits() && (mlBits + llBits > 24)) - BIT_reloadDStream(&seqState->DStream); - - seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ - if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - - { - size_t const pos = seqState->pos + seq.litLength; - seq.match = seqState->base + pos - seq.offset; /* single memory segment */ - if (seq.offset > pos) - seq.match += seqState->gotoDict; /* separate memory segment */ - seqState->pos = pos + seq.matchLength; - } - - /* ANS state update */ - FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ - if (ZSTD_32bits()) - BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ - - return seq; -} - -static seq_t ZSTD_decodeSequenceLong(seqState_t *seqState, unsigned const windowSize) -{ - if (ZSTD_highbit32(windowSize) > STREAM_ACCUMULATOR_MIN) { - return ZSTD_decodeSequenceLong_generic(seqState, 1); - } else { - return ZSTD_decodeSequenceLong_generic(seqState, 0); - } -} - -FORCE_INLINE -size_t ZSTD_execSequenceLong(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base, - const BYTE *const vBase, const BYTE *const dictEnd) -{ - BYTE *const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH; - const BYTE *const iLitEnd = *litPtr + sequence.litLength; - const BYTE *match = sequence.match; - - /* check */ - if (oMatchEnd > oend) - return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit) - return ERROR(corruption_detected); /* over-read beyond lit buffer */ - if (oLitEnd > oend_w) - return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); - - /* copy Literals */ - ZSTD_copy8(op, *litPtr); - if (sequence.litLength > 8) - ZSTD_wildcopy(op + 8, (*litPtr) + 8, - sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ - op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - base)) { - /* offset beyond prefix */ - if (sequence.offset > (size_t)(oLitEnd - vBase)) - return ERROR(corruption_detected); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currPrefixSegment */ - { - size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = base; - if (op > oend_w || sequence.matchLength < MINMATCH) { - U32 i; - for (i = 0; i < sequence.matchLength; ++i) - op[i] = match[i]; - return sequenceLength; - } - } - } - /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ - - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */ - static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTD_copy4(op + 4, match); - match -= sub2; - } else { - ZSTD_copy8(op, match); - } - op += 8; - match += 8; - - if (oMatchEnd > oend - (16 - MINMATCH)) { - if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) - *op++ = *match++; - } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */ - } - return sequenceLength; -} - -static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize) -{ - const BYTE *ip = (const BYTE *)seqStart; - const BYTE *const iend = ip + seqSize; - BYTE *const ostart = (BYTE * const)dst; - BYTE *const oend = ostart + maxDstSize; - BYTE *op = ostart; - const BYTE *litPtr = dctx->litPtr; - const BYTE *const litEnd = litPtr + dctx->litSize; - const BYTE *const base = (const BYTE *)(dctx->base); - const BYTE *const vBase = (const BYTE *)(dctx->vBase); - const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd); - unsigned const windowSize = dctx->fParams.windowSize; - int nbSeq; - - /* Build Decoding Tables */ - { - size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); - if (ZSTD_isError(seqHSize)) - return seqHSize; - ip += seqHSize; - } - - /* Regen sequences */ - if (nbSeq) { -#define STORED_SEQS 4 -#define STOSEQ_MASK (STORED_SEQS - 1) -#define ADVANCED_SEQS 4 - seq_t *sequences = (seq_t *)dctx->entropy.workspace; - int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); - seqState_t seqState; - int seqNb; - ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.workspace) >= sizeof(seq_t) * STORED_SEQS); - dctx->fseEntropy = 1; - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - seqState.prevOffset[i] = dctx->entropy.rep[i]; - } - seqState.base = base; - seqState.pos = (size_t)(op - base); - seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */ - CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected); - FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); - FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); - FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); - - /* prepare in advance */ - for (seqNb = 0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb < seqAdvance; seqNb++) { - sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize); - } - if (seqNb < seqAdvance) - return ERROR(corruption_detected); - - /* decode and decompress */ - for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb < nbSeq; seqNb++) { - seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize); - size_t const oneSeqSize = - ZSTD_execSequenceLong(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd); - if (ZSTD_isError(oneSeqSize)) - return oneSeqSize; - ZSTD_PREFETCH(sequence.match); - sequences[seqNb & STOSEQ_MASK] = sequence; - op += oneSeqSize; - } - if (seqNb < nbSeq) - return ERROR(corruption_detected); - - /* finish queue */ - seqNb -= seqAdvance; - for (; seqNb < nbSeq; seqNb++) { - size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd); - if (ZSTD_isError(oneSeqSize)) - return oneSeqSize; - op += oneSeqSize; - } - - /* save reps for next block */ - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); - } - } - - /* last literal segment */ - { - size_t const lastLLSize = litEnd - litPtr; - if (lastLLSize > (size_t)(oend - op)) - return ERROR(dstSize_tooSmall); - memcpy(op, litPtr, lastLLSize); - op += lastLLSize; - } - - return op - ostart; -} - -static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ /* blockType == blockCompressed */ - const BYTE *ip = (const BYTE *)src; - - if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX) - return ERROR(srcSize_wrong); - - /* Decode literals section */ - { - size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); - if (ZSTD_isError(litCSize)) - return litCSize; - ip += litCSize; - srcSize -= litCSize; - } - if (sizeof(size_t) > 4) /* do not enable prefetching on 32-bits x86, as it's performance detrimental */ - /* likely because of register pressure */ - /* if that's the correct cause, then 32-bits ARM should be affected differently */ - /* it would be good to test this on ARM real hardware, to see if prefetch version improves speed */ - if (dctx->fParams.windowSize > (1 << 23)) - return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize); - return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize); -} - -static void ZSTD_checkContinuity(ZSTD_DCtx *dctx, const void *dst) -{ - if (dst != dctx->previousDstEnd) { /* not contiguous */ - dctx->dictEnd = dctx->previousDstEnd; - dctx->vBase = (const char *)dst - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base)); - dctx->base = dst; - dctx->previousDstEnd = dst; - } -} - -size_t ZSTD_decompressBlock(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - size_t dSize; - ZSTD_checkContinuity(dctx, dst); - dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); - dctx->previousDstEnd = (char *)dst + dSize; - return dSize; -} - -/** ZSTD_insertBlock() : - insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ -size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void *blockStart, size_t blockSize) -{ - ZSTD_checkContinuity(dctx, blockStart); - dctx->previousDstEnd = (const char *)blockStart + blockSize; - return blockSize; -} - -size_t ZSTD_generateNxBytes(void *dst, size_t dstCapacity, BYTE byte, size_t length) -{ - if (length > dstCapacity) - return ERROR(dstSize_tooSmall); - memset(dst, byte, length); - return length; -} - -/** ZSTD_findFrameCompressedSize() : - * compatible with legacy mode - * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame - * `srcSize` must be at least as large as the frame contained - * @return : the compressed size of the frame starting at `src` */ -size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) -{ - if (srcSize >= ZSTD_skippableHeaderSize && (ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { - return ZSTD_skippableHeaderSize + ZSTD_readLE32((const BYTE *)src + 4); - } else { - const BYTE *ip = (const BYTE *)src; - const BYTE *const ipstart = ip; - size_t remainingSize = srcSize; - ZSTD_frameParams fParams; - - size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize); - if (ZSTD_isError(headerSize)) - return headerSize; - - /* Frame Header */ - { - size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize); - if (ZSTD_isError(ret)) - return ret; - if (ret > 0) - return ERROR(srcSize_wrong); - } - - ip += headerSize; - remainingSize -= headerSize; - - /* Loop on each block */ - while (1) { - blockProperties_t blockProperties; - size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); - if (ZSTD_isError(cBlockSize)) - return cBlockSize; - - if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) - return ERROR(srcSize_wrong); - - ip += ZSTD_blockHeaderSize + cBlockSize; - remainingSize -= ZSTD_blockHeaderSize + cBlockSize; - - if (blockProperties.lastBlock) - break; - } - - if (fParams.checksumFlag) { /* Frame content checksum */ - if (remainingSize < 4) - return ERROR(srcSize_wrong); - ip += 4; - remainingSize -= 4; - } - - return ip - ipstart; - } -} - -/*! ZSTD_decompressFrame() : -* @dctx must be properly initialized */ -static size_t ZSTD_decompressFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void **srcPtr, size_t *srcSizePtr) -{ - const BYTE *ip = (const BYTE *)(*srcPtr); - BYTE *const ostart = (BYTE * const)dst; - BYTE *const oend = ostart + dstCapacity; - BYTE *op = ostart; - size_t remainingSize = *srcSizePtr; - - /* check */ - if (remainingSize < ZSTD_frameHeaderSize_min + ZSTD_blockHeaderSize) - return ERROR(srcSize_wrong); - - /* Frame Header */ - { - size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix); - if (ZSTD_isError(frameHeaderSize)) - return frameHeaderSize; - if (remainingSize < frameHeaderSize + ZSTD_blockHeaderSize) - return ERROR(srcSize_wrong); - CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize)); - ip += frameHeaderSize; - remainingSize -= frameHeaderSize; - } - - /* Loop on each block */ - while (1) { - size_t decodedSize; - blockProperties_t blockProperties; - size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); - if (ZSTD_isError(cBlockSize)) - return cBlockSize; - - ip += ZSTD_blockHeaderSize; - remainingSize -= ZSTD_blockHeaderSize; - if (cBlockSize > remainingSize) - return ERROR(srcSize_wrong); - - switch (blockProperties.blockType) { - case bt_compressed: decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend - op, ip, cBlockSize); break; - case bt_raw: decodedSize = ZSTD_copyRawBlock(op, oend - op, ip, cBlockSize); break; - case bt_rle: decodedSize = ZSTD_generateNxBytes(op, oend - op, *ip, blockProperties.origSize); break; - case bt_reserved: - default: return ERROR(corruption_detected); - } - - if (ZSTD_isError(decodedSize)) - return decodedSize; - if (dctx->fParams.checksumFlag) - xxh64_update(&dctx->xxhState, op, decodedSize); - op += decodedSize; - ip += cBlockSize; - remainingSize -= cBlockSize; - if (blockProperties.lastBlock) - break; - } - - if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ - U32 const checkCalc = (U32)xxh64_digest(&dctx->xxhState); - U32 checkRead; - if (remainingSize < 4) - return ERROR(checksum_wrong); - checkRead = ZSTD_readLE32(ip); - if (checkRead != checkCalc) - return ERROR(checksum_wrong); - ip += 4; - remainingSize -= 4; - } - - /* Allow caller to get size read */ - *srcPtr = ip; - *srcSizePtr = remainingSize; - return op - ostart; -} - -static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict); -static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict); - -static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, - const ZSTD_DDict *ddict) -{ - void *const dststart = dst; - - if (ddict) { - if (dict) { - /* programmer error, these two cases should be mutually exclusive */ - return ERROR(GENERIC); - } - - dict = ZSTD_DDictDictContent(ddict); - dictSize = ZSTD_DDictDictSize(ddict); - } - - while (srcSize >= ZSTD_frameHeaderSize_prefix) { - U32 magicNumber; - - magicNumber = ZSTD_readLE32(src); - if (magicNumber != ZSTD_MAGICNUMBER) { - if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { - size_t skippableSize; - if (srcSize < ZSTD_skippableHeaderSize) - return ERROR(srcSize_wrong); - skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize; - if (srcSize < skippableSize) { - return ERROR(srcSize_wrong); - } - - src = (const BYTE *)src + skippableSize; - srcSize -= skippableSize; - continue; - } else { - return ERROR(prefix_unknown); - } - } - - if (ddict) { - /* we were called from ZSTD_decompress_usingDDict */ - ZSTD_refDDict(dctx, ddict); - } else { - /* this will initialize correctly with no dict if dict == NULL, so - * use this in all cases but ddict */ - CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize)); - } - ZSTD_checkContinuity(dctx, dst); - - { - const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, &src, &srcSize); - if (ZSTD_isError(res)) - return res; - /* don't need to bounds check this, ZSTD_decompressFrame will have - * already */ - dst = (BYTE *)dst + res; - dstCapacity -= res; - } - } - - if (srcSize) - return ERROR(srcSize_wrong); /* input not entirely consumed */ - - return (BYTE *)dst - (BYTE *)dststart; -} - -size_t ZSTD_decompress_usingDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize) -{ - return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); -} - -size_t ZSTD_decompressDCtx(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); -} - -/*-************************************** -* Advanced Streaming Decompression API -* Bufferless and synchronous -****************************************/ -size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx *dctx) { return dctx->expected; } - -ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx *dctx) -{ - switch (dctx->stage) { - default: /* should not happen */ - case ZSTDds_getFrameHeaderSize: - case ZSTDds_decodeFrameHeader: return ZSTDnit_frameHeader; - case ZSTDds_decodeBlockHeader: return ZSTDnit_blockHeader; - case ZSTDds_decompressBlock: return ZSTDnit_block; - case ZSTDds_decompressLastBlock: return ZSTDnit_lastBlock; - case ZSTDds_checkChecksum: return ZSTDnit_checksum; - case ZSTDds_decodeSkippableHeader: - case ZSTDds_skipFrame: return ZSTDnit_skippableFrame; - } -} - -int ZSTD_isSkipFrame(ZSTD_DCtx *dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */ - -/** ZSTD_decompressContinue() : -* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) -* or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - /* Sanity check */ - if (srcSize != dctx->expected) - return ERROR(srcSize_wrong); - if (dstCapacity) - ZSTD_checkContinuity(dctx, dst); - - switch (dctx->stage) { - case ZSTDds_getFrameHeaderSize: - if (srcSize != ZSTD_frameHeaderSize_prefix) - return ERROR(srcSize_wrong); /* impossible */ - if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ - memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); - dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */ - dctx->stage = ZSTDds_decodeSkippableHeader; - return 0; - } - dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix); - if (ZSTD_isError(dctx->headerSize)) - return dctx->headerSize; - memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); - if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) { - dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix; - dctx->stage = ZSTDds_decodeFrameHeader; - return 0; - } - dctx->expected = 0; /* not necessary to copy more */ - - case ZSTDds_decodeFrameHeader: - memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); - CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize)); - dctx->expected = ZSTD_blockHeaderSize; - dctx->stage = ZSTDds_decodeBlockHeader; - return 0; - - case ZSTDds_decodeBlockHeader: { - blockProperties_t bp; - size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); - if (ZSTD_isError(cBlockSize)) - return cBlockSize; - dctx->expected = cBlockSize; - dctx->bType = bp.blockType; - dctx->rleSize = bp.origSize; - if (cBlockSize) { - dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; - return 0; - } - /* empty block */ - if (bp.lastBlock) { - if (dctx->fParams.checksumFlag) { - dctx->expected = 4; - dctx->stage = ZSTDds_checkChecksum; - } else { - dctx->expected = 0; /* end of frame */ - dctx->stage = ZSTDds_getFrameHeaderSize; - } - } else { - dctx->expected = 3; /* go directly to next header */ - dctx->stage = ZSTDds_decodeBlockHeader; - } - return 0; - } - case ZSTDds_decompressLastBlock: - case ZSTDds_decompressBlock: { - size_t rSize; - switch (dctx->bType) { - case bt_compressed: rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); break; - case bt_raw: rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); break; - case bt_rle: rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); break; - case bt_reserved: /* should never happen */ - default: return ERROR(corruption_detected); - } - if (ZSTD_isError(rSize)) - return rSize; - if (dctx->fParams.checksumFlag) - xxh64_update(&dctx->xxhState, dst, rSize); - - if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ - if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ - dctx->expected = 4; - dctx->stage = ZSTDds_checkChecksum; - } else { - dctx->expected = 0; /* ends here */ - dctx->stage = ZSTDds_getFrameHeaderSize; - } - } else { - dctx->stage = ZSTDds_decodeBlockHeader; - dctx->expected = ZSTD_blockHeaderSize; - dctx->previousDstEnd = (char *)dst + rSize; - } - return rSize; - } - case ZSTDds_checkChecksum: { - U32 const h32 = (U32)xxh64_digest(&dctx->xxhState); - U32 const check32 = ZSTD_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */ - if (check32 != h32) - return ERROR(checksum_wrong); - dctx->expected = 0; - dctx->stage = ZSTDds_getFrameHeaderSize; - return 0; - } - case ZSTDds_decodeSkippableHeader: { - memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); - dctx->expected = ZSTD_readLE32(dctx->headerBuffer + 4); - dctx->stage = ZSTDds_skipFrame; - return 0; - } - case ZSTDds_skipFrame: { - dctx->expected = 0; - dctx->stage = ZSTDds_getFrameHeaderSize; - return 0; - } - default: - return ERROR(GENERIC); /* impossible */ - } -} - -static size_t ZSTD_refDictContent(ZSTD_DCtx *dctx, const void *dict, size_t dictSize) -{ - dctx->dictEnd = dctx->previousDstEnd; - dctx->vBase = (const char *)dict - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base)); - dctx->base = dict; - dctx->previousDstEnd = (const char *)dict + dictSize; - return 0; -} - -/* ZSTD_loadEntropy() : - * dict : must point at beginning of a valid zstd dictionary - * @return : size of entropy tables read */ -static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t *entropy, const void *const dict, size_t const dictSize) -{ - const BYTE *dictPtr = (const BYTE *)dict; - const BYTE *const dictEnd = dictPtr + dictSize; - - if (dictSize <= 8) - return ERROR(dictionary_corrupted); - dictPtr += 8; /* skip header = magic + dictID */ - - { - size_t const hSize = HUF_readDTableX4_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, entropy->workspace, sizeof(entropy->workspace)); - if (HUF_isError(hSize)) - return ERROR(dictionary_corrupted); - dictPtr += hSize; - } - - { - short offcodeNCount[MaxOff + 1]; - U32 offcodeMaxValue = MaxOff, offcodeLog; - size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr); - if (FSE_isError(offcodeHeaderSize)) - return ERROR(dictionary_corrupted); - if (offcodeLog > OffFSELog) - return ERROR(dictionary_corrupted); - CHECK_E(FSE_buildDTable_wksp(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted); - dictPtr += offcodeHeaderSize; - } - - { - short matchlengthNCount[MaxML + 1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog; - size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr); - if (FSE_isError(matchlengthHeaderSize)) - return ERROR(dictionary_corrupted); - if (matchlengthLog > MLFSELog) - return ERROR(dictionary_corrupted); - CHECK_E(FSE_buildDTable_wksp(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted); - dictPtr += matchlengthHeaderSize; - } - - { - short litlengthNCount[MaxLL + 1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog; - size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr); - if (FSE_isError(litlengthHeaderSize)) - return ERROR(dictionary_corrupted); - if (litlengthLog > LLFSELog) - return ERROR(dictionary_corrupted); - CHECK_E(FSE_buildDTable_wksp(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted); - dictPtr += litlengthHeaderSize; - } - - if (dictPtr + 12 > dictEnd) - return ERROR(dictionary_corrupted); - { - int i; - size_t const dictContentSize = (size_t)(dictEnd - (dictPtr + 12)); - for (i = 0; i < 3; i++) { - U32 const rep = ZSTD_readLE32(dictPtr); - dictPtr += 4; - if (rep == 0 || rep >= dictContentSize) - return ERROR(dictionary_corrupted); - entropy->rep[i] = rep; - } - } - - return dictPtr - (const BYTE *)dict; -} - -static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx *dctx, const void *dict, size_t dictSize) -{ - if (dictSize < 8) - return ZSTD_refDictContent(dctx, dict, dictSize); - { - U32 const magic = ZSTD_readLE32(dict); - if (magic != ZSTD_DICT_MAGIC) { - return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ - } - } - dctx->dictID = ZSTD_readLE32((const char *)dict + 4); - - /* load entropy tables */ - { - size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize); - if (ZSTD_isError(eSize)) - return ERROR(dictionary_corrupted); - dict = (const char *)dict + eSize; - dictSize -= eSize; - } - dctx->litEntropy = dctx->fseEntropy = 1; - - /* reference dictionary content */ - return ZSTD_refDictContent(dctx, dict, dictSize); -} - -size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx *dctx, const void *dict, size_t dictSize) -{ - CHECK_F(ZSTD_decompressBegin(dctx)); - if (dict && dictSize) - CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted); - return 0; -} - -/* ====== ZSTD_DDict ====== */ - -struct ZSTD_DDict_s { - void *dictBuffer; - const void *dictContent; - size_t dictSize; - ZSTD_entropyTables_t entropy; - U32 dictID; - U32 entropyPresent; - ZSTD_customMem cMem; -}; /* typedef'd to ZSTD_DDict within "zstd.h" */ - -size_t ZSTD_DDictWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DDict)); } - -static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict) { return ddict->dictContent; } - -static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict) { return ddict->dictSize; } - -static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict) -{ - ZSTD_decompressBegin(dstDCtx); /* init */ - if (ddict) { /* support refDDict on NULL */ - dstDCtx->dictID = ddict->dictID; - dstDCtx->base = ddict->dictContent; - dstDCtx->vBase = ddict->dictContent; - dstDCtx->dictEnd = (const BYTE *)ddict->dictContent + ddict->dictSize; - dstDCtx->previousDstEnd = dstDCtx->dictEnd; - if (ddict->entropyPresent) { - dstDCtx->litEntropy = 1; - dstDCtx->fseEntropy = 1; - dstDCtx->LLTptr = ddict->entropy.LLTable; - dstDCtx->MLTptr = ddict->entropy.MLTable; - dstDCtx->OFTptr = ddict->entropy.OFTable; - dstDCtx->HUFptr = ddict->entropy.hufTable; - dstDCtx->entropy.rep[0] = ddict->entropy.rep[0]; - dstDCtx->entropy.rep[1] = ddict->entropy.rep[1]; - dstDCtx->entropy.rep[2] = ddict->entropy.rep[2]; - } else { - dstDCtx->litEntropy = 0; - dstDCtx->fseEntropy = 0; - } - } -} - -static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict *ddict) -{ - ddict->dictID = 0; - ddict->entropyPresent = 0; - if (ddict->dictSize < 8) - return 0; - { - U32 const magic = ZSTD_readLE32(ddict->dictContent); - if (magic != ZSTD_DICT_MAGIC) - return 0; /* pure content mode */ - } - ddict->dictID = ZSTD_readLE32((const char *)ddict->dictContent + 4); - - /* load entropy tables */ - CHECK_E(ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted); - ddict->entropyPresent = 1; - return 0; -} - -static ZSTD_DDict *ZSTD_createDDict_advanced(const void *dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem) -{ - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - { - ZSTD_DDict *const ddict = (ZSTD_DDict *)ZSTD_malloc(sizeof(ZSTD_DDict), customMem); - if (!ddict) - return NULL; - ddict->cMem = customMem; - - if ((byReference) || (!dict) || (!dictSize)) { - ddict->dictBuffer = NULL; - ddict->dictContent = dict; - } else { - void *const internalBuffer = ZSTD_malloc(dictSize, customMem); - if (!internalBuffer) { - ZSTD_freeDDict(ddict); - return NULL; - } - memcpy(internalBuffer, dict, dictSize); - ddict->dictBuffer = internalBuffer; - ddict->dictContent = internalBuffer; - } - ddict->dictSize = dictSize; - ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ - /* parse dictionary content */ - { - size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict); - if (ZSTD_isError(errorCode)) { - ZSTD_freeDDict(ddict); - return NULL; - } - } - - return ddict; - } -} - -/*! ZSTD_initDDict() : -* Create a digested dictionary, to start decompression without startup delay. -* `dict` content is copied inside DDict. -* Consequently, `dict` can be released after `ZSTD_DDict` creation */ -ZSTD_DDict *ZSTD_initDDict(const void *dict, size_t dictSize, void *workspace, size_t workspaceSize) -{ - ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); - return ZSTD_createDDict_advanced(dict, dictSize, 1, stackMem); -} - -size_t ZSTD_freeDDict(ZSTD_DDict *ddict) -{ - if (ddict == NULL) - return 0; /* support free on NULL */ - { - ZSTD_customMem const cMem = ddict->cMem; - ZSTD_free(ddict->dictBuffer, cMem); - ZSTD_free(ddict, cMem); - return 0; - } -} - -/*! ZSTD_getDictID_fromDict() : - * Provides the dictID stored within dictionary. - * if @return == 0, the dictionary is not conformant with Zstandard specification. - * It can still be loaded, but as a content-only dictionary. */ -unsigned ZSTD_getDictID_fromDict(const void *dict, size_t dictSize) -{ - if (dictSize < 8) - return 0; - if (ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) - return 0; - return ZSTD_readLE32((const char *)dict + 4); -} - -/*! ZSTD_getDictID_fromDDict() : - * Provides the dictID of the dictionary loaded into `ddict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict) -{ - if (ddict == NULL) - return 0; - return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); -} - -/*! ZSTD_getDictID_fromFrame() : - * Provides the dictID required to decompressed the frame stored within `src`. - * If @return == 0, the dictID could not be decoded. - * This could for one of the following reasons : - * - The frame does not require a dictionary to be decoded (most common case). - * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. - * Note : this use case also happens when using a non-conformant dictionary. - * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). - * - This is not a Zstandard frame. - * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */ -unsigned ZSTD_getDictID_fromFrame(const void *src, size_t srcSize) -{ - ZSTD_frameParams zfp = {0, 0, 0, 0}; - size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize); - if (ZSTD_isError(hError)) - return 0; - return zfp.dictID; -} - -/*! ZSTD_decompress_usingDDict() : -* Decompression using a pre-digested Dictionary -* Use dictionary without significant overhead. */ -size_t ZSTD_decompress_usingDDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_DDict *ddict) -{ - /* pass content and size in case legacy frames are encountered */ - return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, NULL, 0, ddict); -} - -/*===================================== -* Streaming decompression -*====================================*/ - -typedef enum { zdss_init, zdss_loadHeader, zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; - -/* *** Resource management *** */ -struct ZSTD_DStream_s { - ZSTD_DCtx *dctx; - ZSTD_DDict *ddictLocal; - const ZSTD_DDict *ddict; - ZSTD_frameParams fParams; - ZSTD_dStreamStage stage; - char *inBuff; - size_t inBuffSize; - size_t inPos; - size_t maxWindowSize; - char *outBuff; - size_t outBuffSize; - size_t outStart; - size_t outEnd; - size_t blockSize; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */ - size_t lhSize; - ZSTD_customMem customMem; - void *legacyContext; - U32 previousLegacyVersion; - U32 legacyVersion; - U32 hostageByte; -}; /* typedef'd to ZSTD_DStream within "zstd.h" */ - -size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize) -{ - size_t const blockSize = MIN(maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); - size_t const inBuffSize = blockSize; - size_t const outBuffSize = maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; - return ZSTD_DCtxWorkspaceBound() + ZSTD_ALIGN(sizeof(ZSTD_DStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize); -} - -static ZSTD_DStream *ZSTD_createDStream_advanced(ZSTD_customMem customMem) -{ - ZSTD_DStream *zds; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - zds = (ZSTD_DStream *)ZSTD_malloc(sizeof(ZSTD_DStream), customMem); - if (zds == NULL) - return NULL; - memset(zds, 0, sizeof(ZSTD_DStream)); - memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); - zds->dctx = ZSTD_createDCtx_advanced(customMem); - if (zds->dctx == NULL) { - ZSTD_freeDStream(zds); - return NULL; - } - zds->stage = zdss_init; - zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; - return zds; -} - -ZSTD_DStream *ZSTD_initDStream(size_t maxWindowSize, void *workspace, size_t workspaceSize) -{ - ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize); - ZSTD_DStream *zds = ZSTD_createDStream_advanced(stackMem); - if (!zds) { - return NULL; - } - - zds->maxWindowSize = maxWindowSize; - zds->stage = zdss_loadHeader; - zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; - ZSTD_freeDDict(zds->ddictLocal); - zds->ddictLocal = NULL; - zds->ddict = zds->ddictLocal; - zds->legacyVersion = 0; - zds->hostageByte = 0; - - { - size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); - size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; - - zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem); - zds->inBuffSize = blockSize; - zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem); - zds->outBuffSize = neededOutSize; - if (zds->inBuff == NULL || zds->outBuff == NULL) { - ZSTD_freeDStream(zds); - return NULL; - } - } - return zds; -} - -ZSTD_DStream *ZSTD_initDStream_usingDDict(size_t maxWindowSize, const ZSTD_DDict *ddict, void *workspace, size_t workspaceSize) -{ - ZSTD_DStream *zds = ZSTD_initDStream(maxWindowSize, workspace, workspaceSize); - if (zds) { - zds->ddict = ddict; - } - return zds; -} - -size_t ZSTD_freeDStream(ZSTD_DStream *zds) -{ - if (zds == NULL) - return 0; /* support free on null */ - { - ZSTD_customMem const cMem = zds->customMem; - ZSTD_freeDCtx(zds->dctx); - zds->dctx = NULL; - ZSTD_freeDDict(zds->ddictLocal); - zds->ddictLocal = NULL; - ZSTD_free(zds->inBuff, cMem); - zds->inBuff = NULL; - ZSTD_free(zds->outBuff, cMem); - zds->outBuff = NULL; - ZSTD_free(zds, cMem); - return 0; - } -} - -/* *** Initialization *** */ - -size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; } -size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } - -size_t ZSTD_resetDStream(ZSTD_DStream *zds) -{ - zds->stage = zdss_loadHeader; - zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; - zds->legacyVersion = 0; - zds->hostageByte = 0; - return ZSTD_frameHeaderSize_prefix; -} - -/* ***** Decompression ***** */ - -ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize) -{ - size_t const length = MIN(dstCapacity, srcSize); - memcpy(dst, src, length); - return length; -} - -size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inBuffer *input) -{ - const char *const istart = (const char *)(input->src) + input->pos; - const char *const iend = (const char *)(input->src) + input->size; - const char *ip = istart; - char *const ostart = (char *)(output->dst) + output->pos; - char *const oend = (char *)(output->dst) + output->size; - char *op = ostart; - U32 someMoreWork = 1; - - while (someMoreWork) { - switch (zds->stage) { - case zdss_init: - ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */ - /* fall-through */ - - case zdss_loadHeader: { - size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize); - if (ZSTD_isError(hSize)) - return hSize; - if (hSize != 0) { /* need more input */ - size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ - if (toLoad > (size_t)(iend - ip)) { /* not enough input to load full header */ - memcpy(zds->headerBuffer + zds->lhSize, ip, iend - ip); - zds->lhSize += iend - ip; - input->pos = input->size; - return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + - ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ - } - memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); - zds->lhSize = hSize; - ip += toLoad; - break; - } - - /* check for single-pass mode opportunity */ - if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ - && (U64)(size_t)(oend - op) >= zds->fParams.frameContentSize) { - size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend - istart); - if (cSize <= (size_t)(iend - istart)) { - size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend - op, istart, cSize, zds->ddict); - if (ZSTD_isError(decompressedSize)) - return decompressedSize; - ip = istart + cSize; - op += decompressedSize; - zds->dctx->expected = 0; - zds->stage = zdss_init; - someMoreWork = 0; - break; - } - } - - /* Consume header */ - ZSTD_refDDict(zds->dctx, zds->ddict); - { - size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ - CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); - { - size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); - CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer + h1Size, h2Size)); - } - } - - zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); - if (zds->fParams.windowSize > zds->maxWindowSize) - return ERROR(frameParameter_windowTooLarge); - - /* Buffers are preallocated, but double check */ - { - size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); - size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2; - if (zds->inBuffSize < blockSize) { - return ERROR(GENERIC); - } - if (zds->outBuffSize < neededOutSize) { - return ERROR(GENERIC); - } - zds->blockSize = blockSize; - } - zds->stage = zdss_read; - } - /* pass-through */ - - case zdss_read: { - size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); - if (neededInSize == 0) { /* end of frame */ - zds->stage = zdss_init; - someMoreWork = 0; - break; - } - if ((size_t)(iend - ip) >= neededInSize) { /* decode directly from src */ - const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); - size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, - (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), ip, neededInSize); - if (ZSTD_isError(decodedSize)) - return decodedSize; - ip += neededInSize; - if (!decodedSize && !isSkipFrame) - break; /* this was just a header */ - zds->outEnd = zds->outStart + decodedSize; - zds->stage = zdss_flush; - break; - } - if (ip == iend) { - someMoreWork = 0; - break; - } /* no more input */ - zds->stage = zdss_load; - /* pass-through */ - } - - case zdss_load: { - size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); - size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ - size_t loadedSize; - if (toLoad > zds->inBuffSize - zds->inPos) - return ERROR(corruption_detected); /* should never happen */ - loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend - ip); - ip += loadedSize; - zds->inPos += loadedSize; - if (loadedSize < toLoad) { - someMoreWork = 0; - break; - } /* not enough input, wait for more */ - - /* decode loaded input */ - { - const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); - size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, - zds->inBuff, neededInSize); - if (ZSTD_isError(decodedSize)) - return decodedSize; - zds->inPos = 0; /* input is consumed */ - if (!decodedSize && !isSkipFrame) { - zds->stage = zdss_read; - break; - } /* this was just a header */ - zds->outEnd = zds->outStart + decodedSize; - zds->stage = zdss_flush; - /* pass-through */ - } - } - - case zdss_flush: { - size_t const toFlushSize = zds->outEnd - zds->outStart; - size_t const flushedSize = ZSTD_limitCopy(op, oend - op, zds->outBuff + zds->outStart, toFlushSize); - op += flushedSize; - zds->outStart += flushedSize; - if (flushedSize == toFlushSize) { /* flush completed */ - zds->stage = zdss_read; - if (zds->outStart + zds->blockSize > zds->outBuffSize) - zds->outStart = zds->outEnd = 0; - break; - } - /* cannot complete flush */ - someMoreWork = 0; - break; - } - default: - return ERROR(GENERIC); /* impossible */ - } - } - - /* result */ - input->pos += (size_t)(ip - istart); - output->pos += (size_t)(op - ostart); - { - size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx); - if (!nextSrcSizeHint) { /* frame fully decoded */ - if (zds->outEnd == zds->outStart) { /* output fully flushed */ - if (zds->hostageByte) { - if (input->pos >= input->size) { - zds->stage = zdss_read; - return 1; - } /* can't release hostage (not present) */ - input->pos++; /* release hostage */ - } - return 0; - } - if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ - input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ - zds->hostageByte = 1; - } - return 1; - } - nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */ - if (zds->inPos > nextSrcSizeHint) - return ERROR(GENERIC); /* should never happen */ - nextSrcSizeHint -= zds->inPos; /* already loaded*/ - return nextSrcSizeHint; - } -} - -EXPORT_SYMBOL(ZSTD_DCtxWorkspaceBound); -EXPORT_SYMBOL(ZSTD_initDCtx); -EXPORT_SYMBOL(ZSTD_decompressDCtx); -EXPORT_SYMBOL(ZSTD_decompress_usingDict); - -EXPORT_SYMBOL(ZSTD_DDictWorkspaceBound); -EXPORT_SYMBOL(ZSTD_initDDict); -EXPORT_SYMBOL(ZSTD_decompress_usingDDict); - -EXPORT_SYMBOL(ZSTD_DStreamWorkspaceBound); -EXPORT_SYMBOL(ZSTD_initDStream); -EXPORT_SYMBOL(ZSTD_initDStream_usingDDict); -EXPORT_SYMBOL(ZSTD_resetDStream); -EXPORT_SYMBOL(ZSTD_decompressStream); -EXPORT_SYMBOL(ZSTD_DStreamInSize); -EXPORT_SYMBOL(ZSTD_DStreamOutSize); - -EXPORT_SYMBOL(ZSTD_findFrameCompressedSize); -EXPORT_SYMBOL(ZSTD_getFrameContentSize); -EXPORT_SYMBOL(ZSTD_findDecompressedSize); - -EXPORT_SYMBOL(ZSTD_isFrame); -EXPORT_SYMBOL(ZSTD_getDictID_fromDict); -EXPORT_SYMBOL(ZSTD_getDictID_fromDDict); -EXPORT_SYMBOL(ZSTD_getDictID_fromFrame); - -EXPORT_SYMBOL(ZSTD_getFrameParams); -EXPORT_SYMBOL(ZSTD_decompressBegin); -EXPORT_SYMBOL(ZSTD_decompressBegin_usingDict); -EXPORT_SYMBOL(ZSTD_copyDCtx); -EXPORT_SYMBOL(ZSTD_nextSrcSizeToDecompress); -EXPORT_SYMBOL(ZSTD_decompressContinue); -EXPORT_SYMBOL(ZSTD_nextInputType); - -EXPORT_SYMBOL(ZSTD_decompressBlock); -EXPORT_SYMBOL(ZSTD_insertBlock); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_DESCRIPTION("Zstd Decompressor"); diff --git a/contrib/linux-kernel/lib/zstd/entropy_common.c b/contrib/linux-kernel/lib/zstd/entropy_common.c deleted file mode 100644 index 2b0a643c..00000000 --- a/contrib/linux-kernel/lib/zstd/entropy_common.c +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Common functions of New Generation Entropy library - * Copyright (C) 2016, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ - -/* ************************************* -* Dependencies -***************************************/ -#include "error_private.h" /* ERR_*, ERROR */ -#include "fse.h" -#include "huf.h" -#include "mem.h" - -/*=== Version ===*/ -unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } - -/*=== Error Management ===*/ -unsigned FSE_isError(size_t code) { return ERR_isError(code); } - -unsigned HUF_isError(size_t code) { return ERR_isError(code); } - -/*-************************************************************** -* FSE NCount encoding-decoding -****************************************************************/ -size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSVPtr, unsigned *tableLogPtr, const void *headerBuffer, size_t hbSize) -{ - const BYTE *const istart = (const BYTE *)headerBuffer; - const BYTE *const iend = istart + hbSize; - const BYTE *ip = istart; - int nbBits; - int remaining; - int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; - - if (hbSize < 4) - return ERROR(srcSize_wrong); - bitStream = ZSTD_readLE32(ip); - nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ - if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) - return ERROR(tableLog_tooLarge); - bitStream >>= 4; - bitCount = 4; - *tableLogPtr = nbBits; - remaining = (1 << nbBits) + 1; - threshold = 1 << nbBits; - nbBits++; - - while ((remaining > 1) & (charnum <= *maxSVPtr)) { - if (previous0) { - unsigned n0 = charnum; - while ((bitStream & 0xFFFF) == 0xFFFF) { - n0 += 24; - if (ip < iend - 5) { - ip += 2; - bitStream = ZSTD_readLE32(ip) >> bitCount; - } else { - bitStream >>= 16; - bitCount += 16; - } - } - while ((bitStream & 3) == 3) { - n0 += 3; - bitStream >>= 2; - bitCount += 2; - } - n0 += bitStream & 3; - bitCount += 2; - if (n0 > *maxSVPtr) - return ERROR(maxSymbolValue_tooSmall); - while (charnum < n0) - normalizedCounter[charnum++] = 0; - if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) { - ip += bitCount >> 3; - bitCount &= 7; - bitStream = ZSTD_readLE32(ip) >> bitCount; - } else { - bitStream >>= 2; - } - } - { - int const max = (2 * threshold - 1) - remaining; - int count; - - if ((bitStream & (threshold - 1)) < (U32)max) { - count = bitStream & (threshold - 1); - bitCount += nbBits - 1; - } else { - count = bitStream & (2 * threshold - 1); - if (count >= threshold) - count -= max; - bitCount += nbBits; - } - - count--; /* extra accuracy */ - remaining -= count < 0 ? -count : count; /* -1 means +1 */ - normalizedCounter[charnum++] = (short)count; - previous0 = !count; - while (remaining < threshold) { - nbBits--; - threshold >>= 1; - } - - if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) { - ip += bitCount >> 3; - bitCount &= 7; - } else { - bitCount -= (int)(8 * (iend - 4 - ip)); - ip = iend - 4; - } - bitStream = ZSTD_readLE32(ip) >> (bitCount & 31); - } - } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ - if (remaining != 1) - return ERROR(corruption_detected); - if (bitCount > 32) - return ERROR(corruption_detected); - *maxSVPtr = charnum - 1; - - ip += (bitCount + 7) >> 3; - return ip - istart; -} - -/*! HUF_readStats() : - Read compact Huffman tree, saved by HUF_writeCTable(). - `huffWeight` is destination buffer. - `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. - @return : size read from `src` , or an error Code . - Note : Needed by HUF_readCTable() and HUF_readDTableX?() . -*/ -size_t HUF_readStats_wksp(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize, void *workspace, size_t workspaceSize) -{ - U32 weightTotal; - const BYTE *ip = (const BYTE *)src; - size_t iSize; - size_t oSize; - - if (!srcSize) - return ERROR(srcSize_wrong); - iSize = ip[0]; - /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */ - - if (iSize >= 128) { /* special header */ - oSize = iSize - 127; - iSize = ((oSize + 1) / 2); - if (iSize + 1 > srcSize) - return ERROR(srcSize_wrong); - if (oSize >= hwSize) - return ERROR(corruption_detected); - ip += 1; - { - U32 n; - for (n = 0; n < oSize; n += 2) { - huffWeight[n] = ip[n / 2] >> 4; - huffWeight[n + 1] = ip[n / 2] & 15; - } - } - } else { /* header compressed with FSE (normal case) */ - if (iSize + 1 > srcSize) - return ERROR(srcSize_wrong); - oSize = FSE_decompress_wksp(huffWeight, hwSize - 1, ip + 1, iSize, 6, workspace, workspaceSize); /* max (hwSize-1) values decoded, as last one is implied */ - if (FSE_isError(oSize)) - return oSize; - } - - /* collect weight stats */ - memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); - weightTotal = 0; - { - U32 n; - for (n = 0; n < oSize; n++) { - if (huffWeight[n] >= HUF_TABLELOG_MAX) - return ERROR(corruption_detected); - rankStats[huffWeight[n]]++; - weightTotal += (1 << huffWeight[n]) >> 1; - } - } - if (weightTotal == 0) - return ERROR(corruption_detected); - - /* get last non-null symbol weight (implied, total must be 2^n) */ - { - U32 const tableLog = BIT_highbit32(weightTotal) + 1; - if (tableLog > HUF_TABLELOG_MAX) - return ERROR(corruption_detected); - *tableLogPtr = tableLog; - /* determine last weight */ - { - U32 const total = 1 << tableLog; - U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; - if (verif != rest) - return ERROR(corruption_detected); /* last value must be a clean power of 2 */ - huffWeight[oSize] = (BYTE)lastWeight; - rankStats[lastWeight]++; - } - } - - /* check tree construction validity */ - if ((rankStats[1] < 2) || (rankStats[1] & 1)) - return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ - - /* results */ - *nbSymbolsPtr = (U32)(oSize + 1); - return iSize + 1; -} diff --git a/contrib/linux-kernel/lib/zstd/error_private.h b/contrib/linux-kernel/lib/zstd/error_private.h deleted file mode 100644 index 2062ff05..00000000 --- a/contrib/linux-kernel/lib/zstd/error_private.h +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -/* Note : this module is expected to remain private, do not expose it */ - -#ifndef ERROR_H_MODULE -#define ERROR_H_MODULE - -/* **************************************** -* Dependencies -******************************************/ -#include <linux/types.h> /* size_t */ -#include <linux/zstd.h> /* enum list */ - -/* **************************************** -* Compiler-specific -******************************************/ -#define ERR_STATIC static __attribute__((unused)) - -/*-**************************************** -* Customization (error_public.h) -******************************************/ -typedef ZSTD_ErrorCode ERR_enum; -#define PREFIX(name) ZSTD_error_##name - -/*-**************************************** -* Error codes handling -******************************************/ -#define ERROR(name) ((size_t)-PREFIX(name)) - -ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } - -ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) -{ - if (!ERR_isError(code)) - return (ERR_enum)0; - return (ERR_enum)(0 - code); -} - -#endif /* ERROR_H_MODULE */ diff --git a/contrib/linux-kernel/lib/zstd/fse.h b/contrib/linux-kernel/lib/zstd/fse.h deleted file mode 100644 index a694199f..00000000 --- a/contrib/linux-kernel/lib/zstd/fse.h +++ /dev/null @@ -1,575 +0,0 @@ -/* - * FSE : Finite State Entropy codec - * Public Prototypes declaration - * Copyright (C) 2013-2016, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ -#ifndef FSE_H -#define FSE_H - -/*-***************************************** -* Dependencies -******************************************/ -#include <linux/types.h> /* size_t, ptrdiff_t */ - -/*-***************************************** -* FSE_PUBLIC_API : control library symbols visibility -******************************************/ -#define FSE_PUBLIC_API - -/*------ Version ------*/ -#define FSE_VERSION_MAJOR 0 -#define FSE_VERSION_MINOR 9 -#define FSE_VERSION_RELEASE 0 - -#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE -#define FSE_QUOTE(str) #str -#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) -#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) - -#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR * 100 * 100 + FSE_VERSION_MINOR * 100 + FSE_VERSION_RELEASE) -FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ - -/*-***************************************** -* Tool functions -******************************************/ -FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ - -/* Error Management */ -FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ - -/*-***************************************** -* FSE detailed API -******************************************/ -/*! -FSE_compress() does the following: -1. count symbol occurrence from source[] into table count[] -2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) -3. save normalized counters to memory buffer using writeNCount() -4. build encoding table 'CTable' from normalized counters -5. encode the data stream using encoding table 'CTable' - -FSE_decompress() does the following: -1. read normalized counters with readNCount() -2. build decoding table 'DTable' from normalized counters -3. decode the data stream using decoding table 'DTable' - -The following API allows targeting specific sub-functions for advanced tasks. -For example, it's possible to compress several blocks using the same 'CTable', -or to save and provide normalized distribution using external method. -*/ - -/* *** COMPRESSION *** */ -/*! FSE_optimalTableLog(): - dynamically downsize 'tableLog' when conditions are met. - It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. - @return : recommended tableLog (necessarily <= 'maxTableLog') */ -FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); - -/*! FSE_normalizeCount(): - normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) - 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). - @return : tableLog, - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t srcSize, unsigned maxSymbolValue); - -/*! FSE_NCountWriteBound(): - Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. - Typically useful for allocation purpose. */ -FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); - -/*! FSE_writeNCount(): - Compactly save 'normalizedCounter' into 'buffer'. - @return : size of the compressed table, - or an errorCode, which can be tested using FSE_isError(). */ -FSE_PUBLIC_API size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! Constructor and Destructor of FSE_CTable. - Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ -typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ - -/*! FSE_compress_usingCTable(): - Compress `src` using `ct` into `dst` which must be already allocated. - @return : size of compressed data (<= `dstCapacity`), - or 0 if compressed data could not fit into `dst`, - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_compress_usingCTable(void *dst, size_t dstCapacity, const void *src, size_t srcSize, const FSE_CTable *ct); - -/*! -Tutorial : ----------- -The first step is to count all symbols. FSE_count() does this job very fast. -Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. -'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] -maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) -FSE_count() will return the number of occurrence of the most frequent symbol. -This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. -If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). - -The next step is to normalize the frequencies. -FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. -It also guarantees a minimum of 1 to any Symbol with frequency >= 1. -You can use 'tableLog'==0 to mean "use default tableLog value". -If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), -which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). - -The result of FSE_normalizeCount() will be saved into a table, -called 'normalizedCounter', which is a table of signed short. -'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. -The return value is tableLog if everything proceeded as expected. -It is 0 if there is a single symbol within distribution. -If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). - -'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). -'buffer' must be already allocated. -For guaranteed success, buffer size must be at least FSE_headerBound(). -The result of the function is the number of bytes written into 'buffer'. -If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). - -'normalizedCounter' can then be used to create the compression table 'CTable'. -The space required by 'CTable' must be already allocated, using FSE_createCTable(). -You can then use FSE_buildCTable() to fill 'CTable'. -If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). - -'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). -Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' -The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. -If it returns '0', compressed data could not fit into 'dst'. -If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). -*/ - -/* *** DECOMPRESSION *** */ - -/*! FSE_readNCount(): - Read compactly saved 'normalizedCounter' from 'rBuffer'. - @return : size read from 'rBuffer', - or an errorCode, which can be tested using FSE_isError(). - maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ -FSE_PUBLIC_API size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSymbolValuePtr, unsigned *tableLogPtr, const void *rBuffer, size_t rBuffSize); - -/*! Constructor and Destructor of FSE_DTable. - Note that its size depends on 'tableLog' */ -typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ - -/*! FSE_buildDTable(): - Builds 'dt', which must be already allocated, using FSE_createDTable(). - return : 0, or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize); - -/*! FSE_decompress_usingDTable(): - Decompress compressed source `cSrc` of size `cSrcSize` using `dt` - into `dst` which must be already allocated. - @return : size of regenerated data (necessarily <= `dstCapacity`), - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt); - -/*! -Tutorial : ----------- -(Note : these functions only decompress FSE-compressed blocks. - If block is uncompressed, use memcpy() instead - If block is a single repeated byte, use memset() instead ) - -The first step is to obtain the normalized frequencies of symbols. -This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). -'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. -In practice, that means it's necessary to know 'maxSymbolValue' beforehand, -or size the table to handle worst case situations (typically 256). -FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. -The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. -Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. -If there is an error, the function will return an error code, which can be tested using FSE_isError(). - -The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. -This is performed by the function FSE_buildDTable(). -The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). -If there is an error, the function will return an error code, which can be tested using FSE_isError(). - -`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). -`cSrcSize` must be strictly correct, otherwise decompression will fail. -FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). -If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) -*/ - -/* *** Dependency *** */ -#include "bitstream.h" - -/* ***************************************** -* Static allocation -*******************************************/ -/* FSE buffer bounds */ -#define FSE_NCOUNTBOUND 512 -#define FSE_BLOCKBOUND(size) (size + (size >> 7) + 4 /* constant for initial fse states */ ) -#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ - -/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ -#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1 << (maxTableLog - 1)) + ((maxSymbolValue + 1) * 2)) -#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1 << maxTableLog)) - -/* ***************************************** -* FSE advanced API -*******************************************/ -/* FSE_count_wksp() : - * Same as FSE_count(), but using an externally provided scratch buffer. - * `workSpace` size must be table of >= `1024` unsigned - */ -size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace); - -/* FSE_countFast_wksp() : - * Same as FSE_countFast(), but using an externally provided scratch buffer. - * `workSpace` must be a table of minimum `1024` unsigned - */ -size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize, unsigned *workSpace); - -/*! FSE_count_simple - * Same as FSE_countFast(), but does not use any additional memory (not even on stack). - * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`). -*/ -size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize); - -unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); -/**< same as FSE_optimalTableLog(), which used `minus==2` */ - -size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits); -/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ - -size_t FSE_buildCTable_rle(FSE_CTable *ct, unsigned char symbolValue); -/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ - -/* FSE_buildCTable_wksp() : - * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). - * `wkspSize` must be >= `(1<<tableLog)`. - */ -size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, size_t wkspSize); - -size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits); -/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ - -size_t FSE_buildDTable_rle(FSE_DTable *dt, unsigned char symbolValue); -/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ - -size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, unsigned maxLog, void *workspace, size_t workspaceSize); -/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */ - -/* ***************************************** -* FSE symbol compression API -*******************************************/ -/*! - This API consists of small unitary functions, which highly benefit from being inlined. - Hence their body are included in next section. -*/ -typedef struct { - ptrdiff_t value; - const void *stateTable; - const void *symbolTT; - unsigned stateLog; -} FSE_CState_t; - -static void FSE_initCState(FSE_CState_t *CStatePtr, const FSE_CTable *ct); - -static void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *CStatePtr, unsigned symbol); - -static void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *CStatePtr); - -/**< -These functions are inner components of FSE_compress_usingCTable(). -They allow the creation of custom streams, mixing multiple tables and bit sources. - -A key property to keep in mind is that encoding and decoding are done **in reverse direction**. -So the first symbol you will encode is the last you will decode, like a LIFO stack. - -You will need a few variables to track your CStream. They are : - -FSE_CTable ct; // Provided by FSE_buildCTable() -BIT_CStream_t bitStream; // bitStream tracking structure -FSE_CState_t state; // State tracking structure (can have several) - - -The first thing to do is to init bitStream and state. - size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize); - FSE_initCState(&state, ct); - -Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError(); -You can then encode your input data, byte after byte. -FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time. -Remember decoding will be done in reverse direction. - FSE_encodeByte(&bitStream, &state, symbol); - -At any time, you can also add any bit sequence. -Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders - BIT_addBits(&bitStream, bitField, nbBits); - -The above methods don't commit data to memory, they just store it into local register, for speed. -Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). -Writing data to memory is a manual operation, performed by the flushBits function. - BIT_flushBits(&bitStream); - -Your last FSE encoding operation shall be to flush your last state value(s). - FSE_flushState(&bitStream, &state); - -Finally, you must close the bitStream. -The function returns the size of CStream in bytes. -If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible) -If there is an error, it returns an errorCode (which can be tested using FSE_isError()). - size_t size = BIT_closeCStream(&bitStream); -*/ - -/* ***************************************** -* FSE symbol decompression API -*******************************************/ -typedef struct { - size_t state; - const void *table; /* precise table may vary, depending on U16 */ -} FSE_DState_t; - -static void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt); - -static unsigned char FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD); - -static unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr); - -/**< -Let's now decompose FSE_decompress_usingDTable() into its unitary components. -You will decode FSE-encoded symbols from the bitStream, -and also any other bitFields you put in, **in reverse order**. - -You will need a few variables to track your bitStream. They are : - -BIT_DStream_t DStream; // Stream context -FSE_DState_t DState; // State context. Multiple ones are possible -FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable() - -The first thing to do is to init the bitStream. - errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); - -You should then retrieve your initial state(s) -(in reverse flushing order if you have several ones) : - errorCode = FSE_initDState(&DState, &DStream, DTablePtr); - -You can then decode your data, symbol after symbol. -For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'. -Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). - unsigned char symbol = FSE_decodeSymbol(&DState, &DStream); - -You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) -Note : maximum allowed nbBits is 25, for 32-bits compatibility - size_t bitField = BIT_readBits(&DStream, nbBits); - -All above operations only read from local register (which size depends on size_t). -Refueling the register from memory is manually performed by the reload method. - endSignal = FSE_reloadDStream(&DStream); - -BIT_reloadDStream() result tells if there is still some more data to read from DStream. -BIT_DStream_unfinished : there is still some data left into the DStream. -BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. -BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. -BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. - -When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, -to properly detect the exact end of stream. -After each decoded symbol, check if DStream is fully consumed using this simple test : - BIT_reloadDStream(&DStream) >= BIT_DStream_completed - -When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. -Checking if DStream has reached its end is performed by : - BIT_endOfDStream(&DStream); -Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. - FSE_endOfDState(&DState); -*/ - -/* ***************************************** -* FSE unsafe API -*******************************************/ -static unsigned char FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD); -/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ - -/* ***************************************** -* Implementation of inlined functions -*******************************************/ -typedef struct { - int deltaFindState; - U32 deltaNbBits; -} FSE_symbolCompressionTransform; /* total 8 bytes */ - -ZSTD_STATIC void FSE_initCState(FSE_CState_t *statePtr, const FSE_CTable *ct) -{ - const void *ptr = ct; - const U16 *u16ptr = (const U16 *)ptr; - const U32 tableLog = ZSTD_read16(ptr); - statePtr->value = (ptrdiff_t)1 << tableLog; - statePtr->stateTable = u16ptr + 2; - statePtr->symbolTT = ((const U32 *)ct + 1 + (tableLog ? (1 << (tableLog - 1)) : 1)); - statePtr->stateLog = tableLog; -} - -/*! FSE_initCState2() : -* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) -* uses the smallest state value possible, saving the cost of this symbol */ -ZSTD_STATIC void FSE_initCState2(FSE_CState_t *statePtr, const FSE_CTable *ct, U32 symbol) -{ - FSE_initCState(statePtr, ct); - { - const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol]; - const U16 *stateTable = (const U16 *)(statePtr->stateTable); - U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1 << 15)) >> 16); - statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; - statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; - } -} - -ZSTD_STATIC void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *statePtr, U32 symbol) -{ - const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol]; - const U16 *const stateTable = (const U16 *)(statePtr->stateTable); - U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); - BIT_addBits(bitC, statePtr->value, nbBitsOut); - statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; -} - -ZSTD_STATIC void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *statePtr) -{ - BIT_addBits(bitC, statePtr->value, statePtr->stateLog); - BIT_flushBits(bitC); -} - -/* ====== Decompression ====== */ - -typedef struct { - U16 tableLog; - U16 fastMode; -} FSE_DTableHeader; /* sizeof U32 */ - -typedef struct { - unsigned short newState; - unsigned char symbol; - unsigned char nbBits; -} FSE_decode_t; /* size == U32 */ - -ZSTD_STATIC void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt) -{ - const void *ptr = dt; - const FSE_DTableHeader *const DTableH = (const FSE_DTableHeader *)ptr; - DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); - BIT_reloadDStream(bitD); - DStatePtr->table = dt + 1; -} - -ZSTD_STATIC BYTE FSE_peekSymbol(const FSE_DState_t *DStatePtr) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; - return DInfo.symbol; -} - -ZSTD_STATIC void FSE_updateState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.newState + lowBits; -} - -ZSTD_STATIC BYTE FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BIT_readBits(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -/*! FSE_decodeSymbolFast() : - unsafe, only works if no symbol has a probability > 50% */ -ZSTD_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BIT_readBitsFast(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr) { return DStatePtr->state == 0; } - -/* ************************************************************** -* Tuning parameters -****************************************************************/ -/*!MEMORY_USAGE : -* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) -* Increasing memory usage improves compression ratio -* Reduced memory usage can improve speed, due to cache effect -* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#ifndef FSE_MAX_MEMORY_USAGE -#define FSE_MAX_MEMORY_USAGE 14 -#endif -#ifndef FSE_DEFAULT_MEMORY_USAGE -#define FSE_DEFAULT_MEMORY_USAGE 13 -#endif - -/*!FSE_MAX_SYMBOL_VALUE : -* Maximum symbol value authorized. -* Required for proper stack allocation */ -#ifndef FSE_MAX_SYMBOL_VALUE -#define FSE_MAX_SYMBOL_VALUE 255 -#endif - -/* ************************************************************** -* template functions type & suffix -****************************************************************/ -#define FSE_FUNCTION_TYPE BYTE -#define FSE_FUNCTION_EXTENSION -#define FSE_DECODE_TYPE FSE_decode_t - -/* *************************************************************** -* Constants -*****************************************************************/ -#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE - 2) -#define FSE_MAX_TABLESIZE (1U << FSE_MAX_TABLELOG) -#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE - 1) -#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE - 2) -#define FSE_MIN_TABLELOG 5 - -#define FSE_TABLELOG_ABSOLUTE_MAX 15 -#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX -#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" -#endif - -#define FSE_TABLESTEP(tableSize) ((tableSize >> 1) + (tableSize >> 3) + 3) - -#endif /* FSE_H */ diff --git a/contrib/linux-kernel/lib/zstd/fse_compress.c b/contrib/linux-kernel/lib/zstd/fse_compress.c deleted file mode 100644 index 0fe468ed..00000000 --- a/contrib/linux-kernel/lib/zstd/fse_compress.c +++ /dev/null @@ -1,795 +0,0 @@ -/* - * FSE : Finite State Entropy encoder - * Copyright (C) 2013-2015, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#define FORCE_INLINE static __always_inline - -/* ************************************************************** -* Includes -****************************************************************/ -#include "bitstream.h" -#include "fse.h" -#include <linux/compiler.h> -#include <linux/kernel.h> -#include <linux/math64.h> -#include <linux/string.h> /* memcpy, memset */ - -/* ************************************************************** -* Error Management -****************************************************************/ -#define FSE_STATIC_ASSERT(c) \ - { \ - enum { FSE_static_assert = 1 / (int)(!!(c)) }; \ - } /* use only *after* variable declarations */ - -/* ************************************************************** -* Templates -****************************************************************/ -/* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ - -/* safety checks */ -#ifndef FSE_FUNCTION_EXTENSION -#error "FSE_FUNCTION_EXTENSION must be defined" -#endif -#ifndef FSE_FUNCTION_TYPE -#error "FSE_FUNCTION_TYPE must be defined" -#endif - -/* Function names */ -#define FSE_CAT(X, Y) X##Y -#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y) -#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y) - -/* Function templates */ - -/* FSE_buildCTable_wksp() : - * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). - * wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)` - * workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements - */ -size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize) -{ - U32 const tableSize = 1 << tableLog; - U32 const tableMask = tableSize - 1; - void *const ptr = ct; - U16 *const tableU16 = ((U16 *)ptr) + 2; - void *const FSCT = ((U32 *)ptr) + 1 /* header */ + (tableLog ? tableSize >> 1 : 1); - FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT); - U32 const step = FSE_TABLESTEP(tableSize); - U32 highThreshold = tableSize - 1; - - U32 *cumul; - FSE_FUNCTION_TYPE *tableSymbol; - size_t spaceUsed32 = 0; - - cumul = (U32 *)workspace + spaceUsed32; - spaceUsed32 += FSE_MAX_SYMBOL_VALUE + 2; - tableSymbol = (FSE_FUNCTION_TYPE *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(sizeof(FSE_FUNCTION_TYPE) * ((size_t)1 << tableLog), sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - /* CTable header */ - tableU16[-2] = (U16)tableLog; - tableU16[-1] = (U16)maxSymbolValue; - - /* For explanations on how to distribute symbol values over the table : - * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ - - /* symbol start positions */ - { - U32 u; - cumul[0] = 0; - for (u = 1; u <= maxSymbolValue + 1; u++) { - if (normalizedCounter[u - 1] == -1) { /* Low proba symbol */ - cumul[u] = cumul[u - 1] + 1; - tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u - 1); - } else { - cumul[u] = cumul[u - 1] + normalizedCounter[u - 1]; - } - } - cumul[maxSymbolValue + 1] = tableSize + 1; - } - - /* Spread symbols */ - { - U32 position = 0; - U32 symbol; - for (symbol = 0; symbol <= maxSymbolValue; symbol++) { - int nbOccurrences; - for (nbOccurrences = 0; nbOccurrences < normalizedCounter[symbol]; nbOccurrences++) { - tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol; - position = (position + step) & tableMask; - while (position > highThreshold) - position = (position + step) & tableMask; /* Low proba area */ - } - } - - if (position != 0) - return ERROR(GENERIC); /* Must have gone through all positions */ - } - - /* Build table */ - { - U32 u; - for (u = 0; u < tableSize; u++) { - FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */ - tableU16[cumul[s]++] = (U16)(tableSize + u); /* TableU16 : sorted by symbol order; gives next state value */ - } - } - - /* Build Symbol Transformation Table */ - { - unsigned total = 0; - unsigned s; - for (s = 0; s <= maxSymbolValue; s++) { - switch (normalizedCounter[s]) { - case 0: break; - - case -1: - case 1: - symbolTT[s].deltaNbBits = (tableLog << 16) - (1 << tableLog); - symbolTT[s].deltaFindState = total - 1; - total++; - break; - default: { - U32 const maxBitsOut = tableLog - BIT_highbit32(normalizedCounter[s] - 1); - U32 const minStatePlus = normalizedCounter[s] << maxBitsOut; - symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; - symbolTT[s].deltaFindState = total - normalizedCounter[s]; - total += normalizedCounter[s]; - } - } - } - } - - return 0; -} - -/*-************************************************************** -* FSE NCount encoding-decoding -****************************************************************/ -size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) -{ - size_t const maxHeaderSize = (((maxSymbolValue + 1) * tableLog) >> 3) + 3; - return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ -} - -static size_t FSE_writeNCount_generic(void *header, size_t headerBufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, - unsigned writeIsSafe) -{ - BYTE *const ostart = (BYTE *)header; - BYTE *out = ostart; - BYTE *const oend = ostart + headerBufferSize; - int nbBits; - const int tableSize = 1 << tableLog; - int remaining; - int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; - - bitStream = 0; - bitCount = 0; - /* Table Size */ - bitStream += (tableLog - FSE_MIN_TABLELOG) << bitCount; - bitCount += 4; - - /* Init */ - remaining = tableSize + 1; /* +1 for extra accuracy */ - threshold = tableSize; - nbBits = tableLog + 1; - - while (remaining > 1) { /* stops at 1 */ - if (previous0) { - unsigned start = charnum; - while (!normalizedCounter[charnum]) - charnum++; - while (charnum >= start + 24) { - start += 24; - bitStream += 0xFFFFU << bitCount; - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream >> 8); - out += 2; - bitStream >>= 16; - } - while (charnum >= start + 3) { - start += 3; - bitStream += 3 << bitCount; - bitCount += 2; - } - bitStream += (charnum - start) << bitCount; - bitCount += 2; - if (bitCount > 16) { - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream >> 8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } - } - { - int count = normalizedCounter[charnum++]; - int const max = (2 * threshold - 1) - remaining; - remaining -= count < 0 ? -count : count; - count++; /* +1 for extra accuracy */ - if (count >= threshold) - count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ - bitStream += count << bitCount; - bitCount += nbBits; - bitCount -= (count < max); - previous0 = (count == 1); - if (remaining < 1) - return ERROR(GENERIC); - while (remaining < threshold) - nbBits--, threshold >>= 1; - } - if (bitCount > 16) { - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream >> 8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } - } - - /* flush remaining bitStream */ - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream >> 8); - out += (bitCount + 7) / 8; - - if (charnum > maxSymbolValue + 1) - return ERROR(GENERIC); - - return (out - ostart); -} - -size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - if (tableLog > FSE_MAX_TABLELOG) - return ERROR(tableLog_tooLarge); /* Unsupported */ - if (tableLog < FSE_MIN_TABLELOG) - return ERROR(GENERIC); /* Unsupported */ - - if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); - - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); -} - -/*-************************************************************** -* Counting histogram -****************************************************************/ -/*! FSE_count_simple - This function counts byte values within `src`, and store the histogram into table `count`. - It doesn't use any additional memory. - But this function is unsafe : it doesn't check that all values within `src` can fit into `count`. - For this reason, prefer using a table `count` with 256 elements. - @return : count of most numerous element -*/ -size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize) -{ - const BYTE *ip = (const BYTE *)src; - const BYTE *const end = ip + srcSize; - unsigned maxSymbolValue = *maxSymbolValuePtr; - unsigned max = 0; - - memset(count, 0, (maxSymbolValue + 1) * sizeof(*count)); - if (srcSize == 0) { - *maxSymbolValuePtr = 0; - return 0; - } - - while (ip < end) - count[*ip++]++; - - while (!count[maxSymbolValue]) - maxSymbolValue--; - *maxSymbolValuePtr = maxSymbolValue; - - { - U32 s; - for (s = 0; s <= maxSymbolValue; s++) - if (count[s] > max) - max = count[s]; - } - - return (size_t)max; -} - -/* FSE_count_parallel_wksp() : - * Same as FSE_count_parallel(), but using an externally provided scratch buffer. - * `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */ -static size_t FSE_count_parallel_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned checkMax, - unsigned *const workSpace) -{ - const BYTE *ip = (const BYTE *)source; - const BYTE *const iend = ip + sourceSize; - unsigned maxSymbolValue = *maxSymbolValuePtr; - unsigned max = 0; - U32 *const Counting1 = workSpace; - U32 *const Counting2 = Counting1 + 256; - U32 *const Counting3 = Counting2 + 256; - U32 *const Counting4 = Counting3 + 256; - - memset(Counting1, 0, 4 * 256 * sizeof(unsigned)); - - /* safety checks */ - if (!sourceSize) { - memset(count, 0, maxSymbolValue + 1); - *maxSymbolValuePtr = 0; - return 0; - } - if (!maxSymbolValue) - maxSymbolValue = 255; /* 0 == default */ - - /* by stripes of 16 bytes */ - { - U32 cached = ZSTD_read32(ip); - ip += 4; - while (ip < iend - 15) { - U32 c = cached; - cached = ZSTD_read32(ip); - ip += 4; - Counting1[(BYTE)c]++; - Counting2[(BYTE)(c >> 8)]++; - Counting3[(BYTE)(c >> 16)]++; - Counting4[c >> 24]++; - c = cached; - cached = ZSTD_read32(ip); - ip += 4; - Counting1[(BYTE)c]++; - Counting2[(BYTE)(c >> 8)]++; - Counting3[(BYTE)(c >> 16)]++; - Counting4[c >> 24]++; - c = cached; - cached = ZSTD_read32(ip); - ip += 4; - Counting1[(BYTE)c]++; - Counting2[(BYTE)(c >> 8)]++; - Counting3[(BYTE)(c >> 16)]++; - Counting4[c >> 24]++; - c = cached; - cached = ZSTD_read32(ip); - ip += 4; - Counting1[(BYTE)c]++; - Counting2[(BYTE)(c >> 8)]++; - Counting3[(BYTE)(c >> 16)]++; - Counting4[c >> 24]++; - } - ip -= 4; - } - - /* finish last symbols */ - while (ip < iend) - Counting1[*ip++]++; - - if (checkMax) { /* verify stats will fit into destination table */ - U32 s; - for (s = 255; s > maxSymbolValue; s--) { - Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; - if (Counting1[s]) - return ERROR(maxSymbolValue_tooSmall); - } - } - - { - U32 s; - for (s = 0; s <= maxSymbolValue; s++) { - count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; - if (count[s] > max) - max = count[s]; - } - } - - while (!count[maxSymbolValue]) - maxSymbolValue--; - *maxSymbolValuePtr = maxSymbolValue; - return (size_t)max; -} - -/* FSE_countFast_wksp() : - * Same as FSE_countFast(), but using an externally provided scratch buffer. - * `workSpace` size must be table of >= `1024` unsigned */ -size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace) -{ - if (sourceSize < 1500) - return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize); - return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace); -} - -/* FSE_count_wksp() : - * Same as FSE_count(), but using an externally provided scratch buffer. - * `workSpace` size must be table of >= `1024` unsigned */ -size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace) -{ - if (*maxSymbolValuePtr < 255) - return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace); - *maxSymbolValuePtr = 255; - return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace); -} - -/*-************************************************************** -* FSE Compression Code -****************************************************************/ -/*! FSE_sizeof_CTable() : - FSE_CTable is a variable size structure which contains : - `U16 tableLog;` - `U16 maxSymbolValue;` - `U16 nextStateNumber[1 << tableLog];` // This size is variable - `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable -Allocation is manual (C standard does not support variable-size structures). -*/ -size_t FSE_sizeof_CTable(unsigned maxSymbolValue, unsigned tableLog) -{ - if (tableLog > FSE_MAX_TABLELOG) - return ERROR(tableLog_tooLarge); - return FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue) * sizeof(U32); -} - -/* provides the minimum logSize to safely represent a distribution */ -static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) -{ - U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; - U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; - U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; - return minBits; -} - -unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) -{ - U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; - U32 tableLog = maxTableLog; - U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); - if (tableLog == 0) - tableLog = FSE_DEFAULT_TABLELOG; - if (maxBitsSrc < tableLog) - tableLog = maxBitsSrc; /* Accuracy can be reduced */ - if (minBits > tableLog) - tableLog = minBits; /* Need a minimum to safely represent all symbol values */ - if (tableLog < FSE_MIN_TABLELOG) - tableLog = FSE_MIN_TABLELOG; - if (tableLog > FSE_MAX_TABLELOG) - tableLog = FSE_MAX_TABLELOG; - return tableLog; -} - -unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) -{ - return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); -} - -/* Secondary normalization method. - To be used when primary method fails. */ - -static size_t FSE_normalizeM2(short *norm, U32 tableLog, const unsigned *count, size_t total, U32 maxSymbolValue) -{ - short const NOT_YET_ASSIGNED = -2; - U32 s; - U32 distributed = 0; - U32 ToDistribute; - - /* Init */ - U32 const lowThreshold = (U32)(total >> tableLog); - U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); - - for (s = 0; s <= maxSymbolValue; s++) { - if (count[s] == 0) { - norm[s] = 0; - continue; - } - if (count[s] <= lowThreshold) { - norm[s] = -1; - distributed++; - total -= count[s]; - continue; - } - if (count[s] <= lowOne) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } - - norm[s] = NOT_YET_ASSIGNED; - } - ToDistribute = (1 << tableLog) - distributed; - - if ((total / ToDistribute) > lowOne) { - /* risk of rounding to zero */ - lowOne = (U32)((total * 3) / (ToDistribute * 2)); - for (s = 0; s <= maxSymbolValue; s++) { - if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } - } - ToDistribute = (1 << tableLog) - distributed; - } - - if (distributed == maxSymbolValue + 1) { - /* all values are pretty poor; - probably incompressible data (should have already been detected); - find max, then give all remaining points to max */ - U32 maxV = 0, maxC = 0; - for (s = 0; s <= maxSymbolValue; s++) - if (count[s] > maxC) - maxV = s, maxC = count[s]; - norm[maxV] += (short)ToDistribute; - return 0; - } - - if (total == 0) { - /* all of the symbols were low enough for the lowOne or lowThreshold */ - for (s = 0; ToDistribute > 0; s = (s + 1) % (maxSymbolValue + 1)) - if (norm[s] > 0) - ToDistribute--, norm[s]++; - return 0; - } - - { - U64 const vStepLog = 62 - tableLog; - U64 const mid = (1ULL << (vStepLog - 1)) - 1; - U64 const rStep = div_u64((((U64)1 << vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */ - U64 tmpTotal = mid; - for (s = 0; s <= maxSymbolValue; s++) { - if (norm[s] == NOT_YET_ASSIGNED) { - U64 const end = tmpTotal + (count[s] * rStep); - U32 const sStart = (U32)(tmpTotal >> vStepLog); - U32 const sEnd = (U32)(end >> vStepLog); - U32 const weight = sEnd - sStart; - if (weight < 1) - return ERROR(GENERIC); - norm[s] = (short)weight; - tmpTotal = end; - } - } - } - - return 0; -} - -size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t total, unsigned maxSymbolValue) -{ - /* Sanity checks */ - if (tableLog == 0) - tableLog = FSE_DEFAULT_TABLELOG; - if (tableLog < FSE_MIN_TABLELOG) - return ERROR(GENERIC); /* Unsupported size */ - if (tableLog > FSE_MAX_TABLELOG) - return ERROR(tableLog_tooLarge); /* Unsupported size */ - if (tableLog < FSE_minTableLog(total, maxSymbolValue)) - return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ - - { - U32 const rtbTable[] = {0, 473195, 504333, 520860, 550000, 700000, 750000, 830000}; - U64 const scale = 62 - tableLog; - U64 const step = div_u64((U64)1 << 62, (U32)total); /* <== here, one division ! */ - U64 const vStep = 1ULL << (scale - 20); - int stillToDistribute = 1 << tableLog; - unsigned s; - unsigned largest = 0; - short largestP = 0; - U32 lowThreshold = (U32)(total >> tableLog); - - for (s = 0; s <= maxSymbolValue; s++) { - if (count[s] == total) - return 0; /* rle special case */ - if (count[s] == 0) { - normalizedCounter[s] = 0; - continue; - } - if (count[s] <= lowThreshold) { - normalizedCounter[s] = -1; - stillToDistribute--; - } else { - short proba = (short)((count[s] * step) >> scale); - if (proba < 8) { - U64 restToBeat = vStep * rtbTable[proba]; - proba += (count[s] * step) - ((U64)proba << scale) > restToBeat; - } - if (proba > largestP) - largestP = proba, largest = s; - normalizedCounter[s] = proba; - stillToDistribute -= proba; - } - } - if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { - /* corner case, need another normalization method */ - size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); - if (FSE_isError(errorCode)) - return errorCode; - } else - normalizedCounter[largest] += (short)stillToDistribute; - } - - return tableLog; -} - -/* fake FSE_CTable, for raw (uncompressed) input */ -size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits) -{ - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSymbolValue = tableMask; - void *const ptr = ct; - U16 *const tableU16 = ((U16 *)ptr) + 2; - void *const FSCT = ((U32 *)ptr) + 1 /* header */ + (tableSize >> 1); /* assumption : tableLog >= 1 */ - FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT); - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) - return ERROR(GENERIC); /* min size */ - - /* header */ - tableU16[-2] = (U16)nbBits; - tableU16[-1] = (U16)maxSymbolValue; - - /* Build table */ - for (s = 0; s < tableSize; s++) - tableU16[s] = (U16)(tableSize + s); - - /* Build Symbol Transformation Table */ - { - const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits); - for (s = 0; s <= maxSymbolValue; s++) { - symbolTT[s].deltaNbBits = deltaNbBits; - symbolTT[s].deltaFindState = s - 1; - } - } - - return 0; -} - -/* fake FSE_CTable, for rle input (always same symbol) */ -size_t FSE_buildCTable_rle(FSE_CTable *ct, BYTE symbolValue) -{ - void *ptr = ct; - U16 *tableU16 = ((U16 *)ptr) + 2; - void *FSCTptr = (U32 *)ptr + 2; - FSE_symbolCompressionTransform *symbolTT = (FSE_symbolCompressionTransform *)FSCTptr; - - /* header */ - tableU16[-2] = (U16)0; - tableU16[-1] = (U16)symbolValue; - - /* Build table */ - tableU16[0] = 0; - tableU16[1] = 0; /* just in case */ - - /* Build Symbol Transformation Table */ - symbolTT[symbolValue].deltaNbBits = 0; - symbolTT[symbolValue].deltaFindState = 0; - - return 0; -} - -static size_t FSE_compress_usingCTable_generic(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct, const unsigned fast) -{ - const BYTE *const istart = (const BYTE *)src; - const BYTE *const iend = istart + srcSize; - const BYTE *ip = iend; - - BIT_CStream_t bitC; - FSE_CState_t CState1, CState2; - - /* init */ - if (srcSize <= 2) - return 0; - { - size_t const initError = BIT_initCStream(&bitC, dst, dstSize); - if (FSE_isError(initError)) - return 0; /* not enough space available to write a bitstream */ - } - -#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) - - if (srcSize & 1) { - FSE_initCState2(&CState1, ct, *--ip); - FSE_initCState2(&CState2, ct, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - FSE_FLUSHBITS(&bitC); - } else { - FSE_initCState2(&CState2, ct, *--ip); - FSE_initCState2(&CState1, ct, *--ip); - } - - /* join to mod 4 */ - srcSize -= 2; - if ((sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) && (srcSize & 2)) { /* test bit 2 */ - FSE_encodeSymbol(&bitC, &CState2, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - FSE_FLUSHBITS(&bitC); - } - - /* 2 or 4 encoding per loop */ - while (ip > istart) { - - FSE_encodeSymbol(&bitC, &CState2, *--ip); - - if (sizeof(bitC.bitContainer) * 8 < FSE_MAX_TABLELOG * 2 + 7) /* this test must be static */ - FSE_FLUSHBITS(&bitC); - - FSE_encodeSymbol(&bitC, &CState1, *--ip); - - if (sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) { /* this test must be static */ - FSE_encodeSymbol(&bitC, &CState2, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - } - - FSE_FLUSHBITS(&bitC); - } - - FSE_flushCState(&bitC, &CState2); - FSE_flushCState(&bitC, &CState1); - return BIT_closeCStream(&bitC); -} - -size_t FSE_compress_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct) -{ - unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); - - if (fast) - return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); - else - return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); -} - -size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } diff --git a/contrib/linux-kernel/lib/zstd/fse_decompress.c b/contrib/linux-kernel/lib/zstd/fse_decompress.c deleted file mode 100644 index a84300e5..00000000 --- a/contrib/linux-kernel/lib/zstd/fse_decompress.c +++ /dev/null @@ -1,332 +0,0 @@ -/* - * FSE : Finite State Entropy decoder - * Copyright (C) 2013-2015, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#define FORCE_INLINE static __always_inline - -/* ************************************************************** -* Includes -****************************************************************/ -#include "bitstream.h" -#include "fse.h" -#include <linux/compiler.h> -#include <linux/kernel.h> -#include <linux/string.h> /* memcpy, memset */ - -/* ************************************************************** -* Error Management -****************************************************************/ -#define FSE_isError ERR_isError -#define FSE_STATIC_ASSERT(c) \ - { \ - enum { FSE_static_assert = 1 / (int)(!!(c)) }; \ - } /* use only *after* variable declarations */ - -/* check and forward error code */ -#define CHECK_F(f) \ - { \ - size_t const e = f; \ - if (FSE_isError(e)) \ - return e; \ - } - -/* ************************************************************** -* Templates -****************************************************************/ -/* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ - -/* safety checks */ -#ifndef FSE_FUNCTION_EXTENSION -#error "FSE_FUNCTION_EXTENSION must be defined" -#endif -#ifndef FSE_FUNCTION_TYPE -#error "FSE_FUNCTION_TYPE must be defined" -#endif - -/* Function names */ -#define FSE_CAT(X, Y) X##Y -#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y) -#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y) - -/* Function templates */ - -size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize) -{ - void *const tdPtr = dt + 1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ - FSE_DECODE_TYPE *const tableDecode = (FSE_DECODE_TYPE *)(tdPtr); - U16 *symbolNext = (U16 *)workspace; - - U32 const maxSV1 = maxSymbolValue + 1; - U32 const tableSize = 1 << tableLog; - U32 highThreshold = tableSize - 1; - - /* Sanity Checks */ - if (workspaceSize < sizeof(U16) * (FSE_MAX_SYMBOL_VALUE + 1)) - return ERROR(tableLog_tooLarge); - if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) - return ERROR(maxSymbolValue_tooLarge); - if (tableLog > FSE_MAX_TABLELOG) - return ERROR(tableLog_tooLarge); - - /* Init, lay down lowprob symbols */ - { - FSE_DTableHeader DTableH; - DTableH.tableLog = (U16)tableLog; - DTableH.fastMode = 1; - { - S16 const largeLimit = (S16)(1 << (tableLog - 1)); - U32 s; - for (s = 0; s < maxSV1; s++) { - if (normalizedCounter[s] == -1) { - tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; - symbolNext[s] = 1; - } else { - if (normalizedCounter[s] >= largeLimit) - DTableH.fastMode = 0; - symbolNext[s] = normalizedCounter[s]; - } - } - } - memcpy(dt, &DTableH, sizeof(DTableH)); - } - - /* Spread symbols */ - { - U32 const tableMask = tableSize - 1; - U32 const step = FSE_TABLESTEP(tableSize); - U32 s, position = 0; - for (s = 0; s < maxSV1; s++) { - int i; - for (i = 0; i < normalizedCounter[s]; i++) { - tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s; - position = (position + step) & tableMask; - while (position > highThreshold) - position = (position + step) & tableMask; /* lowprob area */ - } - } - if (position != 0) - return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ - } - - /* Build Decoding table */ - { - U32 u; - for (u = 0; u < tableSize; u++) { - FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol); - U16 nextState = symbolNext[symbol]++; - tableDecode[u].nbBits = (BYTE)(tableLog - BIT_highbit32((U32)nextState)); - tableDecode[u].newState = (U16)((nextState << tableDecode[u].nbBits) - tableSize); - } - } - - return 0; -} - -/*-******************************************************* -* Decompression (Byte symbols) -*********************************************************/ -size_t FSE_buildDTable_rle(FSE_DTable *dt, BYTE symbolValue) -{ - void *ptr = dt; - FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr; - void *dPtr = dt + 1; - FSE_decode_t *const cell = (FSE_decode_t *)dPtr; - - DTableH->tableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - -size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits) -{ - void *ptr = dt; - FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr; - void *dPtr = dt + 1; - FSE_decode_t *const dinfo = (FSE_decode_t *)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSV1 = tableMask + 1; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) - return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s = 0; s < maxSV1; s++) { - dinfo[s].newState = 0; - dinfo[s].symbol = (BYTE)s; - dinfo[s].nbBits = (BYTE)nbBits; - } - - return 0; -} - -FORCE_INLINE size_t FSE_decompress_usingDTable_generic(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt, - const unsigned fast) -{ - BYTE *const ostart = (BYTE *)dst; - BYTE *op = ostart; - BYTE *const omax = op + maxDstSize; - BYTE *const olimit = omax - 3; - - BIT_DStream_t bitD; - FSE_DState_t state1; - FSE_DState_t state2; - - /* Init */ - CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize)); - - FSE_initDState(&state1, &bitD, dt); - FSE_initDState(&state2, &bitD, dt); - -#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) - - /* 4 symbols per loop */ - for (; (BIT_reloadDStream(&bitD) == BIT_DStream_unfinished) & (op < olimit); op += 4) { - op[0] = FSE_GETSYMBOL(&state1); - - if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[1] = FSE_GETSYMBOL(&state2); - - if (FSE_MAX_TABLELOG * 4 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */ - { - if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { - op += 2; - break; - } - } - - op[2] = FSE_GETSYMBOL(&state1); - - if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[3] = FSE_GETSYMBOL(&state2); - } - - /* tail */ - /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ - while (1) { - if (op > (omax - 2)) - return ERROR(dstSize_tooSmall); - *op++ = FSE_GETSYMBOL(&state1); - if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) { - *op++ = FSE_GETSYMBOL(&state2); - break; - } - - if (op > (omax - 2)) - return ERROR(dstSize_tooSmall); - *op++ = FSE_GETSYMBOL(&state2); - if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) { - *op++ = FSE_GETSYMBOL(&state1); - break; - } - } - - return op - ostart; -} - -size_t FSE_decompress_usingDTable(void *dst, size_t originalSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt) -{ - const void *ptr = dt; - const FSE_DTableHeader *DTableH = (const FSE_DTableHeader *)ptr; - const U32 fastMode = DTableH->fastMode; - - /* select fast mode (static) */ - if (fastMode) - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); -} - -size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, unsigned maxLog, void *workspace, size_t workspaceSize) -{ - const BYTE *const istart = (const BYTE *)cSrc; - const BYTE *ip = istart; - unsigned tableLog; - unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; - size_t NCountLength; - - FSE_DTable *dt; - short *counting; - size_t spaceUsed32 = 0; - - FSE_STATIC_ASSERT(sizeof(FSE_DTable) == sizeof(U32)); - - dt = (FSE_DTable *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += FSE_DTABLE_SIZE_U32(maxLog); - counting = (short *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(sizeof(short) * (FSE_MAX_SYMBOL_VALUE + 1), sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - /* normal FSE decoding mode */ - NCountLength = FSE_readNCount(counting, &maxSymbolValue, &tableLog, istart, cSrcSize); - if (FSE_isError(NCountLength)) - return NCountLength; - // if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining - // case : NCountLength==cSrcSize */ - if (tableLog > maxLog) - return ERROR(tableLog_tooLarge); - ip += NCountLength; - cSrcSize -= NCountLength; - - CHECK_F(FSE_buildDTable_wksp(dt, counting, maxSymbolValue, tableLog, workspace, workspaceSize)); - - return FSE_decompress_usingDTable(dst, dstCapacity, ip, cSrcSize, dt); /* always return, even if it is an error code */ -} diff --git a/contrib/linux-kernel/lib/zstd/huf.h b/contrib/linux-kernel/lib/zstd/huf.h deleted file mode 100644 index 923218d1..00000000 --- a/contrib/linux-kernel/lib/zstd/huf.h +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Huffman coder, part of New Generation Entropy library - * header file - * Copyright (C) 2013-2016, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ -#ifndef HUF_H_298734234 -#define HUF_H_298734234 - -/* *** Dependencies *** */ -#include <linux/types.h> /* size_t */ - -/* *** Tool functions *** */ -#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ -size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ - -/* Error Management */ -unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ - -/* *** Advanced function *** */ - -/** HUF_compress4X_wksp() : -* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */ -size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, - size_t wkspSize); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */ - -/* *** Dependencies *** */ -#include "mem.h" /* U32 */ - -/* *** Constants *** */ -#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ -#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ -#define HUF_SYMBOLVALUE_MAX 255 - -#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ -#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) -#error "HUF_TABLELOG_MAX is too large !" -#endif - -/* **************************************** -* Static allocation -******************************************/ -/* HUF buffer bounds */ -#define HUF_CTABLEBOUND 129 -#define HUF_BLOCKBOUND(size) (size + (size >> 8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ -#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ - -/* static allocation of HUF's Compression Table */ -#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ - U32 name##hb[maxSymbolValue + 1]; \ - void *name##hv = &(name##hb); \ - HUF_CElt *name = (HUF_CElt *)(name##hv) /* no final ; */ - -/* static allocation of HUF's DTable */ -typedef U32 HUF_DTable; -#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1 << (maxTableLog))) -#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = {((U32)((maxTableLog)-1) * 0x01000001)} -#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = {((U32)(maxTableLog)*0x01000001)} - -/* The workspace must have alignment at least 4 and be at least this large */ -#define HUF_COMPRESS_WORKSPACE_SIZE (6 << 10) -#define HUF_COMPRESS_WORKSPACE_SIZE_U32 (HUF_COMPRESS_WORKSPACE_SIZE / sizeof(U32)) - -/* The workspace must have alignment at least 4 and be at least this large */ -#define HUF_DECOMPRESS_WORKSPACE_SIZE (3 << 10) -#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) - -/* **************************************** -* Advanced decompression functions -******************************************/ -size_t HUF_decompress4X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize); /**< decodes RLE and uncompressed */ -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, - size_t workspaceSize); /**< considers RLE and uncompressed as errors */ -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, - size_t workspaceSize); /**< single-symbol decoder */ -size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, - size_t workspaceSize); /**< double-symbols decoder */ - -/* **************************************** -* HUF detailed API -******************************************/ -/*! -HUF_compress() does the following: -1. count symbol occurrence from source[] into table count[] using FSE_count() -2. (optional) refine tableLog using HUF_optimalTableLog() -3. build Huffman table from count using HUF_buildCTable() -4. save Huffman table to memory buffer using HUF_writeCTable_wksp() -5. encode the data stream using HUF_compress4X_usingCTable() - -The following API allows targeting specific sub-functions for advanced tasks. -For example, it's possible to compress several blocks using the same 'CTable', -or to save and regenerate 'CTable' using external methods. -*/ -/* FSE_count() : find it within "fse.h" */ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ -size_t HUF_writeCTable_wksp(void *dst, size_t maxDstSize, const HUF_CElt *CTable, unsigned maxSymbolValue, unsigned huffLog, void *workspace, size_t workspaceSize); -size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable); - -typedef enum { - HUF_repeat_none, /**< Cannot use the previous table */ - HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, - 4}X_repeat */ - HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ -} HUF_repeat; -/** HUF_compress4X_repeat() : -* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. -* If it uses hufTable it does not modify hufTable or repeat. -* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. -* If preferRepeat then the old table will always be used if valid. */ -size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, - size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, - int preferRepeat); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */ - -/** HUF_buildCTable_wksp() : - * Same as HUF_buildCTable(), but using externally allocated scratch buffer. - * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. - */ -size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize); - -/*! HUF_readStats() : - Read compact Huffman tree, saved by HUF_writeCTable(). - `huffWeight` is destination buffer. - @return : size read from `src` , or an error Code . - Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ -size_t HUF_readStats_wksp(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize, - void *workspace, size_t workspaceSize); - -/** HUF_readCTable() : -* Loading a CTable saved with HUF_writeCTable() */ -size_t HUF_readCTable_wksp(HUF_CElt *CTable, unsigned maxSymbolValue, const void *src, size_t srcSize, void *workspace, size_t workspaceSize); - -/* -HUF_decompress() does the following: -1. select the decompression algorithm (X2, X4) based on pre-computed heuristics -2. build Huffman table from save, using HUF_readDTableXn() -3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable -*/ - -/** HUF_selectDecoder() : -* Tells which decoder is likely to decode faster, -* based on a set of pre-determined metrics. -* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . -* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize); - -size_t HUF_readDTableX2_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize); -size_t HUF_readDTableX4_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize); - -size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); -size_t HUF_decompress4X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); -size_t HUF_decompress4X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); - -/* single stream variants */ - -size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, - size_t wkspSize); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */ -size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable); -/** HUF_compress1X_repeat() : -* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. -* If it uses hufTable it does not modify hufTable or repeat. -* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. -* If preferRepeat then the old table will always be used if valid. */ -size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, - size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, - int preferRepeat); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */ - -size_t HUF_decompress1X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize); -size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, - size_t workspaceSize); /**< single-symbol decoder */ -size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, - size_t workspaceSize); /**< double-symbols decoder */ - -size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, - const HUF_DTable *DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ -size_t HUF_decompress1X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); -size_t HUF_decompress1X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); - -#endif /* HUF_H_298734234 */ diff --git a/contrib/linux-kernel/lib/zstd/huf_compress.c b/contrib/linux-kernel/lib/zstd/huf_compress.c deleted file mode 100644 index 40055a70..00000000 --- a/contrib/linux-kernel/lib/zstd/huf_compress.c +++ /dev/null @@ -1,770 +0,0 @@ -/* - * Huffman encoder, part of New Generation Entropy library - * Copyright (C) 2013-2016, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ - -/* ************************************************************** -* Includes -****************************************************************/ -#include "bitstream.h" -#include "fse.h" /* header compression */ -#include "huf.h" -#include <linux/kernel.h> -#include <linux/string.h> /* memcpy, memset */ - -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUF_STATIC_ASSERT(c) \ - { \ - enum { HUF_static_assert = 1 / (int)(!!(c)) }; \ - } /* use only *after* variable declarations */ -#define CHECK_V_F(e, f) \ - size_t const e = f; \ - if (ERR_isError(e)) \ - return f -#define CHECK_F(f) \ - { \ - CHECK_V_F(_var_err__, f); \ - } - -/* ************************************************************** -* Utils -****************************************************************/ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) -{ - return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); -} - -/* ******************************************************* -* HUF : Huffman block compression -*********************************************************/ -/* HUF_compressWeights() : - * Same as FSE_compress(), but dedicated to huff0's weights compression. - * The use case needs much less stack memory. - * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. - */ -#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 -size_t HUF_compressWeights_wksp(void *dst, size_t dstSize, const void *weightTable, size_t wtSize, void *workspace, size_t workspaceSize) -{ - BYTE *const ostart = (BYTE *)dst; - BYTE *op = ostart; - BYTE *const oend = ostart + dstSize; - - U32 maxSymbolValue = HUF_TABLELOG_MAX; - U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; - - FSE_CTable *CTable; - U32 *count; - S16 *norm; - size_t spaceUsed32 = 0; - - HUF_STATIC_ASSERT(sizeof(FSE_CTable) == sizeof(U32)); - - CTable = (FSE_CTable *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX); - count = (U32 *)workspace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 1; - norm = (S16 *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(sizeof(S16) * (HUF_TABLELOG_MAX + 1), sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - /* init conditions */ - if (wtSize <= 1) - return 0; /* Not compressible */ - - /* Scan input and build symbol stats */ - { - CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize)); - if (maxCount == wtSize) - return 1; /* only a single symbol in src : rle */ - if (maxCount == 1) - return 0; /* each symbol present maximum once => not compressible */ - } - - tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); - CHECK_F(FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue)); - - /* Write table description header */ - { - CHECK_V_F(hSize, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog)); - op += hSize; - } - - /* Compress */ - CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, workspace, workspaceSize)); - { - CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable)); - if (cSize == 0) - return 0; /* not enough space for compressed data */ - op += cSize; - } - - return op - ostart; -} - -struct HUF_CElt_s { - U16 val; - BYTE nbBits; -}; /* typedef'd to HUF_CElt within "huf.h" */ - -/*! HUF_writeCTable_wksp() : - `CTable` : Huffman tree to save, using huf representation. - @return : size of saved CTable */ -size_t HUF_writeCTable_wksp(void *dst, size_t maxDstSize, const HUF_CElt *CTable, U32 maxSymbolValue, U32 huffLog, void *workspace, size_t workspaceSize) -{ - BYTE *op = (BYTE *)dst; - U32 n; - - BYTE *bitsToWeight; - BYTE *huffWeight; - size_t spaceUsed32 = 0; - - bitsToWeight = (BYTE *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(HUF_TABLELOG_MAX + 1, sizeof(U32)) >> 2; - huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX, sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - /* check conditions */ - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) - return ERROR(maxSymbolValue_tooLarge); - - /* convert to weight */ - bitsToWeight[0] = 0; - for (n = 1; n < huffLog + 1; n++) - bitsToWeight[n] = (BYTE)(huffLog + 1 - n); - for (n = 0; n < maxSymbolValue; n++) - huffWeight[n] = bitsToWeight[CTable[n].nbBits]; - - /* attempt weights compression by FSE */ - { - CHECK_V_F(hSize, HUF_compressWeights_wksp(op + 1, maxDstSize - 1, huffWeight, maxSymbolValue, workspace, workspaceSize)); - if ((hSize > 1) & (hSize < maxSymbolValue / 2)) { /* FSE compressed */ - op[0] = (BYTE)hSize; - return hSize + 1; - } - } - - /* write raw values as 4-bits (max : 15) */ - if (maxSymbolValue > (256 - 128)) - return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ - if (((maxSymbolValue + 1) / 2) + 1 > maxDstSize) - return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ - op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue - 1)); - huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ - for (n = 0; n < maxSymbolValue; n += 2) - op[(n / 2) + 1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n + 1]); - return ((maxSymbolValue + 1) / 2) + 1; -} - -size_t HUF_readCTable_wksp(HUF_CElt *CTable, U32 maxSymbolValue, const void *src, size_t srcSize, void *workspace, size_t workspaceSize) -{ - U32 *rankVal; - BYTE *huffWeight; - U32 tableLog = 0; - U32 nbSymbols = 0; - size_t readSize; - size_t spaceUsed32 = 0; - - rankVal = (U32 *)workspace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1; - huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - /* get symbol weights */ - readSize = HUF_readStats_wksp(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize); - if (ERR_isError(readSize)) - return readSize; - - /* check result */ - if (tableLog > HUF_TABLELOG_MAX) - return ERROR(tableLog_tooLarge); - if (nbSymbols > maxSymbolValue + 1) - return ERROR(maxSymbolValue_tooSmall); - - /* Prepare base value per rank */ - { - U32 n, nextRankStart = 0; - for (n = 1; n <= tableLog; n++) { - U32 curr = nextRankStart; - nextRankStart += (rankVal[n] << (n - 1)); - rankVal[n] = curr; - } - } - - /* fill nbBits */ - { - U32 n; - for (n = 0; n < nbSymbols; n++) { - const U32 w = huffWeight[n]; - CTable[n].nbBits = (BYTE)(tableLog + 1 - w); - } - } - - /* fill val */ - { - U16 nbPerRank[HUF_TABLELOG_MAX + 2] = {0}; /* support w=0=>n=tableLog+1 */ - U16 valPerRank[HUF_TABLELOG_MAX + 2] = {0}; - { - U32 n; - for (n = 0; n < nbSymbols; n++) - nbPerRank[CTable[n].nbBits]++; - } - /* determine stating value per rank */ - valPerRank[tableLog + 1] = 0; /* for w==0 */ - { - U16 min = 0; - U32 n; - for (n = tableLog; n > 0; n--) { /* start at n=tablelog <-> w=1 */ - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } - } - /* assign value within rank, symbol order */ - { - U32 n; - for (n = 0; n <= maxSymbolValue; n++) - CTable[n].val = valPerRank[CTable[n].nbBits]++; - } - } - - return readSize; -} - -typedef struct nodeElt_s { - U32 count; - U16 parent; - BYTE byte; - BYTE nbBits; -} nodeElt; - -static U32 HUF_setMaxHeight(nodeElt *huffNode, U32 lastNonNull, U32 maxNbBits) -{ - const U32 largestBits = huffNode[lastNonNull].nbBits; - if (largestBits <= maxNbBits) - return largestBits; /* early exit : no elt > maxNbBits */ - - /* there are several too large elements (at least >= 2) */ - { - int totalCost = 0; - const U32 baseCost = 1 << (largestBits - maxNbBits); - U32 n = lastNonNull; - - while (huffNode[n].nbBits > maxNbBits) { - totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); - huffNode[n].nbBits = (BYTE)maxNbBits; - n--; - } /* n stops at huffNode[n].nbBits <= maxNbBits */ - while (huffNode[n].nbBits == maxNbBits) - n--; /* n end at index of smallest symbol using < maxNbBits */ - - /* renorm totalCost */ - totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ - - /* repay normalized cost */ - { - U32 const noSymbol = 0xF0F0F0F0; - U32 rankLast[HUF_TABLELOG_MAX + 2]; - int pos; - - /* Get pos of last (smallest) symbol per rank */ - memset(rankLast, 0xF0, sizeof(rankLast)); - { - U32 currNbBits = maxNbBits; - for (pos = n; pos >= 0; pos--) { - if (huffNode[pos].nbBits >= currNbBits) - continue; - currNbBits = huffNode[pos].nbBits; /* < maxNbBits */ - rankLast[maxNbBits - currNbBits] = pos; - } - } - - while (totalCost > 0) { - U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; - for (; nBitsToDecrease > 1; nBitsToDecrease--) { - U32 highPos = rankLast[nBitsToDecrease]; - U32 lowPos = rankLast[nBitsToDecrease - 1]; - if (highPos == noSymbol) - continue; - if (lowPos == noSymbol) - break; - { - U32 const highTotal = huffNode[highPos].count; - U32 const lowTotal = 2 * huffNode[lowPos].count; - if (highTotal <= lowTotal) - break; - } - } - /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ - /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ - while ((nBitsToDecrease <= HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) - nBitsToDecrease++; - totalCost -= 1 << (nBitsToDecrease - 1); - if (rankLast[nBitsToDecrease - 1] == noSymbol) - rankLast[nBitsToDecrease - 1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ - huffNode[rankLast[nBitsToDecrease]].nbBits++; - if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ - rankLast[nBitsToDecrease] = noSymbol; - else { - rankLast[nBitsToDecrease]--; - if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits - nBitsToDecrease) - rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ - } - } /* while (totalCost > 0) */ - - while (totalCost < 0) { /* Sometimes, cost correction overshoot */ - if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 - (using maxNbBits) */ - while (huffNode[n].nbBits == maxNbBits) - n--; - huffNode[n + 1].nbBits--; - rankLast[1] = n + 1; - totalCost++; - continue; - } - huffNode[rankLast[1] + 1].nbBits--; - rankLast[1]++; - totalCost++; - } - } - } /* there are several too large elements (at least >= 2) */ - - return maxNbBits; -} - -typedef struct { - U32 base; - U32 curr; -} rankPos; - -static void HUF_sort(nodeElt *huffNode, const U32 *count, U32 maxSymbolValue) -{ - rankPos rank[32]; - U32 n; - - memset(rank, 0, sizeof(rank)); - for (n = 0; n <= maxSymbolValue; n++) { - U32 r = BIT_highbit32(count[n] + 1); - rank[r].base++; - } - for (n = 30; n > 0; n--) - rank[n - 1].base += rank[n].base; - for (n = 0; n < 32; n++) - rank[n].curr = rank[n].base; - for (n = 0; n <= maxSymbolValue; n++) { - U32 const c = count[n]; - U32 const r = BIT_highbit32(c + 1) + 1; - U32 pos = rank[r].curr++; - while ((pos > rank[r].base) && (c > huffNode[pos - 1].count)) - huffNode[pos] = huffNode[pos - 1], pos--; - huffNode[pos].count = c; - huffNode[pos].byte = (BYTE)n; - } -} - -/** HUF_buildCTable_wksp() : - * Same as HUF_buildCTable(), but using externally allocated scratch buffer. - * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. - */ -#define STARTNODE (HUF_SYMBOLVALUE_MAX + 1) -typedef nodeElt huffNodeTable[2 * HUF_SYMBOLVALUE_MAX + 1 + 1]; -size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize) -{ - nodeElt *const huffNode0 = (nodeElt *)workSpace; - nodeElt *const huffNode = huffNode0 + 1; - U32 n, nonNullRank; - int lowS, lowN; - U16 nodeNb = STARTNODE; - U32 nodeRoot; - - /* safety checks */ - if (wkspSize < sizeof(huffNodeTable)) - return ERROR(GENERIC); /* workSpace is not large enough */ - if (maxNbBits == 0) - maxNbBits = HUF_TABLELOG_DEFAULT; - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) - return ERROR(GENERIC); - memset(huffNode0, 0, sizeof(huffNodeTable)); - - /* sort, decreasing order */ - HUF_sort(huffNode, count, maxSymbolValue); - - /* init for parents */ - nonNullRank = maxSymbolValue; - while (huffNode[nonNullRank].count == 0) - nonNullRank--; - lowS = nonNullRank; - nodeRoot = nodeNb + lowS - 1; - lowN = nodeNb; - huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS - 1].count; - huffNode[lowS].parent = huffNode[lowS - 1].parent = nodeNb; - nodeNb++; - lowS -= 2; - for (n = nodeNb; n <= nodeRoot; n++) - huffNode[n].count = (U32)(1U << 30); - huffNode0[0].count = (U32)(1U << 31); /* fake entry, strong barrier */ - - /* create parents */ - while (nodeNb <= nodeRoot) { - U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; - huffNode[n1].parent = huffNode[n2].parent = nodeNb; - nodeNb++; - } - - /* distribute weights (unlimited tree height) */ - huffNode[nodeRoot].nbBits = 0; - for (n = nodeRoot - 1; n >= STARTNODE; n--) - huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1; - for (n = 0; n <= nonNullRank; n++) - huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1; - - /* enforce maxTableLog */ - maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); - - /* fill result into tree (val, nbBits) */ - { - U16 nbPerRank[HUF_TABLELOG_MAX + 1] = {0}; - U16 valPerRank[HUF_TABLELOG_MAX + 1] = {0}; - if (maxNbBits > HUF_TABLELOG_MAX) - return ERROR(GENERIC); /* check fit into table */ - for (n = 0; n <= nonNullRank; n++) - nbPerRank[huffNode[n].nbBits]++; - /* determine stating value per rank */ - { - U16 min = 0; - for (n = maxNbBits; n > 0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } - } - for (n = 0; n <= maxSymbolValue; n++) - tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ - for (n = 0; n <= maxSymbolValue; n++) - tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ - } - - return maxNbBits; -} - -static size_t HUF_estimateCompressedSize(HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue) -{ - size_t nbBits = 0; - int s; - for (s = 0; s <= (int)maxSymbolValue; ++s) { - nbBits += CTable[s].nbBits * count[s]; - } - return nbBits >> 3; -} - -static int HUF_validateCTable(const HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue) -{ - int bad = 0; - int s; - for (s = 0; s <= (int)maxSymbolValue; ++s) { - bad |= (count[s] != 0) & (CTable[s].nbBits == 0); - } - return !bad; -} - -static void HUF_encodeSymbol(BIT_CStream_t *bitCPtr, U32 symbol, const HUF_CElt *CTable) -{ - BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); -} - -size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } - -#define HUF_FLUSHBITS(s) BIT_flushBits(s) - -#define HUF_FLUSHBITS_1(stream) \ - if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \ - HUF_FLUSHBITS(stream) - -#define HUF_FLUSHBITS_2(stream) \ - if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 4 + 7) \ - HUF_FLUSHBITS(stream) - -size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable) -{ - const BYTE *ip = (const BYTE *)src; - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstSize; - BYTE *op = ostart; - size_t n; - BIT_CStream_t bitC; - - /* init */ - if (dstSize < 8) - return 0; /* not enough space to compress */ - { - size_t const initErr = BIT_initCStream(&bitC, op, oend - op); - if (HUF_isError(initErr)) - return 0; - } - - n = srcSize & ~3; /* join to mod 4 */ - switch (srcSize & 3) { - case 3: HUF_encodeSymbol(&bitC, ip[n + 2], CTable); HUF_FLUSHBITS_2(&bitC); - case 2: HUF_encodeSymbol(&bitC, ip[n + 1], CTable); HUF_FLUSHBITS_1(&bitC); - case 1: HUF_encodeSymbol(&bitC, ip[n + 0], CTable); HUF_FLUSHBITS(&bitC); - case 0: - default:; - } - - for (; n > 0; n -= 4) { /* note : n&3==0 at this stage */ - HUF_encodeSymbol(&bitC, ip[n - 1], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n - 2], CTable); - HUF_FLUSHBITS_2(&bitC); - HUF_encodeSymbol(&bitC, ip[n - 3], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n - 4], CTable); - HUF_FLUSHBITS(&bitC); - } - - return BIT_closeCStream(&bitC); -} - -size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable) -{ - size_t const segmentSize = (srcSize + 3) / 4; /* first 3 segments */ - const BYTE *ip = (const BYTE *)src; - const BYTE *const iend = ip + srcSize; - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstSize; - BYTE *op = ostart; - - if (dstSize < 6 + 1 + 1 + 1 + 8) - return 0; /* minimum space to compress successfully */ - if (srcSize < 12) - return 0; /* no saving possible : too small input */ - op += 6; /* jumpTable */ - - { - CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable)); - if (cSize == 0) - return 0; - ZSTD_writeLE16(ostart, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - { - CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable)); - if (cSize == 0) - return 0; - ZSTD_writeLE16(ostart + 2, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - { - CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable)); - if (cSize == 0) - return 0; - ZSTD_writeLE16(ostart + 4, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - { - CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, iend - ip, CTable)); - if (cSize == 0) - return 0; - op += cSize; - } - - return op - ostart; -} - -static size_t HUF_compressCTable_internal(BYTE *const ostart, BYTE *op, BYTE *const oend, const void *src, size_t srcSize, unsigned singleStream, - const HUF_CElt *CTable) -{ - size_t const cSize = - singleStream ? HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); - if (HUF_isError(cSize)) { - return cSize; - } - if (cSize == 0) { - return 0; - } /* uncompressible */ - op += cSize; - /* check compressibility */ - if ((size_t)(op - ostart) >= srcSize - 1) { - return 0; - } - return op - ostart; -} - -/* `workSpace` must a table of at least 1024 unsigned */ -static size_t HUF_compress_internal(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, - unsigned singleStream, void *workSpace, size_t wkspSize, HUF_CElt *oldHufTable, HUF_repeat *repeat, int preferRepeat) -{ - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstSize; - BYTE *op = ostart; - - U32 *count; - size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1); - HUF_CElt *CTable; - size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1); - - /* checks & inits */ - if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize) - return ERROR(GENERIC); - if (!srcSize) - return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */ - if (!dstSize) - return 0; /* cannot fit within dst budget */ - if (srcSize > HUF_BLOCKSIZE_MAX) - return ERROR(srcSize_wrong); /* curr block size limit */ - if (huffLog > HUF_TABLELOG_MAX) - return ERROR(tableLog_tooLarge); - if (!maxSymbolValue) - maxSymbolValue = HUF_SYMBOLVALUE_MAX; - if (!huffLog) - huffLog = HUF_TABLELOG_DEFAULT; - - count = (U32 *)workSpace; - workSpace = (BYTE *)workSpace + countSize; - wkspSize -= countSize; - CTable = (HUF_CElt *)workSpace; - workSpace = (BYTE *)workSpace + CTableSize; - wkspSize -= CTableSize; - - /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */ - if (preferRepeat && repeat && *repeat == HUF_repeat_valid) { - return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable); - } - - /* Scan input and build symbol stats */ - { - CHECK_V_F(largest, FSE_count_wksp(count, &maxSymbolValue, (const BYTE *)src, srcSize, (U32 *)workSpace)); - if (largest == srcSize) { - *ostart = ((const BYTE *)src)[0]; - return 1; - } /* single symbol, rle */ - if (largest <= (srcSize >> 7) + 1) - return 0; /* Fast heuristic : not compressible enough */ - } - - /* Check validity of previous table */ - if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) { - *repeat = HUF_repeat_none; - } - /* Heuristic : use existing table for small inputs */ - if (preferRepeat && repeat && *repeat != HUF_repeat_none) { - return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable); - } - - /* Build Huffman Tree */ - huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); - { - CHECK_V_F(maxBits, HUF_buildCTable_wksp(CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize)); - huffLog = (U32)maxBits; - /* Zero the unused symbols so we can check it for validity */ - memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt)); - } - - /* Write table description header */ - { - CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, CTable, maxSymbolValue, huffLog, workSpace, wkspSize)); - /* Check if using the previous table will be beneficial */ - if (repeat && *repeat != HUF_repeat_none) { - size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue); - size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue); - if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { - return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable); - } - } - /* Use the new table */ - if (hSize + 12ul >= srcSize) { - return 0; - } - op += hSize; - if (repeat) { - *repeat = HUF_repeat_none; - } - if (oldHufTable) { - memcpy(oldHufTable, CTable, CTableSize); - } /* Save the new table */ - } - return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable); -} - -size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace, - size_t wkspSize) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0); -} - -size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace, - size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat, - preferRepeat); -} - -size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace, - size_t wkspSize) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0); -} - -size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace, - size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat, - preferRepeat); -} diff --git a/contrib/linux-kernel/lib/zstd/huf_decompress.c b/contrib/linux-kernel/lib/zstd/huf_decompress.c deleted file mode 100644 index 65264820..00000000 --- a/contrib/linux-kernel/lib/zstd/huf_decompress.c +++ /dev/null @@ -1,960 +0,0 @@ -/* - * Huffman decoder, part of New Generation Entropy library - * Copyright (C) 2013-2016, Yann Collet. - * - * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#define FORCE_INLINE static __always_inline - -/* ************************************************************** -* Dependencies -****************************************************************/ -#include "bitstream.h" /* BIT_* */ -#include "fse.h" /* header compression */ -#include "huf.h" -#include <linux/compiler.h> -#include <linux/kernel.h> -#include <linux/string.h> /* memcpy, memset */ - -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUF_STATIC_ASSERT(c) \ - { \ - enum { HUF_static_assert = 1 / (int)(!!(c)) }; \ - } /* use only *after* variable declarations */ - -/*-***************************/ -/* generic DTableDesc */ -/*-***************************/ - -typedef struct { - BYTE maxTableLog; - BYTE tableType; - BYTE tableLog; - BYTE reserved; -} DTableDesc; - -static DTableDesc HUF_getDTableDesc(const HUF_DTable *table) -{ - DTableDesc dtd; - memcpy(&dtd, table, sizeof(dtd)); - return dtd; -} - -/*-***************************/ -/* single-symbol decoding */ -/*-***************************/ - -typedef struct { - BYTE byte; - BYTE nbBits; -} HUF_DEltX2; /* single-symbol decoding */ - -size_t HUF_readDTableX2_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize) -{ - U32 tableLog = 0; - U32 nbSymbols = 0; - size_t iSize; - void *const dtPtr = DTable + 1; - HUF_DEltX2 *const dt = (HUF_DEltX2 *)dtPtr; - - U32 *rankVal; - BYTE *huffWeight; - size_t spaceUsed32 = 0; - - rankVal = (U32 *)workspace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1; - huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); - /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ - - iSize = HUF_readStats_wksp(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize); - if (HUF_isError(iSize)) - return iSize; - - /* Table header */ - { - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (tableLog > (U32)(dtd.maxTableLog + 1)) - return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ - dtd.tableType = 0; - dtd.tableLog = (BYTE)tableLog; - memcpy(DTable, &dtd, sizeof(dtd)); - } - - /* Calculate starting value for each rank */ - { - U32 n, nextRankStart = 0; - for (n = 1; n < tableLog + 1; n++) { - U32 const curr = nextRankStart; - nextRankStart += (rankVal[n] << (n - 1)); - rankVal[n] = curr; - } - } - - /* fill DTable */ - { - U32 n; - for (n = 0; n < nbSymbols; n++) { - U32 const w = huffWeight[n]; - U32 const length = (1 << w) >> 1; - U32 u; - HUF_DEltX2 D; - D.byte = (BYTE)n; - D.nbBits = (BYTE)(tableLog + 1 - w); - for (u = rankVal[w]; u < rankVal[w] + length; u++) - dt[u] = D; - rankVal[w] += length; - } - } - - return iSize; -} - -static BYTE HUF_decodeSymbolX2(BIT_DStream_t *Dstream, const HUF_DEltX2 *dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ - BYTE const c = dt[val].byte; - BIT_skipBits(Dstream, dt[val].nbBits); - return c; -} - -#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ - if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \ - HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ - if (ZSTD_64bits()) \ - HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -FORCE_INLINE size_t HUF_decodeStreamX2(BYTE *p, BIT_DStream_t *const bitDPtr, BYTE *const pEnd, const HUF_DEltX2 *const dt, const U32 dtLog) -{ - BYTE *const pStart = p; - - /* up to 4 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd - 4)) { - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_1(p, bitDPtr); - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - } - - /* closer to the end */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - - /* no more data to retrieve from bitstream, hence no need to reload */ - while (p < pEnd) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - - return pEnd - pStart; -} - -static size_t HUF_decompress1X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - BYTE *op = (BYTE *)dst; - BYTE *const oend = op + dstSize; - const void *dtPtr = DTable + 1; - const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr; - BIT_DStream_t bitD; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - { - size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); - if (HUF_isError(errorCode)) - return errorCode; - } - - HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); - - /* check */ - if (!BIT_endOfDStream(&bitD)) - return ERROR(corruption_detected); - - return dstSize; -} - -size_t HUF_decompress1X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) - return ERROR(GENERIC); - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - const BYTE *ip = (const BYTE *)cSrc; - - size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, workspace, workspaceSize); - if (HUF_isError(hSize)) - return hSize; - if (hSize >= cSrcSize) - return ERROR(srcSize_wrong); - ip += hSize; - cSrcSize -= hSize; - - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx); -} - -static size_t HUF_decompress4X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - /* Check */ - if (cSrcSize < 10) - return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { - const BYTE *const istart = (const BYTE *)cSrc; - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstSize; - const void *const dtPtr = DTable + 1; - const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = ZSTD_readLE16(istart); - size_t const length2 = ZSTD_readLE16(istart + 2); - size_t const length3 = ZSTD_readLE16(istart + 4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE *const istart1 = istart + 6; /* jumpTable */ - const BYTE *const istart2 = istart1 + length1; - const BYTE *const istart3 = istart2 + length2; - const BYTE *const istart4 = istart3 + length3; - const size_t segmentSize = (dstSize + 3) / 4; - BYTE *const opStart2 = ostart + segmentSize; - BYTE *const opStart3 = opStart2 + segmentSize; - BYTE *const opStart4 = opStart3 + segmentSize; - BYTE *op1 = ostart; - BYTE *op2 = opStart2; - BYTE *op3 = opStart3; - BYTE *op4 = opStart4; - U32 endSignal; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - if (length4 > cSrcSize) - return ERROR(corruption_detected); /* overflow */ - { - size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUF_isError(errorCode)) - return errorCode; - } - { - size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUF_isError(errorCode)) - return errorCode; - } - { - size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUF_isError(errorCode)) - return errorCode; - } - { - size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUF_isError(errorCode)) - return errorCode; - } - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for (; (endSignal == BIT_DStream_unfinished) && (op4 < (oend - 7));) { - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_1(op1, &bitD1); - HUF_DECODE_SYMBOLX2_1(op2, &bitD2); - HUF_DECODE_SYMBOLX2_1(op3, &bitD3); - HUF_DECODE_SYMBOLX2_1(op4, &bitD4); - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_0(op1, &bitD1); - HUF_DECODE_SYMBOLX2_0(op2, &bitD2); - HUF_DECODE_SYMBOLX2_0(op3, &bitD3); - HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) - return ERROR(corruption_detected); - if (op2 > opStart3) - return ERROR(corruption_detected); - if (op3 > opStart4) - return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ - - /* finish bitStreams one by one */ - HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); - - /* check */ - endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endSignal) - return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; - } -} - -size_t HUF_decompress4X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) - return ERROR(GENERIC); - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - const BYTE *ip = (const BYTE *)cSrc; - - size_t const hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, workspace, workspaceSize); - if (HUF_isError(hSize)) - return hSize; - if (hSize >= cSrcSize) - return ERROR(srcSize_wrong); - ip += hSize; - cSrcSize -= hSize; - - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); -} - -/* *************************/ -/* double-symbols decoding */ -/* *************************/ -typedef struct { - U16 sequence; - BYTE nbBits; - BYTE length; -} HUF_DEltX4; /* double-symbols decoding */ - -typedef struct { - BYTE symbol; - BYTE weight; -} sortedSymbol_t; - -/* HUF_fillDTableX4Level2() : - * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ -static void HUF_fillDTableX4Level2(HUF_DEltX4 *DTable, U32 sizeLog, const U32 consumed, const U32 *rankValOrigin, const int minWeight, - const sortedSymbol_t *sortedSymbols, const U32 sortedListSize, U32 nbBitsBaseline, U16 baseSeq) -{ - HUF_DEltX4 DElt; - U32 rankVal[HUF_TABLELOG_MAX + 1]; - - /* get pre-calculated rankVal */ - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill skipped values */ - if (minWeight > 1) { - U32 i, skipSize = rankVal[minWeight]; - ZSTD_writeLE16(&(DElt.sequence), baseSeq); - DElt.nbBits = (BYTE)(consumed); - DElt.length = 1; - for (i = 0; i < skipSize; i++) - DTable[i] = DElt; - } - - /* fill DTable */ - { - U32 s; - for (s = 0; s < sortedListSize; s++) { /* note : sortedSymbols already skipped */ - const U32 symbol = sortedSymbols[s].symbol; - const U32 weight = sortedSymbols[s].weight; - const U32 nbBits = nbBitsBaseline - weight; - const U32 length = 1 << (sizeLog - nbBits); - const U32 start = rankVal[weight]; - U32 i = start; - const U32 end = start + length; - - ZSTD_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); - DElt.nbBits = (BYTE)(nbBits + consumed); - DElt.length = 2; - do { - DTable[i++] = DElt; - } while (i < end); /* since length >= 1 */ - - rankVal[weight] += length; - } - } -} - -typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1]; -typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; - -static void HUF_fillDTableX4(HUF_DEltX4 *DTable, const U32 targetLog, const sortedSymbol_t *sortedList, const U32 sortedListSize, const U32 *rankStart, - rankVal_t rankValOrigin, const U32 maxWeight, const U32 nbBitsBaseline) -{ - U32 rankVal[HUF_TABLELOG_MAX + 1]; - const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ - const U32 minBits = nbBitsBaseline - maxWeight; - U32 s; - - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill DTable */ - for (s = 0; s < sortedListSize; s++) { - const U16 symbol = sortedList[s].symbol; - const U32 weight = sortedList[s].weight; - const U32 nbBits = nbBitsBaseline - weight; - const U32 start = rankVal[weight]; - const U32 length = 1 << (targetLog - nbBits); - - if (targetLog - nbBits >= minBits) { /* enough room for a second symbol */ - U32 sortedRank; - int minWeight = nbBits + scaleLog; - if (minWeight < 1) - minWeight = 1; - sortedRank = rankStart[minWeight]; - HUF_fillDTableX4Level2(DTable + start, targetLog - nbBits, nbBits, rankValOrigin[nbBits], minWeight, sortedList + sortedRank, - sortedListSize - sortedRank, nbBitsBaseline, symbol); - } else { - HUF_DEltX4 DElt; - ZSTD_writeLE16(&(DElt.sequence), symbol); - DElt.nbBits = (BYTE)(nbBits); - DElt.length = 1; - { - U32 const end = start + length; - U32 u; - for (u = start; u < end; u++) - DTable[u] = DElt; - } - } - rankVal[weight] += length; - } -} - -size_t HUF_readDTableX4_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize) -{ - U32 tableLog, maxW, sizeOfSort, nbSymbols; - DTableDesc dtd = HUF_getDTableDesc(DTable); - U32 const maxTableLog = dtd.maxTableLog; - size_t iSize; - void *dtPtr = DTable + 1; /* force compiler to avoid strict-aliasing */ - HUF_DEltX4 *const dt = (HUF_DEltX4 *)dtPtr; - U32 *rankStart; - - rankValCol_t *rankVal; - U32 *rankStats; - U32 *rankStart0; - sortedSymbol_t *sortedSymbol; - BYTE *weightList; - size_t spaceUsed32 = 0; - - HUF_STATIC_ASSERT((sizeof(rankValCol_t) & 3) == 0); - - rankVal = (rankValCol_t *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2; - rankStats = (U32 *)workspace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 1; - rankStart0 = (U32 *)workspace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 2; - sortedSymbol = (sortedSymbol_t *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2; - weightList = (BYTE *)((U32 *)workspace + spaceUsed32); - spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > workspaceSize) - return ERROR(tableLog_tooLarge); - workspace = (U32 *)workspace + spaceUsed32; - workspaceSize -= (spaceUsed32 << 2); - - rankStart = rankStart0 + 1; - memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1)); - - HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ - if (maxTableLog > HUF_TABLELOG_MAX) - return ERROR(tableLog_tooLarge); - /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ - - iSize = HUF_readStats_wksp(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize); - if (HUF_isError(iSize)) - return iSize; - - /* check result */ - if (tableLog > maxTableLog) - return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ - - /* find maxWeight */ - for (maxW = tableLog; rankStats[maxW] == 0; maxW--) { - } /* necessarily finds a solution before 0 */ - - /* Get start index of each weight */ - { - U32 w, nextRankStart = 0; - for (w = 1; w < maxW + 1; w++) { - U32 curr = nextRankStart; - nextRankStart += rankStats[w]; - rankStart[w] = curr; - } - rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ - sizeOfSort = nextRankStart; - } - - /* sort symbols by weight */ - { - U32 s; - for (s = 0; s < nbSymbols; s++) { - U32 const w = weightList[s]; - U32 const r = rankStart[w]++; - sortedSymbol[r].symbol = (BYTE)s; - sortedSymbol[r].weight = (BYTE)w; - } - rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ - } - - /* Build rankVal */ - { - U32 *const rankVal0 = rankVal[0]; - { - int const rescale = (maxTableLog - tableLog) - 1; /* tableLog <= maxTableLog */ - U32 nextRankVal = 0; - U32 w; - for (w = 1; w < maxW + 1; w++) { - U32 curr = nextRankVal; - nextRankVal += rankStats[w] << (w + rescale); - rankVal0[w] = curr; - } - } - { - U32 const minBits = tableLog + 1 - maxW; - U32 consumed; - for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { - U32 *const rankValPtr = rankVal[consumed]; - U32 w; - for (w = 1; w < maxW + 1; w++) { - rankValPtr[w] = rankVal0[w] >> consumed; - } - } - } - } - - HUF_fillDTableX4(dt, maxTableLog, sortedSymbol, sizeOfSort, rankStart0, rankVal, maxW, tableLog + 1); - - dtd.tableLog = (BYTE)maxTableLog; - dtd.tableType = 1; - memcpy(DTable, &dtd, sizeof(dtd)); - return iSize; -} - -static U32 HUF_decodeSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt + val, 2); - BIT_skipBits(DStream, dt[val].nbBits); - return dt[val].length; -} - -static U32 HUF_decodeLastSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt + val, 1); - if (dt[val].length == 1) - BIT_skipBits(DStream, dt[val].nbBits); - else { - if (DStream->bitsConsumed < (sizeof(DStream->bitContainer) * 8)) { - BIT_skipBits(DStream, dt[val].nbBits); - if (DStream->bitsConsumed > (sizeof(DStream->bitContainer) * 8)) - /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ - DStream->bitsConsumed = (sizeof(DStream->bitContainer) * 8); - } - } - return 1; -} - -#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ - if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \ - ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ - if (ZSTD_64bits()) \ - ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -FORCE_INLINE size_t HUF_decodeStreamX4(BYTE *p, BIT_DStream_t *bitDPtr, BYTE *const pEnd, const HUF_DEltX4 *const dt, const U32 dtLog) -{ - BYTE *const pStart = p; - - /* up to 8 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd - (sizeof(bitDPtr->bitContainer) - 1))) { - HUF_DECODE_SYMBOLX4_2(p, bitDPtr); - HUF_DECODE_SYMBOLX4_1(p, bitDPtr); - HUF_DECODE_SYMBOLX4_2(p, bitDPtr); - HUF_DECODE_SYMBOLX4_0(p, bitDPtr); - } - - /* closer to end : up to 2 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd - 2)) - HUF_DECODE_SYMBOLX4_0(p, bitDPtr); - - while (p <= pEnd - 2) - HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ - - if (p < pEnd) - p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); - - return p - pStart; -} - -static size_t HUF_decompress1X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - BIT_DStream_t bitD; - - /* Init */ - { - size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); - if (HUF_isError(errorCode)) - return errorCode; - } - - /* decode */ - { - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstSize; - const void *const dtPtr = DTable + 1; /* force compiler to not use strict-aliasing */ - const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); - } - - /* check */ - if (!BIT_endOfDStream(&bitD)) - return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; -} - -size_t HUF_decompress1X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) - return ERROR(GENERIC); - return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - const BYTE *ip = (const BYTE *)cSrc; - - size_t const hSize = HUF_readDTableX4_wksp(DCtx, cSrc, cSrcSize, workspace, workspaceSize); - if (HUF_isError(hSize)) - return hSize; - if (hSize >= cSrcSize) - return ERROR(srcSize_wrong); - ip += hSize; - cSrcSize -= hSize; - - return HUF_decompress1X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx); -} - -static size_t HUF_decompress4X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - if (cSrcSize < 10) - return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { - const BYTE *const istart = (const BYTE *)cSrc; - BYTE *const ostart = (BYTE *)dst; - BYTE *const oend = ostart + dstSize; - const void *const dtPtr = DTable + 1; - const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = ZSTD_readLE16(istart); - size_t const length2 = ZSTD_readLE16(istart + 2); - size_t const length3 = ZSTD_readLE16(istart + 4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE *const istart1 = istart + 6; /* jumpTable */ - const BYTE *const istart2 = istart1 + length1; - const BYTE *const istart3 = istart2 + length2; - const BYTE *const istart4 = istart3 + length3; - size_t const segmentSize = (dstSize + 3) / 4; - BYTE *const opStart2 = ostart + segmentSize; - BYTE *const opStart3 = opStart2 + segmentSize; - BYTE *const opStart4 = opStart3 + segmentSize; - BYTE *op1 = ostart; - BYTE *op2 = opStart2; - BYTE *op3 = opStart3; - BYTE *op4 = opStart4; - U32 endSignal; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - if (length4 > cSrcSize) - return ERROR(corruption_detected); /* overflow */ - { - size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUF_isError(errorCode)) - return errorCode; - } - { - size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUF_isError(errorCode)) - return errorCode; - } - { - size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUF_isError(errorCode)) - return errorCode; - } - { - size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUF_isError(errorCode)) - return errorCode; - } - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for (; (endSignal == BIT_DStream_unfinished) & (op4 < (oend - (sizeof(bitD4.bitContainer) - 1)));) { - HUF_DECODE_SYMBOLX4_2(op1, &bitD1); - HUF_DECODE_SYMBOLX4_2(op2, &bitD2); - HUF_DECODE_SYMBOLX4_2(op3, &bitD3); - HUF_DECODE_SYMBOLX4_2(op4, &bitD4); - HUF_DECODE_SYMBOLX4_1(op1, &bitD1); - HUF_DECODE_SYMBOLX4_1(op2, &bitD2); - HUF_DECODE_SYMBOLX4_1(op3, &bitD3); - HUF_DECODE_SYMBOLX4_1(op4, &bitD4); - HUF_DECODE_SYMBOLX4_2(op1, &bitD1); - HUF_DECODE_SYMBOLX4_2(op2, &bitD2); - HUF_DECODE_SYMBOLX4_2(op3, &bitD3); - HUF_DECODE_SYMBOLX4_2(op4, &bitD4); - HUF_DECODE_SYMBOLX4_0(op1, &bitD1); - HUF_DECODE_SYMBOLX4_0(op2, &bitD2); - HUF_DECODE_SYMBOLX4_0(op3, &bitD3); - HUF_DECODE_SYMBOLX4_0(op4, &bitD4); - - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) - return ERROR(corruption_detected); - if (op2 > opStart3) - return ERROR(corruption_detected); - if (op3 > opStart4) - return ERROR(corruption_detected); - /* note : op4 already verified within main loop */ - - /* finish bitStreams one by one */ - HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); - - /* check */ - { - U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endCheck) - return ERROR(corruption_detected); - } - - /* decoded size */ - return dstSize; - } -} - -size_t HUF_decompress4X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) - return ERROR(GENERIC); - return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - const BYTE *ip = (const BYTE *)cSrc; - - size_t hSize = HUF_readDTableX4_wksp(dctx, cSrc, cSrcSize, workspace, workspaceSize); - if (HUF_isError(hSize)) - return hSize; - if (hSize >= cSrcSize) - return ERROR(srcSize_wrong); - ip += hSize; - cSrcSize -= hSize; - - return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); -} - -/* ********************************/ -/* Generic decompression selector */ -/* ********************************/ - -size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); - return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) - : HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -} - -size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); - return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) - : HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -} - -typedef struct { - U32 tableTime; - U32 decode256Time; -} algo_time_t; -static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = { - /* single, double, quad */ - {{0, 0}, {1, 1}, {2, 2}}, /* Q==0 : impossible */ - {{0, 0}, {1, 1}, {2, 2}}, /* Q==1 : impossible */ - {{38, 130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ - {{448, 128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ - {{556, 128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ - {{714, 128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ - {{883, 128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ - {{897, 128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ - {{926, 128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ - {{947, 128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ - {{1107, 128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ - {{1177, 128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ - {{1242, 128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ - {{1349, 128}, {2644, 106}, {5260, 106}}, /* Q ==13 : 81-87% */ - {{1455, 128}, {2422, 124}, {4174, 124}}, /* Q ==14 : 87-93% */ - {{722, 128}, {1891, 145}, {1936, 146}}, /* Q ==15 : 93-99% */ -}; - -/** HUF_selectDecoder() : -* Tells which decoder is likely to decode faster, -* based on a set of pre-determined metrics. -* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . -* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize) -{ - /* decoder timing evaluation */ - U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ - U32 const D256 = (U32)(dstSize >> 8); - U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); - U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); - DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */ - - return DTime1 < DTime0; -} - -typedef size_t (*decompressionAlgo)(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize); - -size_t HUF_decompress4X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - /* validation checks */ - if (dstSize == 0) - return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) - return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { - memcpy(dst, cSrc, dstSize); - return dstSize; - } /* not compressed */ - if (cSrcSize == 1) { - memset(dst, *(const BYTE *)cSrc, dstSize); - return dstSize; - } /* RLE */ - - { - U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize) - : HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize); - } -} - -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - /* validation checks */ - if (dstSize == 0) - return ERROR(dstSize_tooSmall); - if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) - return ERROR(corruption_detected); /* invalid */ - - { - U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize) - : HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize); - } -} - -size_t HUF_decompress1X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize) -{ - /* validation checks */ - if (dstSize == 0) - return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) - return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { - memcpy(dst, cSrc, dstSize); - return dstSize; - } /* not compressed */ - if (cSrcSize == 1) { - memset(dst, *(const BYTE *)cSrc, dstSize); - return dstSize; - } /* RLE */ - - { - U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUF_decompress1X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize) - : HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize); - } -} diff --git a/contrib/linux-kernel/lib/zstd/mem.h b/contrib/linux-kernel/lib/zstd/mem.h deleted file mode 100644 index 42a697b7..00000000 --- a/contrib/linux-kernel/lib/zstd/mem.h +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -#ifndef MEM_H_MODULE -#define MEM_H_MODULE - -/*-**************************************** -* Dependencies -******************************************/ -#include <asm/unaligned.h> -#include <linux/string.h> /* memcpy */ -#include <linux/types.h> /* size_t, ptrdiff_t */ - -/*-**************************************** -* Compiler specifics -******************************************/ -#define ZSTD_STATIC static __inline __attribute__((unused)) - -/*-************************************************************** -* Basic Types -*****************************************************************/ -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef int16_t S16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -typedef int64_t S64; -typedef ptrdiff_t iPtrDiff; -typedef uintptr_t uPtrDiff; - -/*-************************************************************** -* Memory I/O -*****************************************************************/ -ZSTD_STATIC unsigned ZSTD_32bits(void) { return sizeof(size_t) == 4; } -ZSTD_STATIC unsigned ZSTD_64bits(void) { return sizeof(size_t) == 8; } - -#if defined(__LITTLE_ENDIAN) -#define ZSTD_LITTLE_ENDIAN 1 -#else -#define ZSTD_LITTLE_ENDIAN 0 -#endif - -ZSTD_STATIC unsigned ZSTD_isLittleEndian(void) { return ZSTD_LITTLE_ENDIAN; } - -ZSTD_STATIC U16 ZSTD_read16(const void *memPtr) { return get_unaligned((const U16 *)memPtr); } - -ZSTD_STATIC U32 ZSTD_read32(const void *memPtr) { return get_unaligned((const U32 *)memPtr); } - -ZSTD_STATIC U64 ZSTD_read64(const void *memPtr) { return get_unaligned((const U64 *)memPtr); } - -ZSTD_STATIC size_t ZSTD_readST(const void *memPtr) { return get_unaligned((const size_t *)memPtr); } - -ZSTD_STATIC void ZSTD_write16(void *memPtr, U16 value) { put_unaligned(value, (U16 *)memPtr); } - -ZSTD_STATIC void ZSTD_write32(void *memPtr, U32 value) { put_unaligned(value, (U32 *)memPtr); } - -ZSTD_STATIC void ZSTD_write64(void *memPtr, U64 value) { put_unaligned(value, (U64 *)memPtr); } - -/*=== Little endian r/w ===*/ - -ZSTD_STATIC U16 ZSTD_readLE16(const void *memPtr) { return get_unaligned_le16(memPtr); } - -ZSTD_STATIC void ZSTD_writeLE16(void *memPtr, U16 val) { put_unaligned_le16(val, memPtr); } - -ZSTD_STATIC U32 ZSTD_readLE24(const void *memPtr) { return ZSTD_readLE16(memPtr) + (((const BYTE *)memPtr)[2] << 16); } - -ZSTD_STATIC void ZSTD_writeLE24(void *memPtr, U32 val) -{ - ZSTD_writeLE16(memPtr, (U16)val); - ((BYTE *)memPtr)[2] = (BYTE)(val >> 16); -} - -ZSTD_STATIC U32 ZSTD_readLE32(const void *memPtr) { return get_unaligned_le32(memPtr); } - -ZSTD_STATIC void ZSTD_writeLE32(void *memPtr, U32 val32) { put_unaligned_le32(val32, memPtr); } - -ZSTD_STATIC U64 ZSTD_readLE64(const void *memPtr) { return get_unaligned_le64(memPtr); } - -ZSTD_STATIC void ZSTD_writeLE64(void *memPtr, U64 val64) { put_unaligned_le64(val64, memPtr); } - -ZSTD_STATIC size_t ZSTD_readLEST(const void *memPtr) -{ - if (ZSTD_32bits()) - return (size_t)ZSTD_readLE32(memPtr); - else - return (size_t)ZSTD_readLE64(memPtr); -} - -ZSTD_STATIC void ZSTD_writeLEST(void *memPtr, size_t val) -{ - if (ZSTD_32bits()) - ZSTD_writeLE32(memPtr, (U32)val); - else - ZSTD_writeLE64(memPtr, (U64)val); -} - -/*=== Big endian r/w ===*/ - -ZSTD_STATIC U32 ZSTD_readBE32(const void *memPtr) { return get_unaligned_be32(memPtr); } - -ZSTD_STATIC void ZSTD_writeBE32(void *memPtr, U32 val32) { put_unaligned_be32(val32, memPtr); } - -ZSTD_STATIC U64 ZSTD_readBE64(const void *memPtr) { return get_unaligned_be64(memPtr); } - -ZSTD_STATIC void ZSTD_writeBE64(void *memPtr, U64 val64) { put_unaligned_be64(val64, memPtr); } - -ZSTD_STATIC size_t ZSTD_readBEST(const void *memPtr) -{ - if (ZSTD_32bits()) - return (size_t)ZSTD_readBE32(memPtr); - else - return (size_t)ZSTD_readBE64(memPtr); -} - -ZSTD_STATIC void ZSTD_writeBEST(void *memPtr, size_t val) -{ - if (ZSTD_32bits()) - ZSTD_writeBE32(memPtr, (U32)val); - else - ZSTD_writeBE64(memPtr, (U64)val); -} - -/* function safe only for comparisons */ -ZSTD_STATIC U32 ZSTD_readMINMATCH(const void *memPtr, U32 length) -{ - switch (length) { - default: - case 4: return ZSTD_read32(memPtr); - case 3: - if (ZSTD_isLittleEndian()) - return ZSTD_read32(memPtr) << 8; - else - return ZSTD_read32(memPtr) >> 8; - } -} - -#endif /* MEM_H_MODULE */ diff --git a/contrib/linux-kernel/lib/zstd/zstd_common.c b/contrib/linux-kernel/lib/zstd/zstd_common.c deleted file mode 100644 index e5f06d77..00000000 --- a/contrib/linux-kernel/lib/zstd/zstd_common.c +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -/*-************************************* -* Dependencies -***************************************/ -#include "error_private.h" -#include "zstd_internal.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */ -#include <linux/kernel.h> - -/*=************************************************************** -* Custom allocator -****************************************************************/ - -#define stack_push(stack, size) \ - ({ \ - void *const ptr = ZSTD_PTR_ALIGN((stack)->ptr); \ - (stack)->ptr = (char *)ptr + (size); \ - (stack)->ptr <= (stack)->end ? ptr : NULL; \ - }) - -ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize) -{ - ZSTD_customMem stackMem = {ZSTD_stackAlloc, ZSTD_stackFree, workspace}; - ZSTD_stack *stack = (ZSTD_stack *)workspace; - /* Verify preconditions */ - if (!workspace || workspaceSize < sizeof(ZSTD_stack) || workspace != ZSTD_PTR_ALIGN(workspace)) { - ZSTD_customMem error = {NULL, NULL, NULL}; - return error; - } - /* Initialize the stack */ - stack->ptr = workspace; - stack->end = (char *)workspace + workspaceSize; - stack_push(stack, sizeof(ZSTD_stack)); - return stackMem; -} - -void *ZSTD_stackAllocAll(void *opaque, size_t *size) -{ - ZSTD_stack *stack = (ZSTD_stack *)opaque; - *size = (BYTE const *)stack->end - (BYTE *)ZSTD_PTR_ALIGN(stack->ptr); - return stack_push(stack, *size); -} - -void *ZSTD_stackAlloc(void *opaque, size_t size) -{ - ZSTD_stack *stack = (ZSTD_stack *)opaque; - return stack_push(stack, size); -} -void ZSTD_stackFree(void *opaque, void *address) -{ - (void)opaque; - (void)address; -} - -void *ZSTD_malloc(size_t size, ZSTD_customMem customMem) { return customMem.customAlloc(customMem.opaque, size); } - -void ZSTD_free(void *ptr, ZSTD_customMem customMem) -{ - if (ptr != NULL) - customMem.customFree(customMem.opaque, ptr); -} diff --git a/contrib/linux-kernel/lib/zstd/zstd_internal.h b/contrib/linux-kernel/lib/zstd/zstd_internal.h deleted file mode 100644 index a0fb83e3..00000000 --- a/contrib/linux-kernel/lib/zstd/zstd_internal.h +++ /dev/null @@ -1,261 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -#ifndef ZSTD_CCOMMON_H_MODULE -#define ZSTD_CCOMMON_H_MODULE - -/*-******************************************************* -* Compiler specifics -*********************************************************/ -#define FORCE_INLINE static __always_inline -#define FORCE_NOINLINE static noinline - -/*-************************************* -* Dependencies -***************************************/ -#include "error_private.h" -#include "mem.h" -#include <linux/compiler.h> -#include <linux/kernel.h> -#include <linux/xxhash.h> -#include <linux/zstd.h> - -/*-************************************* -* shared macros -***************************************/ -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define MAX(a, b) ((a) > (b) ? (a) : (b)) -#define CHECK_F(f) \ - { \ - size_t const errcod = f; \ - if (ERR_isError(errcod)) \ - return errcod; \ - } /* check and Forward error code */ -#define CHECK_E(f, e) \ - { \ - size_t const errcod = f; \ - if (ERR_isError(errcod)) \ - return ERROR(e); \ - } /* check and send Error code */ -#define ZSTD_STATIC_ASSERT(c) \ - { \ - enum { ZSTD_static_assert = 1 / (int)(!!(c)) }; \ - } - -/*-************************************* -* Common constants -***************************************/ -#define ZSTD_OPT_NUM (1 << 12) -#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */ - -#define ZSTD_REP_NUM 3 /* number of repcodes */ -#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */ -#define ZSTD_REP_MOVE (ZSTD_REP_NUM - 1) -#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM) -static const U32 repStartValue[ZSTD_REP_NUM] = {1, 4, 8}; - -#define KB *(1 << 10) -#define MB *(1 << 20) -#define GB *(1U << 30) - -#define BIT7 128 -#define BIT6 64 -#define BIT5 32 -#define BIT4 16 -#define BIT1 2 -#define BIT0 1 - -#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 -static const size_t ZSTD_fcs_fieldSize[4] = {0, 2, 4, 8}; -static const size_t ZSTD_did_fieldSize[4] = {0, 1, 2, 4}; - -#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ -static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; -typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; - -#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ -#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ - -#define HufLog 12 -typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; - -#define LONGNBSEQ 0x7F00 - -#define MINMATCH 3 -#define EQUAL_READ32 4 - -#define Litbits 8 -#define MaxLit ((1 << Litbits) - 1) -#define MaxML 52 -#define MaxLL 35 -#define MaxOff 28 -#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */ -#define MLFSELog 9 -#define LLFSELog 9 -#define OffFSELog 8 - -static const U32 LL_bits[MaxLL + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; -static const S16 LL_defaultNorm[MaxLL + 1] = {4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, -1, -1, -1, -1}; -#define LL_DEFAULTNORMLOG 6 /* for static allocation */ -static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG; - -static const U32 ML_bits[MaxML + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; -static const S16 ML_defaultNorm[MaxML + 1] = {1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1}; -#define ML_DEFAULTNORMLOG 6 /* for static allocation */ -static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG; - -static const S16 OF_defaultNorm[MaxOff + 1] = {1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1}; -#define OF_DEFAULTNORMLOG 5 /* for static allocation */ -static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; - -/*-******************************************* -* Shared functions to include for inlining -*********************************************/ -ZSTD_STATIC void ZSTD_copy8(void *dst, const void *src) { - memcpy(dst, src, 8); -} -/*! ZSTD_wildcopy() : -* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ -#define WILDCOPY_OVERLENGTH 8 -ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length) -{ - const BYTE* ip = (const BYTE*)src; - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + length; - /* Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388. - * Avoid the bad case where the loop only runs once by handling the - * special case separately. This doesn't trigger the bug because it - * doesn't involve pointer/integer overflow. - */ - if (length <= 8) - return ZSTD_copy8(dst, src); - do { - ZSTD_copy8(op, ip); - op += 8; - ip += 8; - } while (op < oend); -} - -/*-******************************************* -* Private interfaces -*********************************************/ -typedef struct ZSTD_stats_s ZSTD_stats_t; - -typedef struct { - U32 off; - U32 len; -} ZSTD_match_t; - -typedef struct { - U32 price; - U32 off; - U32 mlen; - U32 litlen; - U32 rep[ZSTD_REP_NUM]; -} ZSTD_optimal_t; - -typedef struct seqDef_s { - U32 offset; - U16 litLength; - U16 matchLength; -} seqDef; - -typedef struct { - seqDef *sequencesStart; - seqDef *sequences; - BYTE *litStart; - BYTE *lit; - BYTE *llCode; - BYTE *mlCode; - BYTE *ofCode; - U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ - U32 longLengthPos; - /* opt */ - ZSTD_optimal_t *priceTable; - ZSTD_match_t *matchTable; - U32 *matchLengthFreq; - U32 *litLengthFreq; - U32 *litFreq; - U32 *offCodeFreq; - U32 matchLengthSum; - U32 matchSum; - U32 litLengthSum; - U32 litSum; - U32 offCodeSum; - U32 log2matchLengthSum; - U32 log2matchSum; - U32 log2litLengthSum; - U32 log2litSum; - U32 log2offCodeSum; - U32 factor; - U32 staticPrices; - U32 cachedPrice; - U32 cachedLitLength; - const BYTE *cachedLiterals; -} seqStore_t; - -const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx); -void ZSTD_seqToCodes(const seqStore_t *seqStorePtr); -int ZSTD_isSkipFrame(ZSTD_DCtx *dctx); - -/*= Custom memory allocation functions */ -typedef void *(*ZSTD_allocFunction)(void *opaque, size_t size); -typedef void (*ZSTD_freeFunction)(void *opaque, void *address); -typedef struct { - ZSTD_allocFunction customAlloc; - ZSTD_freeFunction customFree; - void *opaque; -} ZSTD_customMem; - -void *ZSTD_malloc(size_t size, ZSTD_customMem customMem); -void ZSTD_free(void *ptr, ZSTD_customMem customMem); - -/*====== stack allocation ======*/ - -typedef struct { - void *ptr; - const void *end; -} ZSTD_stack; - -#define ZSTD_ALIGN(x) ALIGN(x, sizeof(size_t)) -#define ZSTD_PTR_ALIGN(p) PTR_ALIGN(p, sizeof(size_t)) - -ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize); - -void *ZSTD_stackAllocAll(void *opaque, size_t *size); -void *ZSTD_stackAlloc(void *opaque, size_t size); -void ZSTD_stackFree(void *opaque, void *address); - -/*====== common function ======*/ - -ZSTD_STATIC U32 ZSTD_highbit32(U32 val) { return 31 - __builtin_clz(val); } - -/* hidden functions */ - -/* ZSTD_invalidateRepCodes() : - * ensures next compression will not use repcodes from previous block. - * Note : only works with regular variant; - * do not use with extDict variant ! */ -void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx); - -size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx); -size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx); -size_t ZSTD_freeCDict(ZSTD_CDict *cdict); -size_t ZSTD_freeDDict(ZSTD_DDict *cdict); -size_t ZSTD_freeCStream(ZSTD_CStream *zcs); -size_t ZSTD_freeDStream(ZSTD_DStream *zds); - -#endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/contrib/linux-kernel/lib/zstd/zstd_opt.h b/contrib/linux-kernel/lib/zstd/zstd_opt.h deleted file mode 100644 index ecdd7259..00000000 --- a/contrib/linux-kernel/lib/zstd/zstd_opt.h +++ /dev/null @@ -1,1012 +0,0 @@ -/** - * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of https://github.com/facebook/zstd. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License version 2 as published by the - * Free Software Foundation. This program is dual-licensed; you may select - * either version 2 of the GNU General Public License ("GPL") or BSD license - * ("BSD"). - */ - -/* Note : this file is intended to be included within zstd_compress.c */ - -#ifndef ZSTD_OPT_H_91842398743 -#define ZSTD_OPT_H_91842398743 - -#define ZSTD_LITFREQ_ADD 2 -#define ZSTD_FREQ_DIV 4 -#define ZSTD_MAX_PRICE (1 << 30) - -/*-************************************* -* Price functions for optimal parser -***************************************/ -FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t *ssPtr) -{ - ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum + 1); - ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum + 1); - ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum + 1); - ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum + 1); - ssPtr->factor = 1 + ((ssPtr->litSum >> 5) / ssPtr->litLengthSum) + ((ssPtr->litSum << 1) / (ssPtr->litSum + ssPtr->matchSum)); -} - -ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t *ssPtr, const BYTE *src, size_t srcSize) -{ - unsigned u; - - ssPtr->cachedLiterals = NULL; - ssPtr->cachedPrice = ssPtr->cachedLitLength = 0; - ssPtr->staticPrices = 0; - - if (ssPtr->litLengthSum == 0) { - if (srcSize <= 1024) - ssPtr->staticPrices = 1; - - for (u = 0; u <= MaxLit; u++) - ssPtr->litFreq[u] = 0; - for (u = 0; u < srcSize; u++) - ssPtr->litFreq[src[u]]++; - - ssPtr->litSum = 0; - ssPtr->litLengthSum = MaxLL + 1; - ssPtr->matchLengthSum = MaxML + 1; - ssPtr->offCodeSum = (MaxOff + 1); - ssPtr->matchSum = (ZSTD_LITFREQ_ADD << Litbits); - - for (u = 0; u <= MaxLit; u++) { - ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> ZSTD_FREQ_DIV); - ssPtr->litSum += ssPtr->litFreq[u]; - } - for (u = 0; u <= MaxLL; u++) - ssPtr->litLengthFreq[u] = 1; - for (u = 0; u <= MaxML; u++) - ssPtr->matchLengthFreq[u] = 1; - for (u = 0; u <= MaxOff; u++) - ssPtr->offCodeFreq[u] = 1; - } else { - ssPtr->matchLengthSum = 0; - ssPtr->litLengthSum = 0; - ssPtr->offCodeSum = 0; - ssPtr->matchSum = 0; - ssPtr->litSum = 0; - - for (u = 0; u <= MaxLit; u++) { - ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> (ZSTD_FREQ_DIV + 1)); - ssPtr->litSum += ssPtr->litFreq[u]; - } - for (u = 0; u <= MaxLL; u++) { - ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u] >> (ZSTD_FREQ_DIV + 1)); - ssPtr->litLengthSum += ssPtr->litLengthFreq[u]; - } - for (u = 0; u <= MaxML; u++) { - ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u] >> ZSTD_FREQ_DIV); - ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u]; - ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3); - } - ssPtr->matchSum *= ZSTD_LITFREQ_ADD; - for (u = 0; u <= MaxOff; u++) { - ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u] >> ZSTD_FREQ_DIV); - ssPtr->offCodeSum += ssPtr->offCodeFreq[u]; - } - } - - ZSTD_setLog2Prices(ssPtr); -} - -FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t *ssPtr, U32 litLength, const BYTE *literals) -{ - U32 price, u; - - if (ssPtr->staticPrices) - return ZSTD_highbit32((U32)litLength + 1) + (litLength * 6); - - if (litLength == 0) - return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0] + 1); - - /* literals */ - if (ssPtr->cachedLiterals == literals) { - U32 const additional = litLength - ssPtr->cachedLitLength; - const BYTE *literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength; - price = ssPtr->cachedPrice + additional * ssPtr->log2litSum; - for (u = 0; u < additional; u++) - price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]] + 1); - ssPtr->cachedPrice = price; - ssPtr->cachedLitLength = litLength; - } else { - price = litLength * ssPtr->log2litSum; - for (u = 0; u < litLength; u++) - price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]] + 1); - - if (litLength >= 12) { - ssPtr->cachedLiterals = literals; - ssPtr->cachedPrice = price; - ssPtr->cachedLitLength = litLength; - } - } - - /* literal Length */ - { - const BYTE LL_deltaCode = 19; - const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; - price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode] + 1); - } - - return price; -} - -FORCE_INLINE U32 ZSTD_getPrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength, const int ultra) -{ - /* offset */ - U32 price; - BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1); - - if (seqStorePtr->staticPrices) - return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength + 1) + 16 + offCode; - - price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode] + 1); - if (!ultra && offCode >= 20) - price += (offCode - 19) * 2; - - /* match Length */ - { - const BYTE ML_deltaCode = 36; - const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; - price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode] + 1); - } - - return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; -} - -ZSTD_STATIC void ZSTD_updatePrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength) -{ - U32 u; - - /* literals */ - seqStorePtr->litSum += litLength * ZSTD_LITFREQ_ADD; - for (u = 0; u < litLength; u++) - seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; - - /* literal Length */ - { - const BYTE LL_deltaCode = 19; - const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; - seqStorePtr->litLengthFreq[llCode]++; - seqStorePtr->litLengthSum++; - } - - /* match offset */ - { - BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1); - seqStorePtr->offCodeSum++; - seqStorePtr->offCodeFreq[offCode]++; - } - - /* match Length */ - { - const BYTE ML_deltaCode = 36; - const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; - seqStorePtr->matchLengthFreq[mlCode]++; - seqStorePtr->matchLengthSum++; - } - - ZSTD_setLog2Prices(seqStorePtr); -} - -#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ - { \ - while (last_pos < pos) { \ - opt[last_pos + 1].price = ZSTD_MAX_PRICE; \ - last_pos++; \ - } \ - opt[pos].mlen = mlen_; \ - opt[pos].off = offset_; \ - opt[pos].litlen = litlen_; \ - opt[pos].price = price_; \ - } - -/* Update hashTable3 up to ip (excluded) - Assumption : always within prefix (i.e. not within extDict) */ -FORCE_INLINE -U32 ZSTD_insertAndFindFirstIndexHash3(ZSTD_CCtx *zc, const BYTE *ip) -{ - U32 *const hashTable3 = zc->hashTable3; - U32 const hashLog3 = zc->hashLog3; - const BYTE *const base = zc->base; - U32 idx = zc->nextToUpdate3; - const U32 target = zc->nextToUpdate3 = (U32)(ip - base); - const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3); - - while (idx < target) { - hashTable3[ZSTD_hash3Ptr(base + idx, hashLog3)] = idx; - idx++; - } - - return hashTable3[hash3]; -} - -/*-************************************* -* Binary Tree search -***************************************/ -static U32 ZSTD_insertBtAndGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, U32 nbCompares, const U32 mls, U32 extDict, - ZSTD_match_t *matches, const U32 minMatchLen) -{ - const BYTE *const base = zc->base; - const U32 curr = (U32)(ip - base); - const U32 hashLog = zc->params.cParams.hashLog; - const size_t h = ZSTD_hashPtr(ip, hashLog, mls); - U32 *const hashTable = zc->hashTable; - U32 matchIndex = hashTable[h]; - U32 *const bt = zc->chainTable; - const U32 btLog = zc->params.cParams.chainLog - 1; - const U32 btMask = (1U << btLog) - 1; - size_t commonLengthSmaller = 0, commonLengthLarger = 0; - const BYTE *const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; - const BYTE *const dictEnd = dictBase + dictLimit; - const BYTE *const prefixStart = base + dictLimit; - const U32 btLow = btMask >= curr ? 0 : curr - btMask; - const U32 windowLow = zc->lowLimit; - U32 *smallerPtr = bt + 2 * (curr & btMask); - U32 *largerPtr = bt + 2 * (curr & btMask) + 1; - U32 matchEndIdx = curr + 8; - U32 dummy32; /* to be nullified at the end */ - U32 mnum = 0; - - const U32 minMatch = (mls == 3) ? 3 : 4; - size_t bestLength = minMatchLen - 1; - - if (minMatch == 3) { /* HC3 match finder */ - U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(zc, ip); - if (matchIndex3 > windowLow && (curr - matchIndex3 < (1 << 18))) { - const BYTE *match; - size_t currMl = 0; - if ((!extDict) || matchIndex3 >= dictLimit) { - match = base + matchIndex3; - if (match[bestLength] == ip[bestLength]) - currMl = ZSTD_count(ip, match, iLimit); - } else { - match = dictBase + matchIndex3; - if (ZSTD_readMINMATCH(match, MINMATCH) == - ZSTD_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ - currMl = ZSTD_count_2segments(ip + MINMATCH, match + MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; - } - - /* save best solution */ - if (currMl > bestLength) { - bestLength = currMl; - matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex3; - matches[mnum].len = (U32)currMl; - mnum++; - if (currMl > ZSTD_OPT_NUM) - goto update; - if (ip + currMl == iLimit) - goto update; /* best possible, and avoid read overflow*/ - } - } - } - - hashTable[h] = curr; /* Update Hash Table */ - - while (nbCompares-- && (matchIndex > windowLow)) { - U32 *nextPtr = bt + 2 * (matchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - const BYTE *match; - - if ((!extDict) || (matchIndex + matchLength >= dictLimit)) { - match = base + matchIndex; - if (match[matchLength] == ip[matchLength]) { - matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iLimit) + 1; - } - } else { - match = dictBase + matchIndex; - matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iLimit, dictEnd, prefixStart); - if (matchIndex + matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ - } - - if (matchLength > bestLength) { - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - bestLength = matchLength; - matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex; - matches[mnum].len = (U32)matchLength; - mnum++; - if (matchLength > ZSTD_OPT_NUM) - break; - if (ip + matchLength == iLimit) /* equal : no way to know if inf or sup */ - break; /* drop, to guarantee consistency (miss a little bit of compression) */ - } - - if (match[matchLength] < ip[matchLength]) { - /* match is smaller than curr */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { - smallerPtr = &dummy32; - break; - } /* beyond tree size, stop the search */ - smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ - matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */ - } else { - /* match is larger than curr */ - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { - largerPtr = &dummy32; - break; - } /* beyond tree size, stop the search */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - } - } - - *smallerPtr = *largerPtr = 0; - -update: - zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1; - return mnum; -} - -/** Tree updater, providing best match */ -static U32 ZSTD_BtGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls, ZSTD_match_t *matches, - const U32 minMatchLen) -{ - if (ip < zc->base + zc->nextToUpdate) - return 0; /* skipped area */ - ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen); -} - -static U32 ZSTD_BtGetAllMatches_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */ - const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, - ZSTD_match_t *matches, const U32 minMatchLen) -{ - switch (matchLengthSearch) { - case 3: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); - default: - case 4: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); - case 5: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); - case 7: - case 6: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); - } -} - -/** Tree updater, providing best match */ -static U32 ZSTD_BtGetAllMatches_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls, - ZSTD_match_t *matches, const U32 minMatchLen) -{ - if (ip < zc->base + zc->nextToUpdate) - return 0; /* skipped area */ - ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen); -} - -static U32 ZSTD_BtGetAllMatches_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */ - const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, - ZSTD_match_t *matches, const U32 minMatchLen) -{ - switch (matchLengthSearch) { - case 3: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); - default: - case 4: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); - case 5: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); - case 7: - case 6: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); - } -} - -/*-******************************* -* Optimal parser -*********************************/ -FORCE_INLINE -void ZSTD_compressBlock_opt_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra) -{ - seqStore_t *seqStorePtr = &(ctx->seqStore); - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - 8; - const BYTE *const base = ctx->base; - const BYTE *const prefixStart = base + ctx->dictLimit; - - const U32 maxSearches = 1U << ctx->params.cParams.searchLog; - const U32 sufficient_len = ctx->params.cParams.targetLength; - const U32 mls = ctx->params.cParams.searchLength; - const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; - - ZSTD_optimal_t *opt = seqStorePtr->priceTable; - ZSTD_match_t *matches = seqStorePtr->matchTable; - const BYTE *inr; - U32 offset, rep[ZSTD_REP_NUM]; - - /* init */ - ctx->nextToUpdate3 = ctx->nextToUpdate; - ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize); - ip += (ip == prefixStart); - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - rep[i] = ctx->rep[i]; - } - - /* Match Loop */ - while (ip < ilimit) { - U32 cur, match_num, last_pos, litlen, price; - U32 u, mlen, best_mlen, best_off, litLength; - memset(opt, 0, sizeof(ZSTD_optimal_t)); - last_pos = 0; - litlen = (U32)(ip - anchor); - - /* check repCode */ - { - U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor); - for (i = (ip == anchor); i < last_i; i++) { - const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i]; - if ((repCur > 0) && (repCur < (S32)(ip - prefixStart)) && - (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) { - mlen = (U32)ZSTD_count(ip + minMatch, ip + minMatch - repCur, iend) + minMatch; - if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { - best_mlen = mlen; - best_off = i; - cur = 0; - last_pos = 1; - goto _storeSequence; - } - best_off = i - (ip == anchor); - do { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); - if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ - mlen--; - } while (mlen >= minMatch); - } - } - } - - match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch); - - if (!last_pos && !match_num) { - ip++; - continue; - } - - if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) { - best_mlen = matches[match_num - 1].len; - best_off = matches[match_num - 1].off; - cur = 0; - last_pos = 1; - goto _storeSequence; - } - - /* set prices using matches at position = 0 */ - best_mlen = (last_pos) ? last_pos : minMatch; - for (u = 0; u < match_num; u++) { - mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen; - best_mlen = matches[u].len; - while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); - if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ - mlen++; - } - } - - if (last_pos < minMatch) { - ip++; - continue; - } - - /* initialize opt[0] */ - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - opt[0].rep[i] = rep[i]; - } - opt[0].mlen = 1; - opt[0].litlen = litlen; - - /* check further positions */ - for (cur = 1; cur <= last_pos; cur++) { - inr = ip + cur; - - if (opt[cur - 1].mlen == 1) { - litlen = opt[cur - 1].litlen + 1; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen); - } else - price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); - } else { - litlen = 1; - price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1); - } - - if (cur > last_pos || price <= opt[cur].price) - SET_PRICE(cur, 1, 0, litlen, price); - - if (cur == last_pos) - break; - - if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ - continue; - - mlen = opt[cur].mlen; - if (opt[cur].off > ZSTD_REP_MOVE_OPT) { - opt[cur].rep[2] = opt[cur - mlen].rep[1]; - opt[cur].rep[1] = opt[cur - mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; - } else { - opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2]; - opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1]; - opt[cur].rep[0] = - ((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]); - } - - best_mlen = minMatch; - { - U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); - for (i = (opt[cur].mlen != 1); i < last_i; i++) { /* check rep */ - const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; - if ((repCur > 0) && (repCur < (S32)(inr - prefixStart)) && - (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) { - mlen = (U32)ZSTD_count(inr + minMatch, inr + minMatch - repCur, iend) + minMatch; - - if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { - best_mlen = mlen; - best_off = i; - last_pos = cur + 1; - goto _storeSequence; - } - - best_off = i - (opt[cur].mlen != 1); - if (mlen > best_mlen) - best_mlen = mlen; - - do { - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen, - best_off, mlen - MINMATCH, ultra); - } else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra); - } - - if (cur + mlen > last_pos || price <= opt[cur + mlen].price) - SET_PRICE(cur + mlen, mlen, i, litlen, price); - mlen--; - } while (mlen >= minMatch); - } - } - } - - match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen); - - if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) { - best_mlen = matches[match_num - 1].len; - best_off = matches[match_num - 1].off; - last_pos = cur + 1; - goto _storeSequence; - } - - /* set prices using matches at position = cur */ - for (u = 0; u < match_num; u++) { - mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen; - best_mlen = matches[u].len; - - while (mlen <= best_mlen) { - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen, - matches[u].off - 1, mlen - MINMATCH, ultra); - else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra); - } - - if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) - SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); - - mlen++; - } - } - } - - best_mlen = opt[last_pos].mlen; - best_off = opt[last_pos].off; - cur = last_pos - best_mlen; - - /* store sequence */ -_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ - opt[0].mlen = 1; - - while (1) { - mlen = opt[cur].mlen; - offset = opt[cur].off; - opt[cur].mlen = best_mlen; - opt[cur].off = best_off; - best_mlen = mlen; - best_off = offset; - if (mlen > cur) - break; - cur -= mlen; - } - - for (u = 0; u <= last_pos;) { - u += opt[u].mlen; - } - - for (cur = 0; cur < last_pos;) { - mlen = opt[cur].mlen; - if (mlen == 1) { - ip++; - cur++; - continue; - } - offset = opt[cur].off; - cur += mlen; - litLength = (U32)(ip - anchor); - - if (offset > ZSTD_REP_MOVE_OPT) { - rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = offset - ZSTD_REP_MOVE_OPT; - offset--; - } else { - if (offset != 0) { - best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); - if (offset != 1) - rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = best_off; - } - if (litLength == 0) - offset--; - } - - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); - ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); - anchor = ip = ip + mlen; - } - } /* for (cur=0; cur < last_pos; ) */ - - /* Save reps for next block */ - { - int i; - for (i = 0; i < ZSTD_REP_NUM; i++) - ctx->repToConfirm[i] = rep[i]; - } - - /* Last Literals */ - { - size_t const lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -FORCE_INLINE -void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra) -{ - seqStore_t *seqStorePtr = &(ctx->seqStore); - const BYTE *const istart = (const BYTE *)src; - const BYTE *ip = istart; - const BYTE *anchor = istart; - const BYTE *const iend = istart + srcSize; - const BYTE *const ilimit = iend - 8; - const BYTE *const base = ctx->base; - const U32 lowestIndex = ctx->lowLimit; - const U32 dictLimit = ctx->dictLimit; - const BYTE *const prefixStart = base + dictLimit; - const BYTE *const dictBase = ctx->dictBase; - const BYTE *const dictEnd = dictBase + dictLimit; - - const U32 maxSearches = 1U << ctx->params.cParams.searchLog; - const U32 sufficient_len = ctx->params.cParams.targetLength; - const U32 mls = ctx->params.cParams.searchLength; - const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; - - ZSTD_optimal_t *opt = seqStorePtr->priceTable; - ZSTD_match_t *matches = seqStorePtr->matchTable; - const BYTE *inr; - - /* init */ - U32 offset, rep[ZSTD_REP_NUM]; - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - rep[i] = ctx->rep[i]; - } - - ctx->nextToUpdate3 = ctx->nextToUpdate; - ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize); - ip += (ip == prefixStart); - - /* Match Loop */ - while (ip < ilimit) { - U32 cur, match_num, last_pos, litlen, price; - U32 u, mlen, best_mlen, best_off, litLength; - U32 curr = (U32)(ip - base); - memset(opt, 0, sizeof(ZSTD_optimal_t)); - last_pos = 0; - opt[0].litlen = (U32)(ip - anchor); - - /* check repCode */ - { - U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor); - for (i = (ip == anchor); i < last_i; i++) { - const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i]; - const U32 repIndex = (U32)(curr - repCur); - const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *const repMatch = repBase + repIndex; - if ((repCur > 0 && repCur <= (S32)curr) && - (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ - && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) { - /* repcode detected we should take it */ - const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(ip + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch; - - if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { - best_mlen = mlen; - best_off = i; - cur = 0; - last_pos = 1; - goto _storeSequence; - } - - best_off = i - (ip == anchor); - litlen = opt[0].litlen; - do { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); - if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ - mlen--; - } while (mlen >= minMatch); - } - } - } - - match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */ - - if (!last_pos && !match_num) { - ip++; - continue; - } - - { - U32 i; - for (i = 0; i < ZSTD_REP_NUM; i++) - opt[0].rep[i] = rep[i]; - } - opt[0].mlen = 1; - - if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) { - best_mlen = matches[match_num - 1].len; - best_off = matches[match_num - 1].off; - cur = 0; - last_pos = 1; - goto _storeSequence; - } - - best_mlen = (last_pos) ? last_pos : minMatch; - - /* set prices using matches at position = 0 */ - for (u = 0; u < match_num; u++) { - mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen; - best_mlen = matches[u].len; - litlen = opt[0].litlen; - while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); - if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, matches[u].off, litlen, price); - mlen++; - } - } - - if (last_pos < minMatch) { - ip++; - continue; - } - - /* check further positions */ - for (cur = 1; cur <= last_pos; cur++) { - inr = ip + cur; - - if (opt[cur - 1].mlen == 1) { - litlen = opt[cur - 1].litlen + 1; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen); - } else - price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); - } else { - litlen = 1; - price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1); - } - - if (cur > last_pos || price <= opt[cur].price) - SET_PRICE(cur, 1, 0, litlen, price); - - if (cur == last_pos) - break; - - if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ - continue; - - mlen = opt[cur].mlen; - if (opt[cur].off > ZSTD_REP_MOVE_OPT) { - opt[cur].rep[2] = opt[cur - mlen].rep[1]; - opt[cur].rep[1] = opt[cur - mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; - } else { - opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2]; - opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1]; - opt[cur].rep[0] = - ((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]); - } - - best_mlen = minMatch; - { - U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); - for (i = (mlen != 1); i < last_i; i++) { - const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; - const U32 repIndex = (U32)(curr + cur - repCur); - const BYTE *const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE *const repMatch = repBase + repIndex; - if ((repCur > 0 && repCur <= (S32)(curr + cur)) && - (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ - && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) { - /* repcode detected */ - const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(inr + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch; - - if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { - best_mlen = mlen; - best_off = i; - last_pos = cur + 1; - goto _storeSequence; - } - - best_off = i - (opt[cur].mlen != 1); - if (mlen > best_mlen) - best_mlen = mlen; - - do { - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen, - best_off, mlen - MINMATCH, ultra); - } else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra); - } - - if (cur + mlen > last_pos || price <= opt[cur + mlen].price) - SET_PRICE(cur + mlen, mlen, i, litlen, price); - mlen--; - } while (mlen >= minMatch); - } - } - } - - match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); - - if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) { - best_mlen = matches[match_num - 1].len; - best_off = matches[match_num - 1].off; - last_pos = cur + 1; - goto _storeSequence; - } - - /* set prices using matches at position = cur */ - for (u = 0; u < match_num; u++) { - mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen; - best_mlen = matches[u].len; - - while (mlen <= best_mlen) { - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen, - matches[u].off - 1, mlen - MINMATCH, ultra); - else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra); - } - - if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) - SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); - - mlen++; - } - } - } /* for (cur = 1; cur <= last_pos; cur++) */ - - best_mlen = opt[last_pos].mlen; - best_off = opt[last_pos].off; - cur = last_pos - best_mlen; - - /* store sequence */ -_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ - opt[0].mlen = 1; - - while (1) { - mlen = opt[cur].mlen; - offset = opt[cur].off; - opt[cur].mlen = best_mlen; - opt[cur].off = best_off; - best_mlen = mlen; - best_off = offset; - if (mlen > cur) - break; - cur -= mlen; - } - - for (u = 0; u <= last_pos;) { - u += opt[u].mlen; - } - - for (cur = 0; cur < last_pos;) { - mlen = opt[cur].mlen; - if (mlen == 1) { - ip++; - cur++; - continue; - } - offset = opt[cur].off; - cur += mlen; - litLength = (U32)(ip - anchor); - - if (offset > ZSTD_REP_MOVE_OPT) { - rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = offset - ZSTD_REP_MOVE_OPT; - offset--; - } else { - if (offset != 0) { - best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); - if (offset != 1) - rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = best_off; - } - - if (litLength == 0) - offset--; - } - - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); - ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH); - anchor = ip = ip + mlen; - } - } /* for (cur=0; cur < last_pos; ) */ - - /* Save reps for next block */ - { - int i; - for (i = 0; i < ZSTD_REP_NUM; i++) - ctx->repToConfirm[i] = rep[i]; - } - - /* Last Literals */ - { - size_t lastLLSize = iend - anchor; - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - -#endif /* ZSTD_OPT_H_91842398743 */ diff --git a/contrib/linux-kernel/linux.mk b/contrib/linux-kernel/linux.mk new file mode 100644 index 00000000..06bf0790 --- /dev/null +++ b/contrib/linux-kernel/linux.mk @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o +obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o + +ccflags-y += -O3 + +zstd_compress-y := \ + zstd_compress_module.o \ + common/debug.o \ + common/entropy_common.o \ + common/error_private.o \ + common/fse_decompress.o \ + common/zstd_common.o \ + compress/fse_compress.o \ + compress/hist.o \ + compress/huf_compress.o \ + compress/zstd_compress.o \ + compress/zstd_compress_literals.o \ + compress/zstd_compress_sequences.o \ + compress/zstd_compress_superblock.o \ + compress/zstd_double_fast.o \ + compress/zstd_fast.o \ + compress/zstd_lazy.o \ + compress/zstd_ldm.o \ + compress/zstd_opt.o \ + +zstd_decompress-y := \ + zstd_decompress_module.o \ + common/debug.o \ + common/entropy_common.o \ + common/error_private.o \ + common/fse_decompress.o \ + common/zstd_common.o \ + decompress/huf_decompress.o \ + decompress/zstd_ddict.o \ + decompress/zstd_decompress.o \ + decompress/zstd_decompress_block.o \ diff --git a/contrib/linux-kernel/linux_zstd.h b/contrib/linux-kernel/linux_zstd.h new file mode 100644 index 00000000..dcd1ec18 --- /dev/null +++ b/contrib/linux-kernel/linux_zstd.h @@ -0,0 +1,459 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of https://github.com/facebook/zstd. + * An additional grant of patent rights can be found in the PATENTS file in the + * same directory. + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License version 2 as published by the + * Free Software Foundation. This program is dual-licensed; you may select + * either version 2 of the GNU General Public License ("GPL") or BSD license + * ("BSD"). + */ + +#ifndef LINUX_ZSTD_H +#define LINUX_ZSTD_H + +/** + * This is a kernel-style API that wraps the upstream zstd API, which cannot be + * used directly because the symbols aren't exported. It exposes the minimal + * functionality which is currently required by users of zstd in the kernel. + * Expose extra functions from lib/zstd/zstd.h as needed. + */ + +/* ====== Dependency ====== */ +#include <linux/types.h> + +/* ====== Helper Functions ====== */ +/** + * zstd_compress_bound() - maximum compressed size in worst case scenario + * @src_size: The size of the data to compress. + * + * Return: The maximum compressed size in the worst case scenario. + */ +size_t zstd_compress_bound(size_t src_size); + +/** + * zstd_is_error() - tells if a size_t function result is an error code + * @code: The function result to check for error. + * + * Return: Non-zero iff the code is an error. + */ +unsigned int zstd_is_error(size_t code); + +/** + * zstd_get_error_code() - translates an error function result to an error code + * @code: The function result for which zstd_is_error(code) is true. + * + * Return: A unique error code for this error. + */ +int zstd_get_error_code(size_t code); + +/** + * zstd_get_error_name() - translates an error function result to a string + * @code: The function result for which zstd_is_error(code) is true. + * + * Return: An error string corresponding to the error code. + */ +const char *zstd_get_error_name(size_t code); + +/* ====== Parameter Selection ====== */ + +/** + * enum zstd_strategy - zstd compression search strategy + * + * From faster to stronger. + */ +enum zstd_strategy { + zstd_fast = 1, + zstd_dfast = 2, + zstd_greedy = 3, + zstd_lazy = 4, + zstd_lazy2 = 5, + zstd_btlazy2 = 6, + zstd_btopt = 7, + zstd_btultra = 8, + zstd_btultra2 = 9 +}; + +/** + * struct zstd_compression_parameters - zstd compression parameters + * @window_log: Log of the largest match distance. Larger means more + * compression, and more memory needed during decompression. + * @chain_log: Fully searched segment. Larger means more compression, + * slower, and more memory (useless for fast). + * @hash_log: Dispatch table. Larger means more compression, + * slower, and more memory. + * @search_log: Number of searches. Larger means more compression and slower. + * @search_length: Match length searched. Larger means faster decompression, + * sometimes less compression. + * @target_length: Acceptable match size for optimal parser (only). Larger means + * more compression, and slower. + * @strategy: The zstd compression strategy. + */ +struct zstd_compression_parameters { + unsigned int window_log; + unsigned int chain_log; + unsigned int hash_log; + unsigned int search_log; + unsigned int search_length; + unsigned int target_length; + enum zstd_strategy strategy; +}; + +/** + * struct zstd_frame_parameters - zstd frame parameters + * @content_size_flag: Controls whether content size will be present in the + * frame header (when known). + * @checksum_flag: Controls whether a 32-bit checksum is generated at the + * end of the frame for error detection. + * @no_dict_id_flag: Controls whether dictID will be saved into the frame + * header when using dictionary compression. + * + * The default value is all fields set to 0. + */ +struct zstd_frame_parameters { + unsigned int content_size_flag; + unsigned int checksum_flag; + unsigned int no_dict_id_flag; +}; + +/** + * struct zstd_parameters - zstd parameters + * @cparams: The compression parameters. + * @fparams: The frame parameters. + */ +struct zstd_parameters { + struct zstd_compression_parameters cparams; + struct zstd_frame_parameters fparams; +}; + +/** + * zstd_get_params() - returns zstd_parameters for selected level + * @level: The compression level + * @estimated_src_size: The estimated source size to compress or 0 + * if unknown. + * + * Return: The selected zstd_parameters. + */ +struct zstd_parameters zstd_get_params(int level, + unsigned long long estimated_src_size); + +/* ====== Single-pass Compression ====== */ + +typedef struct ZSTD_CCtx_s zstd_cctx; + +/** + * zstd_cctx_workspace_bound() - max memory needed to initialize a zstd_cctx + * @parameters: The compression parameters to be used. + * + * If multiple compression parameters might be used, the caller must call + * zstd_cctx_workspace_bound() for each set of parameters and use the maximum + * size. + * + * Return: A lower bound on the size of the workspace that is passed to + * zstd_init_cctx(). + */ +size_t zstd_cctx_workspace_bound( + const struct zstd_compression_parameters *parameters); + +/** + * zstd_init_cctx() - initialize a zstd compression context + * @workspace: The workspace to emplace the context into. It must outlive + * the returned context. + * @workspace_size: The size of workspace. Use zstd_cctx_workspace_bound() to + * determine how large the workspace must be. + * + * Return: A zstd compression context or NULL on error. + */ +zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size); + +/** + * zstd_compress_cctx() - compress src into dst with the initialized parameters + * @cctx: The context. Must have been initialized with zstd_init_cctx(). + * @dst: The buffer to compress src into. + * @dst_capacity: The size of the destination buffer. May be any size, but + * ZSTD_compressBound(srcSize) is guaranteed to be large enough. + * @src: The data to compress. + * @src_size: The size of the data to compress. + * @parameters: The compression parameters to be used. + * + * Return: The compressed size or an error, which can be checked using + * zstd_is_error(). + */ +size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity, + const void *src, size_t src_size, const struct zstd_parameters *parameters); + +/* ====== Single-pass Decompression ====== */ + +typedef struct ZSTD_DCtx_s zstd_dctx; + +/** + * zstd_dctx_workspace_bound() - max memory needed to initialize a zstd_dctx + * + * Return: A lower bound on the size of the workspace that is passed to + * zstd_init_dctx(). + */ +size_t zstd_dctx_workspace_bound(void); + +/** + * zstd_init_dctx() - initialize a zstd decompression context + * @workspace: The workspace to emplace the context into. It must outlive + * the returned context. + * @workspace_size: The size of workspace. Use zstd_dctx_workspace_bound() to + * determine how large the workspace must be. + * + * Return: A zstd decompression context or NULL on error. + */ +zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size); + +/** + * zstd_decompress_dctx() - decompress zstd compressed src into dst + * @dctx: The decompression context. + * @dst: The buffer to decompress src into. + * @dst_capacity: The size of the destination buffer. Must be at least as large + * as the decompressed size. If the caller cannot upper bound the + * decompressed size, then it's better to use the streaming API. + * @src: The zstd compressed data to decompress. Multiple concatenated + * frames and skippable frames are allowed. + * @src_size: The exact size of the data to decompress. + * + * Return: The decompressed size or an error, which can be checked using + * zstd_is_error(). + */ +size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity, + const void *src, size_t src_size); + +/* ====== Streaming Buffers ====== */ + +/** + * struct zstd_in_buffer - input buffer for streaming + * @src: Start of the input buffer. + * @size: Size of the input buffer. + * @pos: Position where reading stopped. Will be updated. + * Necessarily 0 <= pos <= size. + */ +struct zstd_in_buffer { + const void *src; + size_t size; + size_t pos; +}; + +/** + * struct zstd_out_buffer - output buffer for streaming + * @dst: Start of the output buffer. + * @size: Size of the output buffer. + * @pos: Position where writing stopped. Will be updated. + * Necessarily 0 <= pos <= size. + */ +struct zstd_out_buffer { + void *dst; + size_t size; + size_t pos; +}; + +/* ====== Streaming Compression ====== */ + +typedef struct ZSTD_CCtx_s zstd_cstream; + +/** + * zstd_cstream_workspace_bound() - memory needed to initialize a zstd_cstream + * @cparams: The compression parameters to be used for compression. + * + * Return: A lower bound on the size of the workspace that is passed to + * zstd_init_cstream(). + */ +size_t zstd_cstream_workspace_bound( + const struct zstd_compression_parameters *cparams); + +/** + * zstd_init_cstream() - initialize a zstd streaming compression context + * @parameters The zstd parameters to use for compression. + * @pledged_src_size: If params.fParams.contentSizeFlag == 1 then the caller + * must pass the source size (zero means empty source). + * Otherwise, the caller may optionally pass the source + * size, or zero if unknown. + * @workspace: The workspace to emplace the context into. It must outlive + * the returned context. + * @workspace_size: The size of workspace. + * Use zstd_cstream_workspace_bound(params->cparams) to + * determine how large the workspace must be. + * + * Return: The zstd streaming compression context or NULL on error. + */ +zstd_cstream *zstd_init_cstream(const struct zstd_parameters *parameters, + unsigned long long pledged_src_size, void *workspace, size_t workspace_size); + +/** + * zstd_reset_cstream() - reset the context using parameters from creation + * @cstream: The zstd streaming compression context to reset. + * @pledged_src_size: Optionally the source size, or zero if unknown. + * + * Resets the context using the parameters from creation. Skips dictionary + * loading, since it can be reused. If `pledged_src_size` is non-zero the frame + * content size is always written into the frame header. + * + * Return: Zero or an error, which can be checked using + * zstd_is_error(). + */ +size_t zstd_reset_cstream(zstd_cstream *cstream, + unsigned long long pledged_src_size); + +/** + * zstd_compress_stream() - streaming compress some of input into output + * @cstream: The zstd streaming compression context. + * @output: Destination buffer. `output->pos` is updated to indicate how much + * compressed data was written. + * @input: Source buffer. `input->pos` is updated to indicate how much data + * was read. Note that it may not consume the entire input, in which + * case `input->pos < input->size`, and it's up to the caller to + * present remaining data again. + * + * The `input` and `output` buffers may be any size. Guaranteed to make some + * forward progress if `input` and `output` are not empty. + * + * Return: A hint for the number of bytes to use as the input for the next + * function call or an error, which can be checked using + * zstd_is_error(). + */ +size_t zstd_compress_stream(zstd_cstream *cstream, + struct zstd_out_buffer *output, struct zstd_in_buffer *input); + +/** + * zstd_flush_stream() - flush internal buffers into output + * @cstream: The zstd streaming compression context. + * @output: Destination buffer. `output->pos` is updated to indicate how much + * compressed data was written. + * + * zstd_flush_stream() must be called until it returns 0, meaning all the data + * has been flushed. Since zstd_flush_stream() causes a block to be ended, + * calling it too often will degrade the compression ratio. + * + * Return: The number of bytes still present within internal buffers or an + * error, which can be checked using zstd_is_error(). + */ +size_t zstd_flush_stream(zstd_cstream *cstream, struct zstd_out_buffer *output); + +/** + * zstd_end_stream() - flush internal buffers into output and end the frame + * @cstream: The zstd streaming compression context. + * @output: Destination buffer. `output->pos` is updated to indicate how much + * compressed data was written. + * + * zstd_end_stream() must be called until it returns 0, meaning all the data has + * been flushed and the frame epilogue has been written. + * + * Return: The number of bytes still present within internal buffers or an + * error, which can be checked using zstd_is_error(). + */ +size_t zstd_end_stream(zstd_cstream *cstream, struct zstd_out_buffer *output); + +/* ====== Streaming Decompression ====== */ + +typedef struct ZSTD_DCtx_s zstd_dstream; + +/** + * zstd_dstream_workspace_bound() - memory needed to initialize a zstd_dstream + * @max_window_size: The maximum window size allowed for compressed frames. + * + * Return: A lower bound on the size of the workspace that is passed + * to zstd_init_dstream(). + */ +size_t zstd_dstream_workspace_bound(size_t max_window_size); + +/** + * zstd_init_dstream() - initialize a zstd streaming decompression context + * @max_window_size: The maximum window size allowed for compressed frames. + * @workspace: The workspace to emplace the context into. It must outlive + * the returned context. + * @workspaceSize: The size of workspace. + * Use zstd_dstream_workspace_bound(max_window_size) to + * determine how large the workspace must be. + * + * Return: The zstd streaming decompression context. + */ +zstd_dstream *zstd_init_dstream(size_t max_window_size, void *workspace, + size_t workspace_size); + +/** + * zstd_reset_dstream() - reset the context using parameters from creation + * @dstream: The zstd streaming decompression context to reset. + * + * Resets the context using the parameters from creation. Skips dictionary + * loading, since it can be reused. + * + * Return: Zero or an error, which can be checked using zstd_is_error(). + */ +size_t zstd_reset_dstream(zstd_dstream *dstream); + +/** + * zstd_decompress_stream() - streaming decompress some of input into output + * @dstream: The zstd streaming decompression context. + * @output: Destination buffer. `output.pos` is updated to indicate how much + * decompressed data was written. + * @input: Source buffer. `input.pos` is updated to indicate how much data was + * read. Note that it may not consume the entire input, in which case + * `input.pos < input.size`, and it's up to the caller to present + * remaining data again. + * + * The `input` and `output` buffers may be any size. Guaranteed to make some + * forward progress if `input` and `output` are not empty. + * zstd_decompress_stream() will not consume the last byte of the frame until + * the entire frame is flushed. + * + * Return: Returns 0 iff a frame is completely decoded and fully flushed. + * Otherwise returns a hint for the number of bytes to use as the + * input for the next function call or an error, which can be checked + * using zstd_is_error(). The size hint will never load more than the + * frame. + */ +size_t zstd_decompress_stream(zstd_dstream *dstream, + struct zstd_out_buffer *output, struct zstd_in_buffer *input); + +/* ====== Frame Inspection Functions ====== */ + +/** + * zstd_find_frame_compressed_size() - returns the size of a compressed frame + * @src: Source buffer. It should point to the start of a zstd encoded + * frame or a skippable frame. + * @src_size: The size of the source buffer. It must be at least as large as the + * size of the frame. + * + * Return: The compressed size of the frame pointed to by `src` or an error, + * which can be check with zstd_is_error(). + * Suitable to pass to ZSTD_decompress() or similar functions. + */ +size_t zstd_find_frame_compressed_size(const void *src, size_t src_size); + +/** + * struct zstd_frame_params - zstd frame parameters stored in the frame header + * @frame_content_size: The frame content size, or 0 if not present. + * @window_size: The window size, or 0 if the frame is a skippable frame. + * @dict_id: The dictionary id, or 0 if not present. + * @checksum_flag: Whether a checksum was used. + */ +struct zstd_frame_params { + unsigned long long frame_content_size; + unsigned int window_size; + unsigned int dict_id; + unsigned int checksum_flag; +}; + +/** + * zstd_get_frame_params() - extracts parameters from a zstd or skippable frame + * @params: On success the frame parameters are written here. + * @src: The source buffer. It must point to a zstd or skippable frame. + * @src_size: The size of the source buffer. + * + * Return: 0 on success. If more data is required it returns how many bytes + * must be provided to make forward progress. Otherwise it returns + * an error, which can be checked using zstd_is_error(). + */ +size_t zstd_get_frame_params(struct zstd_frame_params *params, const void *src, + size_t src_size); + +#endif /* LINUX_ZSTD_H */ diff --git a/contrib/linux-kernel/mem.h b/contrib/linux-kernel/mem.h new file mode 100644 index 00000000..54832a6d --- /dev/null +++ b/contrib/linux-kernel/mem.h @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +/*-**************************************** +* Dependencies +******************************************/ +#include <asm/unaligned.h> /* get_unaligned, put_unaligned* */ +#include <linux/compiler.h> /* inline */ +#include <linux/swab.h> /* swab32, swab64 */ +#include <linux/types.h> /* size_t, ptrdiff_t */ +#include "debug.h" /* DEBUG_STATIC_ASSERT */ + +/*-**************************************** +* Compiler specifics +******************************************/ +#define MEM_STATIC static inline + +/*-************************************************************** +* Basic Types +*****************************************************************/ +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef int16_t S16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +typedef int64_t S64; + +/*-************************************************************** +* Memory I/O API +*****************************************************************/ +/*=== Static platform detection ===*/ +MEM_STATIC unsigned MEM_32bits(void); +MEM_STATIC unsigned MEM_64bits(void); +MEM_STATIC unsigned MEM_isLittleEndian(void); + +/*=== Native unaligned read/write ===*/ +MEM_STATIC U16 MEM_read16(const void* memPtr); +MEM_STATIC U32 MEM_read32(const void* memPtr); +MEM_STATIC U64 MEM_read64(const void* memPtr); +MEM_STATIC size_t MEM_readST(const void* memPtr); + +MEM_STATIC void MEM_write16(void* memPtr, U16 value); +MEM_STATIC void MEM_write32(void* memPtr, U32 value); +MEM_STATIC void MEM_write64(void* memPtr, U64 value); + +/*=== Little endian unaligned read/write ===*/ +MEM_STATIC U16 MEM_readLE16(const void* memPtr); +MEM_STATIC U32 MEM_readLE24(const void* memPtr); +MEM_STATIC U32 MEM_readLE32(const void* memPtr); +MEM_STATIC U64 MEM_readLE64(const void* memPtr); +MEM_STATIC size_t MEM_readLEST(const void* memPtr); + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val); +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val); +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val); + +/*=== Big endian unaligned read/write ===*/ +MEM_STATIC U32 MEM_readBE32(const void* memPtr); +MEM_STATIC U64 MEM_readBE64(const void* memPtr); +MEM_STATIC size_t MEM_readBEST(const void* memPtr); + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val); + +/*=== Byteswap ===*/ +MEM_STATIC U32 MEM_swap32(U32 in); +MEM_STATIC U64 MEM_swap64(U64 in); +MEM_STATIC size_t MEM_swapST(size_t in); + +/*-************************************************************** +* Memory I/O Implementation +*****************************************************************/ +MEM_STATIC unsigned MEM_32bits(void) +{ + return sizeof(size_t) == 4; +} + +MEM_STATIC unsigned MEM_64bits(void) +{ + return sizeof(size_t) == 8; +} + +#if defined(__LITTLE_ENDIAN) +#define MEM_LITTLE_ENDIAN 1 +#else +#define MEM_LITTLE_ENDIAN 0 +#endif + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ + return MEM_LITTLE_ENDIAN; +} + +MEM_STATIC U16 MEM_read16(const void *memPtr) +{ + return get_unaligned((const U16 *)memPtr); +} + +MEM_STATIC U32 MEM_read32(const void *memPtr) +{ + return get_unaligned((const U32 *)memPtr); +} + +MEM_STATIC U64 MEM_read64(const void *memPtr) +{ + return get_unaligned((const U64 *)memPtr); +} + +MEM_STATIC size_t MEM_readST(const void *memPtr) +{ + return get_unaligned((const size_t *)memPtr); +} + +MEM_STATIC void MEM_write16(void *memPtr, U16 value) +{ + put_unaligned(value, (U16 *)memPtr); +} + +MEM_STATIC void MEM_write32(void *memPtr, U32 value) +{ + put_unaligned(value, (U32 *)memPtr); +} + +MEM_STATIC void MEM_write64(void *memPtr, U64 value) +{ + put_unaligned(value, (U64 *)memPtr); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void *memPtr) +{ + return get_unaligned_le16(memPtr); +} + +MEM_STATIC void MEM_writeLE16(void *memPtr, U16 val) +{ + put_unaligned_le16(val, memPtr); +} + +MEM_STATIC U32 MEM_readLE24(const void *memPtr) +{ + return MEM_readLE16(memPtr) + (((const BYTE *)memPtr)[2] << 16); +} + +MEM_STATIC void MEM_writeLE24(void *memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE *)memPtr)[2] = (BYTE)(val >> 16); +} + +MEM_STATIC U32 MEM_readLE32(const void *memPtr) +{ + return get_unaligned_le32(memPtr); +} + +MEM_STATIC void MEM_writeLE32(void *memPtr, U32 val32) +{ + put_unaligned_le32(val32, memPtr); +} + +MEM_STATIC U64 MEM_readLE64(const void *memPtr) +{ + return get_unaligned_le64(memPtr); +} + +MEM_STATIC void MEM_writeLE64(void *memPtr, U64 val64) +{ + put_unaligned_le64(val64, memPtr); +} + +MEM_STATIC size_t MEM_readLEST(const void *memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void *memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void *memPtr) +{ + return get_unaligned_be32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void *memPtr, U32 val32) +{ + put_unaligned_be32(val32, memPtr); +} + +MEM_STATIC U64 MEM_readBE64(const void *memPtr) +{ + return get_unaligned_be64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void *memPtr, U64 val64) +{ + put_unaligned_be64(val64, memPtr); +} + +MEM_STATIC size_t MEM_readBEST(const void *memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void *memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + +MEM_STATIC U32 MEM_swap32(U32 in) +{ + return swab32(in); +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ + return swab64(in); +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +#endif /* MEM_H_MODULE */ diff --git a/contrib/linux-kernel/test/.gitignore b/contrib/linux-kernel/test/.gitignore deleted file mode 100644 index 4fc10228..00000000 --- a/contrib/linux-kernel/test/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*Test diff --git a/contrib/linux-kernel/test/DecompressCrash.c b/contrib/linux-kernel/test/DecompressCrash.c deleted file mode 100644 index 2ab7dfe5..00000000 --- a/contrib/linux-kernel/test/DecompressCrash.c +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/* - This program takes a file in input, - performs a zstd round-trip test (compression - decompress) - compares the result with original - and generates a crash (double free) on corruption detection. -*/ - -/*=========================================== -* Dependencies -*==========================================*/ -#include <stddef.h> /* size_t */ -#include <stdlib.h> /* malloc, free, exit */ -#include <stdio.h> /* fprintf */ -#include <linux/zstd.h> - -/*=========================================== -* Macros -*==========================================*/ -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - -static ZSTD_DCtx *dctx = NULL; -void *dws = NULL; -static void* rBuff = NULL; -static size_t buffSize = 0; - -static void crash(int errorCode){ - /* abort if AFL/libfuzzer, exit otherwise */ - #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */ - abort(); - #else - exit(errorCode); - #endif -} - -static void decompressCheck(const void* srcBuff, size_t srcBuffSize) -{ - size_t const neededBuffSize = 20 * srcBuffSize; - - /* Allocate all buffers and contexts if not already allocated */ - if (neededBuffSize > buffSize) { - free(rBuff); - buffSize = 0; - - rBuff = malloc(neededBuffSize); - if (!rBuff) { - fprintf(stderr, "not enough memory ! \n"); - crash(1); - } - buffSize = neededBuffSize; - } - if (!dctx) { - size_t const workspaceSize = ZSTD_DCtxWorkspaceBound(); - dws = malloc(workspaceSize); - if (!dws) { - fprintf(stderr, "not enough memory ! \n"); - crash(1); - } - dctx = ZSTD_initDCtx(dws, workspaceSize); - if (!dctx) { - fprintf(stderr, "not enough memory ! \n"); - crash(1); - } - } - ZSTD_decompressDCtx(dctx, rBuff, buffSize, srcBuff, srcBuffSize); - -#ifndef SKIP_FREE - free(dws); dws = NULL; dctx = NULL; - free(rBuff); rBuff = NULL; - buffSize = 0; -#endif -} - -int LLVMFuzzerTestOneInput(const unsigned char *srcBuff, size_t srcBuffSize) { - decompressCheck(srcBuff, srcBuffSize); - return 0; -} diff --git a/contrib/linux-kernel/test/Makefile b/contrib/linux-kernel/test/Makefile index 8411462c..80bce74d 100644 --- a/contrib/linux-kernel/test/Makefile +++ b/contrib/linux-kernel/test/Makefile @@ -1,43 +1,34 @@ -IFLAGS := -isystem include/ -I ../include/ -I ../lib/zstd/ -isystem googletest/googletest/include -isystem ../../../lib/common/ +LINUX := ../linux +LINUX_ZSTDLIB := $(LINUX)/lib/zstd -SOURCES := $(wildcard ../lib/zstd/*.c) -OBJECTS := $(patsubst %.c,%.o,$(SOURCES)) +CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG +# Don't poison the workspace, it currently doesn't work with static allocation and workspace reuse +CPPFLAGS += -DZSTD_ASAN_DONT_POISON_WORKSPACE -ARFLAGS := rcs -CXXFLAGS += -std=c++11 -g -O3 -Wcast-align -CFLAGS += -g -O3 -Wframe-larger-than=400 -Wcast-align -CPPFLAGS += $(IFLAGS) +LINUX_ZSTD_MODULE := $(wildcard $(LINUX_ZSTDLIB)/*.c) +LINUX_ZSTD_COMMON := $(wildcard $(LINUX_ZSTDLIB)/common/*.c) +LINUX_ZSTD_COMPRESS := $(wildcard $(LINUX_ZSTDLIB)/compress/*.c) +LINUX_ZSTD_DECOMPRESS := $(wildcard $(LINUX_ZSTDLIB)/decompress/*.c) +LINUX_ZSTD_FILES := $(LINUX_ZSTD_MODULE) $(LINUX_ZSTD_COMMON) $(LINUX_ZSTD_COMPRESS) $(LINUX_ZSTD_DECOMPRESS) +LINUX_ZSTD_OBJECTS := $(LINUX_ZSTD_FILES:.c=.o) -../lib/zstd/libzstd.a: $(OBJECTS) +liblinuxzstd.a: $(LINUX_ZSTD_OBJECTS) $(AR) $(ARFLAGS) $@ $^ -DecompressCrash: DecompressCrash.o $(OBJECTS) libFuzzer.a - $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) $^ -o $@ +test: test.c liblinuxzstd.a + $(CC) $(LDFLAGS) $(CPPFLAGS) $(CFLAGS) $^ -o $@ -RoundTripCrash: RoundTripCrash.o $(OBJECTS) ../lib/xxhash.o libFuzzer.a - $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) $^ -o $@ +static_test: static_test.c + $(CC) $(LDFLAGS) $(CPPFLAGS) $(CFLAGS) $^ -o $@ -UserlandTest: UserlandTest.cpp ../lib/zstd/libzstd.a ../lib/xxhash.o - $(CXX) $(CXXFLAGS) $(CPPFLAGS) $^ googletest/build/googlemock/gtest/libgtest.a googletest/build/googlemock/gtest/libgtest_main.a -o $@ - -XXHashUserlandTest: XXHashUserlandTest.cpp ../lib/xxhash.o ../../../lib/common/xxhash.o - $(CXX) $(CXXFLAGS) $(CFLAGS) $(CPPFLAGS) $^ googletest/build/googlemock/gtest/libgtest.a googletest/build/googlemock/gtest/libgtest_main.a -o $@ - -# Install libfuzzer -libFuzzer.a: - @$(RM) -rf Fuzzer - @git clone https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer - @./Fuzzer/build.sh - -# Install googletest -.PHONY: googletest -googletest: - @$(RM) -rf googletest - @git clone https://github.com/google/googletest - @mkdir -p googletest/build - @cd googletest/build && cmake .. && $(MAKE) +run-test: test static_test + ./macro-test.sh + ./test + ./static_test +.PHONY: clean: - $(RM) -f *.{o,a} ../lib/zstd/*.{o,a} ../lib/*.o - $(RM) -f DecompressCrash RoundTripCrash UserlandTest XXHashUserlandTest + $(RM) -f $(LINUX_ZSTDLIB)/**/*.o + $(RM) -f *.o *.a + $(RM) -f test diff --git a/contrib/linux-kernel/test/RoundTripCrash.c b/contrib/linux-kernel/test/RoundTripCrash.c deleted file mode 100644 index 4f968023..00000000 --- a/contrib/linux-kernel/test/RoundTripCrash.c +++ /dev/null @@ -1,162 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/* - This program takes a file in input, - performs a zstd round-trip test (compression - decompress) - compares the result with original - and generates a crash (double free) on corruption detection. -*/ - -/*=========================================== -* Dependencies -*==========================================*/ -#include <stddef.h> /* size_t */ -#include <stdlib.h> /* malloc, free, exit */ -#include <stdio.h> /* fprintf */ -#include <linux/xxhash.h> -#include <linux/zstd.h> - -/*=========================================== -* Macros -*==========================================*/ -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - -static const int kMaxClevel = 22; - -static ZSTD_CCtx *cctx = NULL; -void *cws = NULL; -static ZSTD_DCtx *dctx = NULL; -void *dws = NULL; -static void* cBuff = NULL; -static void* rBuff = NULL; -static size_t buffSize = 0; - - -/** roundTripTest() : -* Compresses `srcBuff` into `compressedBuff`, -* then decompresses `compressedBuff` into `resultBuff`. -* Compression level used is derived from first content byte. -* @return : result of decompression, which should be == `srcSize` -* or an error code if either compression or decompression fails. -* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)` -* for compression to be guaranteed to work */ -static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity, - void* compressedBuff, size_t compressedBuffCapacity, - const void* srcBuff, size_t srcBuffSize) -{ - size_t const hashLength = MIN(128, srcBuffSize); - unsigned const h32 = xxh32(srcBuff, hashLength, 0); - int const cLevel = h32 % kMaxClevel; - ZSTD_parameters const params = ZSTD_getParams(cLevel, srcBuffSize, 0); - size_t const cSize = ZSTD_compressCCtx(cctx, compressedBuff, compressedBuffCapacity, srcBuff, srcBuffSize, params); - if (ZSTD_isError(cSize)) { - fprintf(stderr, "Compression error : %u \n", ZSTD_getErrorCode(cSize)); - return cSize; - } - return ZSTD_decompressDCtx(dctx, resultBuff, resultBuffCapacity, compressedBuff, cSize); -} - - -static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize) -{ - const char* ip1 = (const char*)buff1; - const char* ip2 = (const char*)buff2; - size_t pos; - - for (pos=0; pos<buffSize; pos++) - if (ip1[pos]!=ip2[pos]) - break; - - return pos; -} - -static void crash(int errorCode){ - /* abort if AFL/libfuzzer, exit otherwise */ - #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */ - abort(); - #else - exit(errorCode); - #endif -} - -static void roundTripCheck(const void* srcBuff, size_t srcBuffSize) -{ - size_t const neededBuffSize = ZSTD_compressBound(srcBuffSize); - - /* Allocate all buffers and contexts if not already allocated */ - if (neededBuffSize > buffSize) { - free(cBuff); - free(rBuff); - buffSize = 0; - - cBuff = malloc(neededBuffSize); - rBuff = malloc(neededBuffSize); - if (!cBuff || !rBuff) { - fprintf(stderr, "not enough memory ! \n"); - crash(1); - } - buffSize = neededBuffSize; - } - if (!cctx) { - ZSTD_compressionParameters const params = ZSTD_getCParams(kMaxClevel, 0, 0); - size_t const workspaceSize = ZSTD_CCtxWorkspaceBound(params); - cws = malloc(workspaceSize); - if (!cws) { - fprintf(stderr, "not enough memory ! \n"); - crash(1); - } - cctx = ZSTD_initCCtx(cws, workspaceSize); - if (!cctx) { - fprintf(stderr, "not enough memory ! \n"); - crash(1); - } - } - if (!dctx) { - size_t const workspaceSize = ZSTD_DCtxWorkspaceBound(); - dws = malloc(workspaceSize); - if (!dws) { - fprintf(stderr, "not enough memory ! \n"); - crash(1); - } - dctx = ZSTD_initDCtx(dws, workspaceSize); - if (!dctx) { - fprintf(stderr, "not enough memory ! \n"); - crash(1); - } - } - - { size_t const result = roundTripTest(rBuff, buffSize, cBuff, buffSize, srcBuff, srcBuffSize); - if (ZSTD_isError(result)) { - fprintf(stderr, "roundTripTest error : %u \n", ZSTD_getErrorCode(result)); - crash(1); - } - if (result != srcBuffSize) { - fprintf(stderr, "Incorrect regenerated size : %u != %u\n", (unsigned)result, (unsigned)srcBuffSize); - crash(1); - } - if (checkBuffers(srcBuff, rBuff, srcBuffSize) != srcBuffSize) { - fprintf(stderr, "Silent decoding corruption !!!"); - crash(1); - } - } - -#ifndef SKIP_FREE - free(cws); cws = NULL; cctx = NULL; - free(dws); dws = NULL; dctx = NULL; - free(cBuff); cBuff = NULL; - free(rBuff); rBuff = NULL; - buffSize = 0; -#endif -} - -int LLVMFuzzerTestOneInput(const unsigned char *srcBuff, size_t srcBuffSize) { - roundTripCheck(srcBuff, srcBuffSize); - return 0; -} diff --git a/contrib/linux-kernel/test/UserlandTest.cpp b/contrib/linux-kernel/test/UserlandTest.cpp deleted file mode 100644 index 03058382..00000000 --- a/contrib/linux-kernel/test/UserlandTest.cpp +++ /dev/null @@ -1,565 +0,0 @@ -extern "C" { -#include <linux/zstd.h> -} -#include <gtest/gtest.h> -#include <memory> -#include <string> -#include <iostream> - -using namespace std; - -namespace { -struct WorkspaceDeleter { - void *memory; - - template <typename T> void operator()(T const *) { free(memory); } -}; - -std::unique_ptr<ZSTD_CCtx, WorkspaceDeleter> -createCCtx(ZSTD_compressionParameters cParams) { - size_t const workspaceSize = ZSTD_CCtxWorkspaceBound(cParams); - void *workspace = malloc(workspaceSize); - std::unique_ptr<ZSTD_CCtx, WorkspaceDeleter> cctx{ - ZSTD_initCCtx(workspace, workspaceSize), WorkspaceDeleter{workspace}}; - if (!cctx) { - throw std::runtime_error{"Bad cctx"}; - } - return cctx; -} - -std::unique_ptr<ZSTD_CCtx, WorkspaceDeleter> -createCCtx(int level, unsigned long long estimatedSrcSize = 0, - size_t dictSize = 0) { - auto const cParams = ZSTD_getCParams(level, estimatedSrcSize, dictSize); - return createCCtx(cParams); -} - -std::unique_ptr<ZSTD_DCtx, WorkspaceDeleter> -createDCtx() { - size_t const workspaceSize = ZSTD_DCtxWorkspaceBound(); - void *workspace = malloc(workspaceSize); - std::unique_ptr<ZSTD_DCtx, WorkspaceDeleter> dctx{ - ZSTD_initDCtx(workspace, workspaceSize), WorkspaceDeleter{workspace}}; - if (!dctx) { - throw std::runtime_error{"Bad dctx"}; - } - return dctx; -} - -std::unique_ptr<ZSTD_CDict, WorkspaceDeleter> -createCDict(std::string const& dict, ZSTD_parameters params) { - size_t const workspaceSize = ZSTD_CDictWorkspaceBound(params.cParams); - void *workspace = malloc(workspaceSize); - std::unique_ptr<ZSTD_CDict, WorkspaceDeleter> cdict{ - ZSTD_initCDict(dict.data(), dict.size(), params, workspace, - workspaceSize), - WorkspaceDeleter{workspace}}; - if (!cdict) { - throw std::runtime_error{"Bad cdict"}; - } - return cdict; -} - -std::unique_ptr<ZSTD_CDict, WorkspaceDeleter> -createCDict(std::string const& dict, int level) { - auto const params = ZSTD_getParams(level, 0, dict.size()); - return createCDict(dict, params); -} - -std::unique_ptr<ZSTD_DDict, WorkspaceDeleter> -createDDict(std::string const& dict) { - size_t const workspaceSize = ZSTD_DDictWorkspaceBound(); - void *workspace = malloc(workspaceSize); - std::unique_ptr<ZSTD_DDict, WorkspaceDeleter> ddict{ - ZSTD_initDDict(dict.data(), dict.size(), workspace, workspaceSize), - WorkspaceDeleter{workspace}}; - if (!ddict) { - throw std::runtime_error{"Bad ddict"}; - } - return ddict; -} - -std::unique_ptr<ZSTD_CStream, WorkspaceDeleter> -createCStream(ZSTD_parameters params, unsigned long long pledgedSrcSize = 0) { - size_t const workspaceSize = ZSTD_CStreamWorkspaceBound(params.cParams); - void *workspace = malloc(workspaceSize); - std::unique_ptr<ZSTD_CStream, WorkspaceDeleter> zcs{ - ZSTD_initCStream(params, pledgedSrcSize, workspace, workspaceSize)}; - if (!zcs) { - throw std::runtime_error{"bad cstream"}; - } - return zcs; -} - -std::unique_ptr<ZSTD_CStream, WorkspaceDeleter> -createCStream(ZSTD_compressionParameters cParams, ZSTD_CDict const &cdict, - unsigned long long pledgedSrcSize = 0) { - size_t const workspaceSize = ZSTD_CStreamWorkspaceBound(cParams); - void *workspace = malloc(workspaceSize); - std::unique_ptr<ZSTD_CStream, WorkspaceDeleter> zcs{ - ZSTD_initCStream_usingCDict(&cdict, pledgedSrcSize, workspace, - workspaceSize)}; - if (!zcs) { - throw std::runtime_error{"bad cstream"}; - } - return zcs; -} - -std::unique_ptr<ZSTD_CStream, WorkspaceDeleter> -createCStream(int level, unsigned long long pledgedSrcSize = 0) { - auto const params = ZSTD_getParams(level, pledgedSrcSize, 0); - return createCStream(params, pledgedSrcSize); -} - -std::unique_ptr<ZSTD_DStream, WorkspaceDeleter> -createDStream(size_t maxWindowSize = (1ULL << ZSTD_WINDOWLOG_MAX), - ZSTD_DDict const *ddict = nullptr) { - size_t const workspaceSize = ZSTD_DStreamWorkspaceBound(maxWindowSize); - void *workspace = malloc(workspaceSize); - std::unique_ptr<ZSTD_DStream, WorkspaceDeleter> zds{ - ddict == nullptr - ? ZSTD_initDStream(maxWindowSize, workspace, workspaceSize) - : ZSTD_initDStream_usingDDict(maxWindowSize, ddict, workspace, - workspaceSize)}; - if (!zds) { - throw std::runtime_error{"bad dstream"}; - } - return zds; -} - -std::string compress(ZSTD_CCtx &cctx, std::string const &data, - ZSTD_parameters params, std::string const &dict = "") { - std::string compressed; - compressed.resize(ZSTD_compressBound(data.size())); - size_t const rc = - dict.empty() - ? ZSTD_compressCCtx(&cctx, &compressed[0], compressed.size(), - data.data(), data.size(), params) - : ZSTD_compress_usingDict(&cctx, &compressed[0], compressed.size(), - data.data(), data.size(), dict.data(), - dict.size(), params); - if (ZSTD_isError(rc)) { - throw std::runtime_error{"compression error"}; - } - compressed.resize(rc); - return compressed; -} - -std::string compress(ZSTD_CCtx& cctx, std::string const& data, int level, std::string const& dict = "") { - auto const params = ZSTD_getParams(level, 0, dict.size()); - return compress(cctx, data, params, dict); -} - -std::string decompress(ZSTD_DCtx& dctx, std::string const& compressed, size_t decompressedSize, std::string const& dict = "") { - std::string decompressed; - decompressed.resize(decompressedSize); - size_t const rc = - dict.empty() - ? ZSTD_decompressDCtx(&dctx, &decompressed[0], decompressed.size(), - compressed.data(), compressed.size()) - : ZSTD_decompress_usingDict( - &dctx, &decompressed[0], decompressed.size(), compressed.data(), - compressed.size(), dict.data(), dict.size()); - if (ZSTD_isError(rc)) { - throw std::runtime_error{"decompression error"}; - } - decompressed.resize(rc); - return decompressed; -} - -std::string compress(ZSTD_CCtx& cctx, std::string const& data, ZSTD_CDict& cdict) { - std::string compressed; - compressed.resize(ZSTD_compressBound(data.size())); - size_t const rc = - ZSTD_compress_usingCDict(&cctx, &compressed[0], compressed.size(), - data.data(), data.size(), &cdict); - if (ZSTD_isError(rc)) { - throw std::runtime_error{"compression error"}; - } - compressed.resize(rc); - return compressed; -} - -std::string decompress(ZSTD_DCtx& dctx, std::string const& compressed, size_t decompressedSize, ZSTD_DDict& ddict) { - std::string decompressed; - decompressed.resize(decompressedSize); - size_t const rc = - ZSTD_decompress_usingDDict(&dctx, &decompressed[0], decompressed.size(), - compressed.data(), compressed.size(), &ddict); - if (ZSTD_isError(rc)) { - throw std::runtime_error{"decompression error"}; - } - decompressed.resize(rc); - return decompressed; -} - -std::string compress(ZSTD_CStream& zcs, std::string const& data) { - std::string compressed; - compressed.resize(ZSTD_compressBound(data.size())); - ZSTD_inBuffer in = {data.data(), data.size(), 0}; - ZSTD_outBuffer out = {&compressed[0], compressed.size(), 0}; - while (in.pos != in.size) { - size_t const rc = ZSTD_compressStream(&zcs, &out, &in); - if (ZSTD_isError(rc)) { - throw std::runtime_error{"compress stream failed"}; - } - } - size_t const rc = ZSTD_endStream(&zcs, &out); - if (rc != 0) { - throw std::runtime_error{"compress end failed"}; - } - compressed.resize(out.pos); - return compressed; -} - -std::string decompress(ZSTD_DStream &zds, std::string const &compressed, - size_t decompressedSize) { - std::string decompressed; - decompressed.resize(decompressedSize); - ZSTD_inBuffer in = {compressed.data(), compressed.size(), 0}; - ZSTD_outBuffer out = {&decompressed[0], decompressed.size(), 0}; - while (in.pos != in.size) { - size_t const rc = ZSTD_decompressStream(&zds, &out, &in); - if (ZSTD_isError(rc)) { - throw std::runtime_error{"decompress stream failed"}; - } - } - decompressed.resize(out.pos); - return decompressed; -} - -std::string makeData(size_t size) { - std::string result; - result.reserve(size + 20); - while (result.size() < size) { - result += "Hello world"; - } - return result; -} - -std::string const kData = "Hello world"; -std::string const kPlainDict = makeData(10000); -std::string const kZstdDict{ - "\x37\xA4\x30\xEC\x99\x69\x58\x1C\x21\x10\xD8\x4A\x84\x01\xCC\xF3" - "\x3C\xCF\x9B\x25\xBB\xC9\x6E\xB2\x9B\xEC\x26\xAD\xCF\xDF\x4E\xCD" - "\xF3\x2C\x3A\x21\x84\x10\x42\x08\x21\x01\x33\xF1\x78\x3C\x1E\x8F" - "\xC7\xE3\xF1\x78\x3C\xCF\xF3\xBC\xF7\xD4\x42\x41\x41\x41\x41\x41" - "\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41" - "\x41\x41\x41\x41\xA1\x50\x28\x14\x0A\x85\x42\xA1\x50\x28\x14\x0A" - "\x85\xA2\x28\x8A\xA2\x28\x4A\x29\x7D\x74\xE1\xE1\xE1\xE1\xE1\xE1" - "\xE1\xE1\xE1\xE1\xE1\xE1\xE1\xE1\xE1\xE1\xE1\xE1\xE1\xF1\x78\x3C" - "\x1E\x8F\xC7\xE3\xF1\x78\x9E\xE7\x79\xEF\x01\x01\x00\x00\x00\x04" - "\x00\x00\x00\x08\x00\x00\x00" - "0123456789", - 161}; -} - -TEST(Block, CCtx) { - auto cctx = createCCtx(1); - auto const compressed = compress(*cctx, kData, 1); - auto dctx = createDCtx(); - auto const decompressed = decompress(*dctx, compressed, kData.size()); - EXPECT_EQ(kData, decompressed); -} - -TEST(Block, NoContentSize) { - auto cctx = createCCtx(1); - auto const c = compress(*cctx, kData, 1); - auto const size = ZSTD_findDecompressedSize(c.data(), c.size()); - EXPECT_EQ(ZSTD_CONTENTSIZE_UNKNOWN, size); -} - -TEST(Block, ContentSize) { - auto cctx = createCCtx(1); - auto params = ZSTD_getParams(1, 0, 0); - params.fParams.contentSizeFlag = 1; - auto const c = compress(*cctx, kData, params); - auto const size = ZSTD_findDecompressedSize(c.data(), c.size()); - EXPECT_EQ(kData.size(), size); -} - -TEST(Block, CCtxLevelIncrease) { - std::string c; - auto cctx = createCCtx(22); - auto dctx = createDCtx(); - for (int level = 1; level <= 22; ++level) { - auto compressed = compress(*cctx, kData, level); - auto const decompressed = decompress(*dctx, compressed, kData.size()); - EXPECT_EQ(kData, decompressed); - } -} - -TEST(Block, PlainDict) { - auto cctx = createCCtx(1); - auto const compressed = compress(*cctx, kData, 1, kPlainDict); - auto dctx = createDCtx(); - EXPECT_ANY_THROW(decompress(*dctx, compressed, kData.size())); - auto const decompressed = - decompress(*dctx, compressed, kData.size(), kPlainDict); - EXPECT_EQ(kData, decompressed); -} - -TEST(Block, ZstdDict) { - auto cctx = createCCtx(1); - auto const compressed = compress(*cctx, kData, 1, kZstdDict); - auto dctx = createDCtx(); - EXPECT_ANY_THROW(decompress(*dctx, compressed, kData.size())); - auto const decompressed = - decompress(*dctx, compressed, kData.size(), kZstdDict); - EXPECT_EQ(kData, decompressed); -} - -TEST(Block, PreprocessedPlainDict) { - auto cctx = createCCtx(1); - auto const cdict = createCDict(kPlainDict, 1); - auto const compressed = compress(*cctx, kData, *cdict); - auto dctx = createDCtx(); - auto const ddict = createDDict(kPlainDict); - EXPECT_ANY_THROW(decompress(*dctx, compressed, kData.size())); - auto const decompressed = - decompress(*dctx, compressed, kData.size(), *ddict); - EXPECT_EQ(kData, decompressed); -} - -TEST(Block, PreprocessedZstdDict) { - auto cctx = createCCtx(1); - auto const cdict = createCDict(kZstdDict, 1); - auto const compressed = compress(*cctx, kData, *cdict); - auto dctx = createDCtx(); - auto const ddict = createDDict(kZstdDict); - EXPECT_ANY_THROW(decompress(*dctx, compressed, kData.size())); - auto const decompressed = - decompress(*dctx, compressed, kData.size(), *ddict); - EXPECT_EQ(kData, decompressed); -} - -TEST(Block, ReinitializeCCtx) { - auto cctx = createCCtx(1); - { - auto const compressed = compress(*cctx, kData, 1); - auto dctx = createDCtx(); - auto const decompressed = decompress(*dctx, compressed, kData.size()); - EXPECT_EQ(kData, decompressed); - } - // Create the cctx with the same memory - auto d = cctx.get_deleter(); - auto raw = cctx.release(); - auto params = ZSTD_getParams(1, 0, 0); - cctx.reset( - ZSTD_initCCtx(d.memory, ZSTD_CCtxWorkspaceBound(params.cParams))); - // Repeat - { - auto const compressed = compress(*cctx, kData, 1); - auto dctx = createDCtx(); - auto const decompressed = decompress(*dctx, compressed, kData.size()); - EXPECT_EQ(kData, decompressed); - } -} - -TEST(Block, ReinitializeDCtx) { - auto dctx = createDCtx(); - { - auto cctx = createCCtx(1); - auto const compressed = compress(*cctx, kData, 1); - auto const decompressed = decompress(*dctx, compressed, kData.size()); - EXPECT_EQ(kData, decompressed); - } - // Create the cctx with the same memory - auto d = dctx.get_deleter(); - auto raw = dctx.release(); - dctx.reset(ZSTD_initDCtx(d.memory, ZSTD_DCtxWorkspaceBound())); - // Repeat - { - auto cctx = createCCtx(1); - auto const compressed = compress(*cctx, kData, 1); - auto dctx = createDCtx(); - auto const decompressed = decompress(*dctx, compressed, kData.size()); - EXPECT_EQ(kData, decompressed); - } -} - -TEST(Stream, Basic) { - auto zcs = createCStream(1); - auto const compressed = compress(*zcs, kData); - auto zds = createDStream(); - auto const decompressed = decompress(*zds, compressed, kData.size()); - EXPECT_EQ(kData, decompressed); -} - -TEST(Stream, PlainDict) { - auto params = ZSTD_getParams(1, kData.size(), kPlainDict.size()); - params.cParams.windowLog = 17; - auto cdict = createCDict(kPlainDict, params); - auto zcs = createCStream(params.cParams, *cdict, kData.size()); - auto const compressed = compress(*zcs, kData); - auto const contentSize = - ZSTD_findDecompressedSize(compressed.data(), compressed.size()); - EXPECT_ANY_THROW(decompress(*createDStream(), compressed, kData.size())); - auto ddict = createDDict(kPlainDict); - auto zds = createDStream(1 << 17, ddict.get()); - auto const decompressed = decompress(*zds, compressed, kData.size()); - EXPECT_EQ(kData, decompressed); -} - -TEST(Stream, ZstdDict) { - auto params = ZSTD_getParams(1, 0, 0); - params.cParams.windowLog = 17; - auto cdict = createCDict(kZstdDict, 1); - auto zcs = createCStream(params.cParams, *cdict); - auto const compressed = compress(*zcs, kData); - EXPECT_ANY_THROW(decompress(*createDStream(), compressed, kData.size())); - auto ddict = createDDict(kZstdDict); - auto zds = createDStream(1 << 17, ddict.get()); - auto const decompressed = decompress(*zds, compressed, kData.size()); - EXPECT_EQ(kData, decompressed); -} - -TEST(Stream, ResetCStream) { - auto zcs = createCStream(1); - auto zds = createDStream(); - { - auto const compressed = compress(*zcs, kData); - auto const decompressed = decompress(*zds, compressed, kData.size()); - EXPECT_EQ(kData, decompressed); - } - { - ZSTD_resetCStream(zcs.get(), 0); - auto const compressed = compress(*zcs, kData); - auto const decompressed = decompress(*zds, compressed, kData.size()); - EXPECT_EQ(kData, decompressed); - } -} - -TEST(Stream, ResetDStream) { - auto zcs = createCStream(1); - auto zds = createDStream(); - auto const compressed = compress(*zcs, kData); - EXPECT_ANY_THROW(decompress(*zds, kData, kData.size())); - EXPECT_ANY_THROW(decompress(*zds, compressed, kData.size())); - ZSTD_resetDStream(zds.get()); - auto const decompressed = decompress(*zds, compressed, kData.size()); - EXPECT_EQ(kData, decompressed); -} - -TEST(Stream, Flush) { - auto zcs = createCStream(1); - auto zds = createDStream(); - std::string compressed; - { - compressed.resize(ZSTD_compressBound(kData.size())); - ZSTD_inBuffer in = {kData.data(), kData.size(), 0}; - ZSTD_outBuffer out = {&compressed[0], compressed.size(), 0}; - while (in.pos != in.size) { - size_t const rc = ZSTD_compressStream(zcs.get(), &out, &in); - if (ZSTD_isError(rc)) { - throw std::runtime_error{"compress stream failed"}; - } - } - EXPECT_EQ(0, out.pos); - size_t const rc = ZSTD_flushStream(zcs.get(), &out); - if (rc != 0) { - throw std::runtime_error{"compress end failed"}; - } - compressed.resize(out.pos); - EXPECT_LT(0, out.pos); - } - std::string decompressed; - { - decompressed.resize(kData.size()); - ZSTD_inBuffer in = {compressed.data(), compressed.size(), 0}; - ZSTD_outBuffer out = {&decompressed[0], decompressed.size(), 0}; - while (in.pos != in.size) { - size_t const rc = ZSTD_decompressStream(zds.get(), &out, &in); - if (ZSTD_isError(rc)) { - throw std::runtime_error{"decompress stream failed"}; - } - } - } - EXPECT_EQ(kData, decompressed); -} - -TEST(Stream, DStreamLevelIncrease) { - auto zds = createDStream(); - for (int level = 1; level <= 22; ++level) { - auto zcs = createCStream(level); - auto compressed = compress(*zcs, kData); - ZSTD_resetDStream(zds.get()); - auto const decompressed = decompress(*zds, compressed, kData.size()); - EXPECT_EQ(kData, decompressed); - } -} - -#define TEST_SYMBOL(symbol) \ - do { \ - extern void *__##symbol; \ - EXPECT_NE((void *)0, __##symbol); \ - } while (0) - -TEST(API, Symbols) { - TEST_SYMBOL(ZSTD_CCtxWorkspaceBound); - TEST_SYMBOL(ZSTD_initCCtx); - TEST_SYMBOL(ZSTD_compressCCtx); - TEST_SYMBOL(ZSTD_compress_usingDict); - TEST_SYMBOL(ZSTD_DCtxWorkspaceBound); - TEST_SYMBOL(ZSTD_initDCtx); - TEST_SYMBOL(ZSTD_decompressDCtx); - TEST_SYMBOL(ZSTD_decompress_usingDict); - - TEST_SYMBOL(ZSTD_CDictWorkspaceBound); - TEST_SYMBOL(ZSTD_initCDict); - TEST_SYMBOL(ZSTD_compress_usingCDict); - TEST_SYMBOL(ZSTD_DDictWorkspaceBound); - TEST_SYMBOL(ZSTD_initDDict); - TEST_SYMBOL(ZSTD_decompress_usingDDict); - - TEST_SYMBOL(ZSTD_CStreamWorkspaceBound); - TEST_SYMBOL(ZSTD_initCStream); - TEST_SYMBOL(ZSTD_initCStream_usingCDict); - TEST_SYMBOL(ZSTD_resetCStream); - TEST_SYMBOL(ZSTD_compressStream); - TEST_SYMBOL(ZSTD_flushStream); - TEST_SYMBOL(ZSTD_endStream); - TEST_SYMBOL(ZSTD_CStreamInSize); - TEST_SYMBOL(ZSTD_CStreamOutSize); - TEST_SYMBOL(ZSTD_DStreamWorkspaceBound); - TEST_SYMBOL(ZSTD_initDStream); - TEST_SYMBOL(ZSTD_initDStream_usingDDict); - TEST_SYMBOL(ZSTD_resetDStream); - TEST_SYMBOL(ZSTD_decompressStream); - TEST_SYMBOL(ZSTD_DStreamInSize); - TEST_SYMBOL(ZSTD_DStreamOutSize); - - TEST_SYMBOL(ZSTD_findFrameCompressedSize); - TEST_SYMBOL(ZSTD_getFrameContentSize); - TEST_SYMBOL(ZSTD_findDecompressedSize); - - TEST_SYMBOL(ZSTD_getCParams); - TEST_SYMBOL(ZSTD_getParams); - TEST_SYMBOL(ZSTD_checkCParams); - TEST_SYMBOL(ZSTD_adjustCParams); - - TEST_SYMBOL(ZSTD_isFrame); - TEST_SYMBOL(ZSTD_getDictID_fromDict); - TEST_SYMBOL(ZSTD_getDictID_fromDDict); - TEST_SYMBOL(ZSTD_getDictID_fromFrame); - - TEST_SYMBOL(ZSTD_compressBegin); - TEST_SYMBOL(ZSTD_compressBegin_usingDict); - TEST_SYMBOL(ZSTD_compressBegin_advanced); - TEST_SYMBOL(ZSTD_copyCCtx); - TEST_SYMBOL(ZSTD_compressBegin_usingCDict); - TEST_SYMBOL(ZSTD_compressContinue); - TEST_SYMBOL(ZSTD_compressEnd); - TEST_SYMBOL(ZSTD_getFrameParams); - TEST_SYMBOL(ZSTD_decompressBegin); - TEST_SYMBOL(ZSTD_decompressBegin_usingDict); - TEST_SYMBOL(ZSTD_copyDCtx); - TEST_SYMBOL(ZSTD_nextSrcSizeToDecompress); - TEST_SYMBOL(ZSTD_decompressContinue); - TEST_SYMBOL(ZSTD_nextInputType); - - TEST_SYMBOL(ZSTD_getBlockSizeMax); - TEST_SYMBOL(ZSTD_compressBlock); - TEST_SYMBOL(ZSTD_decompressBlock); - TEST_SYMBOL(ZSTD_insertBlock); -} diff --git a/contrib/linux-kernel/test/XXHashUserlandTest.cpp b/contrib/linux-kernel/test/XXHashUserlandTest.cpp deleted file mode 100644 index f50401a2..00000000 --- a/contrib/linux-kernel/test/XXHashUserlandTest.cpp +++ /dev/null @@ -1,166 +0,0 @@ -extern "C" { -#include <linux/errno.h> -#include <linux/xxhash.h> -} -#include <gtest/gtest.h> -#include <array> -#include <iostream> -#include <memory> -#include <string> -#define XXH_STATIC_LINKING_ONLY -#include <xxhash.h> - -using namespace std; - -namespace { -const std::array<std::string, 11> kTestInputs = { - "", - "0", - "01234", - "0123456789abcde", - "0123456789abcdef", - "0123456789abcdef0", - "0123456789abcdef0123", - "0123456789abcdef0123456789abcde", - "0123456789abcdef0123456789abcdef", - "0123456789abcdef0123456789abcdef0", - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", -}; - -bool testXXH32(const void *input, const size_t length, uint32_t seed) { - return XXH32(input, length, seed) == xxh32(input, length, seed); -} - -bool testXXH64(const void *input, const size_t length, uint32_t seed) { - return XXH64(input, length, seed) == xxh64(input, length, seed); -} - -class XXH32State { - struct xxh32_state kernelState; - XXH32_state_t state; - -public: - explicit XXH32State(const uint32_t seed) { reset(seed); } - XXH32State(XXH32State const& other) noexcept { - xxh32_copy_state(&kernelState, &other.kernelState); - XXH32_copyState(&state, &other.state); - } - XXH32State& operator=(XXH32State const& other) noexcept { - xxh32_copy_state(&kernelState, &other.kernelState); - XXH32_copyState(&state, &other.state); - return *this; - } - - void reset(const uint32_t seed) { - xxh32_reset(&kernelState, seed); - EXPECT_EQ(0, XXH32_reset(&state, seed)); - } - - void update(const void *input, const size_t length) { - EXPECT_EQ(0, xxh32_update(&kernelState, input, length)); - EXPECT_EQ(0, (int)XXH32_update(&state, input, length)); - } - - bool testDigest() const { - return xxh32_digest(&kernelState) == XXH32_digest(&state); - } -}; - -class XXH64State { - struct xxh64_state kernelState; - XXH64_state_t state; - -public: - explicit XXH64State(const uint64_t seed) { reset(seed); } - XXH64State(XXH64State const& other) noexcept { - xxh64_copy_state(&kernelState, &other.kernelState); - XXH64_copyState(&state, &other.state); - } - XXH64State& operator=(XXH64State const& other) noexcept { - xxh64_copy_state(&kernelState, &other.kernelState); - XXH64_copyState(&state, &other.state); - return *this; - } - - void reset(const uint64_t seed) { - xxh64_reset(&kernelState, seed); - EXPECT_EQ(0, XXH64_reset(&state, seed)); - } - - void update(const void *input, const size_t length) { - EXPECT_EQ(0, xxh64_update(&kernelState, input, length)); - EXPECT_EQ(0, (int)XXH64_update(&state, input, length)); - } - - bool testDigest() const { - return xxh64_digest(&kernelState) == XXH64_digest(&state); - } -}; -} - -TEST(Simple, Null) { - EXPECT_TRUE(testXXH32(NULL, 0, 0)); - EXPECT_TRUE(testXXH64(NULL, 0, 0)); -} - -TEST(Stream, Null) { - struct xxh32_state state32; - xxh32_reset(&state32, 0); - EXPECT_EQ(-EINVAL, xxh32_update(&state32, NULL, 0)); - - struct xxh64_state state64; - xxh64_reset(&state64, 0); - EXPECT_EQ(-EINVAL, xxh64_update(&state64, NULL, 0)); -} - -TEST(Simple, TestInputs) { - for (uint32_t seed = 0; seed < 100000; seed = (seed + 1) * 3) { - for (auto const input : kTestInputs) { - EXPECT_TRUE(testXXH32(input.data(), input.size(), seed)); - EXPECT_TRUE(testXXH64(input.data(), input.size(), (uint64_t)seed)); - } - } -} - -TEST(Stream, TestInputs) { - for (uint32_t seed = 0; seed < 100000; seed = (seed + 1) * 3) { - for (auto const input : kTestInputs) { - XXH32State s32(seed); - XXH64State s64(seed); - s32.update(input.data(), input.size()); - s64.update(input.data(), input.size()); - EXPECT_TRUE(s32.testDigest()); - EXPECT_TRUE(s64.testDigest()); - } - } -} - -TEST(Stream, MultipleTestInputs) { - for (uint32_t seed = 0; seed < 100000; seed = (seed + 1) * 3) { - XXH32State s32(seed); - XXH64State s64(seed); - for (auto const input : kTestInputs) { - s32.update(input.data(), input.size()); - s64.update(input.data(), input.size()); - } - EXPECT_TRUE(s32.testDigest()); - EXPECT_TRUE(s64.testDigest()); - } -} - -TEST(Stream, CopyState) { - for (uint32_t seed = 0; seed < 100000; seed = (seed + 1) * 3) { - XXH32State s32(seed); - XXH64State s64(seed); - for (auto const input : kTestInputs) { - auto t32(s32); - t32.update(input.data(), input.size()); - s32 = t32; - auto t64(s64); - t64.update(input.data(), input.size()); - s64 = t64; - } - EXPECT_TRUE(s32.testDigest()); - EXPECT_TRUE(s64.testDigest()); - } -} diff --git a/contrib/linux-kernel/test/include/asm/unaligned.h b/contrib/linux-kernel/test/include/asm/unaligned.h index 4f482812..6576b37e 100644 --- a/contrib/linux-kernel/test/include/asm/unaligned.h +++ b/contrib/linux-kernel/test/include/asm/unaligned.h @@ -2,7 +2,6 @@ #define ASM_UNALIGNED_H #include <assert.h> -#include <linux/string.h> #include <linux/types.h> #define _LITTLE_ENDIAN 1 @@ -33,7 +32,7 @@ static uint64_t _swap64(uint64_t in) static uint16_t get_unaligned_le16(const void* memPtr) { uint16_t val; - memcpy(&val, memPtr, sizeof(val)); + __builtin_memcpy(&val, memPtr, sizeof(val)); if (!_isLittleEndian()) _swap16(val); return val; } @@ -41,7 +40,7 @@ static uint16_t get_unaligned_le16(const void* memPtr) static uint32_t get_unaligned_le32(const void* memPtr) { uint32_t val; - memcpy(&val, memPtr, sizeof(val)); + __builtin_memcpy(&val, memPtr, sizeof(val)); if (!_isLittleEndian()) _swap32(val); return val; } @@ -49,7 +48,7 @@ static uint32_t get_unaligned_le32(const void* memPtr) static uint64_t get_unaligned_le64(const void* memPtr) { uint64_t val; - memcpy(&val, memPtr, sizeof(val)); + __builtin_memcpy(&val, memPtr, sizeof(val)); if (!_isLittleEndian()) _swap64(val); return val; } @@ -57,26 +56,26 @@ static uint64_t get_unaligned_le64(const void* memPtr) static void put_unaligned_le16(uint16_t value, void* memPtr) { if (!_isLittleEndian()) value = _swap16(value); - memcpy(memPtr, &value, sizeof(value)); + __builtin_memcpy(memPtr, &value, sizeof(value)); } static void put_unaligned_le32(uint32_t value, void* memPtr) { if (!_isLittleEndian()) value = _swap32(value); - memcpy(memPtr, &value, sizeof(value)); + __builtin_memcpy(memPtr, &value, sizeof(value)); } static void put_unaligned_le64(uint64_t value, void* memPtr) { if (!_isLittleEndian()) value = _swap64(value); - memcpy(memPtr, &value, sizeof(value)); + __builtin_memcpy(memPtr, &value, sizeof(value)); } /* big endian */ static uint32_t get_unaligned_be32(const void* memPtr) { uint32_t val; - memcpy(&val, memPtr, sizeof(val)); + __builtin_memcpy(&val, memPtr, sizeof(val)); if (_isLittleEndian()) _swap32(val); return val; } @@ -84,7 +83,7 @@ static uint32_t get_unaligned_be32(const void* memPtr) static uint64_t get_unaligned_be64(const void* memPtr) { uint64_t val; - memcpy(&val, memPtr, sizeof(val)); + __builtin_memcpy(&val, memPtr, sizeof(val)); if (_isLittleEndian()) _swap64(val); return val; } @@ -92,13 +91,13 @@ static uint64_t get_unaligned_be64(const void* memPtr) static void put_unaligned_be32(uint32_t value, void* memPtr) { if (_isLittleEndian()) value = _swap32(value); - memcpy(memPtr, &value, sizeof(value)); + __builtin_memcpy(memPtr, &value, sizeof(value)); } static void put_unaligned_be64(uint64_t value, void* memPtr) { if (_isLittleEndian()) value = _swap64(value); - memcpy(memPtr, &value, sizeof(value)); + __builtin_memcpy(memPtr, &value, sizeof(value)); } /* generic */ diff --git a/contrib/linux-kernel/test/include/linux/compiler.h b/contrib/linux-kernel/test/include/linux/compiler.h index 4fb4f42e..b614b27e 100644 --- a/contrib/linux-kernel/test/include/linux/compiler.h +++ b/contrib/linux-kernel/test/include/linux/compiler.h @@ -1,12 +1,17 @@ -#ifndef LINUX_COMPILER_H_ -#define LINUX_COMPILER_H_ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#ifndef LINUX_COMPILER_H +#define LINUX_COMPILER_H -#ifndef __always_inline -# define __always_inline inline +#ifndef inline +#define inline __inline __attribute__((unused)) #endif -#ifndef noinline -# define noinline __attribute__((__noinline__)) #endif - -#endif // LINUX_COMPILER_H_ diff --git a/contrib/linux-kernel/test/include/linux/errno.h b/contrib/linux-kernel/test/include/linux/errno.h index b9db0852..11c54b93 100644 --- a/contrib/linux-kernel/test/include/linux/errno.h +++ b/contrib/linux-kernel/test/include/linux/errno.h @@ -1,6 +1,15 @@ -#ifndef LINUX_ERRNO_H_ -#define LINUX_ERRNO_H_ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#ifndef LINUX_ERRNO_H +#define LINUX_ERRNO_H #define EINVAL 22 -#endif // LINUX_ERRNO_H_ +#endif diff --git a/contrib/linux-kernel/test/include/linux/kernel.h b/contrib/linux-kernel/test/include/linux/kernel.h index 3ef2f7fe..1124f02b 100644 --- a/contrib/linux-kernel/test/include/linux/kernel.h +++ b/contrib/linux-kernel/test/include/linux/kernel.h @@ -1,16 +1,15 @@ -#ifndef LINUX_KERNEL_H_ -#define LINUX_KERNEL_H_ - -#define ALIGN(x, a) ({ \ - typeof(x) const __xe = (x); \ - typeof(a) const __ae = (a); \ - typeof(a) const __m = __ae - 1; \ - typeof(x) const __r = __xe & __m; \ - __xe + (__r ? (__ae - __r) : 0); \ - }) - -#define PTR_ALIGN(p, a) (typeof(p))ALIGN((unsigned long long)(p), (a)) - -#define current Something that doesn't compile :) - -#endif // LINUX_KERNEL_H_ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#ifndef LINUX_KERNEL_H +#define LINUX_KERNEL_H + +#define WARN_ON(x) + +#endif diff --git a/contrib/linux-kernel/test/include/linux/limits.h b/contrib/linux-kernel/test/include/linux/limits.h new file mode 100644 index 00000000..7f8d18d4 --- /dev/null +++ b/contrib/linux-kernel/test/include/linux/limits.h @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#ifndef LINUX_LIMITS_H +#define LINUX_LIMITS_H + +#include <limits.h> + +#endif diff --git a/contrib/linux-kernel/test/include/linux/math64.h b/contrib/linux-kernel/test/include/linux/math64.h index 3d0ae72d..4bc7f4b5 100644 --- a/contrib/linux-kernel/test/include/linux/math64.h +++ b/contrib/linux-kernel/test/include/linux/math64.h @@ -1,11 +1,15 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ #ifndef LINUX_MATH64_H #define LINUX_MATH64_H -#include <stdint.h> - -static uint64_t div_u64(uint64_t n, uint32_t d) -{ - return n / d; -} +#define div_u64(dividend, divisor) ((dividend) / (divisor)) #endif diff --git a/contrib/linux-kernel/test/include/linux/module.h b/contrib/linux-kernel/test/include/linux/module.h index ef514c34..8fd66930 100644 --- a/contrib/linux-kernel/test/include/linux/module.h +++ b/contrib/linux-kernel/test/include/linux/module.h @@ -1,10 +1,18 @@ -#ifndef LINUX_MODULE_H_ -#define LINUX_MODULE_H_ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#ifndef LINUX_MODULE_H +#define LINUX_MODULE_H #define EXPORT_SYMBOL(symbol) \ void* __##symbol = symbol -#define MODULE_LICENSE(license) static char const *const LICENSE = license -#define MODULE_DESCRIPTION(description) \ - static char const *const DESCRIPTION = description +#define MODULE_LICENSE(license) +#define MODULE_DESCRIPTION(description) -#endif // LINUX_MODULE_H_ +#endif diff --git a/contrib/linux-kernel/test/include/linux/printk.h b/contrib/linux-kernel/test/include/linux/printk.h new file mode 100644 index 00000000..2ca2fb3a --- /dev/null +++ b/contrib/linux-kernel/test/include/linux/printk.h @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#ifndef LINUX_PRINTK_H +#define LINUX_PRINTK_H + +#define pr_debug(...) + +#endif diff --git a/contrib/linux-kernel/test/include/linux/stddef.h b/contrib/linux-kernel/test/include/linux/stddef.h new file mode 100644 index 00000000..c00d0654 --- /dev/null +++ b/contrib/linux-kernel/test/include/linux/stddef.h @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#ifndef LINUX_STDDEF_H +#define LINUX_STDDEF_H + +#include <stddef.h> + +#endif diff --git a/contrib/linux-kernel/test/include/linux/string.h b/contrib/linux-kernel/test/include/linux/string.h deleted file mode 100644 index 3b2f5900..00000000 --- a/contrib/linux-kernel/test/include/linux/string.h +++ /dev/null @@ -1 +0,0 @@ -#include <string.h> diff --git a/contrib/linux-kernel/test/include/linux/swab.h b/contrib/linux-kernel/test/include/linux/swab.h new file mode 100644 index 00000000..693b797e --- /dev/null +++ b/contrib/linux-kernel/test/include/linux/swab.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#ifndef LINUX_SWAB_H +#define LINUX_SWAB_H + +#define swab32(x) __builtin_bswap32((x)) +#define swab64(x) __builtin_bswap64((x)) + +#endif diff --git a/contrib/linux-kernel/test/include/linux/types.h b/contrib/linux-kernel/test/include/linux/types.h index c2d4f4b7..6db834b1 100644 --- a/contrib/linux-kernel/test/include/linux/types.h +++ b/contrib/linux-kernel/test/include/linux/types.h @@ -1,2 +1,16 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#ifndef LINUX_TYPES_H +#define LINUX_TYPES_H + #include <stddef.h> #include <stdint.h> + +#endif diff --git a/contrib/linux-kernel/lib/xxhash.c b/contrib/linux-kernel/test/include/linux/xxhash.h index aa61e2a3..0a43bb27 100644 --- a/contrib/linux-kernel/lib/xxhash.c +++ b/contrib/linux-kernel/test/include/linux/xxhash.h @@ -34,16 +34,273 @@ * ("BSD"). * * You can contact the author at: - * - xxHash homepage: http://cyan4973.github.io/xxHash/ + * - xxHash homepage: https://cyan4973.github.io/xxHash/ + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +/* + * Notice extracted from xxHash homepage: + * + * xxHash is an extremely fast Hash algorithm, running at RAM speed limits. + * It also successfully passes all tests from the SMHasher suite. + * + * Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 + * Duo @3GHz) + * + * Name Speed Q.Score Author + * xxHash 5.4 GB/s 10 + * CrapWow 3.2 GB/s 2 Andrew + * MumurHash 3a 2.7 GB/s 10 Austin Appleby + * SpookyHash 2.0 GB/s 10 Bob Jenkins + * SBox 1.4 GB/s 9 Bret Mulvey + * Lookup3 1.2 GB/s 9 Bob Jenkins + * SuperFastHash 1.2 GB/s 1 Paul Hsieh + * CityHash64 1.05 GB/s 10 Pike & Alakuijala + * FNV 0.55 GB/s 5 Fowler, Noll, Vo + * CRC32 0.43 GB/s 9 + * MD5-32 0.33 GB/s 10 Ronald L. Rivest + * SHA1-32 0.28 GB/s 10 + * + * Q.Score is a measure of quality of the hash function. + * It depends on successfully passing SMHasher test set. + * 10 is a perfect score. + * + * A 64-bits version, named xxh64 offers much better speed, + * but for 64-bits applications only. + * Name Speed on 64 bits Speed on 32 bits + * xxh64 13.8 GB/s 1.9 GB/s + * xxh32 6.8 GB/s 6.0 GB/s + */ + +#ifndef XXHASH_H +#define XXHASH_H + +#include <linux/types.h> + +#define XXH_API static inline __attribute__((unused)) +/*-**************************** + * Simple Hash Functions + *****************************/ + +/** + * xxh32() - calculate the 32-bit hash of the input with a given seed. + * + * @input: The data to hash. + * @length: The length of the data to hash. + * @seed: The seed can be used to alter the result predictably. + * + * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s + * + * Return: The 32-bit hash of the data. + */ +XXH_API uint32_t xxh32(const void *input, size_t length, uint32_t seed); + +/** + * xxh64() - calculate the 64-bit hash of the input with a given seed. + * + * @input: The data to hash. + * @length: The length of the data to hash. + * @seed: The seed can be used to alter the result predictably. + * + * This function runs 2x faster on 64-bit systems, but slower on 32-bit systems. + * + * Return: The 64-bit hash of the data. + */ +XXH_API uint64_t xxh64(const void *input, size_t length, uint64_t seed); + +/** + * xxhash() - calculate wordsize hash of the input with a given seed + * @input: The data to hash. + * @length: The length of the data to hash. + * @seed: The seed can be used to alter the result predictably. + * + * If the hash does not need to be comparable between machines with + * different word sizes, this function will call whichever of xxh32() + * or xxh64() is faster. + * + * Return: wordsize hash of the data. + */ + +static inline unsigned long xxhash(const void *input, size_t length, + uint64_t seed) +{ +#if BITS_PER_LONG == 64 + return xxh64(input, length, seed); +#else + return xxh32(input, length, seed); +#endif +} + +/*-**************************** + * Streaming Hash Functions + *****************************/ + +/* + * These definitions are only meant to allow allocation of XXH state + * statically, on stack, or in a struct for example. + * Do not use members directly. + */ + +/** + * struct xxh32_state - private xxh32 state, do not use members directly + */ +struct xxh32_state { + uint32_t total_len_32; + uint32_t large_len; + uint32_t v1; + uint32_t v2; + uint32_t v3; + uint32_t v4; + uint32_t mem32[4]; + uint32_t memsize; +}; + +/** + * struct xxh32_state - private xxh64 state, do not use members directly + */ +struct xxh64_state { + uint64_t total_len; + uint64_t v1; + uint64_t v2; + uint64_t v3; + uint64_t v4; + uint64_t mem64[4]; + uint32_t memsize; +}; + +/** + * xxh32_reset() - reset the xxh32 state to start a new hashing operation + * + * @state: The xxh32 state to reset. + * @seed: Initialize the hash state with this seed. + * + * Call this function on any xxh32_state to prepare for a new hashing operation. + */ +XXH_API void xxh32_reset(struct xxh32_state *state, uint32_t seed); + +/** + * xxh32_update() - hash the data given and update the xxh32 state + * + * @state: The xxh32 state to update. + * @input: The data to hash. + * @length: The length of the data to hash. + * + * After calling xxh32_reset() call xxh32_update() as many times as necessary. + * + * Return: Zero on success, otherwise an error code. + */ +XXH_API int xxh32_update(struct xxh32_state *state, const void *input, size_t length); + +/** + * xxh32_digest() - produce the current xxh32 hash + * + * @state: Produce the current xxh32 hash of this state. + * + * A hash value can be produced at any time. It is still possible to continue + * inserting input into the hash state after a call to xxh32_digest(), and + * generate new hashes later on, by calling xxh32_digest() again. + * + * Return: The xxh32 hash stored in the state. + */ +XXH_API uint32_t xxh32_digest(const struct xxh32_state *state); + +/** + * xxh64_reset() - reset the xxh64 state to start a new hashing operation + * + * @state: The xxh64 state to reset. + * @seed: Initialize the hash state with this seed. + */ +XXH_API void xxh64_reset(struct xxh64_state *state, uint64_t seed); + +/** + * xxh64_update() - hash the data given and update the xxh64 state + * @state: The xxh64 state to update. + * @input: The data to hash. + * @length: The length of the data to hash. + * + * After calling xxh64_reset() call xxh64_update() as many times as necessary. + * + * Return: Zero on success, otherwise an error code. + */ +XXH_API int xxh64_update(struct xxh64_state *state, const void *input, size_t length); + +/** + * xxh64_digest() - produce the current xxh64 hash + * + * @state: Produce the current xxh64 hash of this state. + * + * A hash value can be produced at any time. It is still possible to continue + * inserting input into the hash state after a call to xxh64_digest(), and + * generate new hashes later on, by calling xxh64_digest() again. + * + * Return: The xxh64 hash stored in the state. + */ +XXH_API uint64_t xxh64_digest(const struct xxh64_state *state); + +/*-************************** + * Utils + ***************************/ + +/** + * xxh32_copy_state() - copy the source state into the destination state + * + * @src: The source xxh32 state. + * @dst: The destination xxh32 state. + */ +XXH_API void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src); + +/** + * xxh64_copy_state() - copy the source state into the destination state + * + * @src: The source xxh64 state. + * @dst: The destination xxh64 state. + */ +XXH_API void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src); + +/* + * xxHash - Extremely Fast Hash algorithm + * Copyright (C) 2012-2016, Yann Collet. + * + * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License version 2 as published by the + * Free Software Foundation. This program is dual-licensed; you may select + * either version 2 of the GNU General Public License ("GPL") or BSD license + * ("BSD"). + * + * You can contact the author at: + * - xxHash homepage: https://cyan4973.github.io/xxHash/ * - xxHash source repository: https://github.com/Cyan4973/xxHash */ #include <asm/unaligned.h> #include <linux/errno.h> -#include <linux/compiler.h> #include <linux/kernel.h> #include <linux/module.h> -#include <linux/string.h> #include <linux/xxhash.h> /*-************************************* @@ -76,17 +333,15 @@ static const uint64_t PRIME64_5 = 2870177450012600261ULL; /*-************************** * Utils ***************************/ -void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src) +XXH_API void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src) { - memcpy(dst, src, sizeof(*dst)); + __builtin_memcpy(dst, src, sizeof(*dst)); } -EXPORT_SYMBOL(xxh32_copy_state); -void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src) +XXH_API void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src) { - memcpy(dst, src, sizeof(*dst)); + __builtin_memcpy(dst, src, sizeof(*dst)); } -EXPORT_SYMBOL(xxh64_copy_state); /*-*************************** * Simple Hash Functions @@ -99,7 +354,7 @@ static uint32_t xxh32_round(uint32_t seed, const uint32_t input) return seed; } -uint32_t xxh32(const void *input, const size_t len, const uint32_t seed) +XXH_API uint32_t xxh32(const void *input, const size_t len, const uint32_t seed) { const uint8_t *p = (const uint8_t *)input; const uint8_t *b_end = p + len; @@ -151,7 +406,6 @@ uint32_t xxh32(const void *input, const size_t len, const uint32_t seed) return h32; } -EXPORT_SYMBOL(xxh32); static uint64_t xxh64_round(uint64_t acc, const uint64_t input) { @@ -169,7 +423,7 @@ static uint64_t xxh64_merge_round(uint64_t acc, uint64_t val) return acc; } -uint64_t xxh64(const void *input, const size_t len, const uint64_t seed) +XXH_API uint64_t xxh64(const void *input, const size_t len, const uint64_t seed) { const uint8_t *p = (const uint8_t *)input; const uint8_t *const b_end = p + len; @@ -234,40 +488,37 @@ uint64_t xxh64(const void *input, const size_t len, const uint64_t seed) return h64; } -EXPORT_SYMBOL(xxh64); /*-************************************************** * Advanced Hash Functions ***************************************************/ -void xxh32_reset(struct xxh32_state *statePtr, const uint32_t seed) +XXH_API void xxh32_reset(struct xxh32_state *statePtr, const uint32_t seed) { /* use a local state for memcpy() to avoid strict-aliasing warnings */ struct xxh32_state state; - memset(&state, 0, sizeof(state)); + __builtin_memset(&state, 0, sizeof(state)); state.v1 = seed + PRIME32_1 + PRIME32_2; state.v2 = seed + PRIME32_2; state.v3 = seed + 0; state.v4 = seed - PRIME32_1; - memcpy(statePtr, &state, sizeof(state)); + __builtin_memcpy(statePtr, &state, sizeof(state)); } -EXPORT_SYMBOL(xxh32_reset); -void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed) +XXH_API void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed) { /* use a local state for memcpy() to avoid strict-aliasing warnings */ struct xxh64_state state; - memset(&state, 0, sizeof(state)); + __builtin_memset(&state, 0, sizeof(state)); state.v1 = seed + PRIME64_1 + PRIME64_2; state.v2 = seed + PRIME64_2; state.v3 = seed + 0; state.v4 = seed - PRIME64_1; - memcpy(statePtr, &state, sizeof(state)); + __builtin_memcpy(statePtr, &state, sizeof(state)); } -EXPORT_SYMBOL(xxh64_reset); -int xxh32_update(struct xxh32_state *state, const void *input, const size_t len) +XXH_API int xxh32_update(struct xxh32_state *state, const void *input, const size_t len) { const uint8_t *p = (const uint8_t *)input; const uint8_t *const b_end = p + len; @@ -279,7 +530,7 @@ int xxh32_update(struct xxh32_state *state, const void *input, const size_t len) state->large_len |= (len >= 16) | (state->total_len_32 >= 16); if (state->memsize + len < 16) { /* fill in tmp buffer */ - memcpy((uint8_t *)(state->mem32) + state->memsize, input, len); + __builtin_memcpy((uint8_t *)(state->mem32) + state->memsize, input, len); state->memsize += (uint32_t)len; return 0; } @@ -287,7 +538,7 @@ int xxh32_update(struct xxh32_state *state, const void *input, const size_t len) if (state->memsize) { /* some data left from previous update */ const uint32_t *p32 = state->mem32; - memcpy((uint8_t *)(state->mem32) + state->memsize, input, + __builtin_memcpy((uint8_t *)(state->mem32) + state->memsize, input, 16 - state->memsize); state->v1 = xxh32_round(state->v1, get_unaligned_le32(p32)); @@ -328,15 +579,14 @@ int xxh32_update(struct xxh32_state *state, const void *input, const size_t len) } if (p < b_end) { - memcpy(state->mem32, p, (size_t)(b_end-p)); + __builtin_memcpy(state->mem32, p, (size_t)(b_end-p)); state->memsize = (uint32_t)(b_end-p); } return 0; } -EXPORT_SYMBOL(xxh32_update); -uint32_t xxh32_digest(const struct xxh32_state *state) +XXH_API uint32_t xxh32_digest(const struct xxh32_state *state) { const uint8_t *p = (const uint8_t *)state->mem32; const uint8_t *const b_end = (const uint8_t *)(state->mem32) + @@ -372,9 +622,8 @@ uint32_t xxh32_digest(const struct xxh32_state *state) return h32; } -EXPORT_SYMBOL(xxh32_digest); -int xxh64_update(struct xxh64_state *state, const void *input, const size_t len) +XXH_API int xxh64_update(struct xxh64_state *state, const void *input, const size_t len) { const uint8_t *p = (const uint8_t *)input; const uint8_t *const b_end = p + len; @@ -385,7 +634,7 @@ int xxh64_update(struct xxh64_state *state, const void *input, const size_t len) state->total_len += len; if (state->memsize + len < 32) { /* fill in tmp buffer */ - memcpy(((uint8_t *)state->mem64) + state->memsize, input, len); + __builtin_memcpy(((uint8_t *)state->mem64) + state->memsize, input, len); state->memsize += (uint32_t)len; return 0; } @@ -393,7 +642,7 @@ int xxh64_update(struct xxh64_state *state, const void *input, const size_t len) if (state->memsize) { /* tmp buffer is full */ uint64_t *p64 = state->mem64; - memcpy(((uint8_t *)p64) + state->memsize, input, + __builtin_memcpy(((uint8_t *)p64) + state->memsize, input, 32 - state->memsize); state->v1 = xxh64_round(state->v1, get_unaligned_le64(p64)); @@ -433,15 +682,14 @@ int xxh64_update(struct xxh64_state *state, const void *input, const size_t len) } if (p < b_end) { - memcpy(state->mem64, p, (size_t)(b_end-p)); + __builtin_memcpy(state->mem64, p, (size_t)(b_end-p)); state->memsize = (uint32_t)(b_end - p); } return 0; } -EXPORT_SYMBOL(xxh64_update); -uint64_t xxh64_digest(const struct xxh64_state *state) +XXH_API uint64_t xxh64_digest(const struct xxh64_state *state) { const uint8_t *p = (const uint8_t *)state->mem64; const uint8_t *const b_end = (const uint8_t *)state->mem64 + @@ -494,7 +742,5 @@ uint64_t xxh64_digest(const struct xxh64_state *state) return h64; } -EXPORT_SYMBOL(xxh64_digest); -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_DESCRIPTION("xxHash"); +#endif /* XXHASH_H */ diff --git a/contrib/linux-kernel/test/macro-test.sh b/contrib/linux-kernel/test/macro-test.sh new file mode 100755 index 00000000..c688ac03 --- /dev/null +++ b/contrib/linux-kernel/test/macro-test.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env sh + +set -e + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +INCLUDE_DIR="$SCRIPT_DIR/../linux/include" +LIB_DIR="$SCRIPT_DIR/../linux/lib" + + +print() { + printf '%b' "${*}" +} + +println() { + printf '%b\n' "${*}" +} + +die() { + println "$@" 1>&2 + exit 1 +} + +test_not_present() { + print "Testing that '$1' is not present... " + grep -r $1 "$INCLUDE_DIR" "$LIB_DIR" && die "Fail!" + println "Okay" +} + +println "This test checks that the macro removal process worked as expected" +println "If this test fails, then freestanding.py wasn't able to remove one of these" +println "macros from the source code completely. You'll either need to rewrite the check" +println "or improve freestanding.py." +println "" + +test_not_present "ZSTD_NO_INTRINSICS" +test_not_present "ZSTD_NO_UNUSED_FUNCTIONS" +test_not_present "ZSTD_LEGACY_SUPPORT" +test_not_present "STATIC_BMI2" +test_not_present "ZSTD_NO_INLINE" +test_not_present "ZSTD_DLL_EXPORT" +test_not_present "ZSTD_DLL_IMPORT" +test_not_present "__ICCARM__" +test_not_present "_MSC_VER" +test_not_present "_WIN32" diff --git a/contrib/linux-kernel/test/static_test.c b/contrib/linux-kernel/test/static_test.c new file mode 100644 index 00000000..53c559c0 --- /dev/null +++ b/contrib/linux-kernel/test/static_test.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "decompress_sources.h" +#include <linux/zstd.h> + +#define CONTROL(x) \ + do { \ + if (!(x)) { \ + fprintf(stderr, "%s:%u: %s failed!\n", __FUNCTION__, __LINE__, #x); \ + abort(); \ + } \ + } while (0) + + +static const char kEmptyZstdFrame[] = { + 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x00, 0x01, 0x00, 0x00, 0x99, 0xe9, 0xd8, 0x51 +}; + +static void test_decompress_unzstd() { + fprintf(stderr, "Testing decompress unzstd... "); + { + size_t const wkspSize = zstd_dctx_workspace_bound(); + void* wksp = malloc(wkspSize); + CONTROL(wksp != NULL); + ZSTD_DCtx* dctx = zstd_init_dctx(wksp, wkspSize); + CONTROL(dctx != NULL); + size_t const dSize = zstd_decompress_dctx(dctx, NULL, 0, kEmptyZstdFrame, sizeof(kEmptyZstdFrame)); + CONTROL(!zstd_is_error(dSize)); + CONTROL(dSize == 0); + free(wksp); + } + fprintf(stderr, "Ok\n"); +} + +int main(void) { + test_decompress_unzstd(); + return 0; +} diff --git a/contrib/linux-kernel/test/test.c b/contrib/linux-kernel/test/test.c new file mode 100644 index 00000000..9579976a --- /dev/null +++ b/contrib/linux-kernel/test/test.c @@ -0,0 +1,210 @@ +/* + * Copyright (c) 7-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <linux/zstd.h> + +#define CONTROL(x) \ + do { \ + if (!(x)) { \ + fprintf(stderr, "%s:%u: %s failed!\n", __FUNCTION__, __LINE__, #x); \ + abort(); \ + } \ + } while (0) + +typedef struct { + char *data; + char *data2; + size_t dataSize; + char *comp; + size_t compSize; +} test_data_t; + +test_data_t create_test_data(void) { + test_data_t data; + data.dataSize = 128 * 1024; + data.data = malloc(data.dataSize); + CONTROL(data.data != NULL); + data.data2 = malloc(data.dataSize); + CONTROL(data.data2 != NULL); + data.compSize = zstd_compress_bound(data.dataSize); + data.comp = malloc(data.compSize); + CONTROL(data.comp != NULL); + memset(data.data, 0, data.dataSize); + return data; +} + +static void free_test_data(test_data_t const *data) { + free(data->data); + free(data->data2); + free(data->comp); +} + +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define MAX(a, b) ((a) > (b) ? (a) : (b)) + +static void test_btrfs(test_data_t const *data) { + fprintf(stderr, "testing btrfs use cases... "); + size_t const size = MIN(data->dataSize, 128 * 1024); + for (int level = -1; level < 16; ++level) { + struct zstd_parameters params = zstd_get_params(level, size); + CONTROL(params.cparams.window_log <= 17); + size_t const workspaceSize = + MAX(zstd_cstream_workspace_bound(¶ms.cparams), + zstd_dstream_workspace_bound(size)); + void *workspace = malloc(workspaceSize); + CONTROL(workspace != NULL); + + char const *ip = data->data; + char const *iend = ip + size; + char *op = data->comp; + char *oend = op + data->compSize; + { + zstd_cstream *cctx = zstd_init_cstream(¶ms, size, workspace, workspaceSize); + CONTROL(cctx != NULL); + struct zstd_out_buffer out = {NULL, 0, 0}; + struct zstd_in_buffer in = {NULL, 0, 0}; + for (;;) { + if (in.pos == in.size) { + in.src = ip; + in.size = MIN(4096, iend - ip); + in.pos = 0; + ip += in.size; + } + + if (out.pos == out.size) { + out.dst = op; + out.size = MIN(4096, oend - op); + out.pos = 0; + op += out.size; + } + + if (ip != iend || in.pos < in.size) { + CONTROL(!zstd_is_error(zstd_compress_stream(cctx, &out, &in))); + } else { + size_t const ret = zstd_end_stream(cctx, &out); + CONTROL(!zstd_is_error(ret)); + if (ret == 0) { + break; + } + } + } + op += out.pos; + } + + ip = data->comp; + iend = op; + op = data->data2; + oend = op + size; + { + zstd_dstream *dctx = zstd_init_dstream(1ULL << params.cparams.window_log, workspace, workspaceSize); + CONTROL(dctx != NULL); + struct zstd_out_buffer out = {NULL, 0, 0}; + struct zstd_in_buffer in = {NULL, 0, 0}; + for (;;) { + if (in.pos == in.size) { + in.src = ip; + in.size = MIN(4096, iend - ip); + in.pos = 0; + ip += in.size; + } + + if (out.pos == out.size) { + out.dst = op; + out.size = MIN(4096, oend - op); + out.pos = 0; + op += out.size; + } + + size_t const ret = zstd_decompress_stream(dctx, &out, &in); + CONTROL(!zstd_is_error(ret)); + if (ret == 0) { + break; + } + } + } + CONTROL(op - data->data2 == data->dataSize); + CONTROL(!memcmp(data->data, data->data2, data->dataSize)); + free(workspace); + } + fprintf(stderr, "Ok\n"); +} + +static void test_decompress_unzstd(test_data_t const *data) { + fprintf(stderr, "Testing decompress unzstd... "); + size_t cSize; + { + struct zstd_parameters params = zstd_get_params(19, 0); + size_t const wkspSize = zstd_cctx_workspace_bound(¶ms.cparams); + void* wksp = malloc(wkspSize); + CONTROL(wksp != NULL); + zstd_cctx* cctx = zstd_init_cctx(wksp, wkspSize); + CONTROL(cctx != NULL); + cSize = zstd_compress_cctx(cctx, data->comp, data->compSize, data->data, data->dataSize, ¶ms); + CONTROL(!zstd_is_error(cSize)); + free(wksp); + } + { + size_t const wkspSize = zstd_dctx_workspace_bound(); + void* wksp = malloc(wkspSize); + CONTROL(wksp != NULL); + zstd_dctx* dctx = zstd_init_dctx(wksp, wkspSize); + CONTROL(dctx != NULL); + size_t const dSize = zstd_decompress_dctx(dctx, data->data2, data->dataSize, data->comp, cSize); + CONTROL(!zstd_is_error(dSize)); + CONTROL(dSize == data->dataSize); + CONTROL(!memcmp(data->data, data->data2, data->dataSize)); + free(wksp); + } + fprintf(stderr, "Ok\n"); +} + +static char *g_stack = NULL; + +static void __attribute__((noinline)) use(void *x) { + asm volatile("" : "+r"(x)); +} + +static void __attribute__((noinline)) set_stack() { + + char stack[8192]; + g_stack = stack; + memset(g_stack, 0x33, 8192); + use(g_stack); +} + +static void __attribute__((noinline)) check_stack() { + size_t cleanStack = 0; + while (cleanStack < 8192 && g_stack[cleanStack] == 0x33) { + ++cleanStack; + } + size_t const stackSize = 8192 - cleanStack; + fprintf(stderr, "Maximum stack size: %zu\n", stackSize); + CONTROL(stackSize <= 2048 + 512); +} + +static void test_stack_usage(test_data_t const *data) { + set_stack(); + test_btrfs(data); + test_decompress_unzstd(data); + check_stack(); +} + +int main(void) { + test_data_t data = create_test_data(); + test_btrfs(&data); + test_decompress_unzstd(&data); + test_stack_usage(&data); + free_test_data(&data); + return 0; +} diff --git a/contrib/linux-kernel/xxhash_test.c b/contrib/linux-kernel/xxhash_test.c deleted file mode 100644 index eb0fb1cd..00000000 --- a/contrib/linux-kernel/xxhash_test.c +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/* DO_XXH should be 32 or 64 for xxh32 and xxh64 respectively */ -#define DO_XXH 0 -/* DO_CRC should be 0 or 1 */ -#define DO_CRC 0 -/* Buffer size */ -#define BUFFER_SIZE 4096 - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/fs.h> -#include <linux/uaccess.h> - -#if DO_XXH -#include <linux/xxhash.h> -#endif - -#if DO_CRC -#include <linux/crc32.h> -#endif - -/* Device name to pass to register_chrdev(). */ -#define DEVICE_NAME "xxhash_test" - -/* Dynamically allocated device major number */ -static int device_major; - -/* - * We reuse the same hash state, and thus can hash only one - * file at a time. - */ -static bool device_is_open; - -static uint64_t total_length; - - -#if (DO_XXH == 32) - -#define xxh_state xxh32_state -#define xxh_reset xxh32_reset -#define xxh_update xxh32_update -#define xxh_digest xxh32_digest -#define XXH_FORMAT "XXH32 = 0x%x" - -#elif (DO_XXH == 64) - -#define xxh_state xxh64_state -#define xxh_reset xxh64_reset -#define xxh_update xxh64_update -#define xxh_digest xxh64_digest -#define XXH_FORMAT "XXH64 = 0x%llx" - -#elif DO_XXH - -#error "Invalid value of DO_XXH" - -#endif - -#if DO_XXH - -/* XXH state */ -static struct xxh_state state; - -#endif /* DO_XXH */ - -#if DO_CRC - -static uint32_t crc; - -#endif /* DO_CRC */ - -/* - * Input buffer used to put data coming from userspace. - */ -static uint8_t buffer_in[BUFFER_SIZE]; - -static int xxhash_test_open(struct inode *i, struct file *f) -{ - if (device_is_open) - return -EBUSY; - - device_is_open = true; - - total_length = 0; -#if DO_XXH - xxh_reset(&state, 0); -#endif -#if DO_CRC - crc = 0xFFFFFFFF; -#endif - - printk(KERN_INFO DEVICE_NAME ": opened\n"); - return 0; -} - -static int xxhash_test_release(struct inode *i, struct file *f) -{ - device_is_open = false; - - printk(KERN_INFO DEVICE_NAME ": total_len = %llu\n", total_length); -#if DO_XXH - printk(KERN_INFO DEVICE_NAME ": " XXH_FORMAT "\n", xxh_digest(&state)); -#endif -#if DO_CRC - printk(KERN_INFO DEVICE_NAME ": CRC32 = 0x%08x\n", ~crc); -#endif - printk(KERN_INFO DEVICE_NAME ": closed\n"); - return 0; -} - -/* - * Hash the data given to us from userspace. - */ -static ssize_t xxhash_test_write(struct file *file, const char __user *buf, - size_t size, loff_t *pos) -{ - size_t remaining = size; - - while (remaining > 0) { -#if DO_XXH - int ret; -#endif - size_t const copy_size = min(remaining, sizeof(buffer_in)); - - if (copy_from_user(buffer_in, buf, copy_size)) - return -EFAULT; - buf += copy_size; - remaining -= copy_size; - total_length += copy_size; -#if DO_XXH - if ((ret = xxh_update(&state, buffer_in, copy_size))) { - printk(KERN_INFO DEVICE_NAME ": xxh failure."); - return ret; - } -#endif -#if DO_CRC - crc = crc32(crc, buffer_in, copy_size); -#endif - } - return size; -} -/* register the character device. */ -static int __init xxhash_test_init(void) -{ - static const struct file_operations fileops = { - .owner = THIS_MODULE, - .open = &xxhash_test_open, - .release = &xxhash_test_release, - .write = &xxhash_test_write - }; - - device_major = register_chrdev(0, DEVICE_NAME, &fileops); - if (device_major < 0) { - return device_major; - } - - printk(KERN_INFO DEVICE_NAME ": module loaded\n"); - printk(KERN_INFO DEVICE_NAME ": Create a device node with " - "'mknod " DEVICE_NAME " c %d 0' and write data " - "to it.\n", device_major); - return 0; -} - -static void __exit xxhash_test_exit(void) -{ - unregister_chrdev(device_major, DEVICE_NAME); - printk(KERN_INFO DEVICE_NAME ": module unloaded\n"); -} - -module_init(xxhash_test_init); -module_exit(xxhash_test_exit); - -MODULE_DESCRIPTION("XXHash tester"); -MODULE_VERSION("1.0"); - - -MODULE_LICENSE("Dual BSD/GPL"); diff --git a/contrib/linux-kernel/zstd_compress_module.c b/contrib/linux-kernel/zstd_compress_module.c new file mode 100644 index 00000000..bab79afb --- /dev/null +++ b/contrib/linux-kernel/zstd_compress_module.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> +#include <linux/zstd.h> + +#include "zstd.h" +#include "common/zstd_deps.h" +#include "common/zstd_internal.h" + +static void zstd_check_structs(void) { + /* Check that the structs have the same size. */ + ZSTD_STATIC_ASSERT(sizeof(ZSTD_parameters) == + sizeof(struct zstd_parameters)); + ZSTD_STATIC_ASSERT(sizeof(ZSTD_compressionParameters) == + sizeof(struct zstd_compression_parameters)); + ZSTD_STATIC_ASSERT(sizeof(ZSTD_frameParameters) == + sizeof(struct zstd_frame_parameters)); + /* Zstd guarantees that the layout of the structs never change. Verify it. */ + ZSTD_STATIC_ASSERT(offsetof(ZSTD_parameters, cParams) == + offsetof(struct zstd_parameters, cparams)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_parameters, fParams) == + offsetof(struct zstd_parameters, fparams)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, windowLog) == + offsetof(struct zstd_compression_parameters, window_log)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, chainLog) == + offsetof(struct zstd_compression_parameters, chain_log)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, hashLog) == + offsetof(struct zstd_compression_parameters, hash_log)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, searchLog) == + offsetof(struct zstd_compression_parameters, search_log)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, minMatch) == + offsetof(struct zstd_compression_parameters, search_length)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, targetLength) == + offsetof(struct zstd_compression_parameters, target_length)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, strategy) == + offsetof(struct zstd_compression_parameters, strategy)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_frameParameters, contentSizeFlag) == + offsetof(struct zstd_frame_parameters, content_size_flag)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_frameParameters, checksumFlag) == + offsetof(struct zstd_frame_parameters, checksum_flag)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_frameParameters, noDictIDFlag) == + offsetof(struct zstd_frame_parameters, no_dict_id_flag)); + /* Check that the strategies are the same. This can change. */ + ZSTD_STATIC_ASSERT((int)ZSTD_fast == (int)zstd_fast); + ZSTD_STATIC_ASSERT((int)ZSTD_dfast == (int)zstd_dfast); + ZSTD_STATIC_ASSERT((int)ZSTD_greedy == (int)zstd_greedy); + ZSTD_STATIC_ASSERT((int)ZSTD_lazy == (int)zstd_lazy); + ZSTD_STATIC_ASSERT((int)ZSTD_lazy2 == (int)zstd_lazy2); + ZSTD_STATIC_ASSERT((int)ZSTD_btlazy2 == (int)zstd_btlazy2); + ZSTD_STATIC_ASSERT((int)ZSTD_btopt == (int)zstd_btopt); + ZSTD_STATIC_ASSERT((int)ZSTD_btultra == (int)zstd_btultra); + ZSTD_STATIC_ASSERT((int)ZSTD_btultra2 == (int)zstd_btultra2); + /* Check input buffer */ + ZSTD_STATIC_ASSERT(sizeof(ZSTD_inBuffer) == sizeof(struct zstd_in_buffer)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_inBuffer, src) == + offsetof(struct zstd_in_buffer, src)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_inBuffer, size) == + offsetof(struct zstd_in_buffer, size)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_inBuffer, pos) == + offsetof(struct zstd_in_buffer, pos)); + /* Check output buffer */ + ZSTD_STATIC_ASSERT(sizeof(ZSTD_outBuffer) == + sizeof(struct zstd_out_buffer)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_outBuffer, dst) == + offsetof(struct zstd_out_buffer, dst)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_outBuffer, size) == + offsetof(struct zstd_out_buffer, size)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_outBuffer, pos) == + offsetof(struct zstd_out_buffer, pos)); +} + +size_t zstd_compress_bound(size_t src_size) +{ + return ZSTD_compressBound(src_size); +} +EXPORT_SYMBOL(zstd_compress_bound); + +struct zstd_parameters zstd_get_params(int level, + unsigned long long estimated_src_size) +{ + const ZSTD_parameters params = ZSTD_getParams(level, estimated_src_size, 0); + struct zstd_parameters out; + + /* no-op */ + zstd_check_structs(); + ZSTD_memcpy(&out, ¶ms, sizeof(out)); + return out; +} +EXPORT_SYMBOL(zstd_get_params); + +size_t zstd_cctx_workspace_bound( + const struct zstd_compression_parameters *cparams) +{ + ZSTD_compressionParameters p; + + ZSTD_memcpy(&p, cparams, sizeof(p)); + return ZSTD_estimateCCtxSize_usingCParams(p); +} +EXPORT_SYMBOL(zstd_cctx_workspace_bound); + +zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size) +{ + if (workspace == NULL) + return NULL; + return ZSTD_initStaticCCtx(workspace, workspace_size); +} +EXPORT_SYMBOL(zstd_init_cctx); + +size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity, + const void *src, size_t src_size, const struct zstd_parameters *parameters) +{ + ZSTD_parameters p; + + ZSTD_memcpy(&p, parameters, sizeof(p)); + return ZSTD_compress_advanced(cctx, dst, dst_capacity, src, src_size, NULL, 0, p); +} +EXPORT_SYMBOL(zstd_compress_cctx); + +size_t zstd_cstream_workspace_bound( + const struct zstd_compression_parameters *cparams) +{ + ZSTD_compressionParameters p; + + ZSTD_memcpy(&p, cparams, sizeof(p)); + return ZSTD_estimateCStreamSize_usingCParams(p); +} +EXPORT_SYMBOL(zstd_cstream_workspace_bound); + +zstd_cstream *zstd_init_cstream(const struct zstd_parameters *parameters, + unsigned long long pledged_src_size, void *workspace, size_t workspace_size) +{ + ZSTD_parameters p; + zstd_cstream *cstream; + size_t ret; + + if (workspace == NULL) + return NULL; + + cstream = ZSTD_initStaticCStream(workspace, workspace_size); + if (cstream == NULL) + return NULL; + + /* 0 means unknown in linux zstd API but means 0 in new zstd API */ + if (pledged_src_size == 0) + pledged_src_size = ZSTD_CONTENTSIZE_UNKNOWN; + + ZSTD_memcpy(&p, parameters, sizeof(p)); + ret = ZSTD_initCStream_advanced(cstream, NULL, 0, p, pledged_src_size); + if (ZSTD_isError(ret)) + return NULL; + + return cstream; +} +EXPORT_SYMBOL(zstd_init_cstream); + +size_t zstd_reset_cstream(zstd_cstream *cstream, + unsigned long long pledged_src_size) +{ + return ZSTD_resetCStream(cstream, pledged_src_size); +} +EXPORT_SYMBOL(zstd_reset_cstream); + +size_t zstd_compress_stream(zstd_cstream *cstream, + struct zstd_out_buffer *output, struct zstd_in_buffer *input) +{ + ZSTD_outBuffer o; + ZSTD_inBuffer i; + size_t ret; + + ZSTD_memcpy(&o, output, sizeof(o)); + ZSTD_memcpy(&i, input, sizeof(i)); + ret = ZSTD_compressStream(cstream, &o, &i); + ZSTD_memcpy(output, &o, sizeof(o)); + ZSTD_memcpy(input, &i, sizeof(i)); + return ret; +} +EXPORT_SYMBOL(zstd_compress_stream); + +size_t zstd_flush_stream(zstd_cstream *cstream, struct zstd_out_buffer *output) +{ + ZSTD_outBuffer o; + size_t ret; + + ZSTD_memcpy(&o, output, sizeof(o)); + ret = ZSTD_flushStream(cstream, &o); + ZSTD_memcpy(output, &o, sizeof(o)); + return ret; +} +EXPORT_SYMBOL(zstd_flush_stream); + +size_t zstd_end_stream(zstd_cstream *cstream, struct zstd_out_buffer *output) +{ + ZSTD_outBuffer o; + size_t ret; + + ZSTD_memcpy(&o, output, sizeof(o)); + ret = ZSTD_endStream(cstream, &o); + ZSTD_memcpy(output, &o, sizeof(o)); + return ret; +} +EXPORT_SYMBOL(zstd_end_stream); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("Zstd Compressor"); diff --git a/contrib/linux-kernel/zstd_compress_test.c b/contrib/linux-kernel/zstd_compress_test.c deleted file mode 100644 index dc17adf8..00000000 --- a/contrib/linux-kernel/zstd_compress_test.c +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/* Compression level or 0 to disable */ -#define DO_ZLIB 9 -/* Compression level or 0 to disable */ -#define DO_ZSTD 0 -/* Buffer size */ -#define BUFFER_SIZE 4096 - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/vmalloc.h> -#include <linux/fs.h> -#include <linux/uaccess.h> - -#if DO_ZSTD -#include <linux/zstd.h> -#endif - -#if DO_ZLIB -#include <linux/zlib.h> -#endif - -/* Device name to pass to register_chrdev(). */ -#define DEVICE_NAME "zstd_compress_test" - -/* Dynamically allocated device major number */ -static int device_major; - -/* - * We reuse the same state, and thus can compress only one file at a time. - */ -static bool device_is_open; - - -static void *workspace = NULL; - -/* - * Input buffer used to put data coming from userspace. - */ -static uint8_t buffer_in[BUFFER_SIZE]; -static uint8_t buffer_out[BUFFER_SIZE]; - -static uint64_t uncompressed_len; -static uint64_t compressed_len; - -#if DO_ZSTD - -static ZSTD_CStream *state; - -static ZSTD_inBuffer input = { - .src = buffer_in, - .size = sizeof(buffer_in), - .pos = sizeof(buffer_in), -}; - -static ZSTD_outBuffer output = { - .dst = buffer_out, - .size = sizeof(buffer_out), - .pos = sizeof(buffer_out), -}; - -#endif /* DO_ZSTD */ - -#if DO_ZLIB - -static z_stream state = { - .next_in = buffer_in, - .avail_in = 0, - .total_in = 0, - - .next_out = buffer_out, - .avail_out = sizeof(buffer_out), - .total_out = 0, - - .msg = NULL, - .state = NULL, - .workspace = NULL, -}; - -#endif /* DO_ZLIB */ - -static int zstd_compress_test_open(struct inode *i, struct file *f) -{ - if (device_is_open) - return -EBUSY; - - device_is_open = true; - - uncompressed_len = compressed_len = 0; - -#if DO_ZSTD - if (ZSTD_isError(ZSTD_resetCStream(state, 0))) - return -EIO; -#endif - -#if DO_ZLIB - if (zlib_deflateReset(&state) != Z_OK) - return -EIO; -#endif - - printk(KERN_INFO DEVICE_NAME ": opened\n"); - return 0; -} - -static int zstd_compress_test_release(struct inode *i, struct file *f) -{ - device_is_open = false; - -#if DO_ZSTD - do { - size_t ret; - - output.pos = 0; - ret = ZSTD_endStream(state, &output); - if (ZSTD_isError(ret)) { - printk(KERN_INFO DEVICE_NAME ": zstd end error %u\n", ZSTD_getErrorCode(ret)); - return -EIO; - } - compressed_len += output.pos; - } while (output.pos != output.size); -#endif - -#if DO_ZLIB - for (;;) { - int ret; - - state.next_out = buffer_out; - state.avail_out = sizeof(buffer_out); - ret = zlib_deflate(&state, Z_FINISH); - compressed_len += sizeof(buffer_out) - state.avail_out; - if (ret == Z_STREAM_END) - break; - if (ret != Z_OK) { - printk(KERN_INFO DEVICE_NAME ": zlib end error %d: %s\n", ret, state.msg); - return -EIO; - } - } -#endif - - printk(KERN_INFO DEVICE_NAME ": uncompressed_len = %llu\n", uncompressed_len); - printk(KERN_INFO DEVICE_NAME ": compressed_len = %llu\n", compressed_len); - printk(KERN_INFO DEVICE_NAME ": closed\n"); - return 0; -} - -/* - * Hash the data given to us from userspace. - */ -static ssize_t zstd_compress_test_write(struct file *file, - const char __user *buf, size_t size, loff_t *pos) -{ - size_t remaining = size; - - while (remaining > 0) { - size_t const copy_size = min(remaining, sizeof(buffer_in)); - - if (copy_from_user(buffer_in, buf, copy_size)) - return -EFAULT; - buf += copy_size; - remaining -= copy_size; - uncompressed_len += copy_size; - -#if DO_ZSTD - input.pos = 0; - input.size = copy_size; - while (input.pos != input.size) { - size_t ret; - - output.pos = 0; - ret = ZSTD_compressStream(state, &output, &input); - if (ZSTD_isError(ret)) { - printk(KERN_INFO DEVICE_NAME ": zstd compress error %u\n", ZSTD_getErrorCode(ret)); - return -EIO; - } - compressed_len += output.pos; - } -#endif -#if DO_ZLIB - state.next_in = buffer_in; - state.avail_in = copy_size; - while (state.avail_in > 0) { - int ret; - - state.next_out = buffer_out; - state.avail_out = sizeof(buffer_out); - ret = zlib_deflate(&state, Z_NO_FLUSH); - compressed_len += sizeof(buffer_out) - state.avail_out; - if (ret != Z_OK) { - printk(KERN_INFO DEVICE_NAME ": zlib end error %d: %s\n", ret, state.msg); - return -EIO; - } - } -#endif - } - return size; -} -/* register the character device. */ -static int __init zstd_compress_test_init(void) -{ - static const struct file_operations fileops = { - .owner = THIS_MODULE, - .open = &zstd_compress_test_open, - .release = &zstd_compress_test_release, - .write = &zstd_compress_test_write - }; - size_t workspace_size = 0; -#if DO_ZSTD - ZSTD_parameters params; -#endif - - device_major = register_chrdev(0, DEVICE_NAME, &fileops); - if (device_major < 0) { - return device_major; - } - -#if DO_ZSTD - params = ZSTD_getParams(DO_ZSTD, 0, 0); - workspace_size = ZSTD_CStreamWorkspaceBound(params.cParams); - - if (!(workspace = vmalloc(workspace_size))) - goto fail; - if (!(state = ZSTD_initCStream(params, 0, workspace, workspace_size))) - goto fail; -#endif - -#if DO_ZLIB - workspace_size = zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL); - - if (!(workspace = vmalloc(workspace_size))) - goto fail; - state.workspace = workspace; - if (zlib_deflateInit(&state, DO_ZLIB) != Z_OK) - goto fail; -#endif - - printk(KERN_INFO DEVICE_NAME ": module loaded\n"); - printk(KERN_INFO DEVICE_NAME ": compression requires %zu bytes of memory\n", workspace_size); - printk(KERN_INFO DEVICE_NAME ": Create a device node with " - "'mknod " DEVICE_NAME " c %d 0' and write data " - "to it.\n", device_major); - return 0; - -fail: - printk(KERN_INFO DEVICE_NAME ": failed to load module\n"); - if (workspace) { - vfree(workspace); - workspace = NULL; - } - return -ENOMEM; -} - -static void __exit zstd_compress_test_exit(void) -{ - unregister_chrdev(device_major, DEVICE_NAME); -#if DO_ZLIB - zlib_deflateEnd(&state); -#endif - if (workspace) { - vfree(workspace); - workspace = NULL; - } - printk(KERN_INFO DEVICE_NAME ": module unloaded\n"); -} - -module_init(zstd_compress_test_init); -module_exit(zstd_compress_test_exit); - -MODULE_DESCRIPTION("Zstd compression tester"); -MODULE_VERSION("1.0"); - -MODULE_LICENSE("Dual BSD/GPL"); diff --git a/contrib/linux-kernel/zstd_decompress_module.c b/contrib/linux-kernel/zstd_decompress_module.c new file mode 100644 index 00000000..988fdb57 --- /dev/null +++ b/contrib/linux-kernel/zstd_decompress_module.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> +#include <linux/zstd.h> + +#include "zstd.h" +#include "common/zstd_deps.h" +#include "common/zstd_errors.h" + +/* Common symbols. zstd_compress must depend on zstd_decompress. */ + +unsigned int zstd_is_error(size_t code) +{ + return ZSTD_isError(code); +} +EXPORT_SYMBOL(zstd_is_error); + +int zstd_get_error_code(size_t code) +{ + return ZSTD_getErrorCode(code); +} +EXPORT_SYMBOL(zstd_get_error_code); + +const char *zstd_get_error_name(size_t code) +{ + return ZSTD_getErrorName(code); +} +EXPORT_SYMBOL(zstd_get_error_name); + +/* Decompression symbols. */ + +size_t zstd_dctx_workspace_bound(void) +{ + return ZSTD_estimateDCtxSize(); +} +EXPORT_SYMBOL(zstd_dctx_workspace_bound); + +zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size) +{ + if (workspace == NULL) + return NULL; + return ZSTD_initStaticDCtx(workspace, workspace_size); +} +EXPORT_SYMBOL(zstd_init_dctx); + +size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity, + const void *src, size_t src_size) +{ + return ZSTD_decompressDCtx(dctx, dst, dst_capacity, src, src_size); +} +EXPORT_SYMBOL(zstd_decompress_dctx); + +size_t zstd_dstream_workspace_bound(size_t max_window_size) +{ + return ZSTD_estimateDStreamSize(max_window_size); +} +EXPORT_SYMBOL(zstd_dstream_workspace_bound); + +zstd_dstream *zstd_init_dstream(size_t max_window_size, void *workspace, + size_t workspace_size) +{ + if (workspace == NULL) + return NULL; + (void)max_window_size; + return ZSTD_initStaticDStream(workspace, workspace_size); +} +EXPORT_SYMBOL(zstd_init_dstream); + +size_t zstd_reset_dstream(zstd_dstream *dstream) +{ + return ZSTD_resetDStream(dstream); +} +EXPORT_SYMBOL(zstd_reset_dstream); + +size_t zstd_decompress_stream(zstd_dstream *dstream, + struct zstd_out_buffer *output, struct zstd_in_buffer *input) +{ + ZSTD_outBuffer o; + ZSTD_inBuffer i; + size_t ret; + + ZSTD_memcpy(&o, output, sizeof(o)); + ZSTD_memcpy(&i, input, sizeof(i)); + ret = ZSTD_decompressStream(dstream, &o, &i); + ZSTD_memcpy(output, &o, sizeof(o)); + ZSTD_memcpy(input, &i, sizeof(i)); + return ret; +} +EXPORT_SYMBOL(zstd_decompress_stream); + +size_t zstd_find_frame_compressed_size(const void *src, size_t src_size) +{ + return ZSTD_findFrameCompressedSize(src, src_size); +} +EXPORT_SYMBOL(zstd_find_frame_compressed_size); + +size_t zstd_get_frame_params(struct zstd_frame_params *params, const void *src, + size_t src_size) +{ + ZSTD_frameHeader h; + const size_t ret = ZSTD_getFrameHeader(&h, src, src_size); + + if (ret != 0) + return ret; + + if (h.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) + params->frame_content_size = h.frameContentSize; + else + params->frame_content_size = 0; + + params->window_size = h.windowSize; + params->dict_id = h.dictID; + params->checksum_flag = h.checksumFlag; + + return ret; +} +EXPORT_SYMBOL(zstd_get_frame_params); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("Zstd Decompressor"); diff --git a/contrib/linux-kernel/zstd_decompress_test.c b/contrib/linux-kernel/zstd_decompress_test.c deleted file mode 100644 index f6efddd3..00000000 --- a/contrib/linux-kernel/zstd_decompress_test.c +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - */ - -/* Compression level or 0 to disable */ -#define DO_ZLIB 1 -/* Compression level or 0 to disable */ -#define DO_ZSTD 0 -/* Buffer size */ -#define BUFFER_SIZE 4096 - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/vmalloc.h> -#include <linux/fs.h> -#include <linux/uaccess.h> - -#if DO_ZSTD -#include <linux/zstd.h> -#endif - -#if DO_ZLIB -#include <linux/zlib.h> -#endif - -/* Device name to pass to register_chrdev(). */ -#define DEVICE_NAME "zstd_decompress_test" - -/* Dynamically allocated device major number */ -static int device_major; - -/* - * We reuse the same state, and thus can compress only one file at a time. - */ -static bool device_is_open; - - -static void *workspace = NULL; - -/* - * Input buffer used to put data coming from userspace. - */ -static uint8_t buffer_in[BUFFER_SIZE]; -static uint8_t buffer_out[BUFFER_SIZE]; - -static uint64_t uncompressed_len; -static uint64_t compressed_len; - -#if DO_ZSTD - -static ZSTD_DStream *state; - -static ZSTD_inBuffer input = { - .src = buffer_in, - .size = sizeof(buffer_in), - .pos = sizeof(buffer_in), -}; - -static ZSTD_outBuffer output = { - .dst = buffer_out, - .size = sizeof(buffer_out), - .pos = sizeof(buffer_out), -}; - -#endif /* DO_ZSTD */ - -#if DO_ZLIB - -static z_stream state = { - .next_in = buffer_in, - .avail_in = 0, - .total_in = 0, - - .next_out = buffer_out, - .avail_out = sizeof(buffer_out), - .total_out = 0, - - .msg = NULL, - .state = NULL, - .workspace = NULL, -}; - -#endif /* DO_ZLIB */ - -static int zstd_decompress_test_open(struct inode *i, struct file *f) -{ - if (device_is_open) - return -EBUSY; - - device_is_open = true; - - uncompressed_len = compressed_len = 0; - -#if DO_ZSTD - if (ZSTD_isError(ZSTD_resetDStream(state))) - return -EIO; -#endif - -#if DO_ZLIB - if (zlib_inflateReset(&state) != Z_OK) - return -EIO; -#endif - - printk(KERN_INFO DEVICE_NAME ": opened\n"); - return 0; -} - -static int zstd_decompress_test_release(struct inode *i, struct file *f) -{ - device_is_open = false; - - printk(KERN_INFO DEVICE_NAME ": uncompressed_len = %llu\n", uncompressed_len); - printk(KERN_INFO DEVICE_NAME ": compressed_len = %llu\n", compressed_len); - printk(KERN_INFO DEVICE_NAME ": closed\n"); - return 0; -} - -/* - * Hash the data given to us from userspace. - */ -static ssize_t zstd_decompress_test_write(struct file *file, - const char __user *buf, size_t size, loff_t *pos) -{ - size_t remaining = size; - - while (remaining > 0) { - size_t const copy_size = min(remaining, sizeof(buffer_in)); - - if (copy_from_user(buffer_in, buf, copy_size)) - return -EFAULT; - buf += copy_size; - remaining -= copy_size; - compressed_len += copy_size; - -#if DO_ZSTD - input.pos = 0; - input.size = copy_size; - while (input.pos != input.size) { - size_t ret; - - output.pos = 0; - ret = ZSTD_decompressStream(state, &output, &input); - if (ZSTD_isError(ret)) { - printk(KERN_INFO DEVICE_NAME ": zstd decompress error %u\n", ZSTD_getErrorCode(ret)); - return -EIO; - } - uncompressed_len += output.pos; - } -#endif -#if DO_ZLIB - state.next_in = buffer_in; - state.avail_in = copy_size; - while (state.avail_in > 0) { - int ret; - - state.next_out = buffer_out; - state.avail_out = sizeof(buffer_out); - ret = zlib_inflate(&state, Z_NO_FLUSH); - uncompressed_len += sizeof(buffer_out) - state.avail_out; - if (ret != Z_OK && ret != Z_STREAM_END) { - printk(KERN_INFO DEVICE_NAME ": zlib decompress error %d: %s\n", ret, state.msg); - return -EIO; - } - } -#endif - } - return size; -} -/* register the character device. */ -static int __init zstd_decompress_test_init(void) -{ - static const struct file_operations fileops = { - .owner = THIS_MODULE, - .open = &zstd_decompress_test_open, - .release = &zstd_decompress_test_release, - .write = &zstd_decompress_test_write - }; - size_t workspace_size = 0; -#if DO_ZSTD - ZSTD_parameters params; - size_t max_window_size; -#endif - - device_major = register_chrdev(0, DEVICE_NAME, &fileops); - if (device_major < 0) { - return device_major; - } - -#if DO_ZSTD - params = ZSTD_getParams(DO_ZSTD, 0, 0); - max_window_size = (size_t)1 << params.cParams.windowLog; - workspace_size = ZSTD_DStreamWorkspaceBound(max_window_size); - - if (!(workspace = vmalloc(workspace_size))) - goto fail; - if (!(state = ZSTD_initDStream(max_window_size, workspace, workspace_size))) - goto fail; -#endif - -#if DO_ZLIB - workspace_size = zlib_inflate_workspacesize(); - - if (!(workspace = vmalloc(workspace_size))) - goto fail; - state.workspace = workspace; - if (zlib_inflateInit(&state) != Z_OK) - goto fail; -#endif - - printk(KERN_INFO DEVICE_NAME ": module loaded\n"); - printk(KERN_INFO DEVICE_NAME ": decompression requires %zu bytes of memory\n", workspace_size); - printk(KERN_INFO DEVICE_NAME ": Create a device node with " - "'mknod " DEVICE_NAME " c %d 0' and write data " - "to it.\n", device_major); - return 0; - -fail: - printk(KERN_INFO DEVICE_NAME ": failed to load module\n"); - if (workspace) { - vfree(workspace); - workspace = NULL; - } - return -ENOMEM; -} - -static void __exit zstd_decompress_test_exit(void) -{ - unregister_chrdev(device_major, DEVICE_NAME); -#if DO_ZLIB - zlib_deflateEnd(&state); -#endif - if (workspace) { - vfree(workspace); - workspace = NULL; - } - printk(KERN_INFO DEVICE_NAME ": module unloaded\n"); -} - -module_init(zstd_decompress_test_init); -module_exit(zstd_decompress_test_exit); - -MODULE_DESCRIPTION("Zstd decompression tester"); -MODULE_VERSION("1.0"); - -MODULE_LICENSE("Dual BSD/GPL"); diff --git a/contrib/linux-kernel/zstd_deps.h b/contrib/linux-kernel/zstd_deps.h new file mode 100644 index 00000000..4a6d35f1 --- /dev/null +++ b/contrib/linux-kernel/zstd_deps.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* + * This file provides common libc dependencies that zstd requires. + * The purpose is to allow replacing this file with a custom implementation + * to compile zstd without libc support. + */ + +/* Need: + * NULL + * INT_MAX + * UINT_MAX + * ZSTD_memcpy() + * ZSTD_memset() + * ZSTD_memmove() + */ +#ifndef ZSTD_DEPS_COMMON +#define ZSTD_DEPS_COMMON + +#include <linux/limits.h> +#include <linux/stddef.h> + +#define ZSTD_memcpy(d,s,n) __builtin_memcpy((d),(s),(n)) +#define ZSTD_memmove(d,s,n) __builtin_memmove((d),(s),(n)) +#define ZSTD_memset(d,s,n) __builtin_memset((d),(s),(n)) + +#endif /* ZSTD_DEPS_COMMON */ + +/* + * Define malloc as always failing. That means the user must + * either use ZSTD_customMem or statically allocate memory. + * Need: + * ZSTD_malloc() + * ZSTD_free() + * ZSTD_calloc() + */ +#ifdef ZSTD_DEPS_NEED_MALLOC +#ifndef ZSTD_DEPS_MALLOC +#define ZSTD_DEPS_MALLOC + +#define ZSTD_malloc(s) ({ (void)(s); NULL; }) +#define ZSTD_free(p) ((void)(p)) +#define ZSTD_calloc(n,s) ({ (void)(n); (void)(s); NULL; }) + +#endif /* ZSTD_DEPS_MALLOC */ +#endif /* ZSTD_DEPS_NEED_MALLOC */ + +/* + * Provides 64-bit math support. + * Need: + * U64 ZSTD_div64(U64 dividend, U32 divisor) + */ +#ifdef ZSTD_DEPS_NEED_MATH64 +#ifndef ZSTD_DEPS_MATH64 +#define ZSTD_DEPS_MATH64 + +#include <linux/math64.h> + +static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) { + return div_u64(dividend, divisor); +} + +#endif /* ZSTD_DEPS_MATH64 */ +#endif /* ZSTD_DEPS_NEED_MATH64 */ + +/* + * This is only requested when DEBUGLEVEL >= 1, meaning + * it is disabled in production. + * Need: + * assert() + */ +#ifdef ZSTD_DEPS_NEED_ASSERT +#ifndef ZSTD_DEPS_ASSERT +#define ZSTD_DEPS_ASSERT + +#include <linux/kernel.h> + +#define assert(x) WARN_ON((x)) + +#endif /* ZSTD_DEPS_ASSERT */ +#endif /* ZSTD_DEPS_NEED_ASSERT */ + +/* + * This is only requested when DEBUGLEVEL >= 2, meaning + * it is disabled in production. + * Need: + * ZSTD_DEBUG_PRINT() + */ +#ifdef ZSTD_DEPS_NEED_IO +#ifndef ZSTD_DEPS_IO +#define ZSTD_DEPS_IO + +#include <linux/printk.h> + +#define ZSTD_DEBUG_PRINT(...) pr_debug(__VA_ARGS__) + +#endif /* ZSTD_DEPS_IO */ +#endif /* ZSTD_DEPS_NEED_IO */ + +/* + * Only requested when MSAN is enabled. + * Need: + * intptr_t + */ +#ifdef ZSTD_DEPS_NEED_STDINT +#ifndef ZSTD_DEPS_STDINT +#define ZSTD_DEPS_STDINT + +/* + * The Linux Kernel doesn't provide intptr_t, only uintptr_t, which + * is an unsigned long. + */ +typedef long intptr_t; + +#endif /* ZSTD_DEPS_STDINT */ +#endif /* ZSTD_DEPS_NEED_STDINT */ diff --git a/contrib/pzstd/Logging.h b/contrib/pzstd/Logging.h index 16a63932..beb160b6 100644 --- a/contrib/pzstd/Logging.h +++ b/contrib/pzstd/Logging.h @@ -13,10 +13,10 @@ namespace pzstd { -constexpr int ERROR = 1; -constexpr int INFO = 2; -constexpr int DEBUG = 3; -constexpr int VERBOSE = 4; +constexpr int kLogError = 1; +constexpr int kLogInfo = 2; +constexpr int kLogDebug = 3; +constexpr int kLogVerbose = 4; class Logger { std::mutex mutex_; diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp index 652187c3..ce142ad8 100644 --- a/contrib/pzstd/Pzstd.cpp +++ b/contrib/pzstd/Pzstd.cpp @@ -94,12 +94,12 @@ static std::uint64_t handleOneInput(const Options &options, if (!options.decompress) { double ratio = static_cast<double>(bytesWritten) / static_cast<double>(bytesRead + !bytesRead); - state.log(INFO, "%-20s :%6.2f%% (%6" PRIu64 " => %6" PRIu64 + state.log(kLogInfo, "%-20s :%6.2f%% (%6" PRIu64 " => %6" PRIu64 " bytes, %s)\n", inputFileName.c_str(), ratio * 100, bytesRead, bytesWritten, outputFileName.c_str()); } else { - state.log(INFO, "%-20s: %" PRIu64 " bytes \n", + state.log(kLogInfo, "%-20s: %" PRIu64 " bytes \n", inputFileName.c_str(),bytesWritten); } } @@ -139,12 +139,12 @@ static FILE *openOutputFile(const Options &options, auto outputFd = std::fopen(outputFile.c_str(), "rb"); if (outputFd != nullptr) { std::fclose(outputFd); - if (!state.log.logsAt(INFO)) { + if (!state.log.logsAt(kLogInfo)) { state.errorHolder.setError("Output file exists"); return nullptr; } state.log( - INFO, + kLogInfo, "pzstd: %s already exists; do you wish to overwrite (y/n) ? ", outputFile.c_str()); int c = getchar(); @@ -170,7 +170,7 @@ int pzstdMain(const Options &options) { auto printErrorGuard = makeScopeGuard([&] { if (state.errorHolder.hasError()) { returnCode = 1; - state.log(ERROR, "pzstd: %s: %s.\n", input.c_str(), + state.log(kLogError, "pzstd: %s: %s.\n", input.c_str(), state.errorHolder.getError().c_str()); } }); @@ -388,7 +388,7 @@ std::uint64_t asyncCompressChunks( // Break the input up into chunks of size `step` and compress each chunk // independently. size_t step = calculateStep(size, numThreads, params); - state.log(DEBUG, "Chosen frame size: %zu\n", step); + state.log(kLogDebug, "Chosen frame size: %zu\n", step); auto status = FileStatus::Continue; while (status == FileStatus::Continue && !state.errorHolder.hasError()) { // Make a new input queue that we will put the chunk's input data into. @@ -403,7 +403,7 @@ std::uint64_t asyncCompressChunks( }); // Pass the output queue to the writer thread. chunks.push(std::move(out)); - state.log(VERBOSE, "%s\n", "Starting a new frame"); + state.log(kLogVerbose, "%s\n", "Starting a new frame"); // Fill the input queue for the compression job we just started status = readData(*in, ZSTD_CStreamInSize(), step, fd, &bytesRead); } @@ -540,14 +540,14 @@ std::uint64_t asyncDecompressFrames( if (frameSize == 0) { // We hit a non SkippableFrame ==> not compressed by pzstd or corrupted // Pass the rest of the source to this decompression task - state.log(VERBOSE, "%s\n", + state.log(kLogVerbose, "%s\n", "Input not in pzstd format, falling back to serial decompression"); while (status == FileStatus::Continue && !state.errorHolder.hasError()) { status = readData(*in, chunkSize, chunkSize, fd, &totalBytesRead); } break; } - state.log(VERBOSE, "Decompressing a frame of size %zu", frameSize); + state.log(kLogVerbose, "Decompressing a frame of size %zu", frameSize); // Fill the input queue for the decompression job we just started status = readData(*in, chunkSize, frameSize, fd, &totalBytesRead); } @@ -573,7 +573,7 @@ std::uint64_t writeFile( bool decompress) { auto& errorHolder = state.errorHolder; auto lineClearGuard = makeScopeGuard([&state] { - state.log.clear(INFO); + state.log.clear(kLogInfo); }); std::uint64_t bytesWritten = 0; std::shared_ptr<BufferWorkQueue> out; @@ -602,7 +602,7 @@ std::uint64_t writeFile( return bytesWritten; } bytesWritten += buffer.size(); - state.log.update(INFO, "Written: %u MB ", + state.log.update(kLogInfo, "Written: %u MB ", static_cast<std::uint32_t>(bytesWritten >> 20)); } } diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h index 79d1fcca..033adefa 100644 --- a/contrib/pzstd/Pzstd.h +++ b/contrib/pzstd/Pzstd.h @@ -41,7 +41,7 @@ class SharedState { auto parameters = options.determineParameters(); cStreamPool.reset(new ResourcePool<ZSTD_CStream>{ [this, parameters]() -> ZSTD_CStream* { - this->log(VERBOSE, "%s\n", "Creating new ZSTD_CStream"); + this->log(kLogVerbose, "%s\n", "Creating new ZSTD_CStream"); auto zcs = ZSTD_createCStream(); if (zcs) { auto err = ZSTD_initCStream_advanced( @@ -59,7 +59,7 @@ class SharedState { } else { dStreamPool.reset(new ResourcePool<ZSTD_DStream>{ [this]() -> ZSTD_DStream* { - this->log(VERBOSE, "%s\n", "Creating new ZSTD_DStream"); + this->log(kLogVerbose, "%s\n", "Creating new ZSTD_DStream"); auto zds = ZSTD_createDStream(); if (zds) { auto err = ZSTD_initDStream(zds); diff --git a/contrib/seekable_format/tests/.gitignore b/contrib/seekable_format/tests/.gitignore new file mode 100644 index 00000000..f831eaf3 --- /dev/null +++ b/contrib/seekable_format/tests/.gitignore @@ -0,0 +1 @@ +seekable_tests diff --git a/contrib/seekable_format/tests/Makefile b/contrib/seekable_format/tests/Makefile new file mode 100644 index 00000000..b00657f8 --- /dev/null +++ b/contrib/seekable_format/tests/Makefile @@ -0,0 +1,38 @@ +# ################################################################ +# Copyright (c) 2017-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ + +# This Makefile presumes libzstd is built, using `make` in / or /lib/ + +ZSTDLIB_PATH = ../../../lib +ZSTDLIB_NAME = libzstd.a +ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME) + +CPPFLAGS += -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common + +CFLAGS ?= -O3 +CFLAGS += -g + +SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB) + +.PHONY: default clean test + +default: seekable_tests + +test: seekable_tests + ./seekable_tests + +$(ZSTDLIB): + $(MAKE) -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME) + +seekable_tests : seekable_tests.c $(SEEKABLE_OBJS) + +clean: + @rm -f core *.o tmp* result* *.zst \ + seekable_tests + @echo Cleaning completed diff --git a/contrib/seekable_format/tests/seekable_tests.c b/contrib/seekable_format/tests/seekable_tests.c new file mode 100644 index 00000000..f2556b51 --- /dev/null +++ b/contrib/seekable_format/tests/seekable_tests.c @@ -0,0 +1,63 @@ +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> + +#include "zstd_seekable.h" + +/* Basic unit tests for zstd seekable format */ +int main(int argc, const char** argv) +{ + unsigned testNb = 1; + printf("Beginning zstd seekable format tests...\n"); + printf("Test %u - check that seekable decompress does not hang: ", testNb++); + { /* Github issue #2335 */ + const size_t compressed_size = 17; + const uint8_t compressed_data[17] = { + '^', + '*', + 'M', + '\x18', + '\t', + '\x00', + '\x00', + '\x00', + '\x00', + '\x00', + '\x00', + '\x00', + ';', + (uint8_t)('\xb1'), + (uint8_t)('\xea'), + (uint8_t)('\x92'), + (uint8_t)('\x8f'), + }; + const size_t uncompressed_size = 32; + uint8_t uncompressed_data[32]; + + ZSTD_seekable* stream = ZSTD_seekable_create(); + size_t status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size); + if (ZSTD_isError(status)) { + ZSTD_seekable_free(stream); + goto _test_error; + } + + const size_t offset = 2; + /* Should return an error, but not hang */ + status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset); + if (!ZSTD_isError(status)) { + ZSTD_seekable_free(stream); + goto _test_error; + } + + ZSTD_seekable_free(stream); + } + printf("Success!\n"); + + /* TODO: Add more tests */ + printf("Finished tests\n"); + return 0; + +_test_error: + printf("test failed! Exiting..\n"); + return 1; +} diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c index abfd1e90..cc5c8595 100644 --- a/contrib/seekable_format/zstdseek_decompress.c +++ b/contrib/seekable_format/zstdseek_decompress.c @@ -79,6 +79,8 @@ #define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define ZSTD_SEEKABLE_NO_OUTPUT_PROGRESS_MAX 16 + /* Special-case callbacks for FILE* and in-memory modes, so that we can treat * them the same way as the advanced API */ static int ZSTD_seekable_read_FILE(void* opaque, void* buffer, size_t n) @@ -380,6 +382,7 @@ size_t ZSTD_seekable_initAdvanced(ZSTD_seekable* zs, ZSTD_seekable_customFile sr size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsigned long long offset) { U32 targetFrame = ZSTD_seekable_offsetToFrameIndex(zs, offset); + U32 noOutputProgressCount = 0; do { /* check if we can continue from a previous decompress job */ if (targetFrame != zs->curFrame || offset != zs->decompressedOffset) { @@ -398,6 +401,7 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign size_t toRead; ZSTD_outBuffer outTmp; size_t prevOutPos; + size_t forwardProgress; if (zs->decompressedOffset < offset) { /* dummy decompressions until we get to the target offset */ outTmp = (ZSTD_outBuffer){zs->outBuff, MIN(SEEKABLE_BUFF_SIZE, offset - zs->decompressedOffset), 0}; @@ -415,7 +419,15 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign XXH64_update(&zs->xxhState, (BYTE*)outTmp.dst + prevOutPos, outTmp.pos - prevOutPos); } - zs->decompressedOffset += outTmp.pos - prevOutPos; + forwardProgress = outTmp.pos - prevOutPos; + if (forwardProgress == 0) { + if (noOutputProgressCount++ > ZSTD_SEEKABLE_NO_OUTPUT_PROGRESS_MAX) { + return ERROR(seekableIO); + } + } else { + noOutputProgressCount = 0; + } + zs->decompressedOffset += forwardProgress; if (toRead == 0) { /* frame complete */ diff --git a/contrib/single_file_libs/.gitignore b/contrib/single_file_libs/.gitignore index 054333f9..8c1bc71e 100644 --- a/contrib/single_file_libs/.gitignore +++ b/contrib/single_file_libs/.gitignore @@ -1,4 +1,9 @@ + +# build artifacts zstddeclib.c zstdenclib.c zstd.c zstd.h + +# test artifacts +temp* diff --git a/contrib/single_file_libs/build_decoder_test.sh b/contrib/single_file_libs/build_decoder_test.sh index 74e3f1c5..48d017fc 100755 --- a/contrib/single_file_libs/build_decoder_test.sh +++ b/contrib/single_file_libs/build_decoder_test.sh @@ -6,6 +6,56 @@ OUT_FILE="tempbin" # Optional temporary compiled WebAssembly OUT_WASM="temp.wasm" +# Source files to compile using Emscripten. +IN_FILES="examples/emscripten.c" + +# Emscripten build using emcc. +emscripten_emcc_build() { + # Compile the the same example as above + CC_FLAGS="-Wall -Wextra -Wshadow -Werror -Os -g0 -flto" + emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES + # Did compilation work? + if [ $? -ne 0 ]; then + echo "Compiling ${IN_FILES}: FAILED" + exit 1 + fi + echo "Compiling ${IN_FILES}: PASSED" + rm -f $OUT_WASM +} + +# Emscripten build using docker. +emscripten_docker_build() { + docker container run --rm \ + --volume $PWD:/code \ + --workdir /code \ + emscripten/emsdk:latest \ + emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES + # Did compilation work? + if [ $? -ne 0 ]; then + echo "Compiling ${IN_FILES} (using docker): FAILED" + exit 1 + fi + echo "Compiling ${IN_FILES} (using docker): PASSED" + rm -f $OUT_WASM +} + +# Try Emscripten build using emcc or docker. +try_emscripten_build() { + which emcc > /dev/null + if [ $? -eq 0 ]; then + emscripten_emcc_build + return $? + fi + + which docker > /dev/null + if [ $? -eq 0 ]; then + emscripten_docker_build + return $? + fi + + echo "(Skipping Emscripten test)" +} + # Amalgamate the sources ./create_single_file_decoder.sh # Did combining work? @@ -16,7 +66,7 @@ fi echo "Single file decoder creation script: PASSED" # Compile the generated output -cc -Wall -Wextra -Werror -Os -g0 -o $OUT_FILE examples/simple.c +cc -Wall -Wextra -Wshadow -Werror -Os -g0 -o $OUT_FILE examples/simple.c # Did compilation work? if [ $? -ne 0 ]; then echo "Compiling simple.c: FAILED" @@ -35,21 +85,7 @@ if [ $retVal -ne 0 ]; then fi echo "Running simple.c: PASSED" -# Is Emscripten available? -which emcc > /dev/null -if [ $? -ne 0 ]; then - echo "(Skipping Emscripten test)" -else - # Compile the Emscripten example - CC_FLAGS="-Wall -Wextra -Werror -Os -g0 -flto --llvm-lto 3 -lGL -DNDEBUG=1" - emcc $CC_FLAGS -s WASM=1 -o $OUT_WASM examples/emscripten.c - # Did compilation work? - if [ $? -ne 0 ]; then - echo "Compiling emscripten.c: FAILED" - exit 1 - fi - echo "Compiling emscripten.c: PASSED" - rm -f $OUT_WASM -fi +# Try Emscripten build if emcc or docker command is available. +try_emscripten_build exit 0 diff --git a/contrib/single_file_libs/build_library_test.sh b/contrib/single_file_libs/build_library_test.sh index 3b8b16a0..7fb99656 100755 --- a/contrib/single_file_libs/build_library_test.sh +++ b/contrib/single_file_libs/build_library_test.sh @@ -9,6 +9,56 @@ OUT_FILE="tempbin" # Optional temporary compiled WebAssembly OUT_WASM="temp.wasm" +# Source files to compile using Emscripten. +IN_FILES="zstd.c examples/roundtrip.c" + +# Emscripten build using emcc. +emscripten_emcc_build() { + # Compile the the same example as above + CC_FLAGS="-Wall -Wextra -Wshadow -Werror -Os -g0 -flto" + emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES + # Did compilation work? + if [ $? -ne 0 ]; then + echo "Compiling ${IN_FILES}: FAILED" + exit 1 + fi + echo "Compiling ${IN_FILES}: PASSED" + rm -f $OUT_WASM +} + +# Emscripten build using docker. +emscripten_docker_build() { + docker container run --rm \ + --volume $PWD:/code \ + --workdir /code \ + emscripten/emsdk:latest \ + emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES + # Did compilation work? + if [ $? -ne 0 ]; then + echo "Compiling ${IN_FILES} (using docker): FAILED" + exit 1 + fi + echo "Compiling ${IN_FILES} (using docker): PASSED" + rm -f $OUT_WASM +} + +# Try Emscripten build using emcc or docker. +try_emscripten_build() { + which emcc > /dev/null + if [ $? -eq 0 ]; then + emscripten_emcc_build + return $? + fi + + which docker > /dev/null + if [ $? -eq 0 ]; then + emscripten_docker_build + return $? + fi + + echo "(Skipping Emscripten test)" +} + # Amalgamate the sources ./create_single_file_library.sh # Did combining work? @@ -22,7 +72,7 @@ echo "Single file library creation script: PASSED" cp "$ZSTD_SRC_ROOT/zstd.h" zstd.h # Compile the generated output -cc -Wall -Wextra -Werror -pthread -I. -Os -g0 -o $OUT_FILE zstd.c examples/roundtrip.c +cc -Wall -Wextra -Werror -Wshadow -pthread -I. -Os -g0 -o $OUT_FILE zstd.c examples/roundtrip.c # Did compilation work? if [ $? -ne 0 ]; then echo "Compiling roundtrip.c: FAILED" @@ -41,21 +91,7 @@ if [ $retVal -ne 0 ]; then fi echo "Running roundtrip.c: PASSED" -# Is Emscripten available? -which emcc > /dev/null -if [ $? -ne 0 ]; then - echo "(Skipping Emscripten test)" -else - # Compile the the same example as above - CC_FLAGS="-Wall -Wextra -Werror -Os -g0 -flto" - emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM zstd.c examples/roundtrip.c - # Did compilation work? - if [ $? -ne 0 ]; then - echo "Compiling emscripten.c: FAILED" - exit 1 - fi - echo "Compiling emscripten.c: PASSED" - rm -f $OUT_WASM -fi +# Try Emscripten build if emcc or docker command is available. +try_emscripten_build exit 0 diff --git a/contrib/single_file_libs/zstd-in.c b/contrib/single_file_libs/zstd-in.c index cac80554..4f0fb56a 100644 --- a/contrib/single_file_libs/zstd-in.c +++ b/contrib/single_file_libs/zstd-in.c @@ -39,13 +39,15 @@ #undef XXH_INLINE_ALL #define XXH_INLINE_ALL #define ZSTD_LEGACY_SUPPORT 0 -#define ZSTD_LIB_DICTBUILDER 0 -#define ZSTD_LIB_DEPRECATED 0 -#define ZSTD_NOBENCH #ifndef __EMSCRIPTEN__ #define ZSTD_MULTITHREAD #endif +/* Include zstd_deps.h first with all the options we need enabled. */ +#define ZSTD_DEPS_NEED_MALLOC +#define ZSTD_DEPS_NEED_MATH64 +#include "common/zstd_deps.h" + #include "common/debug.c" #include "common/entropy_common.c" #include "common/error_private.c" @@ -74,3 +76,8 @@ #include "decompress/zstd_ddict.c" #include "decompress/zstd_decompress.c" #include "decompress/zstd_decompress_block.c" + +#include "dictBuilder/cover.c" +#include "dictBuilder/divsufsort.c" +#include "dictBuilder/fastcover.c" +#include "dictBuilder/zdict.c" diff --git a/contrib/single_file_libs/zstddeclib-in.c b/contrib/single_file_libs/zstddeclib-in.c index a9083dd3..f461b555 100644 --- a/contrib/single_file_libs/zstddeclib-in.c +++ b/contrib/single_file_libs/zstddeclib-in.c @@ -37,11 +37,12 @@ #undef XXH_INLINE_ALL #define XXH_INLINE_ALL #define ZSTD_LEGACY_SUPPORT 0 -#define ZSTD_LIB_COMPRESSION 0 -#define ZSTD_LIB_DEPRECATED 0 -#define ZSTD_NOBENCH #define ZSTD_STRIP_ERROR_STRINGS +/* Include zstd_deps.h first with all the options we need enabled. */ +#define ZSTD_DEPS_NEED_MALLOC +#include "common/zstd_deps.h" + #include "common/debug.c" #include "common/entropy_common.c" #include "common/error_private.c" diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md index fc61726f..0af6bf91 100644 --- a/doc/zstd_compression_format.md +++ b/doc/zstd_compression_format.md @@ -3,7 +3,7 @@ Zstandard Compression Format ### Notices -Copyright (c) 2016-present Yann Collet, Facebook, Inc. +Copyright (c) 2016-2020 Yann Collet, Facebook, Inc. Permission is granted to copy and distribute this document for any purpose and without charge, @@ -16,7 +16,7 @@ Distribution of this document is unlimited. ### Version -0.3.5 (13/11/19) +0.3.7 (2020-12-09) Introduction @@ -291,21 +291,10 @@ Format is __little-endian__. It's allowed to represent a small ID (for example `13`) with a large 4-bytes dictionary ID, even if it is less efficient. -_Reserved ranges :_ -Within private environments, any `Dictionary_ID` can be used. - -However, for frames and dictionaries distributed in public space, -`Dictionary_ID` must be attributed carefully. -Rules for public environment are not yet decided, -but the following ranges are reserved for some future registrar : -- low range : `<= 32767` -- high range : `>= (1 << 31)` - -Outside of these ranges, any value of `Dictionary_ID` -which is both `>= 32768` and `< (1<<31)` can be used freely, -even in public environment. - - +A value of `0` has same meaning as no `Dictionary_ID`, +in which case the frame may or may not need a dictionary to be decoded, +and the ID of such a dictionary is not specified. +The decoder must know this information by other means. #### `Frame_Content_Size` @@ -389,7 +378,7 @@ __`Block_Size`__ The upper 21 bits of `Block_Header` represent the `Block_Size`. When `Block_Type` is `Compressed_Block` or `Raw_Block`, -`Block_Size` is the size of `Block_Content` (hence excluding `Block_Header`). +`Block_Size` is the size of `Block_Content` (hence excluding `Block_Header`). When `Block_Type` is `RLE_Block`, since `Block_Content`’s size is always 1, `Block_Size` represents the number of times this byte must be repeated. @@ -929,38 +918,41 @@ Note that blocks which are not `Compressed_Block` are skipped, they do not contr ###### Offset updates rules -The newest offset takes the lead in offset history, -shifting others back by one rank, -up to the previous rank of the new offset _if it was present in history_. - -__Examples__ : - -In the common case, when new offset is not part of history : -`Repeated_Offset3` = `Repeated_Offset2` -`Repeated_Offset2` = `Repeated_Offset1` -`Repeated_Offset1` = `NewOffset` - -When the new offset _is_ part of history, there may be specific adjustments. - -When `NewOffset` == `Repeated_Offset1`, offset history remains actually unmodified. - -When `NewOffset` == `Repeated_Offset2`, -`Repeated_Offset1` and `Repeated_Offset2` ranks are swapped. -`Repeated_Offset3` is unmodified. - -When `NewOffset` == `Repeated_Offset3`, -there is actually no difference with the common case : -all offsets are shifted by one rank, -`NewOffset` (== `Repeated_Offset3`) becomes the new `Repeated_Offset1`. - -Also worth mentioning, the specific corner case when `offset_value` == 3, -and the literal length of the current sequence is zero. -In which case , `NewOffset` = `Repeated_Offset1` - 1_byte. -Here also, from an offset history update perspective, it's just a common case : -`Repeated_Offset3` = `Repeated_Offset2` -`Repeated_Offset2` = `Repeated_Offset1` -`Repeated_Offset1` = `NewOffset` ( == `Repeated_Offset1` - 1_byte ) - +During the execution of the sequences of a `Compressed_Block`, the +`Repeated_Offsets`' values are kept up to date, so that they always represent +the three most-recently used offsets. In order to achieve that, they are +updated after executing each sequence in the following way: + +When the sequence's `offset_value` does not refer to one of the +`Repeated_Offsets`--when it has value greater than 3, or when it has value 3 +and the sequence's `literals_length` is zero--the `Repeated_Offsets`' values +are shifted back one, and `Repeated_Offset1` takes on the value of the +just-used offset. + +Otherwise, when the sequence's `offset_value` refers to one of the +`Repeated_Offsets`--when it has value 1 or 2, or when it has value 3 and the +sequence's `literals_length` is non-zero--the `Repeated_Offsets` are re-ordered +so that `Repeated_Offset1` takes on the value of the used Repeated_Offset, and +the existing values are pushed back from the first `Repeated_Offset` through to +the `Repeated_Offset` selected by the `offset_value`. This effectively performs +a single-stepped wrapping rotation of the values of these offsets, so that +their order again reflects the recency of their use. + +The following table shows the values of the `Repeated_Offsets` as a series of +sequences are applied to them: + +| `offset_value` | `literals_length` | `Repeated_Offset1` | `Repeated_Offset2` | `Repeated_Offset3` | Comment | +|:--------------:|:-----------------:|:------------------:|:------------------:|:------------------:|:-----------------------:| +| | | 1 | 4 | 8 | starting values | +| 1114 | 11 | 1111 | 1 | 4 | non-repeat | +| 1 | 22 | 1111 | 1 | 4 | repeat 1; no change | +| 2225 | 22 | 2222 | 1111 | 1 | non-repeat | +| 1114 | 111 | 1111 | 2222 | 1111 | non-repeat | +| 3336 | 33 | 3333 | 1111 | 2222 | non-repeat | +| 2 | 22 | 1111 | 3333 | 2222 | repeat 2; swap 1 & 2 | +| 3 | 33 | 2222 | 1111 | 3333 | repeat 3; rotate 3 to 1 | +| 3 | 0 | 2221 | 2222 | 1111 | insert resolved offset | +| 1 | 0 | 2222 | 2221 | 3333 | repeat 2 | Skippable Frames @@ -1429,13 +1421,17 @@ __`Dictionary_ID`__ : 4 bytes, stored in __little-endian__ format. It's used by decoders to check if they use the correct dictionary. _Reserved ranges :_ - If the frame is going to be distributed in a private environment, - any `Dictionary_ID` can be used. - However, for public distribution of compressed frames, - the following ranges are reserved and shall not be used : +If the dictionary is going to be distributed in a public environment, +the following ranges of `Dictionary_ID` are reserved for some future registrar +and shall not be used : + + - low range : <= 32767 + - high range : >= (2^31) + +Outside of these ranges, any value of `Dictionary_ID` +which is both `>= 32768` and `< (1<<31)` can be used freely, +even in public environment. - - low range : <= 32767 - - high range : >= (2^31) __`Entropy_Tables`__ : follow the same format as tables in [compressed blocks]. See the relevant [FSE](#fse-table-description) @@ -1447,7 +1443,7 @@ __`Entropy_Tables`__ : follow the same format as tables in [compressed blocks]. Repeat distribution mode for sequence decoding. It's finally followed by 3 offset values, populating recent offsets (instead of using `{1,4,8}`), stored in order, 4-bytes __little-endian__ each, for a total of 12 bytes. - Each recent offset must have a value < dictionary size. + Each recent offset must have a value <= dictionary content size, and cannot equal 0. __`Content`__ : The rest of the dictionary is its content. The content act as a "past" in front of data to compress or decompress, @@ -1455,7 +1451,7 @@ __`Content`__ : The rest of the dictionary is its content. As long as the amount of data decoded from this frame is less than or equal to `Window_Size`, sequence commands may specify offsets longer than the total length of decoded output so far to reference back to the - dictionary, even parts of the dictionary with offsets larger than `Window_Size`. + dictionary, even parts of the dictionary with offsets larger than `Window_Size`. After the total output has surpassed `Window_Size` however, this is no longer allowed and the dictionary is no longer accessible. @@ -1673,6 +1669,8 @@ or at least provide a meaningful error code explaining for which reason it canno Version changes --------------- +- 0.3.7 : clarifications for Repeat_Offsets +- 0.3.6 : clarifications for Dictionary_ID - 0.3.5 : clarifications for Block_Maximum_Size - 0.3.4 : clarifications for FSE decoding table - 0.3.3 : clarifications for field Block_Size diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index fe58f78c..cb5ded07 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1,10 +1,10 @@ <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"> -<title>zstd 1.4.5 Manual</title> +<title>zstd 1.4.7 Manual</title> </head> <body> -<h1>zstd 1.4.5 Manual</h1> +<h1>zstd 1.4.7 Manual</h1> <hr> <a name="Contents"></a><h2>Contents</h2> <ol> @@ -27,16 +27,10 @@ <li><a href="#Chapter17">Advanced compression functions</a></li> <li><a href="#Chapter18">Advanced decompression functions</a></li> <li><a href="#Chapter19">Advanced streaming functions</a></li> -<li><a href="#Chapter20">! ZSTD_initCStream_usingDict() :</a></li> -<li><a href="#Chapter21">! ZSTD_initCStream_advanced() :</a></li> -<li><a href="#Chapter22">! ZSTD_initCStream_usingCDict() :</a></li> -<li><a href="#Chapter23">! ZSTD_initCStream_usingCDict_advanced() :</a></li> -<li><a href="#Chapter24">This function is deprecated, and is equivalent to:</a></li> -<li><a href="#Chapter25">This function is deprecated, and is equivalent to:</a></li> -<li><a href="#Chapter26">Buffer-less and synchronous inner streaming functions</a></li> -<li><a href="#Chapter27">Buffer-less streaming compression (synchronous mode)</a></li> -<li><a href="#Chapter28">Buffer-less streaming decompression (synchronous mode)</a></li> -<li><a href="#Chapter29">Block level API</a></li> +<li><a href="#Chapter20">Buffer-less and synchronous inner streaming functions</a></li> +<li><a href="#Chapter21">Buffer-less streaming compression (synchronous mode)</a></li> +<li><a href="#Chapter22">Buffer-less streaming decompression (synchronous mode)</a></li> +<li><a href="#Chapter23">Block level API</a></li> </ol> <hr> <a name="Chapter1"></a><h2>Introduction</h2><pre> @@ -72,8 +66,14 @@ <a name="Chapter2"></a><h2>Version</h2><pre></pre> -<pre><b>unsigned ZSTD_versionNumber(void); </b>/**< to check runtime library version */<b> -</b></pre><BR> +<pre><b>unsigned ZSTD_versionNumber(void); +</b><p> Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). +</p></pre><BR> + +<pre><b>const char* ZSTD_versionString(void); +</b><p> Return runtime library version, like "1.4.5". Requires v1.3.0+. +</p></pre><BR> + <a name="Chapter3"></a><h2>Simple API</h2><pre></pre> <pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity, @@ -277,7 +277,9 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); * for large inputs, by finding large matches at long distance. * It increases memory usage and window size. * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB - * except when expressly set to a different value. */ + * except when expressly set to a different value. + * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and + * compression strategy >= ZSTD_btopt (== compression level 16+) */ ZSTD_c_ldmHashLog=161, </b>/* Size of the table for long distance matching, as a power of 2.<b> * Larger values increase memory usage and compression ratio, * but decrease compression speed. @@ -308,16 +310,20 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); ZSTD_c_dictIDFlag=202, </b>/* When applicable, dictionary's ID is written into frame header (default:1) */<b> </b>/* multi-threading parameters */<b> - </b>/* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).<b> - * They return an error otherwise. */ + </b>/* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).<b> + * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. + * In a situation where it's unknown if the linked library supports multi-threading or not, + * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. + */ ZSTD_c_nbWorkers=400, </b>/* Select how many threads will be spawned to compress in parallel.<b> - * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() : + * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, - * while compression work is performed in parallel, within worker threads. + * while compression is performed in parallel, within worker thread(s). * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). * More workers improve speed, but also increase memory usage. - * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */ + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, + * compression is performed inside Caller's thread, and all invocations are blocking */ ZSTD_c_jobSize=401, </b>/* Size of a compression job. This value is enforced only when nbWorkers >= 1.<b> * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. * 0 means default, which is dynamically determined based on compression parameters. @@ -346,6 +352,11 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); * ZSTD_c_literalCompressionMode * ZSTD_c_targetCBlockSize * ZSTD_c_srcSizeHint + * ZSTD_c_enableDedicatedDictSearch + * ZSTD_c_stableInBuffer + * ZSTD_c_stableOutBuffer + * ZSTD_c_blockDelimiters + * ZSTD_c_validateSequences * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -356,7 +367,12 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam5=1002, ZSTD_c_experimentalParam6=1003, - ZSTD_c_experimentalParam7=1004 + ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam8=1005, + ZSTD_c_experimentalParam9=1006, + ZSTD_c_experimentalParam10=1007, + ZSTD_c_experimentalParam11=1008, + ZSTD_c_experimentalParam12=1009 } ZSTD_cParameter; </b></pre><BR> <pre><b>typedef struct { @@ -456,11 +472,13 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); * At the time of this writing, they include : * ZSTD_d_format * ZSTD_d_stableOutBuffer + * ZSTD_d_forceIgnoreChecksum * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly */ ZSTD_d_experimentalParam1=1000, - ZSTD_d_experimentalParam2=1001 + ZSTD_d_experimentalParam2=1001, + ZSTD_d_experimentalParam3=1002 } ZSTD_dParameter; </b></pre><BR> @@ -591,8 +609,9 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs); - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) - output->pos must be <= dstCapacity, input->pos must be <= srcSize - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + - endOp must be a valid directive - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. - - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available, + - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, and then immediately returns, just indicating that there is some data remaining to be flushed. The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. @@ -895,21 +914,40 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); <BR></pre> <pre><b>typedef struct { - unsigned int matchPos; </b>/* Match pos in dst */<b> - </b>/* If seqDef.offset > 3, then this is seqDef.offset - 3<b> - * If seqDef.offset < 3, then this is the corresponding repeat offset - * But if seqDef.offset < 3 and litLength == 0, this is the - * repeat offset before the corresponding repeat offset - * And if seqDef.offset == 3 and litLength == 0, this is the - * most recent repeat offset - 1 - */ - unsigned int offset; - unsigned int litLength; </b>/* Literal length */<b> - unsigned int matchLength; </b>/* Match length */<b> - </b>/* 0 when seq not rep and seqDef.offset otherwise<b> - * when litLength == 0 this will be <= 4, otherwise <= 3 like normal - */ - unsigned int rep; + unsigned int offset; </b>/* The offset of the match. (NOT the same as the offset code)<b> + * If offset == 0 and matchLength == 0, this sequence represents the last + * literals in the block of litLength size. + */ + + unsigned int litLength; </b>/* Literal length of the sequence. */<b> + unsigned int matchLength; </b>/* Match length of the sequence. */<b> + + </b>/* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.<b> + * In this case, we will treat the sequence as a marker for a block boundary. + */ + + unsigned int rep; </b>/* Represents which repeat offset is represented by the field 'offset'.<b> + * Ranges from [0, 3]. + * + * Repeat offsets are essentially previous offsets from previous sequences sorted in + * recency order. For more detail, see doc/zstd_compression_format.md + * + * If rep == 0, then 'offset' does not contain a repeat offset. + * If rep > 0: + * If litLength != 0: + * rep == 1 --> offset == repeat_offset_1 + * rep == 2 --> offset == repeat_offset_2 + * rep == 3 --> offset == repeat_offset_3 + * If litLength == 0: + * rep == 1 --> offset == repeat_offset_2 + * rep == 2 --> offset == repeat_offset_3 + * rep == 3 --> offset == repeat_offset_1 - 1 + * + * Note: This field is optional. ZSTD_generateSequences() will calculate the value of + * 'rep', but repeat offsets do not necessarily need to be calculated from an external + * sequence provider's perspective. For example, ZSTD_compressSequences() does not + * use this 'rep' field at all (as of now). + */ } ZSTD_Sequence; </b></pre><BR> <pre><b>typedef struct { @@ -952,6 +990,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); } ZSTD_format_e; </b></pre><BR> <pre><b>typedef enum { + </b>/* Note: this enum controls ZSTD_d_forceIgnoreChecksum */<b> + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1 +} ZSTD_forceIgnoreChecksum_e; +</b></pre><BR> +<pre><b>typedef enum { </b>/* Note: this enum and the behavior it controls are effectively internal<b> * implementation details of the compressor. They are expected to continue * to evolve and should be considered only in the context of extremely @@ -1045,12 +1089,69 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); or an error code (if srcSize is too small) </p></pre><BR> -<pre><b>size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, - size_t outSeqsSize, const void* src, size_t srcSize); -</b><p> Extract sequences from the sequence store +<pre><b>typedef enum { + ZSTD_sf_noBlockDelimiters = 0, </b>/* Representation of ZSTD_Sequence has no block delimiters, sequences only */<b> + ZSTD_sf_explicitBlockDelimiters = 1 </b>/* Representation of ZSTD_Sequence contains explicit block delimiters */<b> +} ZSTD_sequenceFormat_e; +</b></pre><BR> +<pre><b></b><p> Generate sequences using ZSTD_compress2, given a source buffer. + + Each block will end with a dummy sequence + with offset == 0, matchLength == 0, and litLength == length of last literals. + litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) + simply acts as a block delimiter. + zc can be used to insert custom compression params. This function invokes ZSTD_compress2 - @return : number of sequences extracted + + The output of this function can be fed into ZSTD_compressSequences() with CCtx + setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters + @return : number of sequences generated + +</p></pre><BR> + +<pre><b>size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); +</b><p> Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals + by merging them into into the literals of the next sequence. + + As such, the final generated result has no explicit representation of block boundaries, + and the final last literals segment is not represented in the sequences. + + The output of this function can be fed into ZSTD_compressSequences() with CCtx + setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters + @return : number of sequences left after merging + +</p></pre><BR> + +<pre><b>size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); +</b><p> Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. + If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) + The entire source is compressed into a single frame. + + The compression behavior changes based on cctx params. In particular: + If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain + no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on + the block size derived from the cctx, and sequences may be split. This is the default setting. + + If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain + block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + + If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined + behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for + specifics regarding offset/matchlength requirements) then the function will bail out and return an error. + + In addition to the two adjustable experimental params, there are other important cctx params. + - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. + - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. + - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset + is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + + Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. + Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, + and cannot emit an RLE block that disagrees with the repcode history + @return : final compressed size or a ZSTD error. </p></pre><BR> @@ -1141,7 +1242,11 @@ ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); <pre><b>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); typedef void (*ZSTD_freeFunction) (void* opaque, void* address); typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; -static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< this constant defers to stdlib's functions */<b> +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< this constant defers to stdlib's functions */<b> </b><p> These prototypes make it possible to pass your own allocation/free functions. ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones. @@ -1158,6 +1263,12 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef </p></pre><BR> +<pre><b>unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); +</b><p> Provides the dictID of the dictionary loaded into `cdict`. + If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + Non-conformant dictionaries can still be loaded, but as content-only dictionaries. +</p></pre><BR> + <pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); </b><p> @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. `estimatedSrcSize` value is optional, select 0 if not known @@ -1264,8 +1375,10 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); <pre><b>size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); </b><p> Similar to ZSTD_CCtx_setParameter. Set one compression parameter, selected by enum ZSTD_cParameter. - Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams(). - @result : 0, or an error code (which can be tested with ZSTD_isError()). + Parameters must be applied to a ZSTD_CCtx using + ZSTD_CCtx_setParametersUsingCCtxParams(). + @result : a code representing success or failure (which can be tested with + ZSTD_isError()). </p></pre><BR> @@ -1342,6 +1455,13 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); </p></pre><BR> +<pre><b>size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); +</b><p> Get the requested decompression parameter value, selected by enum ZSTD_dParameter, + and store it into int* value. + @return : 0, or an error code (which can be tested with ZSTD_isError()). + +</p></pre><BR> + <pre><b>size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); </b><p> Instruct the decoder context about what kind of data to decode next. This instruction is mandatory to decode data without a fully-formed header, @@ -1365,24 +1485,29 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); redundant functions will be deprecated, and then at some point removed. <BR></pre> -<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre></b>/**! ZSTD_initCStream_srcSize() :<b> - * This function is deprecated, and equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) - * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * - * pledgedSrcSize must be correct. If it is not known at init time, use - * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, - * "0" also disables frame content size field. It may be enabled in the future. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -size_t +<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre></pre></b><BR> +<pre><b>size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, - int compressionLevel, - unsigned long long pledgedSrcSize); -</pre></b><BR> -<a name="Chapter20"></a><h2>! ZSTD_initCStream_usingDict() :</h2><pre> This function is deprecated, and is equivalent to: + int compressionLevel, + unsigned long long pledgedSrcSize); +</b><p> This function is deprecated, and equivalent to: + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + + pledgedSrcSize must be correct. If it is not known at init time, use + ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + "0" also disables frame content size field. It may be enabled in the future. + Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + +</p></pre><BR> + +<pre><b>size_t +ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); +</b><p> This function is deprecated, and is equivalent to: ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); @@ -1393,9 +1518,14 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x -<BR></pre> +</p></pre><BR> -<a name="Chapter21"></a><h2>! ZSTD_initCStream_advanced() :</h2><pre> This function is deprecated, and is approximately equivalent to: +<pre><b>size_t +ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); +</b><p> This function is deprecated, and is approximately equivalent to: ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); // Pseudocode: Set each zstd parameter and leave the rest as-is. for ((param, value) : params) { @@ -1409,18 +1539,24 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x -<BR></pre> +</p></pre><BR> -<a name="Chapter22"></a><h2>! ZSTD_initCStream_usingCDict() :</h2><pre> This function is deprecated, and equivalent to: +<pre><b>size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); +</b><p> This function is deprecated, and equivalent to: ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); ZSTD_CCtx_refCDict(zcs, cdict); note : cdict will just be referenced, and must outlive compression session Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x -<BR></pre> +</p></pre><BR> -<a name="Chapter23"></a><h2>! ZSTD_initCStream_usingCDict_advanced() :</h2><pre> This function is DEPRECATED, and is approximately equivalent to: +<pre><b>size_t +ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); +</b><p> This function is DEPRECATED, and is approximately equivalent to: ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); // Pseudocode: Set each zstd frame parameter and leave the rest as-is. for ((fParam, value) : fParams) { @@ -1434,7 +1570,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, value ZSTD_CONTENTSIZE_UNKNOWN. Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x -<BR></pre> +</p></pre><BR> <pre><b>size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); </b><p> This function is deprecated, and is equivalent to: @@ -1477,42 +1613,44 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, </p></pre><BR> -<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre></b>/**<b> - * This function is deprecated, and is equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); - * - * note: no dictionary will be used if dict == NULL or dictSize < 8 - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); -</pre></b><BR> -<a name="Chapter24"></a><h2>This function is deprecated, and is equivalent to:</h2><pre> +<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre></pre></b><BR> +<pre><b>size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); +</b><p> + ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + + note: no dictionary will be used if dict == NULL or dictSize < 8 + Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + +</p></pre><BR> + +<pre><b>size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); +</b><p> ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); ZSTD_DCtx_refDDict(zds, ddict); note : ddict is referenced, it must outlive decompression session Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x -<BR></pre> +</p></pre><BR> -<a name="Chapter25"></a><h2>This function is deprecated, and is equivalent to:</h2><pre> +<pre><b>size_t ZSTD_resetDStream(ZSTD_DStream* zds); +</b><p> ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); re-use decompression parameters from previous init; saves dictionary loading Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x -<BR></pre> +</p></pre><BR> -<a name="Chapter26"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre> +<a name="Chapter20"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre> This is an advanced API, giving full control over buffer management, for users which need direct control over memory. But it's also a complex one, with several restrictions, documented below. Prefer normal streaming API for an easier experience. <BR></pre> -<a name="Chapter27"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre> +<a name="Chapter21"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre> A ZSTD_CCtx object is required to track streaming operations. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. ZSTD_CCtx object can be re-used multiple times within successive compression operations. @@ -1548,7 +1686,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b> size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b> </pre></b><BR> -<a name="Chapter28"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> +<a name="Chapter22"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> A ZSTD_DCtx object is required to track streaming operations. Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. A ZSTD_DCtx object can be re-used multiple times. @@ -1644,7 +1782,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long <pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; </b></pre><BR> -<a name="Chapter29"></a><h2>Block level API</h2><pre></pre> +<a name="Chapter23"></a><h2>Block level API</h2><pre></pre> <pre><b></b><p> Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. diff --git a/examples/Makefile b/examples/Makefile index 1ae6bce8..f5e3274b 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -8,15 +8,15 @@ # You may select, at your option, one of the above-listed licenses. # ################################################################ -# This Makefile presumes libzstd is installed, using `sudo make install` +LIBDIR =../lib +CPPFLAGS += -I$(LIBDIR) +LIB = $(LIBDIR)/libzstd.a -CPPFLAGS += -I../lib -LIB = ../lib/libzstd.a - -.PHONY: default all clean test +.PHONY: default default: all +.PHONY: all all: simple_compression simple_decompression \ multiple_simple_compression\ dictionary_compression dictionary_decompression \ @@ -24,37 +24,39 @@ all: simple_compression simple_decompression \ multiple_streaming_compression streaming_memory_usage $(LIB) : - $(MAKE) -C ../lib libzstd.a + $(MAKE) -C $(LIBDIR) libzstd.a + +simple_compression.o: common.h +simple_compression : $(LIB) -simple_compression : simple_compression.c common.h $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ +simple_decompression.o: common.h +simple_decompression : $(LIB) -simple_decompression : simple_decompression.c common.h $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ +multiple_simple_compression.o: common.h +multiple_simple_compression : $(LIB) -multiple_simple_compression : multiple_simple_compression.c common.h $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ +dictionary_compression.o: common.h +dictionary_compression : $(LIB) -dictionary_compression : dictionary_compression.c common.h $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ +dictionary_decompression.o: common.h +dictionary_decompression : $(LIB) -dictionary_decompression : dictionary_decompression.c common.h $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ +streaming_compression.o: common.h +streaming_compression : $(LIB) -streaming_compression : streaming_compression.c common.h $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ +multiple_streaming_compression.o: common.h +multiple_streaming_compression : $(LIB) -multiple_streaming_compression : multiple_streaming_compression.c common.h $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ +streaming_decompression.o: common.h +streaming_decompression : $(LIB) -streaming_decompression : streaming_decompression.c common.h $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ +streaming_memory_usage.o: common.h +streaming_memory_usage : $(LIB) -streaming_memory_usage : streaming_memory_usage.c $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ +.PHONY:clean clean: - @rm -f core *.o tmp* result* *.zst \ + @$(RM) core *.o tmp* result* *.zst \ simple_compression simple_decompression \ multiple_simple_compression \ dictionary_compression dictionary_decompression \ @@ -62,6 +64,7 @@ clean: multiple_streaming_compression streaming_memory_usage @echo Cleaning completed +.PHONY:test test: all cp README.md tmp cp Makefile tmp2 diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c index f0f1065b..045437f2 100644 --- a/examples/streaming_compression.c +++ b/examples/streaming_compression.c @@ -39,6 +39,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve */ CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel) ); CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) ); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 4); /* This loop read from the input file, compresses that entire chunk, * and writes all output produced to the output file. diff --git a/examples/streaming_compression_thread_pool.c b/examples/streaming_compression_thread_pool.c new file mode 100644 index 00000000..22c3b2ef --- /dev/null +++ b/examples/streaming_compression_thread_pool.c @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2020, Martin Liska, SUSE, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#include <stdio.h> // printf +#include <stdlib.h> // free +#include <string.h> // memset, strcat, strlen +#include <zstd.h> // presumes zstd library is installed +#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() +#include <pthread.h> + +typedef struct compress_args +{ + const char *fname; + char *outName; + int cLevel; +#if defined(ZSTD_STATIC_LINKING_ONLY) + ZSTD_threadPool *pool; +#endif +} compress_args_t; + +static void *compressFile_orDie(void *data) +{ + compress_args_t *args = (compress_args_t *)data; + fprintf (stderr, "Starting compression of %s with level %d\n", args->fname, args->cLevel); + /* Open the input and output files. */ + FILE* const fin = fopen_orDie(args->fname, "rb"); + FILE* const fout = fopen_orDie(args->outName, "wb"); + /* Create the input and output buffers. + * They may be any size, but we recommend using these functions to size them. + * Performance will only suffer significantly for very tiny buffers. + */ + size_t const buffInSize = ZSTD_CStreamInSize(); + void* const buffIn = malloc_orDie(buffInSize); + size_t const buffOutSize = ZSTD_CStreamOutSize(); + void* const buffOut = malloc_orDie(buffOutSize); + + /* Create the context. */ + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + CHECK(cctx != NULL, "ZSTD_createCCtx() failed!"); + +#if defined(ZSTD_STATIC_LINKING_ONLY) + size_t r = ZSTD_CCtx_refThreadPool(cctx, args->pool); + CHECK(r == 0, "ZSTD_CCtx_refThreadPool failed!"); +#endif + + /* Set any parameters you want. + * Here we set the compression level, and enable the checksum. + */ + CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, args->cLevel) ); + CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) ); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 16); + + /* This loop read from the input file, compresses that entire chunk, + * and writes all output produced to the output file. + */ + size_t const toRead = buffInSize; + for (;;) { + size_t read = fread_orDie(buffIn, toRead, fin); + /* Select the flush mode. + * If the read may not be finished (read == toRead) we use + * ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end. + * Zstd optimizes the case where the first flush mode is ZSTD_e_end, + * since it knows it is compressing the entire source in one pass. + */ + int const lastChunk = (read < toRead); + ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue; + /* Set the input buffer to what we just read. + * We compress until the input buffer is empty, each time flushing the + * output. + */ + ZSTD_inBuffer input = { buffIn, read, 0 }; + int finished; + do { + /* Compress into the output buffer and write all of the output to + * the file so we can reuse the buffer next iteration. + */ + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode); + CHECK_ZSTD(remaining); + fwrite_orDie(buffOut, output.pos, fout); + /* If we're on the last chunk we're finished when zstd returns 0, + * which means its consumed all the input AND finished the frame. + * Otherwise, we're finished when we've consumed all the input. + */ + finished = lastChunk ? (remaining == 0) : (input.pos == input.size); + } while (!finished); + CHECK(input.pos == input.size, + "Impossible: zstd only returns 0 when the input is completely consumed!"); + + if (lastChunk) { + break; + } + } + + fprintf (stderr, "Finishing compression of %s\n", args->outName); + + ZSTD_freeCCtx(cctx); + fclose_orDie(fout); + fclose_orDie(fin); + free(buffIn); + free(buffOut); + free(args->outName); + + return NULL; +} + + +static char* createOutFilename_orDie(const char* filename) +{ + size_t const inL = strlen(filename); + size_t const outL = inL + 5; + void* const outSpace = malloc_orDie(outL); + memset(outSpace, 0, outL); + strcat(outSpace, filename); + strcat(outSpace, ".zst"); + return (char*)outSpace; +} + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc<=3) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s POOL_SIZE LEVEL FILES\n", exeName); + return 1; + } + + int pool_size = atoi (argv[1]); + CHECK(pool_size != 0, "can't parse POOL_SIZE!"); + + int level = atoi (argv[2]); + CHECK(level != 0, "can't parse LEVEL!"); + + argc -= 3; + argv += 3; + +#if defined(ZSTD_STATIC_LINKING_ONLY) + ZSTD_threadPool *pool = ZSTD_createThreadPool (pool_size); + CHECK(pool != NULL, "ZSTD_createThreadPool() failed!"); + fprintf (stderr, "Using shared thread pool of size %d\n", pool_size); +#else + fprintf (stderr, "All threads use its own thread pool\n"); +#endif + + pthread_t *threads = malloc_orDie(argc * sizeof(pthread_t)); + compress_args_t *args = malloc_orDie(argc * sizeof(compress_args_t)); + + for (unsigned i = 0; i < argc; i++) + { + args[i].fname = argv[i]; + args[i].outName = createOutFilename_orDie(args[i].fname); + args[i].cLevel = level; +#if defined(ZSTD_STATIC_LINKING_ONLY) + args[i].pool = pool; +#endif + + pthread_create (&threads[i], NULL, compressFile_orDie, &args[i]); + } + + for (unsigned i = 0; i < argc; i++) + pthread_join (threads[i], NULL); + +#if defined(ZSTD_STATIC_LINKING_ONLY) + ZSTD_freeThreadPool (pool); +#endif + + return 0; +} diff --git a/lib/Makefile b/lib/Makefile index 7c6dff02..869d7663 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -8,15 +8,16 @@ # You may select, at your option, one of the above-listed licenses. # ################################################################ -Q = $(if $(filter 1,$(V) $(VERBOSE)),,@) +.PHONY: default +default: lib-release -# When cross-compiling from linux to windows, you might -# need to specify this as "Windows." Fedora build fails -# without it. -# -# Note: mingw-w64 build from linux to windows does not -# fail on other tested distros (ubuntu, debian) even -# without manually specifying the TARGET_SYSTEM. +# define silent mode as default (verbose mode with V=1 or VERBOSE=1) +$(V)$(VERBOSE).SILENT: + +# When cross-compiling from linux to windows, +# one might need to specify TARGET_SYSTEM as "Windows." +# Building from Fedora fails without it. +# (but Ubuntu and Debian don't need to set anything) TARGET_SYSTEM ?= $(OS) # Version numbers @@ -31,24 +32,49 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT)) VERSION?= $(LIBVER) CCVER := $(shell $(CC) --version) -CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_ +# ZSTD_LIB_MINIFY is a helper variable that +# configures a bunch of other variables to space-optimized defaults. +ZSTD_LIB_MINIFY ?= 0 +ifneq ($(ZSTD_LIB_MINIFY), 0) + HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0) + ZSTD_LEGACY_SUPPORT ?= 0 + ZSTD_LIB_DEPRECATED ?= 0 + HUF_FORCE_DECOMPRESS_X1 ?= 1 + ZSTD_FORCE_DECOMPRESS_SHORT ?= 1 + ZSTD_NO_INLINE ?= 1 + ZSTD_STRIP_ERROR_STRINGS ?= 1 +ifneq ($(HAVE_CC_OZ), 0) + # Some compilers (clang) support an even more space-optimized setting. + CFLAGS += -Oz +else + CFLAGS += -Os +endif + CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \ + -DDYNAMIC_BMI2=0 -DNDEBUG +else + CFLAGS += -O3 +endif + +DEBUGLEVEL ?= 0 +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL) ifeq ($(TARGET_SYSTEM),Windows_NT) # MinGW assumed -CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting + CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef -Wpointer-arith \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wredundant-decls -Wmissing-prototypes -Wc++-compat -CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) GREP_OPTIONS ?= ifeq ($HAVE_COLORNEVER, 1) -GREP_OPTIONS += --color=never + GREP_OPTIONS += --color=never endif GREP = grep $(GREP_OPTIONS) +SED_ERE_OPT ?= -E ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c)) ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c)) @@ -58,30 +84,7 @@ ZDEPR_FILES := $(sort $(wildcard deprecated/*.c)) ZSTD_FILES := $(ZSTDCOMMON_FILES) ifeq ($(findstring GCC,$(CCVER)),GCC) -decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize -endif - -# This is a helper variable that configures a bunch of other variables to new, -# space-optimized defaults. -ZSTD_LIB_MINIFY ?= 0 -ifneq ($(ZSTD_LIB_MINIFY), 0) - HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0) - ZSTD_LEGACY_SUPPORT ?= 0 - ZSTD_LIB_DEPRECATED ?= 0 - HUF_FORCE_DECOMPRESS_X1 ?= 1 - ZSTD_FORCE_DECOMPRESS_SHORT ?= 1 - ZSTD_NO_INLINE ?= 1 - ZSTD_STRIP_ERROR_STRINGS ?= 1 - ifneq ($(HAVE_CC_OZ), 0) - # Some compilers (clang) support an even more space-optimized setting. - CFLAGS += -Oz - else - CFLAGS += -Os - endif - CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \ - -DDYNAMIC_BMI2=0 -DNDEBUG -else - CFLAGS += -O3 +decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize endif # Modules @@ -103,116 +106,187 @@ ZSTD_NO_INLINE ?= 0 ZSTD_STRIP_ERROR_STRINGS ?= 0 ifeq ($(ZSTD_LIB_COMPRESSION), 0) - ZSTD_LIB_DICTBUILDER = 0 - ZSTD_LIB_DEPRECATED = 0 + ZSTD_LIB_DICTBUILDER = 0 + ZSTD_LIB_DEPRECATED = 0 endif ifeq ($(ZSTD_LIB_DECOMPRESSION), 0) - ZSTD_LEGACY_SUPPORT = 0 - ZSTD_LIB_DEPRECATED = 0 + ZSTD_LEGACY_SUPPORT = 0 + ZSTD_LIB_DEPRECATED = 0 endif ifneq ($(ZSTD_LIB_COMPRESSION), 0) - ZSTD_FILES += $(ZSTDCOMP_FILES) + ZSTD_FILES += $(ZSTDCOMP_FILES) endif ifneq ($(ZSTD_LIB_DECOMPRESSION), 0) - ZSTD_FILES += $(ZSTDDECOMP_FILES) + ZSTD_FILES += $(ZSTDDECOMP_FILES) endif ifneq ($(ZSTD_LIB_DEPRECATED), 0) - ZSTD_FILES += $(ZDEPR_FILES) + ZSTD_FILES += $(ZDEPR_FILES) endif ifneq ($(ZSTD_LIB_DICTBUILDER), 0) - ZSTD_FILES += $(ZDICT_FILES) + ZSTD_FILES += $(ZDICT_FILES) endif ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0) - CFLAGS += -DHUF_FORCE_DECOMPRESS_X1 + CFLAGS += -DHUF_FORCE_DECOMPRESS_X1 endif ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0) - CFLAGS += -DHUF_FORCE_DECOMPRESS_X2 + CFLAGS += -DHUF_FORCE_DECOMPRESS_X2 endif ifneq ($(ZSTD_FORCE_DECOMPRESS_SHORT), 0) - CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT + CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT endif ifneq ($(ZSTD_FORCE_DECOMPRESS_LONG), 0) - CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG + CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG endif ifneq ($(ZSTD_NO_INLINE), 0) - CFLAGS += -DZSTD_NO_INLINE + CFLAGS += -DZSTD_NO_INLINE endif ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0) - CFLAGS += -DZSTD_STRIP_ERROR_STRINGS + CFLAGS += -DZSTD_STRIP_ERROR_STRINGS endif ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0) - CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API + CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API endif ifneq ($(ZSTD_LEGACY_SUPPORT), 0) ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) - ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') + ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') endif endif CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) -ZSTD_OBJ := $(patsubst %.c,%.o,$(ZSTD_FILES)) +ZSTD_LOCAL_SRC := $(notdir $(ZSTD_FILES)) +ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_SRC:.c=.o) + +ZSTD_SUBDIR := common compress decompress dictBuilder legacy deprecated +vpath %.c $(ZSTD_SUBDIR) + +UNAME := $(shell uname) + +ifndef BUILD_DIR +ifeq ($(UNAME), Darwin) + HASH ?= md5 +else ifeq ($(UNAME), FreeBSD) + HASH ?= gmd5sum +else ifeq ($(UNAME), OpenBSD) + HASH ?= md5 +endif +HASH ?= md5sum + +HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " " ) +HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0) +ifeq ($(HAVE_HASH),0) + $(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags) + BUILD_DIR := obj/generic_noconf +endif +endif # BUILD_DIR + # macOS linker doesn't support -soname, and use different extension # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html -ifeq ($(shell uname), Darwin) - SHARED_EXT = dylib - SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) - SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) - SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) +ifeq ($(UNAME), Darwin) + SHARED_EXT = dylib + SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) + SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) + SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) else - SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR) - SHARED_EXT = so - SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR) - SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) + SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR) + SHARED_EXT = so + SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR) + SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) endif +SET_CACHE_DIRECTORY = \ + $(MAKE) --no-print-directory $@ \ + BUILD_DIR=obj/$(HASH_DIR) \ + CPPFLAGS="$(CPPFLAGS)" \ + CFLAGS="$(CFLAGS)" \ + LDFLAGS="$(LDFLAGS)" -.PHONY: default lib-all all clean install uninstall -default: lib-release +.PHONY: lib-all all clean install uninstall # alias lib-all: all all: lib -libzstd.a: ARFLAGS = rcs -libzstd.a: $(ZSTD_OBJ) +.PHONY: libzstd.a # must be run every time + +ifndef BUILD_DIR +# determine BUILD_DIR from compilation flags + +libzstd.a: + $(SET_CACHE_DIRECTORY) + +else +# BUILD_DIR is defined + +ZSTD_STATLIB_DIR := $(BUILD_DIR)/static +ZSTD_STATLIB := $(ZSTD_STATLIB_DIR)/libzstd.a +ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/,$(ZSTD_LOCAL_OBJ)) +$(ZSTD_STATLIB): ARFLAGS = rcs +$(ZSTD_STATLIB): | $(ZSTD_STATLIB_DIR) +$(ZSTD_STATLIB): $(ZSTD_STATLIB_OBJ) @echo compiling static library - $(Q)$(AR) $(ARFLAGS) $@ $^ + $(AR) $(ARFLAGS) $@ $^ + +libzstd.a: $(ZSTD_STATLIB) + cp -f $< $@ + +endif ifneq (,$(filter Windows%,$(TARGET_SYSTEM))) -LIBZSTD = dll\libzstd.dll +LIBZSTD = dll/libzstd.dll $(LIBZSTD): $(ZSTD_FILES) @echo compiling dynamic library $(LIBVER) - $(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.dll.a -shared $^ -o $@ + $(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll/libzstd.dll.a -shared $^ -o $@ -else +else # not Windows LIBZSTD = libzstd.$(SHARED_EXT_VER) -$(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden -$(LIBZSTD): $(ZSTD_FILES) +.PHONY: $(LIBZSTD) # must be run every time +$(LIBZSTD): CFLAGS += -fPIC +$(LIBZSTD): LDFLAGS += -shared -fvisibility=hidden + +ifndef BUILD_DIR +# determine BUILD_DIR from compilation flags + +$(LIBZSTD): + $(SET_CACHE_DIRECTORY) + +else +# BUILD_DIR is defined + +ZSTD_DYNLIB_DIR := $(BUILD_DIR)/dynamic +ZSTD_DYNLIB := $(ZSTD_DYNLIB_DIR)/$(LIBZSTD) +ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/,$(ZSTD_LOCAL_OBJ)) + +$(ZSTD_DYNLIB): | $(ZSTD_DYNLIB_DIR) +$(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ) @echo compiling dynamic library $(LIBVER) - $(Q)$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ + $(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ @echo creating versioned links - $(Q)ln -sf $@ libzstd.$(SHARED_EXT_MAJOR) - $(Q)ln -sf $@ libzstd.$(SHARED_EXT) + ln -sf $@ libzstd.$(SHARED_EXT_MAJOR) + ln -sf $@ libzstd.$(SHARED_EXT) -endif +$(LIBZSTD): $(ZSTD_DYNLIB) + cp -f $< $@ + +endif # ifndef BUILD_DIR +endif # if windows .PHONY: libzstd libzstd : $(LIBZSTD) @@ -220,18 +294,42 @@ libzstd : $(LIBZSTD) .PHONY: lib lib : libzstd.a libzstd -.PHONY: lib-mt + +# note : do not define lib-mt or lib-release as .PHONY +# make does not consider implicit pattern rule for .PHONY target + %-mt : CPPFLAGS += -DZSTD_MULTITHREAD %-mt : LDFLAGS += -pthread %-mt : % @echo multi-threading build completed -.PHONY: lib-release %-release : DEBUGFLAGS := %-release : % @echo release build completed +# Generate .h dependencies automatically + +DEPFLAGS = -MT $@ -MMD -MP -MF + +$(ZSTD_DYNLIB_DIR)/%.o : %.c $(ZSTD_DYNLIB_DIR)/%.d | $(ZSTD_DYNLIB_DIR) + @echo CC $@ + $(COMPILE.c) $(DEPFLAGS) $(ZSTD_DYNLIB_DIR)/$*.d $(OUTPUT_OPTION) $< + +$(ZSTD_STATLIB_DIR)/%.o : %.c $(ZSTD_STATLIB_DIR)/%.d | $(ZSTD_STATLIB_DIR) + @echo CC $@ + $(COMPILE.c) $(DEPFLAGS) $(ZSTD_STATLIB_DIR)/$*.d $(OUTPUT_OPTION) $< + +MKDIR ?= mkdir +$(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR): + $(MKDIR) -p $@ + +DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(ZSTD_STATLIB_OBJ:.o=.d) +$(DEPFILES): + +include $(wildcard $(DEPFILES)) + + # Special case : building library in single-thread mode _and_ without zstdmt_compress.c ZSTDMT_FILES = compress/zstdmt_compress.c ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(ZSTD_FILES)) @@ -239,22 +337,24 @@ libzstd-nomt: LDFLAGS += -shared -fPIC -fvisibility=hidden libzstd-nomt: $(ZSTD_NOMT_FILES) @echo compiling single-thread dynamic library $(LIBVER) @echo files : $(ZSTD_NOMT_FILES) - $(Q)$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ + $(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ clean: - $(Q)$(RM) -r *.dSYM # macOS-specific - $(Q)$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc - $(Q)$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt* - $(Q)$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o + $(RM) -r *.dSYM # macOS-specific + $(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc + $(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt* + $(RM) -r obj/* @echo Cleaning library completed #----------------------------------------------------------------------------- # make install is validated only for below listed environments #----------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) +ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) all: libzstd.pc +HAS_EXPLICIT_EXEC_PREFIX := $(if $(or $(EXEC_PREFIX),$(exec_prefix)),1,) + DESTDIR ?= # directory variables : GNU conventions prefer lowercase # see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html @@ -268,35 +368,28 @@ LIBDIR ?= $(libdir) includedir ?= $(PREFIX)/include INCLUDEDIR ?= $(includedir) -PCLIBDIR ?= $(shell echo "$(LIBDIR)" | sed -n -E -e "s@^$(EXEC_PREFIX)(/|$$)@@p") -PCINCDIR ?= $(shell echo "$(INCLUDEDIR)" | sed -n -E -e "s@^$(PREFIX)(/|$$)@@p") +PCINCDIR := $(patsubst $(PREFIX)%,%,$(INCLUDEDIR)) +PCLIBDIR := $(patsubst $(EXEC_PREFIX)%,%,$(LIBDIR)) -ifeq (,$(PCLIBDIR)) -# Additional prefix check is required, since the empty string is technically a -# valid PCLIBDIR -ifeq (,$(shell echo "$(LIBDIR)" | sed -n -E -e "\\@^$(EXEC_PREFIX)(/|$$)@ p")) -$(error configured libdir ($(LIBDIR)) is outside of prefix ($(PREFIX)), can't generate pkg-config file) -endif -endif +# If we successfully stripped off a prefix, we'll add a reference to the +# relevant pc variable. +PCINCPREFIX := $(if $(findstring $(INCLUDEDIR),$(PCINCDIR)),,$${prefix}) +PCLIBPREFIX := $(if $(findstring $(LIBDIR),$(PCLIBDIR)),,$${exec_prefix}) -ifeq (,$(PCINCDIR)) -# Additional prefix check is required, since the empty string is technically a -# valid PCINCDIR -ifeq (,$(shell echo "$(INCLUDEDIR)" | sed -n -E -e "\\@^$(PREFIX)(/|$$)@ p")) -$(error configured includedir ($(INCLUDEDIR)) is outside of exec_prefix ($(EXEC_PREFIX)), can't generate pkg-config file) -endif -endif +# If no explicit EXEC_PREFIX was set by the caller, write it out as a reference +# to PREFIX, rather than as a resolved value. +PCEXEC_PREFIX := $(if $(HAS_EXPLICIT_EXEC_PREFIX),$(EXEC_PREFIX),$${prefix}) -ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly)) -PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig +ifneq (,$(filter $(UNAME),FreeBSD NetBSD DragonFly)) + PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig else -PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig + PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig endif -ifneq (,$(filter $(shell uname),SunOS)) -INSTALL ?= ginstall +ifneq (,$(filter $(UNAME),SunOS)) + INSTALL ?= ginstall else -INSTALL ?= install + INSTALL ?= install endif INSTALL_PROGRAM ?= $(INSTALL) @@ -306,9 +399,11 @@ INSTALL_DATA ?= $(INSTALL) -m 644 libzstd.pc: libzstd.pc: libzstd.pc.in @echo creating pkgconfig - $(Q)@sed -E -e 's|@PREFIX@|$(PREFIX)|' \ - -e 's|@LIBDIR@|$(PCLIBDIR)|' \ - -e 's|@INCLUDEDIR@|$(PCINCDIR)|' \ + @sed $(SED_ERE_OPT) \ + -e 's|@PREFIX@|$(PREFIX)|' \ + -e 's|@EXEC_PREFIX@|$(PCEXEC_PREFIX)|' \ + -e 's|@INCLUDEDIR@|$(PCINCPREFIX)$(PCINCDIR)|' \ + -e 's|@LIBDIR@|$(PCLIBPREFIX)$(PCLIBDIR)|' \ -e 's|@VERSION@|$(VERSION)|' \ $< >$@ @@ -316,39 +411,41 @@ install: install-pc install-static install-shared install-includes @echo zstd static and shared library installed install-pc: libzstd.pc - $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ - $(Q)$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/ + [ -e $(DESTDIR)$(PKGCONFIGDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ + $(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/ -install-static: libzstd.a +install-static: + # only generate libzstd.a if it's not already present + [ -e libzstd.a ] || $(MAKE) libzstd.a-release + [ -e $(DESTDIR)$(LIBDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/ @echo Installing static library - $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/ - $(Q)$(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR) + $(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR) -install-shared: libzstd +install-shared: + # only generate libzstd.so if it's not already present + [ -e $(LIBZSTD) ] || $(MAKE) libzstd-release + [ -e $(DESTDIR)$(LIBDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/ @echo Installing shared library - $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/ - $(Q)$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR) - $(Q)ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) - $(Q)ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) + $(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR) + ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) + ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) install-includes: + [ -e $(DESTDIR)$(INCLUDEDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/ @echo Installing includes - $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/ - $(Q)$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR) - $(Q)$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR) - $(Q)$(INSTALL_DATA) deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR) # prototypes generate deprecation warnings - $(Q)$(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR) + $(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR) + $(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR) + $(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR) uninstall: - $(Q)$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a - $(Q)$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) - $(Q)$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) - $(Q)$(RM) $(DESTDIR)$(LIBDIR)/$(LIBZSTD) - $(Q)$(RM) $(DESTDIR)$(PKGCONFIGDIR)/libzstd.pc - $(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h - $(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h - $(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions - $(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h + $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a + $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) + $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) + $(RM) $(DESTDIR)$(LIBDIR)/$(LIBZSTD) + $(RM) $(DESTDIR)$(PKGCONFIGDIR)/libzstd.pc + $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h + $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h + $(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd libraries successfully uninstalled endif diff --git a/lib/README.md b/lib/README.md index 6ccffb13..db9170a7 100644 --- a/lib/README.md +++ b/lib/README.md @@ -143,6 +143,14 @@ The file structure is designed to make this selection manually achievable for an Setting this macro will either force to generate the BMI2 dispatcher (1) or prevent it (0). It overrides automatic detection. +- The build macro `ZSTD_NO_UNUSED_FUNCTIONS` can be defined to hide the definitions of functions + that zstd does not use. Not all unused functions are hidden, but they can be if needed. + Currently, this macro will hide function definitions in FSE and HUF that use an excessive + amount of stack space. + +- The build macro `ZSTD_NO_INTRINSICS` can be defined to disable all explicit intrinsics. + Compiler builtins are still used. + #### Windows : using MinGW+MSYS to create DLL @@ -160,6 +168,26 @@ file it should be linked with `dll\libzstd.dll`. For example: The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`. +#### Advanced Build options + +The build system requires a hash function in order to +separate object files created with different compilation flags. +By default, it tries to use `md5sum` or equivalent. +The hash function can be manually switched by setting the `HASH` variable. +For example : `make HASH=xxhsum` +The hash function needs to generate at least 64-bit using hexadecimal format. +When no hash function is found, +the Makefile just generates all object files into the same default directory, +irrespective of compilation flags. +This functionality only matters if `libzstd` is compiled multiple times +with different build flags. + +The build directory, where object files are stored +can also be manually controlled using variable `BUILD_DIR`, +for example `make BUILD_DIR=objectDir/v1`. +In which case, the hash function doesn't matter. + + #### Deprecated API Obsolete API on their way out are stored in directory `lib/deprecated`. diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h index 37b99c01..d9a27301 100644 --- a/lib/common/bitstream.h +++ b/lib/common/bitstream.h @@ -17,7 +17,6 @@ #if defined (__cplusplus) extern "C" { #endif - /* * This API consists of small unitary functions, which must be inlined for best performance. * Since link-time-optimization is not available for all compilers, @@ -36,10 +35,12 @@ extern "C" { /*========================================= * Target specific =========================================*/ -#if defined(__BMI__) && defined(__GNUC__) -# include <immintrin.h> /* support for bextr (experimental) */ -#elif defined(__ICCARM__) -# include <intrinsics.h> +#ifndef ZSTD_NO_INTRINSICS +# if defined(__BMI__) && defined(__GNUC__) +# include <immintrin.h> /* support for bextr (experimental) */ +# elif defined(__ICCARM__) +# include <intrinsics.h> +# endif #endif #define STREAM_ACCUMULATOR_MIN_32 25 @@ -141,8 +142,12 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val) assert(val != 0); { # if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - return _BitScanReverse ( &r, val ) ? (unsigned)r : 0; +# if STATIC_BMI2 == 1 + return _lzcnt_u32(val) ^ 31; +# else + unsigned long r = 0; + return _BitScanReverse(&r, val) ? (unsigned)r : 0; +# endif # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ return __builtin_clz (val) ^ 31; # elif defined(__ICCARM__) /* IAR Intrinsic */ @@ -198,7 +203,7 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) { - MEM_STATIC_ASSERT(BIT_MASK_SIZE == 32); + DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); assert(nbBits < BIT_MASK_SIZE); assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; @@ -271,7 +276,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) */ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) { - if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } bitD->start = (const char*)srcBuffer; bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); @@ -317,12 +322,12 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si return srcSize; } -MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start) { return bitContainer >> start; } -MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) { U32 const regMask = sizeof(bitContainer)*8 - 1; /* if start > regMask, bitstream is corrupted, and result is undefined */ @@ -330,10 +335,14 @@ MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 co return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; } -MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) { +#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 + return _bzhi_u64(bitContainer, nbBits); +#else assert(nbBits < BIT_MASK_SIZE); return bitContainer & BIT_mask[nbBits]; +#endif } /*! BIT_lookBits() : @@ -342,7 +351,7 @@ MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) * On 32-bits, maxNbBits==24. * On 64-bits, maxNbBits==56. * @return : value extracted */ -MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) { /* arbitrate between double-shift and shift+mask */ #if 1 @@ -365,7 +374,7 @@ MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); } -MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; } @@ -374,7 +383,7 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) * Read (consume) next n bits from local register and update. * Pay attention to not read more than nbBits contained into local register. * @return : extracted value. */ -MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) { size_t const value = BIT_lookBits(bitD, nbBits); BIT_skipBits(bitD, nbBits); diff --git a/lib/common/compiler.h b/lib/common/compiler.h index 95e94835..3e454f38 100644 --- a/lib/common/compiler.h +++ b/lib/common/compiler.h @@ -39,6 +39,17 @@ #endif /** + On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). + This explictly marks such functions as __cdecl so that the code will still compile + if a CC other than __cdecl has been made the default. +*/ +#if defined(_MSC_VER) +# define WIN_CDECL __cdecl +#else +# define WIN_CDECL +#endif + +/** * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant * parameters. They must be inlined for the compiler to eliminate the constant * branches. @@ -114,12 +125,12 @@ # include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ # define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) # define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) -# elif defined(__aarch64__) -# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) -# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) # elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) # define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) # define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) +# elif defined(__aarch64__) +# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) +# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) # else # define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ # define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ @@ -172,4 +183,106 @@ # pragma warning(disable : 4324) /* disable: C4324: padded structure */ #endif +/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/ +#ifndef STATIC_BMI2 +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) +# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 +# define STATIC_BMI2 1 +# endif +# endif +#endif + +#ifndef STATIC_BMI2 + #define STATIC_BMI2 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* detects whether we are being compiled under msan */ +#ifndef ZSTD_MEMORY_SANITIZER +# if __has_feature(memory_sanitizer) +# define ZSTD_MEMORY_SANITIZER 1 +# else +# define ZSTD_MEMORY_SANITIZER 0 +# endif +#endif + +#if ZSTD_MEMORY_SANITIZER +/* Not all platforms that support msan provide sanitizers/msan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include <stddef.h> /* size_t */ +#define ZSTD_DEPS_NEED_STDINT +#include "zstd_deps.h" /* intptr_t */ + +/* Make memory region fully initialized (without changing its contents). */ +void __msan_unpoison(const volatile void *a, size_t size); + +/* Make memory region fully uninitialized (without changing its contents). + This is a legacy interface that does not update origin information. Use + __msan_allocated_memory() instead. */ +void __msan_poison(const volatile void *a, size_t size); + +/* Returns the offset of the first (at least partially) poisoned byte in the + memory range, or -1 if the whole range is good. */ +intptr_t __msan_test_shadow(const volatile void *x, size_t size); +#endif + +/* detects whether we are being compiled under asan */ +#ifndef ZSTD_ADDRESS_SANITIZER +# if __has_feature(address_sanitizer) +# define ZSTD_ADDRESS_SANITIZER 1 +# elif defined(__SANITIZE_ADDRESS__) +# define ZSTD_ADDRESS_SANITIZER 1 +# else +# define ZSTD_ADDRESS_SANITIZER 0 +# endif +#endif + +#if ZSTD_ADDRESS_SANITIZER +/* Not all platforms that support asan provide sanitizers/asan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include <stddef.h> /* size_t */ + +/** + * Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable. + * + * This memory must be previously allocated by your program. Instrumented + * code is forbidden from accessing addresses in this region until it is + * unpoisoned. This function is not guaranteed to poison the entire region - + * it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan + * alignment restrictions. + * + * \note This function is not thread-safe because no two threads can poison or + * unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_poison_memory_region(void const volatile *addr, size_t size); + +/** + * Marks a memory region (<c>[addr, addr+size)</c>) as addressable. + * + * This memory must be previously allocated by your program. Accessing + * addresses in this region is allowed until this region is poisoned again. + * This function could unpoison a super-region of <c>[addr, addr+size)</c> due + * to ASan alignment restrictions. + * + * \note This function is not thread-safe because no two threads can + * poison or unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); +#endif + #endif /* ZSTD_COMPILER_H */ diff --git a/lib/common/cpu.h b/lib/common/cpu.h index 6e8a974f..cb210593 100644 --- a/lib/common/cpu.h +++ b/lib/common/cpu.h @@ -16,8 +16,6 @@ * https://github.com/facebook/folly/blob/master/folly/CpuId.h */ -#include <string.h> - #include "mem.h" #ifdef _MSC_VER diff --git a/lib/common/debug.h b/lib/common/debug.h index ac622488..8b573436 100644 --- a/lib/common/debug.h +++ b/lib/common/debug.h @@ -51,15 +51,6 @@ extern "C" { #endif -/* DEBUGFILE can be defined externally, - * typically through compiler command line. - * note : currently useless. - * Value must be stderr or stdout */ -#ifndef DEBUGFILE -# define DEBUGFILE stderr -#endif - - /* recommended values for DEBUGLEVEL : * 0 : release mode, no debug, all run-time checks disabled * 1 : enables assert() only, no display @@ -76,7 +67,8 @@ extern "C" { */ #if (DEBUGLEVEL>=1) -# include <assert.h> +# define ZSTD_DEPS_NEED_ASSERT +# include "zstd_deps.h" #else # ifndef assert /* assert may be already defined, due to prior #include <assert.h> */ # define assert(condition) ((void)0) /* disable assert (default) */ @@ -84,7 +76,8 @@ extern "C" { #endif #if (DEBUGLEVEL>=2) -# include <stdio.h> +# define ZSTD_DEPS_NEED_IO +# include "zstd_deps.h" extern int g_debuglevel; /* the variable is only declared, it actually lives in debug.c, and is shared by the whole process. @@ -92,14 +85,14 @@ extern int g_debuglevel; /* the variable is only declared, It's useful when enabling very verbose levels on selective conditions (such as position in src) */ -# define RAWLOG(l, ...) { \ - if (l<=g_debuglevel) { \ - fprintf(stderr, __VA_ARGS__); \ +# define RAWLOG(l, ...) { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__VA_ARGS__); \ } } -# define DEBUGLOG(l, ...) { \ - if (l<=g_debuglevel) { \ - fprintf(stderr, __FILE__ ": " __VA_ARGS__); \ - fprintf(stderr, " \n"); \ +# define DEBUGLOG(l, ...) { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \ + ZSTD_DEBUG_PRINT(" \n"); \ } } #else # define RAWLOG(l, ...) {} /* disabled */ diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c index 9d3e4e8e..f9fcb1ac 100644 --- a/lib/common/entropy_common.c +++ b/lib/common/entropy_common.c @@ -38,8 +38,31 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } /*-************************************************************** * FSE NCount encoding-decoding ****************************************************************/ -size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, - const void* headerBuffer, size_t hbSize) +static U32 FSE_ctz(U32 val) +{ + assert(val != 0); + { +# if defined(_MSC_VER) /* Visual */ + unsigned long r=0; + return _BitScanForward(&r, val) ? (unsigned)r : 0; +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ + return __builtin_ctz(val); +# elif defined(__ICCARM__) /* IAR Intrinsic */ + return __CTZ(val); +# else /* Software version */ + U32 count = 0; + while ((val & 1) == 0) { + val >>= 1; + ++count; + } + return count; +# endif + } +} + +FORCE_INLINE_TEMPLATE +size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) { const BYTE* const istart = (const BYTE*) headerBuffer; const BYTE* const iend = istart + hbSize; @@ -50,23 +73,23 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t U32 bitStream; int bitCount; unsigned charnum = 0; + unsigned const maxSV1 = *maxSVPtr + 1; int previous0 = 0; - if (hbSize < 4) { - /* This function only works when hbSize >= 4 */ - char buffer[4]; - memset(buffer, 0, sizeof(buffer)); - memcpy(buffer, headerBuffer, hbSize); + if (hbSize < 8) { + /* This function only works when hbSize >= 8 */ + char buffer[8] = {0}; + ZSTD_memcpy(buffer, headerBuffer, hbSize); { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr, buffer, sizeof(buffer)); if (FSE_isError(countSize)) return countSize; if (countSize > hbSize) return ERROR(corruption_detected); return countSize; } } - assert(hbSize >= 4); + assert(hbSize >= 8); /* init */ - memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */ + ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */ bitStream = MEM_readLE32(ip); nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); @@ -77,36 +100,58 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t threshold = 1<<nbBits; nbBits++; - while ((remaining>1) & (charnum<=*maxSVPtr)) { + for (;;) { if (previous0) { - unsigned n0 = charnum; - while ((bitStream & 0xFFFF) == 0xFFFF) { - n0 += 24; - if (ip < iend-5) { - ip += 2; - bitStream = MEM_readLE32(ip) >> bitCount; + /* Count the number of repeats. Each time the + * 2-bit repeat code is 0b11 there is another + * repeat. + * Avoid UB by setting the high bit to 1. + */ + int repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + while (repeats >= 12) { + charnum += 3 * 12; + if (LIKELY(ip <= iend-7)) { + ip += 3; } else { - bitStream >>= 16; - bitCount += 16; - } } - while ((bitStream & 3) == 3) { - n0 += 3; - bitStream >>= 2; - bitCount += 2; + bitCount -= (int)(8 * (iend - 7 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; } - n0 += bitStream & 3; + charnum += 3 * repeats; + bitStream >>= 2 * repeats; + bitCount += 2 * repeats; + + /* Add the final repeat which isn't 0b11. */ + assert((bitStream & 3) < 3); + charnum += bitStream & 3; bitCount += 2; - if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); - while (charnum < n0) normalizedCounter[charnum++] = 0; - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + + /* This is an error, but break and return an error + * at the end, because returning out of a loop makes + * it harder for the compiler to optimize. + */ + if (charnum >= maxSV1) break; + + /* We don't need to set the normalized count to 0 + * because we already memset the whole buffer to 0. + */ + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { assert((bitCount >> 3) <= 3); /* For first condition to work */ ip += bitCount>>3; bitCount &= 7; - bitStream = MEM_readLE32(ip) >> bitCount; } else { - bitStream >>= 2; - } } - { int const max = (2*threshold-1) - remaining; + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } + { + int const max = (2*threshold-1) - remaining; int count; if ((bitStream & (threshold-1)) < (U32)max) { @@ -119,24 +164,43 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t } count--; /* extra accuracy */ - remaining -= count < 0 ? -count : count; /* -1 means +1 */ + /* When it matters (small blocks), this is a + * predictable branch, because we don't use -1. + */ + if (count >= 0) { + remaining -= count; + } else { + assert(count == -1); + remaining += count; + } normalizedCounter[charnum++] = (short)count; previous0 = !count; - while (remaining < threshold) { - nbBits--; - threshold >>= 1; + + assert(threshold > 1); + if (remaining < threshold) { + /* This branch can be folded into the + * threshold update condition because we + * know that threshold > 1. + */ + if (remaining <= 1) break; + nbBits = BIT_highbit32(remaining) + 1; + threshold = 1 << (nbBits - 1); } + if (charnum >= maxSV1) break; - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { ip += bitCount>>3; bitCount &= 7; } else { bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; ip = iend - 4; } - bitStream = MEM_readLE32(ip) >> (bitCount & 31); - } } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ + bitStream = MEM_readLE32(ip) >> bitCount; + } } if (remaining != 1) return ERROR(corruption_detected); + /* Only possible when there are too many zeros. */ + if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall); if (bitCount > 32) return ERROR(corruption_detected); *maxSVPtr = charnum-1; @@ -144,6 +208,43 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t return ip-istart; } +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_readNCount_body_default( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +#if DYNAMIC_BMI2 +TARGET_ATTRIBUTE("bmi2") static size_t FSE_readNCount_body_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} +#endif + +size_t FSE_readNCount_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); + } +#endif + (void)bmi2; + return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +size_t FSE_readNCount( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0); +} + /*! HUF_readStats() : Read compact Huffman tree, saved by HUF_writeCTable(). @@ -156,6 +257,17 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize) { + U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int bmi2) +{ U32 weightTotal; const BYTE* ip = (const BYTE*) src; size_t iSize; @@ -163,7 +275,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, if (!srcSize) return ERROR(srcSize_wrong); iSize = ip[0]; - /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + /* ZSTD_memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) { /* special header */ oSize = iSize - 127; @@ -177,14 +289,14 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, huffWeight[n+1] = ip[n/2] & 15; } } } else { /* header compressed with FSE (normal case) */ - FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */ if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */ + /* max (hwSize-1) values decoded, as last one is implied */ + oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2); if (FSE_isError(oSize)) return oSize; } /* collect weight stats */ - memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); + ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); weightTotal = 0; { U32 n; for (n=0; n<oSize; n++) { if (huffWeight[n] >= HUF_TABLELOG_MAX) return ERROR(corruption_detected); @@ -214,3 +326,37 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, *nbSymbolsPtr = (U32)(oSize+1); return iSize+1; } + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +static TARGET_ATTRIBUTE("bmi2") size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1); +} +#endif + +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); + } +#endif + (void)bmi2; + return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); +} diff --git a/lib/common/error_private.c b/lib/common/error_private.c index cd437529..45bba530 100644 --- a/lib/common/error_private.c +++ b/lib/common/error_private.c @@ -48,6 +48,7 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; + case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; case PREFIX(maxCode): default: return notErrorCode; } diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 982cf8e9..71b37b8d 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -21,7 +21,7 @@ extern "C" { /* **************************************** * Dependencies ******************************************/ -#include <stddef.h> /* size_t */ +#include "zstd_deps.h" /* size_t */ #include "zstd_errors.h" /* enum list */ diff --git a/lib/common/fse.h b/lib/common/fse.h index ff54e70e..83a07847 100644 --- a/lib/common/fse.h +++ b/lib/common/fse.h @@ -23,7 +23,7 @@ extern "C" { /*-***************************************** * Dependencies ******************************************/ -#include <stddef.h> /* size_t, ptrdiff_t */ +#include "zstd_deps.h" /* size_t, ptrdiff_t */ /*-***************************************** @@ -137,10 +137,16 @@ FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize /*! FSE_normalizeCount(): normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + useLowProbCount is a boolean parameter which trades off compressed size for + faster header decoding. When it is set to 1, the compressed data will be slightly + smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be + faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0 + is a good default, since header deserialization makes a big speed difference. + Otherwise, useLowProbCount=1 is a good default, since the speed difference is small. @return : tableLog, or an errorCode, which can be tested using FSE_isError() */ FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, - const unsigned* count, size_t srcSize, unsigned maxSymbolValue); + const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount); /*! FSE_NCountWriteBound(): Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. @@ -228,6 +234,13 @@ FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); +/*! FSE_readNCount_bmi2(): + * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise. + */ +FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize, int bmi2); + /*! Constructor and Destructor of FSE_DTable. Note that its size depends on 'tableLog' */ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ @@ -288,12 +301,12 @@ If there is an error, the function will return an error code, which can be teste *******************************************/ /* FSE buffer bounds */ #define FSE_NCOUNTBOUND 512 -#define FSE_BLOCKBOUND(size) (size + (size>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) +#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) #define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ /* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ -#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) -#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog)) +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog))) /* or use the size to malloc() space directly. Pay attention to alignment restrictions though */ #define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable)) @@ -309,9 +322,9 @@ unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsi /* FSE_compress_wksp() : * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). - * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. + * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. */ -#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) +#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); @@ -322,18 +335,29 @@ size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); /* FSE_buildCTable_wksp() : * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). - * `wkspSize` must be >= `(1<<tableLog)`. + * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog)`. */ +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * (maxSymbolValue + 2) + (1ull << tableLog)) size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); +#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8) +#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned)) +FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); +/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ + size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits); /**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue); /**< build a fake FSE_DTable, designed to always generate the same symbolValue */ -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog); -/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */ +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue)) +#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) +size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize); +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */ + +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); +/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */ typedef enum { FSE_repeat_none, /**< Cannot use the previous table */ @@ -644,6 +668,9 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) #ifndef FSE_DEFAULT_MEMORY_USAGE # define FSE_DEFAULT_MEMORY_USAGE 13 #endif +#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE) +# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE" +#endif /*!FSE_MAX_SYMBOL_VALUE : * Maximum symbol value authorized. @@ -677,7 +704,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) # error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" #endif -#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) +#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) #endif /* FSE_STATIC_LINKING_ONLY */ diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c index bcc2223c..c164430f 100644 --- a/lib/common/fse_decompress.c +++ b/lib/common/fse_decompress.c @@ -16,13 +16,14 @@ /* ************************************************************** * Includes ****************************************************************/ -#include <stdlib.h> /* malloc, free, qsort */ -#include <string.h> /* memcpy, memset */ +#include "debug.h" /* assert */ #include "bitstream.h" #include "compiler.h" #define FSE_STATIC_LINKING_ONLY #include "fse.h" #include "error_private.h" +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* ************************************************************** @@ -59,25 +60,27 @@ FSE_DTable* FSE_createDTable (unsigned tableLog) { if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); + return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); } void FSE_freeDTable (FSE_DTable* dt) { - free(dt); + ZSTD_free(dt); } -size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) { void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); - U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; + U16* symbolNext = (U16*)workSpace; + BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1); U32 const maxSV1 = maxSymbolValue + 1; U32 const tableSize = 1 << tableLog; U32 highThreshold = tableSize-1; /* Sanity Checks */ + if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge); if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); @@ -95,11 +98,57 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; symbolNext[s] = normalizedCounter[s]; } } } - memcpy(dt, &DTableH, sizeof(DTableH)); + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); } /* Spread symbols */ - { U32 const tableMask = tableSize-1; + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s<maxSV1; ++s, sv += add) { + int i; + int const n = normalizedCounter[s]; + MEM_write64(spread + pos, sv); + for (i = 8; i < n; i += 8) { + MEM_write64(spread + pos + i, sv); + } + pos += n; + } + } + /* Now we spread those positions across the table. + * The benefit of doing it in two stages is that we avoid the the + * variable size inner loop, which caused lots of branch misses. + * Now we can run through all the positions without any branch misses. + * We unroll the loop twice, since that is what emperically worked best. + */ + { + size_t position = 0; + size_t s; + size_t const unroll = 2; + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableDecode[uPosition].symbol = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); + } + } else { + U32 const tableMask = tableSize-1; U32 const step = FSE_TABLESTEP(tableSize); U32 s, position = 0; for (s=0; s<maxSV1; s++) { @@ -124,6 +173,11 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned return 0; } +size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) +{ + return FSE_buildDTable_internal(dt, normalizedCounter, maxSymbolValue, tableLog, workSpace, wkspSize); +} + #ifndef FSE_COMMONDEFS_ONLY @@ -251,36 +305,89 @@ size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, } -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog) +size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0); +} + +FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( + void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize, + unsigned maxLog, void* workSpace, size_t wkspSize, + int bmi2) { const BYTE* const istart = (const BYTE*)cSrc; const BYTE* ip = istart; short counting[FSE_MAX_SYMBOL_VALUE+1]; unsigned tableLog; unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + FSE_DTable* const dtable = (FSE_DTable*)workSpace; /* normal FSE decoding mode */ - size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); + size_t const NCountLength = FSE_readNCount_bmi2(counting, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); if (FSE_isError(NCountLength)) return NCountLength; - /* if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); */ /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */ if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + assert(NCountLength <= cSrcSize); ip += NCountLength; cSrcSize -= NCountLength; - CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) ); + if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); + workSpace = dtable + FSE_DTABLE_SIZE_U32(tableLog); + wkspSize -= FSE_DTABLE_SIZE(tableLog); + + CHECK_F( FSE_buildDTable_internal(dtable, counting, maxSymbolValue, tableLog, workSpace, wkspSize) ); + + { + const void* ptr = dtable; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 1); + return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 0); + } +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +TARGET_ATTRIBUTE("bmi2") static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); +} +#endif - return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */ +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); + } +#endif + (void)bmi2; + return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); } typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; +#ifndef ZSTD_NO_UNUSED_FUNCTIONS +size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { + U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)]; + return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp)); +} + size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) { - DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ - return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG); + /* Static analyzer seems unable to understand this table will be properly initialized later */ + U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; + return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp)); } - +#endif #endif /* FSE_COMMONDEFS_ONLY */ diff --git a/lib/common/huf.h b/lib/common/huf.h index ef432685..1afef90c 100644 --- a/lib/common/huf.h +++ b/lib/common/huf.h @@ -20,7 +20,7 @@ extern "C" { #define HUF_H_298734234 /* *** Dependencies *** */ -#include <stddef.h> /* size_t */ +#include "zstd_deps.h" /* size_t */ /* *** library symbols visibility *** */ @@ -111,6 +111,8 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, /* *** Dependencies *** */ #include "mem.h" /* U32 */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" /* *** Constants *** */ @@ -133,12 +135,16 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, #define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ /* static allocation of HUF's Compression Table */ +/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */ +struct HUF_CElt_s { + U16 val; + BYTE nbBits; +}; /* typedef'd to HUF_CElt */ +typedef struct HUF_CElt_s HUF_CElt; /* consider it an incomplete type */ #define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue)+1) /* Use tables of U32, for proper alignment */ #define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32)) #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ - U32 name##hb[HUF_CTABLE_SIZE_U32(maxSymbolValue)]; \ - void* name##hv = &(name##hb); \ - HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ + HUF_CElt name[HUF_CTABLE_SIZE_U32(maxSymbolValue)] /* no final ; */ /* static allocation of HUF's DTable */ typedef U32 HUF_DTable; @@ -184,7 +190,6 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, * or to save and regenerate 'CTable' using external methods. */ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); @@ -226,6 +231,19 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize); +/*! HUF_readStats_wksp() : + * Same as HUF_readStats() but takes an external workspace which must be + * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1) +#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workspace, size_t wkspSize, + int bmi2); + /** HUF_readCTable() : * Loading a CTable saved with HUF_writeCTable() */ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); @@ -332,6 +350,9 @@ size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstS #endif size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +#endif #endif /* HUF_STATIC_LINKING_ONLY */ diff --git a/lib/common/mem.h b/lib/common/mem.h index 89c8aea7..4728ef76 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -18,8 +18,10 @@ extern "C" { /*-**************************************** * Dependencies ******************************************/ -#include <stddef.h> /* size_t, ptrdiff_t */ -#include <string.h> /* memcpy */ +#include <stddef.h> /* size_t, ptrdiff_t */ +#include "compiler.h" /* __has_builtin */ +#include "debug.h" /* DEBUG_STATIC_ASSERT */ +#include "zstd_deps.h" /* ZSTD_memcpy */ /*-**************************************** @@ -39,93 +41,15 @@ extern "C" { # define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ #endif -#ifndef __has_builtin -# define __has_builtin(x) 0 /* compat. with non-clang compilers */ -#endif - -/* code only tested on 32 and 64 bits systems */ -#define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; } -MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } - -/* detects whether we are being compiled under msan */ -#if defined (__has_feature) -# if __has_feature(memory_sanitizer) -# define MEMORY_SANITIZER 1 -# endif -#endif - -#if defined (MEMORY_SANITIZER) -/* Not all platforms that support msan provide sanitizers/msan_interface.h. - * We therefore declare the functions we need ourselves, rather than trying to - * include the header file... */ - -#include <stdint.h> /* intptr_t */ - -/* Make memory region fully initialized (without changing its contents). */ -void __msan_unpoison(const volatile void *a, size_t size); - -/* Make memory region fully uninitialized (without changing its contents). - This is a legacy interface that does not update origin information. Use - __msan_allocated_memory() instead. */ -void __msan_poison(const volatile void *a, size_t size); - -/* Returns the offset of the first (at least partially) poisoned byte in the - memory range, or -1 if the whole range is good. */ -intptr_t __msan_test_shadow(const volatile void *x, size_t size); -#endif - -/* detects whether we are being compiled under asan */ -#if defined (__has_feature) -# if __has_feature(address_sanitizer) -# define ADDRESS_SANITIZER 1 -# endif -#elif defined(__SANITIZE_ADDRESS__) -# define ADDRESS_SANITIZER 1 -#endif - -#if defined (ADDRESS_SANITIZER) -/* Not all platforms that support asan provide sanitizers/asan_interface.h. - * We therefore declare the functions we need ourselves, rather than trying to - * include the header file... */ - -/** - * Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable. - * - * This memory must be previously allocated by your program. Instrumented - * code is forbidden from accessing addresses in this region until it is - * unpoisoned. This function is not guaranteed to poison the entire region - - * it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan - * alignment restrictions. - * - * \note This function is not thread-safe because no two threads can poison or - * unpoison memory in the same memory region simultaneously. - * - * \param addr Start of memory region. - * \param size Size of memory region. */ -void __asan_poison_memory_region(void const volatile *addr, size_t size); - -/** - * Marks a memory region (<c>[addr, addr+size)</c>) as addressable. - * - * This memory must be previously allocated by your program. Accessing - * addresses in this region is allowed until this region is poisoned again. - * This function could unpoison a super-region of <c>[addr, addr+size)</c> due - * to ASan alignment restrictions. - * - * \note This function is not thread-safe because no two threads can - * poison or unpoison memory in the same memory region simultaneously. - * - * \param addr Start of memory region. - * \param size Size of memory region. */ -void __asan_unpoison_memory_region(void const volatile *addr, size_t size); -#endif - - /*-************************************************************** * Basic Types *****************************************************************/ #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include <stdint.h> +# if defined(_AIX) +# include <inttypes.h> +# else +# include <stdint.h> /* intptr_t */ +# endif typedef uint8_t BYTE; typedef uint16_t U16; typedef int16_t S16; @@ -157,7 +81,53 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); /*-************************************************************** -* Memory I/O +* Memory I/O API +*****************************************************************/ +/*=== Static platform detection ===*/ +MEM_STATIC unsigned MEM_32bits(void); +MEM_STATIC unsigned MEM_64bits(void); +MEM_STATIC unsigned MEM_isLittleEndian(void); + +/*=== Native unaligned read/write ===*/ +MEM_STATIC U16 MEM_read16(const void* memPtr); +MEM_STATIC U32 MEM_read32(const void* memPtr); +MEM_STATIC U64 MEM_read64(const void* memPtr); +MEM_STATIC size_t MEM_readST(const void* memPtr); + +MEM_STATIC void MEM_write16(void* memPtr, U16 value); +MEM_STATIC void MEM_write32(void* memPtr, U32 value); +MEM_STATIC void MEM_write64(void* memPtr, U64 value); + +/*=== Little endian unaligned read/write ===*/ +MEM_STATIC U16 MEM_readLE16(const void* memPtr); +MEM_STATIC U32 MEM_readLE24(const void* memPtr); +MEM_STATIC U32 MEM_readLE32(const void* memPtr); +MEM_STATIC U64 MEM_readLE64(const void* memPtr); +MEM_STATIC size_t MEM_readLEST(const void* memPtr); + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val); +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val); +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val); + +/*=== Big endian unaligned read/write ===*/ +MEM_STATIC U32 MEM_readBE32(const void* memPtr); +MEM_STATIC U64 MEM_readBE64(const void* memPtr); +MEM_STATIC size_t MEM_readBEST(const void* memPtr); + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val); + +/*=== Byteswap ===*/ +MEM_STATIC U32 MEM_swap32(U32 in); +MEM_STATIC U64 MEM_swap64(U64 in); +MEM_STATIC size_t MEM_swapST(size_t in); + + +/*-************************************************************** +* Memory I/O Implementation *****************************************************************/ /* MEM_FORCE_MEMORY_ACCESS : * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. @@ -236,37 +206,37 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = MEM_STATIC U16 MEM_read16(const void* memPtr) { - U16 val; memcpy(&val, memPtr, sizeof(val)); return val; + U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; } MEM_STATIC U32 MEM_read32(const void* memPtr) { - U32 val; memcpy(&val, memPtr, sizeof(val)); return val; + U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; } MEM_STATIC U64 MEM_read64(const void* memPtr) { - U64 val; memcpy(&val, memPtr, sizeof(val)); return val; + U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; } MEM_STATIC size_t MEM_readST(const void* memPtr) { - size_t val; memcpy(&val, memPtr, sizeof(val)); return val; + size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; } MEM_STATIC void MEM_write16(void* memPtr, U16 value) { - memcpy(memPtr, &value, sizeof(value)); + ZSTD_memcpy(memPtr, &value, sizeof(value)); } MEM_STATIC void MEM_write32(void* memPtr, U32 value) { - memcpy(memPtr, &value, sizeof(value)); + ZSTD_memcpy(memPtr, &value, sizeof(value)); } MEM_STATIC void MEM_write64(void* memPtr, U64 value) { - memcpy(memPtr, &value, sizeof(value)); + ZSTD_memcpy(memPtr, &value, sizeof(value)); } #endif /* MEM_FORCE_MEMORY_ACCESS */ @@ -445,6 +415,9 @@ MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) MEM_writeBE64(memPtr, (U64)val); } +/* code only tested on 32 and 64 bits systems */ +MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + #if defined (__cplusplus) } diff --git a/lib/common/pool.c b/lib/common/pool.c index aa4b4de0..4c1b8337 100644 --- a/lib/common/pool.c +++ b/lib/common/pool.c @@ -10,9 +10,9 @@ /* ====== Dependencies ======= */ -#include <stddef.h> /* size_t */ +#include "zstd_deps.h" /* size_t */ #include "debug.h" /* assert */ -#include "zstd_internal.h" /* ZSTD_malloc, ZSTD_free */ +#include "zstd_internal.h" /* ZSTD_customMalloc, ZSTD_customFree */ #include "pool.h" /* ====== Compiler specifics ====== */ @@ -105,6 +105,10 @@ static void* POOL_thread(void* opaque) { assert(0); /* Unreachable */ } +POOL_ctx* ZSTD_createThreadPool(size_t numThreads) { + return POOL_create (numThreads, 0); +} + POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); } @@ -115,14 +119,14 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, /* Check parameters */ if (!numThreads) { return NULL; } /* Allocate the context and zero initialize */ - ctx = (POOL_ctx*)ZSTD_calloc(sizeof(POOL_ctx), customMem); + ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem); if (!ctx) { return NULL; } /* Initialize the job queue. * It needs one extra space since one space is wasted to differentiate * empty and full queues. */ ctx->queueSize = queueSize + 1; - ctx->queue = (POOL_job*)ZSTD_malloc(ctx->queueSize * sizeof(POOL_job), customMem); + ctx->queue = (POOL_job*)ZSTD_customMalloc(ctx->queueSize * sizeof(POOL_job), customMem); ctx->queueHead = 0; ctx->queueTail = 0; ctx->numThreadsBusy = 0; @@ -136,7 +140,7 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, } ctx->shutdown = 0; /* Allocate space for the thread handles */ - ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem); + ctx->threads = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), customMem); ctx->threadCapacity = 0; ctx->customMem = customMem; /* Check for errors */ @@ -179,12 +183,14 @@ void POOL_free(POOL_ctx *ctx) { ZSTD_pthread_mutex_destroy(&ctx->queueMutex); ZSTD_pthread_cond_destroy(&ctx->queuePushCond); ZSTD_pthread_cond_destroy(&ctx->queuePopCond); - ZSTD_free(ctx->queue, ctx->customMem); - ZSTD_free(ctx->threads, ctx->customMem); - ZSTD_free(ctx, ctx->customMem); + ZSTD_customFree(ctx->queue, ctx->customMem); + ZSTD_customFree(ctx->threads, ctx->customMem); + ZSTD_customFree(ctx, ctx->customMem); } - +void ZSTD_freeThreadPool (ZSTD_threadPool* pool) { + POOL_free (pool); +} size_t POOL_sizeof(POOL_ctx *ctx) { if (ctx==NULL) return 0; /* supports sizeof NULL */ @@ -203,11 +209,11 @@ static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads) return 0; } /* numThreads > threadCapacity */ - { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); + { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); if (!threadPool) return 1; /* replace existing thread pool */ - memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool)); - ZSTD_free(ctx->threads, ctx->customMem); + ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool)); + ZSTD_customFree(ctx->threads, ctx->customMem); ctx->threads = threadPool; /* Initialize additional threads */ { size_t threadId; @@ -301,7 +307,7 @@ int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) struct POOL_ctx_s { int dummy; }; -static POOL_ctx g_ctx; +static POOL_ctx g_poolCtx; POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); @@ -311,11 +317,11 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM (void)numThreads; (void)queueSize; (void)customMem; - return &g_ctx; + return &g_poolCtx; } void POOL_free(POOL_ctx* ctx) { - assert(!ctx || ctx == &g_ctx); + assert(!ctx || ctx == &g_poolCtx); (void)ctx; } @@ -337,7 +343,7 @@ int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) { size_t POOL_sizeof(POOL_ctx* ctx) { if (ctx==NULL) return 0; /* supports sizeof NULL */ - assert(ctx == &g_ctx); + assert(ctx == &g_poolCtx); return sizeof(*ctx); } diff --git a/lib/common/pool.h b/lib/common/pool.h index 259bafc9..63954ca6 100644 --- a/lib/common/pool.h +++ b/lib/common/pool.h @@ -16,7 +16,7 @@ extern "C" { #endif -#include <stddef.h> /* size_t */ +#include "zstd_deps.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */ #include "../zstd.h" diff --git a/lib/common/threading.c b/lib/common/threading.c index e2edb313..92cf57c1 100644 --- a/lib/common/threading.c +++ b/lib/common/threading.c @@ -78,11 +78,12 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr) #if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32) -#include <stdlib.h> +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr) { - *mutex = (pthread_mutex_t*)malloc(sizeof(pthread_mutex_t)); + *mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t)); if (!*mutex) return 1; return pthread_mutex_init(*mutex, attr); @@ -94,14 +95,14 @@ int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex) return 0; { int const ret = pthread_mutex_destroy(*mutex); - free(*mutex); + ZSTD_free(*mutex); return ret; } } int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr) { - *cond = (pthread_cond_t*)malloc(sizeof(pthread_cond_t)); + *cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t)); if (!*cond) return 1; return pthread_cond_init(*cond, attr); @@ -113,7 +114,7 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) return 0; { int const ret = pthread_cond_destroy(*cond); - free(*cond); + ZSTD_free(*cond); return ret; } } diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c index 597de18f..e708df3c 100644 --- a/lib/common/xxhash.c +++ b/lib/common/xxhash.c @@ -77,14 +77,12 @@ * Includes & Memory related functions ***************************************/ /* Modify the local functions below should you wish to use some other memory routines */ -/* for malloc(), free() */ -#include <stdlib.h> -#include <stddef.h> /* size_t */ -static void* XXH_malloc(size_t s) { return malloc(s); } -static void XXH_free (void* p) { free(p); } -/* for memcpy() */ -#include <string.h> -static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } +/* for ZSTD_malloc(), ZSTD_free() */ +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* size_t, ZSTD_malloc, ZSTD_free, ZSTD_memcpy */ +static void* XXH_malloc(size_t s) { return ZSTD_malloc(s); } +static void XXH_free (void* p) { ZSTD_free(p); } +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_memcpy(dest,src,size); } #ifndef XXH_STATIC_LINKING_ONLY # define XXH_STATIC_LINKING_ONLY @@ -95,49 +93,13 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp /* ************************************* * Compiler Specific Options ***************************************/ -#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# define INLINE_KEYWORD inline -#else -# define INLINE_KEYWORD -#endif - -#if defined(__GNUC__) || defined(__ICCARM__) -# define FORCE_INLINE_ATTR __attribute__((always_inline)) -#elif defined(_MSC_VER) -# define FORCE_INLINE_ATTR __forceinline -#else -# define FORCE_INLINE_ATTR -#endif - -#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR - - -#ifdef _MSC_VER -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif +#include "compiler.h" /* ************************************* * Basic Types ***************************************/ -#ifndef MEM_MODULE -# define MEM_MODULE -# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -# else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */ -# endif -#endif - +#include "mem.h" /* BYTE, U32, U64, size_t */ #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) @@ -163,14 +125,14 @@ static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } static U32 XXH_read32(const void* memPtr) { U32 val; - memcpy(&val, memPtr, sizeof(val)); + ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; } static U64 XXH_read64(const void* memPtr) { U64 val; - memcpy(&val, memPtr, sizeof(val)); + ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; } @@ -307,12 +269,12 @@ XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } ****************************/ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState) { - memcpy(dstState, srcState, sizeof(*dstState)); + ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); } XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState) { - memcpy(dstState, srcState, sizeof(*dstState)); + ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); } @@ -554,12 +516,12 @@ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) { XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ + ZSTD_memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ state.v1 = seed + PRIME32_1 + PRIME32_2; state.v2 = seed + PRIME32_2; state.v3 = seed + 0; state.v4 = seed - PRIME32_1; - memcpy(statePtr, &state, sizeof(state)); + ZSTD_memcpy(statePtr, &state, sizeof(state)); return XXH_OK; } @@ -567,12 +529,12 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int s XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) { XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ + ZSTD_memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ state.v1 = seed + PRIME64_1 + PRIME64_2; state.v2 = seed + PRIME64_2; state.v3 = seed + 0; state.v4 = seed - PRIME64_1; - memcpy(statePtr, &state, sizeof(state)); + ZSTD_memcpy(statePtr, &state, sizeof(state)); return XXH_OK; } @@ -843,14 +805,14 @@ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t { XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - memcpy(dst, &hash, sizeof(*dst)); + ZSTD_memcpy(dst, &hash, sizeof(*dst)); } XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) { XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - memcpy(dst, &hash, sizeof(*dst)); + ZSTD_memcpy(dst, &hash, sizeof(*dst)); } XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) diff --git a/lib/common/xxhash.h b/lib/common/xxhash.h index 4207eba8..eceb55d5 100644 --- a/lib/common/xxhash.h +++ b/lib/common/xxhash.h @@ -55,7 +55,7 @@ extern "C" { /* **************************** * Definitions ******************************/ -#include <stddef.h> /* size_t */ +#include "zstd_deps.h" typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 91fe3323..939e9f08 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -13,8 +13,8 @@ /*-************************************* * Dependencies ***************************************/ -#include <stdlib.h> /* malloc, calloc, free */ -#include <string.h> /* memset */ +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ #include "error_private.h" #include "zstd_internal.h" @@ -53,31 +53,31 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString /*=************************************************************** * Custom allocator ****************************************************************/ -void* ZSTD_malloc(size_t size, ZSTD_customMem customMem) +void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) { if (customMem.customAlloc) return customMem.customAlloc(customMem.opaque, size); - return malloc(size); + return ZSTD_malloc(size); } -void* ZSTD_calloc(size_t size, ZSTD_customMem customMem) +void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) { if (customMem.customAlloc) { /* calloc implemented as malloc+memset; * not as efficient as calloc, but next best guess for custom malloc */ void* const ptr = customMem.customAlloc(customMem.opaque, size); - memset(ptr, 0, size); + ZSTD_memset(ptr, 0, size); return ptr; } - return calloc(1, size); + return ZSTD_calloc(1, size); } -void ZSTD_free(void* ptr, ZSTD_customMem customMem) +void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) { if (ptr!=NULL) { if (customMem.customFree) customMem.customFree(customMem.opaque, ptr); else - free(ptr); + ZSTD_free(ptr); } } diff --git a/lib/common/zstd_deps.h b/lib/common/zstd_deps.h new file mode 100644 index 00000000..0fb8b781 --- /dev/null +++ b/lib/common/zstd_deps.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This file provides common libc dependencies that zstd requires. + * The purpose is to allow replacing this file with a custom implementation + * to compile zstd without libc support. + */ + +/* Need: + * NULL + * INT_MAX + * UINT_MAX + * ZSTD_memcpy() + * ZSTD_memset() + * ZSTD_memmove() + */ +#ifndef ZSTD_DEPS_COMMON +#define ZSTD_DEPS_COMMON + +#include <limits.h> +#include <stddef.h> +#include <string.h> + +#if defined(__GNUC__) && __GNUC__ >= 4 +# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l)) +#else +# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) memset((p),(v),(l)) +#endif + +#endif /* ZSTD_DEPS_COMMON */ + +/* Need: + * ZSTD_malloc() + * ZSTD_free() + * ZSTD_calloc() + */ +#ifdef ZSTD_DEPS_NEED_MALLOC +#ifndef ZSTD_DEPS_MALLOC +#define ZSTD_DEPS_MALLOC + +#include <stdlib.h> + +#define ZSTD_malloc(s) malloc(s) +#define ZSTD_calloc(n,s) calloc((n), (s)) +#define ZSTD_free(p) free((p)) + +#endif /* ZSTD_DEPS_MALLOC */ +#endif /* ZSTD_DEPS_NEED_MALLOC */ + +/* + * Provides 64-bit math support. + * Need: + * U64 ZSTD_div64(U64 dividend, U32 divisor) + */ +#ifdef ZSTD_DEPS_NEED_MATH64 +#ifndef ZSTD_DEPS_MATH64 +#define ZSTD_DEPS_MATH64 + +#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor)) + +#endif /* ZSTD_DEPS_MATH64 */ +#endif /* ZSTD_DEPS_NEED_MATH64 */ + +/* Need: + * assert() + */ +#ifdef ZSTD_DEPS_NEED_ASSERT +#ifndef ZSTD_DEPS_ASSERT +#define ZSTD_DEPS_ASSERT + +#include <assert.h> + +#endif /* ZSTD_DEPS_ASSERT */ +#endif /* ZSTD_DEPS_NEED_ASSERT */ + +/* Need: + * ZSTD_DEBUG_PRINT() + */ +#ifdef ZSTD_DEPS_NEED_IO +#ifndef ZSTD_DEPS_IO +#define ZSTD_DEPS_IO + +#include <stdio.h> +#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__) + +#endif /* ZSTD_DEPS_IO */ +#endif /* ZSTD_DEPS_NEED_IO */ + +/* Only requested when <stdint.h> is known to be present. + * Need: + * intptr_t + */ +#ifdef ZSTD_DEPS_NEED_STDINT +#ifndef ZSTD_DEPS_STDINT +#define ZSTD_DEPS_STDINT + +#include <stdint.h> + +#endif /* ZSTD_DEPS_STDINT */ +#endif /* ZSTD_DEPS_NEED_STDINT */ diff --git a/lib/common/zstd_errors.h b/lib/common/zstd_errors.h index 998398e7..6d0d0030 100644 --- a/lib/common/zstd_errors.h +++ b/lib/common/zstd_errors.h @@ -77,6 +77,7 @@ typedef enum { ZSTD_error_frameIndex_tooLarge = 100, ZSTD_error_seekableIO = 102, ZSTD_error_dstBuffer_wrong = 104, + ZSTD_error_srcBuffer_wrong = 105, ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ } ZSTD_ErrorCode; diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 3bc7e55a..0991f20a 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -19,7 +19,7 @@ /*-************************************* * Dependencies ***************************************/ -#ifdef __aarch64__ +#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON) #include <arm_neon.h> #endif #include "compiler.h" @@ -139,7 +139,7 @@ void _force_has_format_string(const char *format, ...) { #define ZSTD_REP_NUM 3 /* number of repcodes */ #define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) -static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; +static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; #define KB *(1 <<10) #define MB *(1 <<20) @@ -153,13 +153,13 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; #define BIT0 1 #define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 -static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; -static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; +static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; #define ZSTD_FRAMEIDSIZE 4 /* magic number size */ #define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ -static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; #define ZSTD_FRAMECHECKSUMSIZE 4 @@ -186,61 +186,75 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy #define OffFSELog 8 #define MaxFSELog MAX(MAX(MLFSELog, LLFSELog), OffFSELog) -static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 2, 2, 3, 3, - 4, 6, 7, 8, 9,10,11,12, - 13,14,15,16 }; -static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, - 2, 3, 2, 1, 1, 1, 1, 1, - -1,-1,-1,-1 }; +#define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */ +/* Each table cannot take more than #symbols * FSELog bits */ +#define ZSTD_MAX_FSE_HEADERS_SIZE (((MaxML + 1) * MLFSELog + (MaxLL + 1) * LLFSELog + (MaxOff + 1) * OffFSELog + 7) / 8) + +static UNUSED_ATTR const U32 LL_bits[MaxLL+1] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, + 4, 6, 7, 8, 9,10,11,12, + 13,14,15,16 +}; +static UNUSED_ATTR const S16 LL_defaultNorm[MaxLL+1] = { + 4, 3, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, + 2, 3, 2, 1, 1, 1, 1, 1, + -1,-1,-1,-1 +}; #define LL_DEFAULTNORMLOG 6 /* for static allocation */ -static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG; - -static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 2, 2, 3, 3, - 4, 4, 5, 7, 8, 9,10,11, - 12,13,14,15,16 }; -static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, - 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1,-1,-1, - -1,-1,-1,-1,-1 }; +static UNUSED_ATTR const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG; + +static UNUSED_ATTR const U32 ML_bits[MaxML+1] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 7, 8, 9,10,11, + 12,13,14,15,16 +}; +static UNUSED_ATTR const S16 ML_defaultNorm[MaxML+1] = { + 1, 4, 3, 2, 2, 2, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1 +}; #define ML_DEFAULTNORMLOG 6 /* for static allocation */ -static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG; - -static const S16 OF_defaultNorm[DefaultMaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, - 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - -1,-1,-1,-1,-1 }; +static UNUSED_ATTR const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG; + +static UNUSED_ATTR const S16 OF_defaultNorm[DefaultMaxOff+1] = { + 1, 1, 1, 1, 1, 1, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + -1,-1,-1,-1,-1 +}; #define OF_DEFAULTNORMLOG 5 /* for static allocation */ -static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; +static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; /*-******************************************* * Shared functions to include for inlining *********************************************/ static void ZSTD_copy8(void* dst, const void* src) { -#ifdef __aarch64__ +#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON) vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src)); #else - memcpy(dst, src, 8); + ZSTD_memcpy(dst, src, 8); #endif } #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } static void ZSTD_copy16(void* dst, const void* src) { -#ifdef __aarch64__ +#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON) vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); #else - memcpy(dst, src, 16); + ZSTD_memcpy(dst, src, 16); #endif } #define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } @@ -255,13 +269,13 @@ typedef enum { } ZSTD_overlap_e; /*! ZSTD_wildcopy() : - * Custom version of memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0) + * Custom version of ZSTD_memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0) * @param ovtype controls the overlap detection * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart. * The src buffer must be before the dst buffer. */ -MEM_STATIC FORCE_INLINE_ATTR +MEM_STATIC FORCE_INLINE_ATTR void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype) { ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; @@ -284,14 +298,16 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e * one COPY16() in the first call. Then, do two calls per loop since * at that point it is more likely to have a high trip count. */ -#ifndef __aarch64__ +#ifdef __aarch64__ do { COPY16(op, ip); } while (op < oend); #else - COPY16(op, ip); - if (op >= oend) return; + ZSTD_copy16(op, ip); + if (16 >= length) return; + op += 16; + ip += 16; do { COPY16(op, ip); COPY16(op, ip); @@ -305,7 +321,7 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, { size_t const length = MIN(dstCapacity, srcSize); if (length > 0) { - memcpy(dst, src, length); + ZSTD_memcpy(dst, src, length); } return length; } @@ -320,28 +336,39 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, * In which case, resize it down to free some memory */ #define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 +/* Controls whether the input/output buffer is buffered or stable. */ +typedef enum { + ZSTD_bm_buffered = 0, /* Buffer the input/output */ + ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ +} ZSTD_bufferMode_e; + /*-******************************************* * Private declarations *********************************************/ typedef struct seqDef_s { - U32 offset; + U32 offset; /* Offset code of the sequence */ U16 litLength; U16 matchLength; } seqDef; typedef struct { seqDef* sequencesStart; - seqDef* sequences; + seqDef* sequences; /* ptr to end of sequences */ BYTE* litStart; - BYTE* lit; + BYTE* lit; /* ptr to end of literals */ BYTE* llCode; BYTE* mlCode; BYTE* ofCode; size_t maxNbSeq; size_t maxNbLit; - U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ - U32 longLengthPos; + + /* longLengthPos and longLengthID to allow us to represent either a single litLength or matchLength + * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment + * the existing value of the litLength or matchLength by 0x10000. + */ + U32 longLengthID; /* 0 == no longLength; 1 == Represent the long literal; 2 == Represent the long match; */ + U32 longLengthPos; /* Index of the sequence to apply long length modification to */ } seqStore_t; typedef struct { @@ -384,9 +411,9 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBu void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ /* custom memory allocation functions */ -void* ZSTD_malloc(size_t size, ZSTD_customMem customMem); -void* ZSTD_calloc(size_t size, ZSTD_customMem customMem); -void ZSTD_free(void* ptr, ZSTD_customMem customMem); +void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem); +void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem); +void ZSTD_customFree(void* ptr, ZSTD_customMem customMem); MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ @@ -394,8 +421,12 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus assert(val != 0); { # if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - return _BitScanReverse(&r, val) ? (unsigned)r : 0; +# if STATIC_BMI2 == 1 + return _lzcnt_u32(val)^31; +# else + unsigned long r=0; + return _BitScanReverse(&r, val) ? (unsigned)r : 0; +# endif # elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ return __builtin_clz (val) ^ 31; # elif defined(__ICCARM__) /* IAR Intrinsic */ diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index a4275981..304a82b3 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -15,8 +15,6 @@ /* ************************************************************** * Includes ****************************************************************/ -#include <stdlib.h> /* malloc, free, qsort */ -#include <string.h> /* memcpy, memset */ #include "../common/compiler.h" #include "../common/mem.h" /* U32, U16, etc. */ #include "../common/debug.h" /* assert, DEBUGLOG */ @@ -25,6 +23,9 @@ #define FSE_STATIC_LINKING_ONLY #include "../common/fse.h" #include "../common/error_private.h" +#define ZSTD_DEPS_NEED_MALLOC +#define ZSTD_DEPS_NEED_MATH64 +#include "../common/zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */ /* ************************************************************** @@ -74,13 +75,15 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ; FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); U32 const step = FSE_TABLESTEP(tableSize); - U32 cumul[FSE_MAX_SYMBOL_VALUE+2]; - FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace; + U32* cumul = (U32*)workSpace; + FSE_FUNCTION_TYPE* tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSymbolValue + 2)); + U32 highThreshold = tableSize-1; + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* Must be 4 byte aligned */ + if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge); /* CTable header */ - if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge); tableU16[-2] = (U16) tableLog; tableU16[-1] = (U16) maxSymbolValue; assert(tableLog < 16); /* required for threshold strategy to work */ @@ -89,7 +92,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ #ifdef __clang_analyzer__ - memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ + ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ #endif /* symbol start positions */ @@ -168,12 +171,13 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, return 0; } - +#ifndef ZSTD_NO_UNUSED_FUNCTIONS size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */ return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol)); } +#endif @@ -307,10 +311,10 @@ FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) size_t size; if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); - return (FSE_CTable*)malloc(size); + return (FSE_CTable*)ZSTD_malloc(size); } -void FSE_freeCTable (FSE_CTable* ct) { free(ct); } +void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); } /* provides the minimum logSize to safely represent a distribution */ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) @@ -341,11 +345,10 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); } - /* Secondary normalization method. To be used when primary method fails. */ -static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue) +static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount) { short const NOT_YET_ASSIGNED = -2; U32 s; @@ -362,7 +365,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, continue; } if (count[s] <= lowThreshold) { - norm[s] = -1; + norm[s] = lowProbCount; distributed++; total -= count[s]; continue; @@ -414,7 +417,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, { U64 const vStepLog = 62 - tableLog; U64 const mid = (1ULL << (vStepLog-1)) - 1; - U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */ + U64 const rStep = ZSTD_div64((((U64)1<<vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */ U64 tmpTotal = mid; for (s=0; s<=maxSymbolValue; s++) { if (norm[s]==NOT_YET_ASSIGNED) { @@ -431,10 +434,9 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, return 0; } - size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t total, - unsigned maxSymbolValue) + unsigned maxSymbolValue, unsigned useLowProbCount) { /* Sanity checks */ if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; @@ -443,8 +445,9 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; + short const lowProbCount = useLowProbCount ? -1 : 1; U64 const scale = 62 - tableLog; - U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */ + U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */ U64 const vStep = 1ULL<<(scale-20); int stillToDistribute = 1<<tableLog; unsigned s; @@ -456,7 +459,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, if (count[s] == total) return 0; /* rle special case */ if (count[s] == 0) { normalizedCounter[s]=0; continue; } if (count[s] <= lowThreshold) { - normalizedCounter[s] = -1; + normalizedCounter[s] = lowProbCount; stillToDistribute--; } else { short proba = (short)((count[s]*step) >> scale); @@ -470,7 +473,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, } } if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { /* corner case, need another normalization method */ - size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); + size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount); if (FSE_isError(errorCode)) return errorCode; } else normalizedCounter[largest] += (short)stillToDistribute; @@ -625,6 +628,7 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize, size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } +#ifndef ZSTD_NO_UNUSED_FUNCTIONS /* FSE_compress_wksp() : * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). * `wkspSize` size must be `(1<<tableLog)`. @@ -643,7 +647,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable)); /* init conditions */ - if (wkspSize < FSE_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge); + if (wkspSize < FSE_COMPRESS_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge); if (srcSize <= 1) return 0; /* Not compressible */ if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE; if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG; @@ -656,7 +660,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src } tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); - CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) ); + CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue, /* useLowProbCount */ srcSize >= 2048) ); /* Write table description header */ { CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); @@ -678,13 +682,16 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src typedef struct { FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; - BYTE scratchBuffer[1 << FSE_MAX_TABLELOG]; + union { + U32 hist_wksp[HIST_WKSP_SIZE_U32]; + BYTE scratchBuffer[1 << FSE_MAX_TABLELOG]; + } workspace; } fseWkspMax_t; size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) { fseWkspMax_t scratchBuffer; - DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */ + DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_COMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */ if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer)); } @@ -693,6 +700,6 @@ size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcS { return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); } - +#endif #endif /* FSE_COMMONDEFS_ONLY */ diff --git a/lib/compress/hist.c b/lib/compress/hist.c index 61e08c79..a9659d11 100644 --- a/lib/compress/hist.c +++ b/lib/compress/hist.c @@ -34,7 +34,7 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, unsigned maxSymbolValue = *maxSymbolValuePtr; unsigned largestCount=0; - memset(count, 0, (maxSymbolValue+1) * sizeof(*count)); + ZSTD_memset(count, 0, (maxSymbolValue+1) * sizeof(*count)); if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } while (ip<end) { @@ -60,9 +60,9 @@ typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e; * this design makes better use of OoO cpus, * and is noticeably faster when some values are heavily repeated. * But it needs some additional workspace for intermediate tables. - * `workSpace` size must be a table of size >= HIST_WKSP_SIZE_U32. + * `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32. * @return : largest histogram frequency, - * or an error code (notably when histogram would be larger than *maxSymbolValuePtr). */ + * or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */ static size_t HIST_count_parallel_wksp( unsigned* count, unsigned* maxSymbolValuePtr, const void* source, size_t sourceSize, @@ -71,22 +71,21 @@ static size_t HIST_count_parallel_wksp( { const BYTE* ip = (const BYTE*)source; const BYTE* const iend = ip+sourceSize; - unsigned maxSymbolValue = *maxSymbolValuePtr; + size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count); unsigned max=0; U32* const Counting1 = workSpace; U32* const Counting2 = Counting1 + 256; U32* const Counting3 = Counting2 + 256; U32* const Counting4 = Counting3 + 256; - memset(workSpace, 0, 4*256*sizeof(unsigned)); - /* safety checks */ + assert(*maxSymbolValuePtr <= 255); if (!sourceSize) { - memset(count, 0, maxSymbolValue + 1); + ZSTD_memset(count, 0, countSize); *maxSymbolValuePtr = 0; return 0; } - if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */ + ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned)); /* by stripes of 16 bytes */ { U32 cached = MEM_read32(ip); ip += 4; @@ -118,21 +117,18 @@ static size_t HIST_count_parallel_wksp( /* finish last symbols */ while (ip<iend) Counting1[*ip++]++; - if (check) { /* verify stats will fit into destination table */ - U32 s; for (s=255; s>maxSymbolValue; s--) { - Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; - if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall); - } } - { U32 s; - if (maxSymbolValue > 255) maxSymbolValue = 255; - for (s=0; s<=maxSymbolValue; s++) { - count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; - if (count[s] > max) max = count[s]; + for (s=0; s<256; s++) { + Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; + if (Counting1[s] > max) max = Counting1[s]; } } - while (!count[maxSymbolValue]) maxSymbolValue--; - *maxSymbolValuePtr = maxSymbolValue; + { unsigned maxSymbolValue = 255; + while (!Counting1[maxSymbolValue]) maxSymbolValue--; + if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall); + *maxSymbolValuePtr = maxSymbolValue; + ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */ + } return (size_t)max; } @@ -152,14 +148,6 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace); } -/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ -size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize) -{ - unsigned tmpCounters[HIST_WKSP_SIZE_U32]; - return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters)); -} - /* HIST_count_wksp() : * Same as HIST_count(), but using an externally provided scratch buffer. * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */ @@ -175,9 +163,19 @@ size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize); } +#ifndef ZSTD_NO_UNUSED_FUNCTIONS +/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize) +{ + unsigned tmpCounters[HIST_WKSP_SIZE_U32]; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters)); +} + size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize) { unsigned tmpCounters[HIST_WKSP_SIZE_U32]; return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters)); } +#endif diff --git a/lib/compress/hist.h b/lib/compress/hist.h index 77e3ec4f..fb9ead68 100644 --- a/lib/compress/hist.h +++ b/lib/compress/hist.h @@ -14,7 +14,7 @@ ****************************************************************** */ /* --- dependencies --- */ -#include <stddef.h> /* size_t */ +#include "../common/zstd_deps.h" /* size_t */ /* --- simple histogram functions --- */ diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index 54687986..abbcc319 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -23,8 +23,7 @@ /* ************************************************************** * Includes ****************************************************************/ -#include <string.h> /* memcpy, memset */ -#include <stdio.h> /* printf (debug) */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ #include "../common/compiler.h" #include "../common/bitstream.h" #include "hist.h" @@ -70,7 +69,7 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; - BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER]; + BYTE scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)]; unsigned count[HUF_TABLELOG_MAX+1]; S16 norm[HUF_TABLELOG_MAX+1]; @@ -85,7 +84,7 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight } tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); - CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) ); + CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) ); /* Write table description header */ { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), norm, maxSymbolValue, tableLog) ); @@ -103,11 +102,6 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight } -struct HUF_CElt_s { - U16 val; - BYTE nbBits; -}; /* typedef'd to HUF_CElt within "huf.h" */ - /*! HUF_writeCTable() : `CTable` : Huffman tree to save, using huf representation. @return : size of saved CTable */ @@ -156,6 +150,7 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void /* get symbol weights */ CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize)); + *hasZeroWeights = (rankVal[0] > 0); /* check result */ if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); @@ -164,16 +159,14 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void /* Prepare base value per rank */ { U32 n, nextRankStart = 0; for (n=1; n<=tableLog; n++) { - U32 current = nextRankStart; + U32 curr = nextRankStart; nextRankStart += (rankVal[n] << (n-1)); - rankVal[n] = current; + rankVal[n] = curr; } } /* fill nbBits */ - *hasZeroWeights = 0; { U32 n; for (n=0; n<nbSymbols; n++) { const U32 w = huffWeight[n]; - *hasZeroWeights |= (w == 0); CTable[n].nbBits = (BYTE)(tableLog + 1 - w) & -(w != 0); } } @@ -212,32 +205,63 @@ typedef struct nodeElt_s { BYTE nbBits; } nodeElt; +/** + * HUF_setMaxHeight(): + * Enforces maxNbBits on the Huffman tree described in huffNode. + * + * It sets all nodes with nbBits > maxNbBits to be maxNbBits. Then it adjusts + * the tree to so that it is a valid canonical Huffman tree. + * + * @pre The sum of the ranks of each symbol == 2^largestBits, + * where largestBits == huffNode[lastNonNull].nbBits. + * @post The sum of the ranks of each symbol == 2^largestBits, + * where largestBits is the return value <= maxNbBits. + * + * @param huffNode The Huffman tree modified in place to enforce maxNbBits. + * @param lastNonNull The symbol with the lowest count in the Huffman tree. + * @param maxNbBits The maximum allowed number of bits, which the Huffman tree + * may not respect. After this function the Huffman tree will + * respect maxNbBits. + * @return The maximum number of bits of the Huffman tree after adjustment, + * necessarily no more than maxNbBits. + */ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) { const U32 largestBits = huffNode[lastNonNull].nbBits; - if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */ + /* early exit : no elt > maxNbBits, so the tree is already valid. */ + if (largestBits <= maxNbBits) return largestBits; /* there are several too large elements (at least >= 2) */ { int totalCost = 0; const U32 baseCost = 1 << (largestBits - maxNbBits); int n = (int)lastNonNull; + /* Adjust any ranks > maxNbBits to maxNbBits. + * Compute totalCost, which is how far the sum of the ranks is + * we are over 2^largestBits after adjust the offending ranks. + */ while (huffNode[n].nbBits > maxNbBits) { totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); huffNode[n].nbBits = (BYTE)maxNbBits; - n --; - } /* n stops at huffNode[n].nbBits <= maxNbBits */ - while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */ + n--; + } + /* n stops at huffNode[n].nbBits <= maxNbBits */ + assert(huffNode[n].nbBits <= maxNbBits); + /* n end at index of smallest symbol using < maxNbBits */ + while (huffNode[n].nbBits == maxNbBits) --n; - /* renorm totalCost */ - totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ + /* renorm totalCost from 2^largestBits to 2^maxNbBits + * note : totalCost is necessarily a multiple of baseCost */ + assert((totalCost & (baseCost - 1)) == 0); + totalCost >>= (largestBits - maxNbBits); + assert(totalCost > 0); /* repay normalized cost */ { U32 const noSymbol = 0xF0F0F0F0; U32 rankLast[HUF_TABLELOG_MAX+2]; - /* Get pos of last (smallest) symbol per rank */ - memset(rankLast, 0xF0, sizeof(rankLast)); + /* Get pos of last (smallest = lowest cum. count) symbol per rank */ + ZSTD_memset(rankLast, 0xF0, sizeof(rankLast)); { U32 currentNbBits = maxNbBits; int pos; for (pos=n ; pos >= 0; pos--) { @@ -247,34 +271,65 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) } } while (totalCost > 0) { + /* Try to reduce the next power of 2 above totalCost because we + * gain back half the rank. + */ U32 nBitsToDecrease = BIT_highbit32((U32)totalCost) + 1; for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { U32 const highPos = rankLast[nBitsToDecrease]; U32 const lowPos = rankLast[nBitsToDecrease-1]; if (highPos == noSymbol) continue; + /* Decrease highPos if no symbols of lowPos or if it is + * not cheaper to remove 2 lowPos than highPos. + */ if (lowPos == noSymbol) break; { U32 const highTotal = huffNode[highPos].count; U32 const lowTotal = 2 * huffNode[lowPos].count; if (highTotal <= lowTotal) break; } } /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ + assert(rankLast[nBitsToDecrease] != noSymbol || nBitsToDecrease == 1); /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) - nBitsToDecrease ++; + nBitsToDecrease++; + assert(rankLast[nBitsToDecrease] != noSymbol); + /* Increase the number of bits to gain back half the rank cost. */ totalCost -= 1 << (nBitsToDecrease-1); + huffNode[rankLast[nBitsToDecrease]].nbBits++; + + /* Fix up the new rank. + * If the new rank was empty, this symbol is now its smallest. + * Otherwise, this symbol will be the largest in the new rank so no adjustment. + */ if (rankLast[nBitsToDecrease-1] == noSymbol) - rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ - huffNode[rankLast[nBitsToDecrease]].nbBits ++; + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; + /* Fix up the old rank. + * If the symbol was at position 0, meaning it was the highest weight symbol in the tree, + * it must be the only symbol in its rank, so the old rank now has no symbols. + * Otherwise, since the Huffman nodes are sorted by count, the previous position is now + * the smallest node in the rank. If the previous position belongs to a different rank, + * then the rank is now empty. + */ if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ rankLast[nBitsToDecrease] = noSymbol; else { rankLast[nBitsToDecrease]--; if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ - } } /* while (totalCost > 0) */ - + } + } /* while (totalCost > 0) */ + + /* If we've removed too much weight, then we have to add it back. + * To avoid overshooting again, we only adjust the smallest rank. + * We take the largest nodes from the lowest rank 0 and move them + * to rank 1. There's guaranteed to be enough rank 0 symbols because + * TODO. + */ while (totalCost < 0) { /* Sometimes, cost correction overshoot */ - if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ + /* special case : no rank 1 symbol (using maxNbBits-1); + * let's create one from largest rank 0 (using maxNbBits). + */ + if (rankLast[1] == noSymbol) { while (huffNode[n].nbBits == maxNbBits) n--; huffNode[n+1].nbBits--; assert(n >= 0); @@ -285,14 +340,16 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) huffNode[ rankLast[1] + 1 ].nbBits--; rankLast[1]++; totalCost ++; - } } } /* there are several too large elements (at least >= 2) */ + } + } /* repay normalized cost */ + } /* there are several too large elements (at least >= 2) */ return maxNbBits; } typedef struct { U32 base; - U32 current; + U32 curr; } rankPos; typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32]; @@ -304,21 +361,45 @@ typedef struct { rankPos rankPosition[RANK_POSITION_TABLE_SIZE]; } HUF_buildCTable_wksp_tables; +/** + * HUF_sort(): + * Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order. + * + * @param[out] huffNode Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled. + * Must have (maxSymbolValue + 1) entries. + * @param[in] count Histogram of the symbols. + * @param[in] maxSymbolValue Maximum symbol value. + * @param rankPosition This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries. + */ static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValue, rankPos* rankPosition) { - U32 n; - - memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE); - for (n=0; n<=maxSymbolValue; n++) { - U32 r = BIT_highbit32(count[n] + 1); - rankPosition[r].base ++; + int n; + int const maxSymbolValue1 = (int)maxSymbolValue + 1; + + /* Compute base and set curr to base. + * For symbol s let lowerRank = BIT_highbit32(count[n]+1) and rank = lowerRank + 1. + * Then 2^lowerRank <= count[n]+1 <= 2^rank. + * We attribute each symbol to lowerRank's base value, because we want to know where + * each rank begins in the output, so for rank R we want to count ranks R+1 and above. + */ + ZSTD_memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE); + for (n = 0; n < maxSymbolValue1; ++n) { + U32 lowerRank = BIT_highbit32(count[n] + 1); + rankPosition[lowerRank].base++; } - for (n=30; n>0; n--) rankPosition[n-1].base += rankPosition[n].base; - for (n=0; n<32; n++) rankPosition[n].current = rankPosition[n].base; - for (n=0; n<=maxSymbolValue; n++) { + assert(rankPosition[RANK_POSITION_TABLE_SIZE - 1].base == 0); + for (n = RANK_POSITION_TABLE_SIZE - 1; n > 0; --n) { + rankPosition[n-1].base += rankPosition[n].base; + rankPosition[n-1].curr = rankPosition[n-1].base; + } + /* Sort */ + for (n = 0; n < maxSymbolValue1; ++n) { U32 const c = count[n]; U32 const r = BIT_highbit32(c+1) + 1; - U32 pos = rankPosition[r].current++; + U32 pos = rankPosition[r].curr++; + /* Insert into the correct position in the rank. + * We have at most 256 symbols, so this insertion should be fine. + */ while ((pos > rankPosition[r].base) && (c > huffNode[pos-1].count)) { huffNode[pos] = huffNode[pos-1]; pos--; @@ -335,28 +416,20 @@ static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValu */ #define STARTNODE (HUF_SYMBOLVALUE_MAX+1) -size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize) +/* HUF_buildTree(): + * Takes the huffNode array sorted by HUF_sort() and builds an unlimited-depth Huffman tree. + * + * @param huffNode The array sorted by HUF_sort(). Builds the Huffman tree in this array. + * @param maxSymbolValue The maximum symbol value. + * @return The smallest node in the Huffman tree (by count). + */ +static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) { - HUF_buildCTable_wksp_tables* const wksp_tables = (HUF_buildCTable_wksp_tables*)workSpace; - nodeElt* const huffNode0 = wksp_tables->huffNodeTbl; - nodeElt* const huffNode = huffNode0+1; + nodeElt* const huffNode0 = huffNode - 1; int nonNullRank; int lowS, lowN; int nodeNb = STARTNODE; int n, nodeRoot; - - /* safety checks */ - if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ - if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) - return ERROR(workSpace_tooSmall); - if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) - return ERROR(maxSymbolValue_tooLarge); - memset(huffNode0, 0, sizeof(huffNodeTable)); - - /* sort, decreasing order */ - HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); - /* init for parents */ nonNullRank = (int)maxSymbolValue; while(huffNode[nonNullRank].count == 0) nonNullRank--; @@ -383,42 +456,72 @@ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbo for (n=0; n<=nonNullRank; n++) huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + return nonNullRank; +} + +/** + * HUF_buildCTableFromTree(): + * Build the CTable given the Huffman tree in huffNode. + * + * @param[out] CTable The output Huffman CTable. + * @param huffNode The Huffman tree. + * @param nonNullRank The last and smallest node in the Huffman tree. + * @param maxSymbolValue The maximum symbol value. + * @param maxNbBits The exact maximum number of bits used in the Huffman tree. + */ +static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, int nonNullRank, U32 maxSymbolValue, U32 maxNbBits) +{ + /* fill result into ctable (val, nbBits) */ + int n; + U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + int const alphabetSize = (int)(maxSymbolValue + 1); + for (n=0; n<=nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine starting value per rank */ + { U16 min = 0; + for (n=(int)maxNbBits; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + for (n=0; n<alphabetSize; n++) + CTable[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ + for (n=0; n<alphabetSize; n++) + CTable[n].val = valPerRank[CTable[n].nbBits]++; /* assign value within rank, symbol order */ +} + +size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize) +{ + HUF_buildCTable_wksp_tables* const wksp_tables = (HUF_buildCTable_wksp_tables*)workSpace; + nodeElt* const huffNode0 = wksp_tables->huffNodeTbl; + nodeElt* const huffNode = huffNode0+1; + int nonNullRank; + + /* safety checks */ + if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) + return ERROR(workSpace_tooSmall); + if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) + return ERROR(maxSymbolValue_tooLarge); + ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable)); + + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); + + /* build tree */ + nonNullRank = HUF_buildTree(huffNode, maxSymbolValue); + /* enforce maxTableLog */ maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); + if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ - /* fill result into tree (val, nbBits) */ - { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; - U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; - int const alphabetSize = (int)(maxSymbolValue + 1); - if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ - for (n=0; n<=nonNullRank; n++) - nbPerRank[huffNode[n].nbBits]++; - /* determine stating value per rank */ - { U16 min = 0; - for (n=(int)maxNbBits; n>0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } } - for (n=0; n<alphabetSize; n++) - tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ - for (n=0; n<alphabetSize; n++) - tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ - } + HUF_buildCTableFromTree(tree, huffNode, nonNullRank, maxSymbolValue, maxNbBits); return maxNbBits; } -/** HUF_buildCTable() : - * @return : maxNbBits - * Note : count is used before tree is written, so they can safely overlap - */ -size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits) -{ - HUF_buildCTable_wksp_tables workspace; - return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, &workspace, sizeof(workspace)); -} - size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { size_t nbBits = 0; @@ -695,7 +798,7 @@ HUF_compress_internal (void* dst, size_t dstSize, CHECK_F(maxBits); huffLog = (U32)maxBits; /* Zero unused symbols in CTable, so we can check it for validity */ - memset(table->CTable + (maxSymbolValue + 1), 0, + ZSTD_memset(table->CTable + (maxSymbolValue + 1), 0, sizeof(table->CTable) - ((maxSymbolValue + 1) * sizeof(HUF_CElt))); } @@ -716,7 +819,7 @@ HUF_compress_internal (void* dst, size_t dstSize, op += hSize; if (repeat) { *repeat = HUF_repeat_none; } if (oldHufTable) - memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */ + ZSTD_memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */ } return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, @@ -747,14 +850,6 @@ size_t HUF_compress1X_repeat (void* dst, size_t dstSize, repeat, preferRepeat, bmi2); } -size_t HUF_compress1X (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - unsigned workSpace[HUF_WORKSPACE_SIZE_U32]; - return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); -} - /* HUF_compress4X_repeat(): * compress input using 4 streams. * provide workspace to generate compression tables */ @@ -784,6 +879,25 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize, hufTable, repeat, preferRepeat, bmi2); } +#ifndef ZSTD_NO_UNUSED_FUNCTIONS +/** HUF_buildCTable() : + * @return : maxNbBits + * Note : count is used before tree is written, so they can safely overlap + */ +size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits) +{ + HUF_buildCTable_wksp_tables workspace; + return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, &workspace, sizeof(workspace)); +} + +size_t HUF_compress1X (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + unsigned workSpace[HUF_WORKSPACE_SIZE_U32]; + return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); +} + size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog) @@ -796,3 +910,4 @@ size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSi { return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT); } +#endif diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3f963b1c..eb7780cf 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -11,8 +11,7 @@ /*-************************************* * Dependencies ***************************************/ -#include <limits.h> /* INT_MAX */ -#include <string.h> /* memset */ +#include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */ #include "../common/cpu.h" #include "../common/mem.h" #include "hist.h" /* HIST_countFast_wksp */ @@ -30,6 +29,19 @@ #include "zstd_ldm.h" #include "zstd_compress_superblock.h" +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * COMPRESS_HEAPMODE : + * Select how default decompression function ZSTD_compress() allocates its context, + * on stack (0, default), or into heap (1). + * Note that functions with explicit context such as ZSTD_compressCCtx() are unaffected. + */ +#ifndef ZSTD_COMPRESS_HEAPMODE +# define ZSTD_COMPRESS_HEAPMODE 0 +#endif + /*-************************************* * Helper functions @@ -52,6 +64,7 @@ size_t ZSTD_compressBound(size_t srcSize) { struct ZSTD_CDict_s { const void* dictContent; size_t dictContentSize; + ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */ U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ ZSTD_cwksp workspace; ZSTD_matchState_t matchState; @@ -69,7 +82,7 @@ ZSTD_CCtx* ZSTD_createCCtx(void) static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) { assert(cctx != NULL); - memset(cctx, 0, sizeof(*cctx)); + ZSTD_memset(cctx, 0, sizeof(*cctx)); cctx->customMem = memManager; cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); @@ -82,8 +95,8 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) { ZSTD_STATIC_ASSERT(zcss_init==0); ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); - if (!customMem.customAlloc ^ !customMem.customFree) return NULL; - { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_customMalloc(sizeof(ZSTD_CCtx), customMem); if (!cctx) return NULL; ZSTD_initCCtx(cctx, customMem); return cctx; @@ -96,20 +109,20 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) ZSTD_CCtx* cctx; if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ - ZSTD_cwksp_init(&ws, workspace, workspaceSize); + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); if (cctx == NULL) return NULL; - memset(cctx, 0, sizeof(ZSTD_CCtx)); + ZSTD_memset(cctx, 0, sizeof(ZSTD_CCtx)); ZSTD_cwksp_move(&cctx->workspace, &ws); cctx->staticSize = workspaceSize; /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ - if (!ZSTD_cwksp_check_available(&cctx->workspace, HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); - cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, HUF_WORKSPACE_SIZE); + cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE); cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); return cctx; } @@ -119,10 +132,10 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) */ static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx) { - ZSTD_free(cctx->localDict.dictBuffer, cctx->customMem); + ZSTD_customFree(cctx->localDict.dictBuffer, cctx->customMem); ZSTD_freeCDict(cctx->localDict.cdict); - memset(&cctx->localDict, 0, sizeof(cctx->localDict)); - memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); + ZSTD_memset(&cctx->localDict, 0, sizeof(cctx->localDict)); + ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); cctx->cdict = NULL; } @@ -153,7 +166,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); ZSTD_freeCCtxContent(cctx); if (!cctxInWorkspace) { - ZSTD_free(cctx, cctx->customMem); + ZSTD_customFree(cctx, cctx->customMem); } } return 0; @@ -189,15 +202,32 @@ size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) /* private API call, for dictBuilder only */ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } +/* Returns 1 if compression parameters are such that we should + * enable long distance matching (wlog >= 27, strategy >= btopt). + * Returns 0 otherwise. + */ +static U32 ZSTD_CParams_shouldEnableLdm(const ZSTD_compressionParameters* const cParams) { + return cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27; +} + static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( ZSTD_compressionParameters cParams) { ZSTD_CCtx_params cctxParams; - memset(&cctxParams, 0, sizeof(cctxParams)); + /* should not matter, as all cParams are presumed properly defined */ + ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT); cctxParams.cParams = cParams; - cctxParams.compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */ + + if (ZSTD_CParams_shouldEnableLdm(&cParams)) { + DEBUGLOG(4, "ZSTD_makeCCtxParamsFromCParams(): Including LDM into cctx params"); + cctxParams.ldmParams.enableLdm = 1; + /* LDM is enabled by default for optimal parser and window size >= 128MB */ + ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams); + assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog); + assert(cctxParams.ldmParams.hashRateLog < 32); + } + assert(!ZSTD_checkCParams(cParams)); - cctxParams.fParams.contentSizeFlag = 1; return cctxParams; } @@ -205,13 +235,12 @@ static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced( ZSTD_customMem customMem) { ZSTD_CCtx_params* params; - if (!customMem.customAlloc ^ !customMem.customFree) return NULL; - params = (ZSTD_CCtx_params*)ZSTD_calloc( + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + params = (ZSTD_CCtx_params*)ZSTD_customCalloc( sizeof(ZSTD_CCtx_params), customMem); if (!params) { return NULL; } + ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); params->customMem = customMem; - params->compressionLevel = ZSTD_CLEVEL_DEFAULT; - params->fParams.contentSizeFlag = 1; return params; } @@ -223,7 +252,7 @@ ZSTD_CCtx_params* ZSTD_createCCtxParams(void) size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) { if (params == NULL) { return 0; } - ZSTD_free(params, params->customMem); + ZSTD_customFree(params, params->customMem); return 0; } @@ -234,7 +263,7 @@ size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) { RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); - memset(cctxParams, 0, sizeof(*cctxParams)); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); cctxParams->compressionLevel = compressionLevel; cctxParams->fParams.contentSizeFlag = 1; return 0; @@ -244,7 +273,7 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete { RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); - memset(cctxParams, 0, sizeof(*cctxParams)); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); assert(!ZSTD_checkCParams(params.cParams)); cctxParams->cParams = params.cParams; cctxParams->fParams = params.fParams; @@ -354,6 +383,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) #endif return bounds; + case ZSTD_c_enableDedicatedDictSearch: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + case ZSTD_c_enableLongDistanceMatching: bounds.lowerBound = 0; bounds.upperBound = 1; @@ -397,7 +431,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) return bounds; case ZSTD_c_forceAttachDict: - ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy); + ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceLoad); bounds.lowerBound = ZSTD_dictDefaultAttach; bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ return bounds; @@ -418,6 +452,22 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) bounds.upperBound = ZSTD_SRCSIZEHINT_MAX; return bounds; + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + + case ZSTD_c_blockDelimiters: + bounds.lowerBound = (int)ZSTD_sf_noBlockDelimiters; + bounds.upperBound = (int)ZSTD_sf_explicitBlockDelimiters; + return bounds; + + case ZSTD_c_validateSequences: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + default: bounds.error = ERROR(parameter_unsupported); return bounds; @@ -465,6 +515,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_jobSize: case ZSTD_c_overlapLog: case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: case ZSTD_c_enableLongDistanceMatching: case ZSTD_c_ldmHashLog: case ZSTD_c_ldmMinMatch: @@ -474,6 +525,10 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_literalCompressionMode: case ZSTD_c_targetCBlockSize: case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: default: return 0; } @@ -515,12 +570,17 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_jobSize: case ZSTD_c_overlapLog: case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: case ZSTD_c_enableLongDistanceMatching: case ZSTD_c_ldmHashLog: case ZSTD_c_ldmMinMatch: case ZSTD_c_ldmBucketSizeLog: case ZSTD_c_targetCBlockSize: case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); @@ -541,9 +601,10 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, case ZSTD_c_compressionLevel : { FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); - if (value) { /* 0 : does not change current level */ + if (value == 0) + CCtxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ + else CCtxParams->compressionLevel = value; - } if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel; return 0; /* return type (size_t) cannot represent negative values */ } @@ -667,6 +728,10 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, return CCtxParams->rsyncable; #endif + case ZSTD_c_enableDedicatedDictSearch : + CCtxParams->enableDedicatedDictSearch = (value!=0); + return CCtxParams->enableDedicatedDictSearch; + case ZSTD_c_enableLongDistanceMatching : CCtxParams->ldmParams.enableLdm = (value!=0); return CCtxParams->ldmParams.enableLdm; @@ -707,6 +772,26 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, CCtxParams->srcSizeHint = value; return CCtxParams->srcSizeHint; + case ZSTD_c_stableInBuffer: + BOUNDCHECK(ZSTD_c_stableInBuffer, value); + CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->inBufferMode; + + case ZSTD_c_stableOutBuffer: + BOUNDCHECK(ZSTD_c_stableOutBuffer, value); + CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->outBufferMode; + + case ZSTD_c_blockDelimiters: + BOUNDCHECK(ZSTD_c_blockDelimiters, value); + CCtxParams->blockDelimiters = (ZSTD_sequenceFormat_e)value; + return CCtxParams->blockDelimiters; + + case ZSTD_c_validateSequences: + BOUNDCHECK(ZSTD_c_validateSequences, value); + CCtxParams->validateSequences = value; + return CCtxParams->validateSequences; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } } @@ -794,6 +879,9 @@ size_t ZSTD_CCtxParams_getParameter( *value = CCtxParams->rsyncable; break; #endif + case ZSTD_c_enableDedicatedDictSearch : + *value = CCtxParams->enableDedicatedDictSearch; + break; case ZSTD_c_enableLongDistanceMatching : *value = CCtxParams->ldmParams.enableLdm; break; @@ -815,6 +903,18 @@ size_t ZSTD_CCtxParams_getParameter( case ZSTD_c_srcSizeHint : *value = (int)CCtxParams->srcSizeHint; break; + case ZSTD_c_stableInBuffer : + *value = (int)CCtxParams->inBufferMode; + break; + case ZSTD_c_stableOutBuffer : + *value = (int)CCtxParams->outBufferMode; + break; + case ZSTD_c_blockDelimiters : + *value = (int)CCtxParams->blockDelimiters; + break; + case ZSTD_c_validateSequences : + *value = (int)CCtxParams->validateSequences; + break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } return 0; @@ -850,6 +950,14 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo return 0; } +static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams( + int const compressionLevel, + size_t const dictSize); +static int ZSTD_dedicatedDictSearch_isSupported( + const ZSTD_compressionParameters* cParams); +static void ZSTD_dedicatedDictSearch_revertCParams( + ZSTD_compressionParameters* cParams); + /** * Initializes the local dict using the requested parameters. * NOTE: This does not use the pledged src size, because it may be used for more @@ -858,8 +966,6 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) { ZSTD_localDict* const dl = &cctx->localDict; - ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams( - &cctx->requestedParams, ZSTD_CONTENTSIZE_UNKNOWN, dl->dictSize); if (dl->dict == NULL) { /* No local dictionary. */ assert(dl->dictBuffer == NULL); @@ -876,12 +982,12 @@ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) assert(cctx->cdict == NULL); assert(cctx->prefixDict.dict == NULL); - dl->cdict = ZSTD_createCDict_advanced( + dl->cdict = ZSTD_createCDict_advanced2( dl->dict, dl->dictSize, ZSTD_dlm_byRef, dl->dictContentType, - cParams, + &cctx->requestedParams, cctx->customMem); RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed"); cctx->cdict = dl->cdict; @@ -894,8 +1000,6 @@ size_t ZSTD_CCtx_loadDictionary_advanced( { RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, "Can't load a dictionary when ctx is not in init stage."); - RETURN_ERROR_IF(cctx->staticSize, memory_allocation, - "no malloc for static CCtx"); DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); ZSTD_clearAllDicts(cctx); /* in case one already exists */ if (dict == NULL || dictSize == 0) /* no dictionary mode */ @@ -903,9 +1007,12 @@ size_t ZSTD_CCtx_loadDictionary_advanced( if (dictLoadMethod == ZSTD_dlm_byRef) { cctx->localDict.dict = dict; } else { - void* dictBuffer = ZSTD_malloc(dictSize, cctx->customMem); + void* dictBuffer; + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "no malloc for static CCtx"); + dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); RETURN_ERROR_IF(!dictBuffer, memory_allocation, "NULL pointer!"); - memcpy(dictBuffer, dict, dictSize); + ZSTD_memcpy(dictBuffer, dict, dictSize); cctx->localDict.dictBuffer = dictBuffer; cctx->localDict.dict = dictBuffer; } @@ -938,6 +1045,14 @@ size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) return 0; } +size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a pool when ctx not in init stage."); + cctx->pool = pool; + return 0; +} + size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) { return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent); @@ -1022,24 +1137,73 @@ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) return hashLog - btScale; } +/** ZSTD_dictAndWindowLog() : + * Returns an adjusted window log that is large enough to fit the source and the dictionary. + * The zstd format says that the entire dictionary is valid if one byte of the dictionary + * is within the window. So the hashLog and chainLog should be large enough to reference both + * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing + * the hashLog and windowLog. + * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN. + */ +static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize) +{ + const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX; + /* No dictionary ==> No change */ + if (dictSize == 0) { + return windowLog; + } + assert(windowLog <= ZSTD_WINDOWLOG_MAX); + assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */ + { + U64 const windowSize = 1ULL << windowLog; + U64 const dictAndWindowSize = dictSize + windowSize; + /* If the window size is already large enough to fit both the source and the dictionary + * then just use the window size. Otherwise adjust so that it fits the dictionary and + * the window. + */ + if (windowSize >= dictSize + srcSize) { + return windowLog; /* Window size large enough already */ + } else if (dictAndWindowSize >= maxWindowSize) { + return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */ + } else { + return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1; + } + } +} + /** ZSTD_adjustCParams_internal() : * optimize `cPar` for a specified input (`srcSize` and `dictSize`). * mostly downsize to reduce memory consumption and initialization latency. * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. + * `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`. * note : `srcSize==0` means 0! * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ static ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, - size_t dictSize) + size_t dictSize, + ZSTD_cParamMode_e mode) { - static const U64 minSrcSize = 513; /* (1<<9) + 1 */ - static const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); + const U64 minSrcSize = 513; /* (1<<9) + 1 */ + const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); assert(ZSTD_checkCParams(cPar)==0); if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN) srcSize = minSrcSize; + switch (mode) { + case ZSTD_cpm_noAttachDict: + case ZSTD_cpm_unknown: + case ZSTD_cpm_createCDict: + break; + case ZSTD_cpm_attachDict: + dictSize = 0; + break; + default: + assert(0); + break; + } + /* resize windowLog if input is small enough, to use less memory */ if ( (srcSize < maxWindowResize) && (dictSize < maxWindowResize) ) { @@ -1049,10 +1213,11 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, ZSTD_highbit32(tSize-1) + 1; if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; } - if (cPar.hashLog > cPar.windowLog+1) cPar.hashLog = cPar.windowLog+1; - { U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); - if (cycleLog > cPar.windowLog) - cPar.chainLog -= (cycleLog - cPar.windowLog); + { U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize); + U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); + if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1; + if (cycleLog > dictAndWindowLog) + cPar.chainLog -= (cycleLog - dictAndWindowLog); } if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) @@ -1068,31 +1233,38 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar, { cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; - return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize); + return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown); } -static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize); -static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize); +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + +static void ZSTD_overrideCParams( + ZSTD_compressionParameters* cParams, + const ZSTD_compressionParameters* overrides) +{ + if (overrides->windowLog) cParams->windowLog = overrides->windowLog; + if (overrides->hashLog) cParams->hashLog = overrides->hashLog; + if (overrides->chainLog) cParams->chainLog = overrides->chainLog; + if (overrides->searchLog) cParams->searchLog = overrides->searchLog; + if (overrides->minMatch) cParams->minMatch = overrides->minMatch; + if (overrides->targetLength) cParams->targetLength = overrides->targetLength; + if (overrides->strategy) cParams->strategy = overrides->strategy; +} ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( - const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize) + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) { ZSTD_compressionParameters cParams; if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { srcSizeHint = CCtxParams->srcSizeHint; } - cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize); + cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; - if (CCtxParams->cParams.windowLog) cParams.windowLog = CCtxParams->cParams.windowLog; - if (CCtxParams->cParams.hashLog) cParams.hashLog = CCtxParams->cParams.hashLog; - if (CCtxParams->cParams.chainLog) cParams.chainLog = CCtxParams->cParams.chainLog; - if (CCtxParams->cParams.searchLog) cParams.searchLog = CCtxParams->cParams.searchLog; - if (CCtxParams->cParams.minMatch) cParams.minMatch = CCtxParams->cParams.minMatch; - if (CCtxParams->cParams.targetLength) cParams.targetLength = CCtxParams->cParams.targetLength; - if (CCtxParams->cParams.strategy) cParams.strategy = CCtxParams->cParams.strategy; + ZSTD_overrideCParams(&cParams, &CCtxParams->cParams); assert(!ZSTD_checkCParams(cParams)); /* srcSizeHint == 0 means 0 */ - return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize); + return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode); } static size_t @@ -1123,45 +1295,61 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, return tableSpace + optSpace; } -size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) +static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( + const ZSTD_compressionParameters* cParams, + const ldmParams_t* ldmParams, + const int isStatic, + const size_t buffInSize, + const size_t buffOutSize, + const U64 pledgedSrcSize) { - RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); - { ZSTD_compressionParameters const cParams = - ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); - U32 const divider = (cParams.minMatch==3) ? 3 : 4; - size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) - + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) - + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); - size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE); - size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); - size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1); + size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << cParams->windowLog), pledgedSrcSize)); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); + U32 const divider = (cParams->minMatch==3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); + size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); + size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, /* forCCtx */ 1); - size_t const ldmSpace = ZSTD_ldm_getTableSize(params->ldmParams); - size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq)); + size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams); + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize); + size_t const ldmSeqSpace = ldmParams->enableLdm ? + ZSTD_cwksp_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; - /* estimateCCtxSize is for one-shot compression. So no buffers should - * be needed. However, we still allocate two 0-sized buffers, which can - * take space under ASAN. */ - size_t const bufferSpace = ZSTD_cwksp_alloc_size(0) - + ZSTD_cwksp_alloc_size(0); - size_t const cctxSpace = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)); + size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + + ZSTD_cwksp_alloc_size(buffOutSize); - size_t const neededSpace = - cctxSpace + - entropySpace + - blockStateSpace + - ldmSpace + - ldmSeqSpace + - matchStateSize + - tokenSpace + - bufferSpace; + size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; - DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); - return neededSpace; - } + size_t const neededSpace = + cctxSpace + + entropySpace + + blockStateSpace + + ldmSpace + + ldmSeqSpace + + matchStateSize + + tokenSpace + + bufferSpace; + + DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); + return neededSpace; +} + +size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + /* estimateCCtxSize is for one-shot compression. So no buffers should + * be needed. However, we still allocate two 0-sized buffers, which can + * take space under ASAN. */ + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN); } size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) @@ -1172,7 +1360,7 @@ size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) { - ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0); + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); return ZSTD_estimateCCtxSize_usingCParams(cParams); } @@ -1191,15 +1379,18 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) { RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); { ZSTD_compressionParameters const cParams = - ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0); - size_t const CCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params); + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); - size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize; - size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; - size_t const streamingSize = ZSTD_cwksp_alloc_size(inBuffSize) - + ZSTD_cwksp_alloc_size(outBuffSize); - - return CCtxSize + streamingSize; + size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) + ? ((size_t)1 << cParams.windowLog) + blockSize + : 0; + size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, inBuffSize, outBuffSize, + ZSTD_CONTENTSIZE_UNKNOWN); } } @@ -1211,7 +1402,7 @@ size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) { - ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0); + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); return ZSTD_estimateCStreamSize_usingCParams(cParams); } @@ -1305,16 +1496,6 @@ static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) } /** - * Indicates whether this compression proceeds directly from user-provided - * source buffer to user-provided destination buffer (ZSTDb_not_buffered), or - * whether the context needs to buffer the input/output (ZSTDb_buffered). - */ -typedef enum { - ZSTDb_not_buffered, - ZSTDb_buffered -} ZSTD_buffered_policy_e; - -/** * Controls, for this matchState reset, whether the tables need to be cleared / * prepared for the coming compression (ZSTDcrp_makeClean), or whether the * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a @@ -1441,45 +1622,32 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) - + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) - + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); - size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0; - size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; - size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); + size_t const buffOutSize = (zbuff == ZSTDb_buffered && params.outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + size_t const buffInSize = (zbuff == ZSTDb_buffered && params.inBufferMode == ZSTD_bm_buffered) + ? windowSize + blockSize + : 0; size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); - ZSTD_indexResetPolicy_e needsIndexReset = zc->initialized ? ZSTDirp_continue : ZSTDirp_reset; + int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); + ZSTD_indexResetPolicy_e needsIndexReset = + (!indexTooClose && zc->initialized) ? ZSTDirp_continue : ZSTDirp_reset; - if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { - needsIndexReset = ZSTDirp_reset; - } + size_t const neededSpace = + ZSTD_estimateCCtxSize_usingCCtxParams_internal( + ¶ms.cParams, ¶ms.ldmParams, zc->staticSize != 0, + buffInSize, buffOutSize, pledgedSrcSize); + FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); /* Check if workspace is large enough, alloc a new one if needed */ - { size_t const cctxSpace = zc->staticSize ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; - size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE); - size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); - size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + ZSTD_cwksp_alloc_size(buffOutSize); - size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams); - size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(maxNbLdmSeq * sizeof(rawSeq)); - - size_t const neededSpace = - cctxSpace + - entropySpace + - blockStateSpace + - ldmSpace + - ldmSeqSpace + - matchStateSize + - tokenSpace + - bufferSpace; - + { int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); - DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", - neededSpace>>10, matchStateSize>>10, bufferSpace>>10); + DEBUGLOG(4, "Need %zu B workspace", neededSpace); DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); if (workspaceTooSmall || workspaceWasteful) { @@ -1503,7 +1671,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); - zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, HUF_WORKSPACE_SIZE); + zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE); RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); } } @@ -1534,6 +1702,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->seqStore.maxNbLit = blockSize; /* buffers */ + zc->bufferedPolicy = zbuff; zc->inBuffSize = buffInSize; zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); zc->outBuffSize = buffOutSize; @@ -1546,7 +1715,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ((size_t)1) << (params.ldmParams.hashLog - params.ldmParams.bucketSizeLog); zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, ldmBucketSize); - memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); + ZSTD_memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); } /* sequences storage */ @@ -1570,7 +1739,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* TODO: avoid memset? */ size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); - memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); + ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); zc->maxNbLdmSequences = maxNbLdmSeq; @@ -1579,6 +1748,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->ldmState.loadedDictEnd = 0; } + /* Due to alignment, when reusing a workspace, we can actually consume + * up to 3 extra bytes for alignment. See the comments in zstd_cwksp.h + */ + assert(ZSTD_cwksp_used(ws) >= neededSpace && + ZSTD_cwksp_used(ws) <= neededSpace + 3); + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); zc->initialized = 1; @@ -1618,12 +1793,14 @@ static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, U64 pledgedSrcSize) { size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; - return ( pledgedSrcSize <= cutoff - || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN - || params->attachDictPref == ZSTD_dictForceAttach ) - && params->attachDictPref != ZSTD_dictForceCopy - && !params->forceWindow; /* dictMatchState isn't correctly - * handled in _enforceMaxDist */ + int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch; + return dedicatedDictSearch + || ( ( pledgedSrcSize <= cutoff + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || params->attachDictPref == ZSTD_dictForceAttach ) + && params->attachDictPref != ZSTD_dictForceCopy + && !params->forceWindow ); /* dictMatchState isn't correctly + * handled in _enforceMaxDist */ } static size_t @@ -1633,17 +1810,24 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { - { const ZSTD_compressionParameters* const cdict_cParams = &cdict->matchState.cParams; + { + ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; unsigned const windowLog = params.cParams.windowLog; assert(windowLog != 0); /* Resize working context table params for input only, since the dict * has its own tables. */ - /* pledgeSrcSize == 0 means 0! */ - params.cParams = ZSTD_adjustCParams_internal(*cdict_cParams, pledgedSrcSize, 0); + /* pledgedSrcSize == 0 means 0! */ + + if (cdict->matchState.dedicatedDictSearch) { + ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams); + } + + params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, + cdict->dictContentSize, ZSTD_cpm_attachDict); params.cParams.windowLog = windowLog; FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, ZSTDcrp_makeClean, zbuff), ""); - assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); + assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); } { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc @@ -1670,7 +1854,7 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, cctx->dictID = cdict->dictID; /* copy block state */ - memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); return 0; } @@ -1683,6 +1867,8 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, { const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; + assert(!cdict->matchState.dedicatedDictSearch); + DEBUGLOG(4, "copying dictionary into context"); { unsigned const windowLog = params.cParams.windowLog; @@ -1703,10 +1889,10 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, { size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog); size_t const hSize = (size_t)1 << cdict_cParams->hashLog; - memcpy(cctx->blockState.matchState.hashTable, + ZSTD_memcpy(cctx->blockState.matchState.hashTable, cdict->matchState.hashTable, hSize * sizeof(U32)); - memcpy(cctx->blockState.matchState.chainTable, + ZSTD_memcpy(cctx->blockState.matchState.chainTable, cdict->matchState.chainTable, chainSize * sizeof(U32)); } @@ -1715,7 +1901,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, { int const h3log = cctx->blockState.matchState.hashLog3; size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; assert(cdict->matchState.hashLog3 == 0); - memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); + ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); } ZSTD_cwksp_mark_tables_clean(&cctx->workspace); @@ -1731,7 +1917,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, cctx->dictID = cdict->dictID; /* copy block state */ - memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); return 0; } @@ -1775,7 +1961,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong, "Can't copy a ctx that's not in init stage."); - memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); + ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); { ZSTD_CCtx_params params = dstCCtx->requestedParams; /* Copy only compression parameters related to tables. */ params.cParams = srcCCtx->appliedParams.cParams; @@ -1797,13 +1983,13 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, int const h3log = srcCCtx->blockState.matchState.hashLog3; size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; - memcpy(dstCCtx->blockState.matchState.hashTable, + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable, srcCCtx->blockState.matchState.hashTable, hSize * sizeof(U32)); - memcpy(dstCCtx->blockState.matchState.chainTable, + ZSTD_memcpy(dstCCtx->blockState.matchState.chainTable, srcCCtx->blockState.matchState.chainTable, chainSize * sizeof(U32)); - memcpy(dstCCtx->blockState.matchState.hashTable3, + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable3, srcCCtx->blockState.matchState.hashTable3, h3Size * sizeof(U32)); } @@ -1821,7 +2007,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, dstCCtx->dictID = srcCCtx->dictID; /* copy block state */ - memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock)); + ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock)); return 0; } @@ -1834,7 +2020,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) { ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; - ZSTD_buffered_policy_e const zbuff = (ZSTD_buffered_policy_e)(srcCCtx->inBuffSize>0); + ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy; ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN); @@ -1861,7 +2047,7 @@ ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerVa assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ assert(size < (1U<<31)); /* can be casted to int */ -#if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) /* To validate that the table re-use logic is sound, and that we don't * access table space that we haven't cleaned, we re-"poison" the table * space every time we mark it dirty. @@ -1958,10 +2144,10 @@ static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams) return (cctxParams->targetCBlockSize != 0); } -/* ZSTD_compressSequences_internal(): +/* ZSTD_entropyCompressSequences_internal(): * actually compresses both literals and sequences */ MEM_STATIC size_t -ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, +ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr, const ZSTD_entropyCTables_t* prevEntropy, ZSTD_entropyCTables_t* nextEntropy, const ZSTD_CCtx_params* cctxParams, @@ -1971,7 +2157,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, { const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; ZSTD_strategy const strategy = cctxParams->cParams.strategy; - unsigned count[MaxSeq+1]; + unsigned* count = (unsigned*)entropyWorkspace; FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; @@ -1987,8 +2173,12 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, BYTE* seqHead; BYTE* lastNCount = NULL; - DEBUGLOG(5, "ZSTD_compressSequences_internal (nbSeq=%zu)", nbSeq); + entropyWorkspace = count + (MaxSeq + 1); + entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); + + DEBUGLOG(4, "ZSTD_entropyCompressSequences_internal (nbSeq=%zu)", nbSeq); ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog))); + assert(entropyWkspSize >= HUF_WORKSPACE_SIZE); /* Compress literals */ { const BYTE* const literals = seqStorePtr->litStart; @@ -2023,7 +2213,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, assert(op <= oend); if (nbSeq==0) { /* Copy the old tables over as if we repeated them */ - memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); + ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); return (size_t)(op - ostart); } @@ -2148,7 +2338,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, } MEM_STATIC size_t -ZSTD_compressSequences(seqStore_t* seqStorePtr, +ZSTD_entropyCompressSequences(seqStore_t* seqStorePtr, const ZSTD_entropyCTables_t* prevEntropy, ZSTD_entropyCTables_t* nextEntropy, const ZSTD_CCtx_params* cctxParams, @@ -2157,7 +2347,7 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr, void* entropyWorkspace, size_t entropyWkspSize, int bmi2) { - size_t const cSize = ZSTD_compressSequences_internal( + size_t const cSize = ZSTD_entropyCompressSequences_internal( seqStorePtr, prevEntropy, nextEntropy, cctxParams, dst, dstCapacity, entropyWorkspace, entropyWkspSize, bmi2); @@ -2167,13 +2357,13 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr, */ if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) return 0; /* block not compressed */ - FORWARD_IF_ERROR(cSize, "ZSTD_compressSequences_internal failed"); + FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSequences_internal failed"); /* Check compressibility */ { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); if (cSize >= maxCSize) return 0; /* block not compressed */ } - + DEBUGLOG(4, "ZSTD_entropyCompressSequences() cSize: %zu\n", cSize); return cSize; } @@ -2182,7 +2372,7 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr, * assumption : strat is a valid strategy */ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode) { - static const ZSTD_blockCompressor blockCompressor[3][ZSTD_STRATEGY_MAX+1] = { + static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { { ZSTD_compressBlock_fast /* default for 0 */, ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, @@ -2212,7 +2402,17 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo ZSTD_compressBlock_btlazy2_dictMatchState, ZSTD_compressBlock_btopt_dictMatchState, ZSTD_compressBlock_btultra_dictMatchState, - ZSTD_compressBlock_btultra_dictMatchState } + ZSTD_compressBlock_btultra_dictMatchState }, + { NULL /* default for 0 */, + NULL, + NULL, + ZSTD_compressBlock_greedy_dedicatedDictSearch, + ZSTD_compressBlock_lazy_dedicatedDictSearch, + ZSTD_compressBlock_lazy2_dedicatedDictSearch, + NULL, + NULL, + NULL, + NULL } }; ZSTD_blockCompressor selectedCompressor; ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); @@ -2226,7 +2426,7 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr, const BYTE* anchor, size_t lastLLSize) { - memcpy(seqStorePtr->lit, anchor, lastLLSize); + ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; } @@ -2247,7 +2447,11 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) /* Assert that we have correctly flushed the ctx params into the ms's copy */ ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { - ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); + if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { + ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); + } else { + ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); + } return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ } ZSTD_resetSeqStore(&(zc->seqStore)); @@ -2263,10 +2467,10 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) /* limited update after a very long match */ { const BYTE* const base = ms->window.base; const BYTE* const istart = (const BYTE*)src; - const U32 current = (U32)(istart-base); + const U32 curr = (U32)(istart-base); if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */ - if (current > ms->nextToUpdate + 384) - ms->nextToUpdate = current - MIN(192, (U32)(current - ms->nextToUpdate - 384)); + if (curr > ms->nextToUpdate + 384) + ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); } /* select and store sequences */ @@ -2286,7 +2490,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) src, srcSize); assert(zc->externSeqStore.pos <= zc->externSeqStore.size); } else if (zc->appliedParams.ldmParams.enableLdm) { - rawSeqStore_t ldmSeqStore = {NULL, 0, 0, 0}; + rawSeqStore_t ldmSeqStore = kNullRawSeqStore; ldmSeqStore.seq = zc->ldmSequences; ldmSeqStore.capacity = zc->maxNbLdmSequences; @@ -2303,6 +2507,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) assert(ldmSeqStore.pos == ldmSeqStore.size); } else { /* not long range mode */ ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, dictMode); + ms->ldmSeqStore = NULL; lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); } { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; @@ -2314,17 +2519,25 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) { const seqStore_t* seqStore = ZSTD_getSeqStore(zc); - const seqDef* seqs = seqStore->sequencesStart; - size_t seqsSize = seqStore->sequences - seqs; + const seqDef* seqStoreSeqs = seqStore->sequencesStart; + size_t seqStoreSeqSize = seqStore->sequences - seqStoreSeqs; + size_t seqStoreLiteralsSize = (size_t)(seqStore->lit - seqStore->litStart); + size_t literalsRead = 0; + size_t lastLLSize; ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; - size_t i; size_t position; int repIdx; + size_t i; + repcodes_t updatedRepcodes; assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); - for (i = 0, position = 0; i < seqsSize; ++i) { - outSeqs[i].offset = seqs[i].offset; - outSeqs[i].litLength = seqs[i].litLength; - outSeqs[i].matchLength = seqs[i].matchLength + MINMATCH; + /* Ensure we have enough space for last literals "sequence" */ + assert(zc->seqCollector.maxSequences >= seqStoreSeqSize + 1); + ZSTD_memcpy(updatedRepcodes.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); + for (i = 0; i < seqStoreSeqSize; ++i) { + U32 rawOffset = seqStoreSeqs[i].offset - ZSTD_REP_NUM; + outSeqs[i].litLength = seqStoreSeqs[i].litLength; + outSeqs[i].matchLength = seqStoreSeqs[i].matchLength + MINMATCH; + outSeqs[i].rep = 0; if (i == seqStore->longLengthPos) { if (seqStore->longLengthID == 1) { @@ -2334,39 +2547,44 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) } } - if (outSeqs[i].offset <= ZSTD_REP_NUM) { - outSeqs[i].rep = outSeqs[i].offset; - repIdx = (unsigned int)i - outSeqs[i].offset; - - if (outSeqs[i].litLength == 0) { - if (outSeqs[i].offset < 3) { - --repIdx; + if (seqStoreSeqs[i].offset <= ZSTD_REP_NUM) { + /* Derive the correct offset corresponding to a repcode */ + outSeqs[i].rep = seqStoreSeqs[i].offset; + if (outSeqs[i].litLength != 0) { + rawOffset = updatedRepcodes.rep[outSeqs[i].rep - 1]; + } else { + if (outSeqs[i].rep == 3) { + rawOffset = updatedRepcodes.rep[0] - 1; } else { - repIdx = (unsigned int)i - 1; + rawOffset = updatedRepcodes.rep[outSeqs[i].rep]; } - ++outSeqs[i].rep; - } - assert(repIdx >= -3); - outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1]; - if (outSeqs[i].rep == 4) { - --outSeqs[i].offset; } - } else { - outSeqs[i].offset -= ZSTD_REP_NUM; } - - position += outSeqs[i].litLength; - outSeqs[i].matchPos = (unsigned int)position; - position += outSeqs[i].matchLength; + outSeqs[i].offset = rawOffset; + /* seqStoreSeqs[i].offset == offCode+1, and ZSTD_updateRep() expects offCode + so we provide seqStoreSeqs[i].offset - 1 */ + updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, + seqStoreSeqs[i].offset - 1, + seqStoreSeqs[i].litLength == 0); + literalsRead += outSeqs[i].litLength; } - zc->seqCollector.seqIndex += seqsSize; + /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. + * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker + * for the block boundary, according to the API. + */ + assert(seqStoreLiteralsSize >= literalsRead); + lastLLSize = seqStoreLiteralsSize - literalsRead; + outSeqs[i].litLength = (U32)lastLLSize; + outSeqs[i].matchLength = outSeqs[i].offset = outSeqs[i].rep = 0; + seqStoreSeqSize++; + zc->seqCollector.seqIndex += seqStoreSeqSize; } -size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, - size_t outSeqsSize, const void* src, size_t srcSize) +size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize) { const size_t dstCapacity = ZSTD_compressBound(srcSize); - void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); + void* dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem); SeqCollector seqCollector; RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!"); @@ -2378,16 +2596,47 @@ size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, zc->seqCollector = seqCollector; ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); - ZSTD_free(dst, ZSTD_defaultCMem); + ZSTD_customFree(dst, ZSTD_defaultCMem); return zc->seqCollector.seqIndex; } -/* Returns true if the given block is a RLE block */ -static int ZSTD_isRLE(const BYTE *ip, size_t length) { +size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize) { + size_t in = 0; + size_t out = 0; + for (; in < seqsSize; ++in) { + if (sequences[in].offset == 0 && sequences[in].matchLength == 0) { + if (in != seqsSize - 1) { + sequences[in+1].litLength += sequences[in].litLength; + } + } else { + sequences[out] = sequences[in]; + ++out; + } + } + return out; +} + +/* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */ +static int ZSTD_isRLE(const BYTE* src, size_t length) { + const BYTE* ip = src; + const BYTE value = ip[0]; + const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL); + const size_t unrollSize = sizeof(size_t) * 4; + const size_t unrollMask = unrollSize - 1; + const size_t prefixLength = length & unrollMask; size_t i; - if (length < 2) return 1; - for (i = 1; i < length; ++i) { - if (ip[0] != ip[i]) return 0; + size_t u; + if (length == 1) return 1; + /* Check if prefix is RLE first before using unrolled loop */ + if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { + return 0; + } + for (i = prefixLength; i != length; i += unrollSize) { + for (u = 0; u < unrollSize; u += sizeof(size_t)) { + if (MEM_readST(ip + i + u) != valueST) { + return 0; + } + } } return 1; } @@ -2434,18 +2683,25 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, if (zc->seqCollector.collectSequences) { ZSTD_copyBlockSequences(zc); + ZSTD_confirmRepcodesAndEntropyTables(zc); return 0; } /* encode sequences and literals */ - cSize = ZSTD_compressSequences(&zc->seqStore, + cSize = ZSTD_entropyCompressSequences(&zc->seqStore, &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, &zc->appliedParams, dst, dstCapacity, srcSize, - zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, zc->bmi2); + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + return 0; + } + + if (frame && /* We don't want to emit our first block as a RLE even if it qualifies because * doing so will cause the decoder (cli only) to throw a "should consume all input error." @@ -2593,7 +2849,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); - DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); + DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); if (cctx->appliedParams.fParams.checksumFlag && srcSize) XXH64_update(&cctx->xxhState, src, srcSize); @@ -2673,7 +2929,6 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, "dst buf is too small to fit worst-case frame header size."); DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); - if (params->format == ZSTD_f_zstd1) { MEM_writeLE32(dst, ZSTD_MAGICNUMBER); pos = 4; @@ -2725,6 +2980,7 @@ size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSe cctx->externSeqStore.size = nbSeq; cctx->externSeqStore.capacity = nbSeq; cctx->externSeqStore.pos = 0; + cctx->externSeqStore.posInSequence = 0; return 0; } @@ -2862,8 +3118,12 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, case ZSTD_greedy: case ZSTD_lazy: case ZSTD_lazy2: - if (chunk >= HASH_READ_SIZE) + if (chunk >= HASH_READ_SIZE && ms->dedicatedDictSearch) { + assert(chunk == remaining); /* must load everything in one go */ + ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, ichunk-HASH_READ_SIZE); + } else if (chunk >= HASH_READ_SIZE) { ZSTD_insertAndFindFirstIndex(ms, ichunk-HASH_READ_SIZE); + } break; case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ @@ -2887,22 +3147,28 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, /* Dictionaries that assign zero probability to symbols that show up causes problems - when FSE encoding. Refuse dictionaries that assign zero probability to symbols - that we may encounter during compression. - NOTE: This behavior is not standard and could be improved in the future. */ -static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) { + * when FSE encoding. Mark dictionaries with zero probability symbols as FSE_repeat_check + * and only dictionaries with 100% valid symbols can be assumed valid. + */ +static FSE_repeat ZSTD_dictNCountRepeat(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) +{ U32 s; - RETURN_ERROR_IF(dictMaxSymbolValue < maxSymbolValue, dictionary_corrupted, "dict fse tables don't have all symbols"); + if (dictMaxSymbolValue < maxSymbolValue) { + return FSE_repeat_check; + } for (s = 0; s <= maxSymbolValue; ++s) { - RETURN_ERROR_IF(normalizedCounter[s] == 0, dictionary_corrupted, "dict fse tables don't have all symbols"); + if (normalizedCounter[s] == 0) { + return FSE_repeat_check; + } } - return 0; + return FSE_repeat_valid; } size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, - short* offcodeNCount, unsigned* offcodeMaxValue, const void* const dict, size_t dictSize) { + short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff; const BYTE* dictPtr = (const BYTE*)dict; /* skip magic num and dict ID */ const BYTE* const dictEnd = dictPtr + dictSize; dictPtr += 8; @@ -2924,16 +3190,16 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, } { unsigned offcodeLog; - size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); - /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ /* fill all offset symbols to avoid garbage at end of table */ RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( bs->entropy.fse.offcodeCTable, offcodeNCount, MaxOff, offcodeLog, workspace, HUF_WORKSPACE_SIZE)), dictionary_corrupted, ""); + /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ dictPtr += offcodeHeaderSize; } @@ -2942,13 +3208,12 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); - /* Every match length code must have non-zero probability */ - FORWARD_IF_ERROR( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML), ""); RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( bs->entropy.fse.matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, workspace, HUF_WORKSPACE_SIZE)), dictionary_corrupted, ""); + bs->entropy.fse.matchlength_repeatMode = ZSTD_dictNCountRepeat(matchlengthNCount, matchlengthMaxValue, MaxML); dictPtr += matchlengthHeaderSize; } @@ -2957,13 +3222,12 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); - /* Every literal length code must have non-zero probability */ - FORWARD_IF_ERROR( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL), ""); RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( bs->entropy.fse.litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, workspace, HUF_WORKSPACE_SIZE)), dictionary_corrupted, ""); + bs->entropy.fse.litlength_repeatMode = ZSTD_dictNCountRepeat(litlengthNCount, litlengthMaxValue, MaxLL); dictPtr += litlengthHeaderSize; } @@ -2973,6 +3237,22 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, bs->rep[2] = MEM_readLE32(dictPtr+8); dictPtr += 12; + { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + U32 offcodeMax = MaxOff; + if (dictContentSize <= ((U32)-1) - 128 KB) { + U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ + offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ + } + /* All offset values <= dictContentSize + 128 KB must be representable for a valid table */ + bs->entropy.fse.offcode_repeatMode = ZSTD_dictNCountRepeat(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)); + + /* All repCodes must be <= dictContentSize and != 0 */ + { U32 u; + for (u=0; u<3; u++) { + RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, ""); + RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); + } } } + return dictPtr - (const BYTE*)dict; } @@ -2995,8 +3275,6 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, { const BYTE* dictPtr = (const BYTE*)dict; const BYTE* const dictEnd = dictPtr + dictSize; - short offcodeNCount[MaxOff+1]; - unsigned offcodeMaxValue = MaxOff; size_t dictID; size_t eSize; @@ -3005,32 +3283,16 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr + 4 /* skip magic number */ ); - eSize = ZSTD_loadCEntropy(bs, workspace, offcodeNCount, &offcodeMaxValue, dict, dictSize); + eSize = ZSTD_loadCEntropy(bs, workspace, dict, dictSize); FORWARD_IF_ERROR(eSize, "ZSTD_loadCEntropy failed"); dictPtr += eSize; - { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); - U32 offcodeMax = MaxOff; - if (dictContentSize <= ((U32)-1) - 128 KB) { - U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ - offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ - } - /* All offset values <= dictContentSize + 128 KB must be representable */ - FORWARD_IF_ERROR(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)), ""); - /* All repCodes must be <= dictContentSize and != 0*/ - { U32 u; - for (u=0; u<3; u++) { - RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, ""); - RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); - } } - - bs->entropy.fse.offcode_repeatMode = FSE_repeat_valid; - bs->entropy.fse.matchlength_repeatMode = FSE_repeat_valid; - bs->entropy.fse.litlength_repeatMode = FSE_repeat_valid; + { + size_t const dictContentSize = (size_t)(dictEnd - dictPtr); FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( ms, NULL, ws, params, dictPtr, dictContentSize, dtlm), ""); - return dictID; } + return dictID; } /** ZSTD_compress_insertDictionary() : @@ -3074,7 +3336,7 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, } #define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) -#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6) +#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL) /*! ZSTD_compressBegin_internal() : * @return : 0, or an error code */ @@ -3106,7 +3368,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, - cdict->dictContentSize, dictContentType, dtlm, + cdict->dictContentSize, cdict->dictContentType, dtlm, cctx->entropyWorkspace) : ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, @@ -3153,7 +3415,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); + ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, ¶ms); DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); @@ -3234,7 +3496,6 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, return cSize + endResult; } - static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -3287,7 +3548,7 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0); + ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict); ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, ¶ms); DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); assert(params.fParams.contentSizeFlag == 1); @@ -3309,10 +3570,17 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, int compressionLevel) { size_t result; +#if ZSTD_COMPRESS_HEAPMODE + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + RETURN_ERROR_IF(!cctx, memory_allocation, "ZSTD_createCCtx failed"); + result = ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel); + ZSTD_freeCCtx(cctx); +#else ZSTD_CCtx ctxBody; ZSTD_initCCtx(&ctxBody, ZSTD_defaultCMem); result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); ZSTD_freeCCtxContent(&ctxBody); /* can't free ctxBody itself, as it's on stack; free only heap content */ +#endif return result; } @@ -3335,7 +3603,7 @@ size_t ZSTD_estimateCDictSize_advanced( size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) { - ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); } @@ -3353,20 +3621,25 @@ static size_t ZSTD_initCDict_internal( const void* dictBuffer, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, - ZSTD_compressionParameters cParams) + ZSTD_CCtx_params params) { DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType); - assert(!ZSTD_checkCParams(cParams)); - cdict->matchState.cParams = cParams; + assert(!ZSTD_checkCParams(params.cParams)); + cdict->matchState.cParams = params.cParams; + cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch; + if (cdict->matchState.dedicatedDictSearch && dictSize > ZSTD_CHUNKSIZE_MAX) { + cdict->matchState.dedicatedDictSearch = 0; + } if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { cdict->dictContent = dictBuffer; } else { void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!"); cdict->dictContent = internalBuffer; - memcpy(internalBuffer, dictBuffer, dictSize); + ZSTD_memcpy(internalBuffer, dictBuffer, dictSize); } cdict->dictContentSize = dictSize; + cdict->dictContentType = dictContentType; cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); @@ -3376,18 +3649,15 @@ static size_t ZSTD_initCDict_internal( FORWARD_IF_ERROR(ZSTD_reset_matchState( &cdict->matchState, &cdict->workspace, - &cParams, + ¶ms.cParams, ZSTDcrp_makeClean, ZSTDirp_reset, ZSTD_resetTarget_CDict), ""); /* (Maybe) load the dictionary * Skips loading the dictionary if it is < 8 bytes. */ - { ZSTD_CCtx_params params; - memset(¶ms, 0, sizeof(params)); - params.compressionLevel = ZSTD_CLEVEL_DEFAULT; + { params.compressionLevel = ZSTD_CLEVEL_DEFAULT; params.fParams.contentSizeFlag = 1; - params.cParams = cParams; { size_t const dictID = ZSTD_compress_insertDictionary( &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, ¶ms, cdict->dictContent, cdict->dictContentSize, @@ -3401,13 +3671,11 @@ static size_t ZSTD_initCDict_internal( return 0; } -ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, +static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams, ZSTD_customMem customMem) { - DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (unsigned)dictContentType); - if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; { size_t const workspaceSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + @@ -3415,16 +3683,16 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); - void* const workspace = ZSTD_malloc(workspaceSize, customMem); + void* const workspace = ZSTD_customMalloc(workspaceSize, customMem); ZSTD_cwksp ws; ZSTD_CDict* cdict; if (!workspace) { - ZSTD_free(workspace, customMem); + ZSTD_customFree(workspace, customMem); return NULL; } - ZSTD_cwksp_init(&ws, workspace, workspaceSize); + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_dynamic_alloc); cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); assert(cdict != NULL); @@ -3432,35 +3700,94 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, cdict->customMem = customMem; cdict->compressionLevel = 0; /* signals advanced API usage */ - if (ZSTD_isError( ZSTD_initCDict_internal(cdict, - dictBuffer, dictSize, - dictLoadMethod, dictContentType, - cParams) )) { - ZSTD_freeCDict(cdict); - return NULL; - } - return cdict; } } +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params cctxParams; + ZSTD_memset(&cctxParams, 0, sizeof(cctxParams)); + ZSTD_CCtxParams_init(&cctxParams, 0); + cctxParams.cParams = cParams; + cctxParams.customMem = customMem; + return ZSTD_createCDict_advanced2( + dictBuffer, dictSize, + dictLoadMethod, dictContentType, + &cctxParams, customMem); +} + +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* originalCctxParams, + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params cctxParams = *originalCctxParams; + ZSTD_compressionParameters cParams; + ZSTD_CDict* cdict; + + DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType); + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + + if (cctxParams.enableDedicatedDictSearch) { + cParams = ZSTD_dedicatedDictSearch_getCParams( + cctxParams.compressionLevel, dictSize); + ZSTD_overrideCParams(&cParams, &cctxParams.cParams); + } else { + cParams = ZSTD_getCParamsFromCCtxParams( + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + } + + if (!ZSTD_dedicatedDictSearch_isSupported(&cParams)) { + /* Fall back to non-DDSS params */ + cctxParams.enableDedicatedDictSearch = 0; + cParams = ZSTD_getCParamsFromCCtxParams( + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + } + + cctxParams.cParams = cParams; + + cdict = ZSTD_createCDict_advanced_internal(dictSize, + dictLoadMethod, cctxParams.cParams, + customMem); + + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + dictLoadMethod, dictContentType, + cctxParams) )) { + ZSTD_freeCDict(cdict); + return NULL; + } + + return cdict; +} + ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); - ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); if (cdict) - cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; return cdict; } ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); - return ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; } size_t ZSTD_freeCDict(ZSTD_CDict* cdict) @@ -3470,7 +3797,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict); ZSTD_cwksp_free(&cdict->workspace, cMem); if (!cdictInWorkspace) { - ZSTD_free(cdict, cMem); + ZSTD_customFree(cdict, cMem); } return 0; } @@ -3503,12 +3830,13 @@ const ZSTD_CDict* ZSTD_initStaticCDict( + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + matchStateSize; ZSTD_CDict* cdict; + ZSTD_CCtx_params params; if ((size_t)workspace & 7) return NULL; /* 8-aligned */ { ZSTD_cwksp ws; - ZSTD_cwksp_init(&ws, workspace, workspaceSize); + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); if (cdict == NULL) return NULL; ZSTD_cwksp_move(&cdict->workspace, &ws); @@ -3518,10 +3846,13 @@ const ZSTD_CDict* ZSTD_initStaticCDict( (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); if (workspaceSize < neededSize) return NULL; + ZSTD_CCtxParams_init(¶ms, 0); + params.cParams = cParams; + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, dictLoadMethod, dictContentType, - cParams) )) + params) )) return NULL; return cdict; @@ -3533,6 +3864,17 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict) return cdict->matchState.cParams; } +/*! ZSTD_getDictID_fromCDict() : + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; + return cdict->dictID; +} + + /* ZSTD_compressBegin_usingCDict_advanced() : * cdict must be != NULL */ size_t ZSTD_compressBegin_usingCDict_advanced( @@ -3640,32 +3982,12 @@ size_t ZSTD_CStreamOutSize(void) return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } -static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx, - const void* const dict, size_t const dictSize, ZSTD_dictContentType_e const dictContentType, - const ZSTD_CDict* const cdict, - ZSTD_CCtx_params params, unsigned long long const pledgedSrcSize) +static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize) { - DEBUGLOG(4, "ZSTD_resetCStream_internal"); - /* Finalize the compression parameters */ - params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, pledgedSrcSize, dictSize); - /* params are supposed to be fully validated at this point */ - assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); - assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - - FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, - dict, dictSize, dictContentType, ZSTD_dtlm_fast, - cdict, - ¶ms, pledgedSrcSize, - ZSTDb_buffered) , ""); - - cctx->inToCompress = 0; - cctx->inBuffPos = 0; - cctx->inBuffTarget = cctx->blockSize - + (cctx->blockSize == pledgedSrcSize); /* for small input: avoid automatic flush on reaching end of block, since it would require to add a 3-bytes null block to end frame */ - cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0; - cctx->streamStage = zcss_load; - cctx->frameEnded = 0; - return 0; /* ready to go */ + if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) + return ZSTD_cpm_attachDict; + else + return ZSTD_cpm_noAttachDict; } /* ZSTD_resetCStream(): @@ -3815,12 +4137,17 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, /* check expectations */ DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode); - assert(zcs->inBuff != NULL); - assert(zcs->inBuffSize > 0); - assert(zcs->outBuff != NULL); - assert(zcs->outBuffSize > 0); + if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { + assert(zcs->inBuff != NULL); + assert(zcs->inBuffSize > 0); + } + if (zcs->appliedParams.outBufferMode == ZSTD_bm_buffered) { + assert(zcs->outBuff != NULL); + assert(zcs->outBuffSize > 0); + } assert(output->pos <= output->size); assert(input->pos <= input->size); + assert((U32)flushMode <= (U32)ZSTD_e_end); while (someMoreWork) { switch(zcs->streamStage) @@ -3830,7 +4157,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, case zcss_load: if ( (flushMode == ZSTD_e_end) - && ((size_t)(oend-op) >= ZSTD_compressBound(iend-ip)) /* enough dstCapacity */ + && ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */ + || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */ && (zcs->inBuffPos == 0) ) { /* shortcut to compression pass directly into output buffer */ size_t const cSize = ZSTD_compressEnd(zcs, @@ -3843,8 +4171,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); someMoreWork = 0; break; } - /* complete loading into inBuffer */ - { size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; + /* complete loading into inBuffer in buffered mode */ + if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { + size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; size_t const loaded = ZSTD_limitCopy( zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip); @@ -3864,31 +4193,49 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, } /* compress current block (note : this stage cannot be stopped in the middle) */ DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); - { void* cDst; + { int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered); + void* cDst; size_t cSize; - size_t const iSize = zcs->inBuffPos - zcs->inToCompress; size_t oSize = oend-op; - unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); - if (oSize >= ZSTD_compressBound(iSize)) + size_t const iSize = inputBuffered + ? zcs->inBuffPos - zcs->inToCompress + : MIN((size_t)(iend - ip), zcs->blockSize); + if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) cDst = op; /* compress into output buffer, to skip flush stage */ else cDst = zcs->outBuff, oSize = zcs->outBuffSize; - cSize = lastBlock ? - ZSTD_compressEnd(zcs, cDst, oSize, - zcs->inBuff + zcs->inToCompress, iSize) : - ZSTD_compressContinue(zcs, cDst, oSize, - zcs->inBuff + zcs->inToCompress, iSize); - FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); - zcs->frameEnded = lastBlock; - /* prepare next block */ - zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; - if (zcs->inBuffTarget > zcs->inBuffSize) - zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; - DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", - (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); - if (!lastBlock) - assert(zcs->inBuffTarget <= zcs->inBuffSize); - zcs->inToCompress = zcs->inBuffPos; + if (inputBuffered) { + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); + cSize = lastBlock ? + ZSTD_compressEnd(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize); + FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); + zcs->frameEnded = lastBlock; + /* prepare next block */ + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + if (zcs->inBuffTarget > zcs->inBuffSize) + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; + DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", + (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); + if (!lastBlock) + assert(zcs->inBuffTarget <= zcs->inBuffSize); + zcs->inToCompress = zcs->inBuffPos; + } else { + unsigned const lastBlock = (ip + iSize == iend); + assert(flushMode == ZSTD_e_end /* Already validated */); + cSize = lastBlock ? + ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) : + ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize); + /* Consume the input prior to error checking to mirror buffered mode. */ + if (iSize > 0) + ip += iSize; + FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); + zcs->frameEnded = lastBlock; + if (lastBlock) + assert(ip == iend); + } if (cDst == op) { /* no need to flush */ op += cSize; if (zcs->frameEnded) { @@ -3905,6 +4252,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, /* fall-through */ case zcss_flush: DEBUGLOG(5, "flush stage"); + assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered); { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op), zcs->outBuff + zcs->outBuffFlushedSize, toFlush); @@ -3959,6 +4307,116 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf return ZSTD_nextInputSizeHint_MTorST(zcs); } +/* After a compression call set the expected input/output buffer. + * This is validated at the start of the next compression call. + */ +static void ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, ZSTD_outBuffer const* output, ZSTD_inBuffer const* input) +{ + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + cctx->expectedInBuffer = *input; + } + if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { + cctx->expectedOutBufferSize = output->size - output->pos; + } +} + +/* Validate that the input/output buffers match the expectations set by + * ZSTD_setBufferExpectations. + */ +static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx, + ZSTD_outBuffer const* output, + ZSTD_inBuffer const* input, + ZSTD_EndDirective endOp) +{ + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + ZSTD_inBuffer const expect = cctx->expectedInBuffer; + if (expect.src != input->src || expect.pos != input->pos || expect.size != input->size) + RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer enabled but input differs!"); + if (endOp != ZSTD_e_end) + RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer can only be used with ZSTD_e_end!"); + } + if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { + size_t const outBufferSize = output->size - output->pos; + if (cctx->expectedOutBufferSize != outBufferSize) + RETURN_ERROR(dstBuffer_wrong, "ZSTD_c_stableOutBuffer enabled but output size differs!"); + } + return 0; +} + +static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, + ZSTD_EndDirective endOp, + size_t inSize) { + ZSTD_CCtx_params params = cctx->requestedParams; + ZSTD_prefixDict const prefixDict = cctx->prefixDict; + FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ + ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ + assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ + if (cctx->cdict) + params.compressionLevel = cctx->cdict->compressionLevel; /* let cdict take priority in terms of compression level */ + DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); + if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-fix pledgedSrcSize */ + { + size_t const dictSize = prefixDict.dict + ? prefixDict.dictSize + : (cctx->cdict ? cctx->cdict->dictContentSize : 0); + ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1); + params.cParams = ZSTD_getCParamsFromCCtxParams( + ¶ms, cctx->pledgedSrcSizePlusOne-1, + dictSize, mode); + } + + if (ZSTD_CParams_shouldEnableLdm(¶ms.cParams)) { + /* Enable LDM by default for optimal parser and window size >= 128MB */ + DEBUGLOG(4, "LDM enabled by default (window size >= 128MB, strategy >= btopt)"); + params.ldmParams.enableLdm = 1; + } + +#ifdef ZSTD_MULTITHREAD + if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { + params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ + } + if (params.nbWorkers > 0) { + /* mt context creation */ + if (cctx->mtctx == NULL) { + DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", + params.nbWorkers); + cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem, cctx->pool); + RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation, "NULL pointer!"); + } + /* mt compression */ + DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); + FORWARD_IF_ERROR( ZSTDMT_initCStream_internal( + cctx->mtctx, + prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, + cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , ""); + cctx->streamStage = zcss_load; + cctx->appliedParams = params; + } else +#endif + { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1; + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, + prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, ZSTD_dtlm_fast, + cctx->cdict, + ¶ms, pledgedSrcSize, + ZSTDb_buffered) , ""); + assert(cctx->appliedParams.nbWorkers == 0); + cctx->inToCompress = 0; + cctx->inBuffPos = 0; + if (cctx->appliedParams.inBufferMode == ZSTD_bm_buffered) { + /* for small input: avoid automatic flush on reaching end of block, since + * it would require to add a 3-bytes null block to end frame + */ + cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize); + } else { + cctx->inBuffTarget = 0; + } + cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0; + cctx->streamStage = zcss_load; + cctx->frameEnded = 0; + } + return 0; +} size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_outBuffer* output, @@ -3967,82 +4425,65 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, { DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp); /* check conditions */ - RETURN_ERROR_IF(output->pos > output->size, GENERIC, "invalid buffer"); - RETURN_ERROR_IF(input->pos > input->size, GENERIC, "invalid buffer"); - assert(cctx!=NULL); + RETURN_ERROR_IF(output->pos > output->size, dstSize_tooSmall, "invalid output buffer"); + RETURN_ERROR_IF(input->pos > input->size, srcSize_wrong, "invalid input buffer"); + RETURN_ERROR_IF((U32)endOp > (U32)ZSTD_e_end, parameter_outOfBound, "invalid endDirective"); + assert(cctx != NULL); /* transparent initialization stage */ if (cctx->streamStage == zcss_init) { - ZSTD_CCtx_params params = cctx->requestedParams; - ZSTD_prefixDict const prefixDict = cctx->prefixDict; - FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ - memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ - assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ - DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); - if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1; /* auto-fix pledgedSrcSize */ - params.cParams = ZSTD_getCParamsFromCCtxParams( - &cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); - - -#ifdef ZSTD_MULTITHREAD - if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { - params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ - } - if (params.nbWorkers > 0) { - /* mt context creation */ - if (cctx->mtctx == NULL) { - DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", - params.nbWorkers); - cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem); - RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation, "NULL pointer!"); - } - /* mt compression */ - DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); - FORWARD_IF_ERROR( ZSTDMT_initCStream_internal( - cctx->mtctx, - prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, - cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , ""); - cctx->streamStage = zcss_load; - cctx->appliedParams.nbWorkers = params.nbWorkers; - } else -#endif - { FORWARD_IF_ERROR( ZSTD_resetCStream_internal(cctx, - prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, - cctx->cdict, - params, cctx->pledgedSrcSizePlusOne-1) , ""); - assert(cctx->streamStage == zcss_load); - assert(cctx->appliedParams.nbWorkers == 0); - } } + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, input->size), "CompressStream2 initialization failed"); + ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ + } /* end of transparent initialization stage */ + FORWARD_IF_ERROR(ZSTD_checkBufferStability(cctx, output, input, endOp), "invalid buffers"); /* compression stage */ #ifdef ZSTD_MULTITHREAD if (cctx->appliedParams.nbWorkers > 0) { - int const forceMaxProgress = (endOp == ZSTD_e_flush || endOp == ZSTD_e_end); size_t flushMin; - assert(forceMaxProgress || endOp == ZSTD_e_continue /* Protection for a new flush type */); if (cctx->cParamsChanged) { ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams); cctx->cParamsChanged = 0; } - do { + for (;;) { + size_t const ipos = input->pos; + size_t const opos = output->pos; flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); if ( ZSTD_isError(flushMin) || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); } FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed"); - } while (forceMaxProgress && flushMin != 0 && output->pos < output->size); + + if (endOp == ZSTD_e_continue) { + /* We only require some progress with ZSTD_e_continue, not maximal progress. + * We're done if we've consumed or produced any bytes, or either buffer is + * full. + */ + if (input->pos != ipos || output->pos != opos || input->pos == input->size || output->pos == output->size) + break; + } else { + assert(endOp == ZSTD_e_flush || endOp == ZSTD_e_end); + /* We require maximal progress. We're done when the flush is complete or the + * output buffer is full. + */ + if (flushMin == 0 || output->pos == output->size) + break; + } + } DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic"); /* Either we don't require maximum forward progress, we've finished the * flush, or we are out of output space. */ - assert(!forceMaxProgress || flushMin == 0 || output->pos == output->size); + assert(endOp == ZSTD_e_continue || flushMin == 0 || output->pos == output->size); + ZSTD_setBufferExpectations(cctx, output, input); return flushMin; } #endif FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , ""); DEBUGLOG(5, "completed ZSTD_compressStream2"); + ZSTD_setBufferExpectations(cctx, output, input); return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ } @@ -4065,14 +4506,22 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { + ZSTD_bufferMode_e const originalInBufferMode = cctx->requestedParams.inBufferMode; + ZSTD_bufferMode_e const originalOutBufferMode = cctx->requestedParams.outBufferMode; DEBUGLOG(4, "ZSTD_compress2 (srcSize=%u)", (unsigned)srcSize); ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + /* Enable stable input/output buffers. */ + cctx->requestedParams.inBufferMode = ZSTD_bm_stable; + cctx->requestedParams.outBufferMode = ZSTD_bm_stable; { size_t oPos = 0; size_t iPos = 0; size_t const result = ZSTD_compressStream2_simpleArgs(cctx, dst, dstCapacity, &oPos, src, srcSize, &iPos, ZSTD_e_end); + /* Reset to the original values. */ + cctx->requestedParams.inBufferMode = originalInBufferMode; + cctx->requestedParams.outBufferMode = originalOutBufferMode; FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed"); if (result != 0) { /* compression not completed, due to lack of output space */ assert(oPos == dstCapacity); @@ -4083,6 +4532,409 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, } } +typedef struct { + U32 idx; /* Index in array of ZSTD_Sequence */ + U32 posInSequence; /* Position within sequence at idx */ + size_t posInSrc; /* Number of bytes given by sequences provided so far */ +} ZSTD_sequencePosition; + +/* Returns a ZSTD error code if sequence is not valid */ +static size_t ZSTD_validateSequence(U32 offCode, U32 matchLength, + size_t posInSrc, U32 windowLog, size_t dictSize, U32 minMatch) { + size_t offsetBound; + U32 windowSize = 1 << windowLog; + /* posInSrc represents the amount of data the the decoder would decode up to this point. + * As long as the amount of data decoded is less than or equal to window size, offsets may be + * larger than the total length of output decoded in order to reference the dict, even larger than + * window size. After output surpasses windowSize, we're limited to windowSize offsets again. + */ + offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; + RETURN_ERROR_IF(offCode > offsetBound + ZSTD_REP_MOVE, corruption_detected, "Offset too large!"); + RETURN_ERROR_IF(matchLength < minMatch, corruption_detected, "Matchlength too small"); + return 0; +} + +/* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ +static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) { + U32 offCode = rawOffset + ZSTD_REP_MOVE; + U32 repCode = 0; + + if (!ll0 && rawOffset == rep[0]) { + repCode = 1; + } else if (rawOffset == rep[1]) { + repCode = 2 - ll0; + } else if (rawOffset == rep[2]) { + repCode = 3 - ll0; + } else if (ll0 && rawOffset == rep[0] - 1) { + repCode = 3; + } + if (repCode) { + /* ZSTD_storeSeq expects a number in the range [0, 2] to represent a repcode */ + offCode = repCode - 1; + } + return offCode; +} + +/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of + * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter. + */ +static size_t ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize) { + U32 idx = seqPos->idx; + BYTE const* ip = (BYTE const*)(src); + const BYTE* const iend = ip + blockSize; + repcodes_t updatedRepcodes; + U32 dictSize; + U32 litLength; + U32 matchLength; + U32 ll0; + U32 offCode; + + if (cctx->cdict) { + dictSize = (U32)cctx->cdict->dictContentSize; + } else if (cctx->prefixDict.dict) { + dictSize = (U32)cctx->prefixDict.dictSize; + } else { + dictSize = 0; + } + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + for (; (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0) && idx < inSeqsSize; ++idx) { + litLength = inSeqs[idx].litLength; + matchLength = inSeqs[idx].matchLength; + ll0 = litLength == 0; + offCode = ZSTD_finalizeOffCode(inSeqs[idx].offset, updatedRepcodes.rep, ll0); + updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); + + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); + if (cctx->appliedParams.validateSequences) { + seqPos->posInSrc += litLength + matchLength; + FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, + cctx->appliedParams.cParams.minMatch), + "Sequence validation failed"); + } + RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH); + ip += matchLength + litLength; + } + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + + if (inSeqs[idx].litLength) { + DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength); + ZSTD_storeLastLiterals(&cctx->seqStore, ip, inSeqs[idx].litLength); + ip += inSeqs[idx].litLength; + seqPos->posInSrc += inSeqs[idx].litLength; + } + RETURN_ERROR_IF(ip != iend, corruption_detected, "Blocksize doesn't agree with block delimiter!"); + seqPos->idx = idx+1; + return 0; +} + +/* Returns the number of bytes to move the current read position back by. Only non-zero + * if we ended up splitting a sequence. Otherwise, it may return a ZSTD error if something + * went wrong. + * + * This function will attempt to scan through blockSize bytes represented by the sequences + * in inSeqs, storing any (partial) sequences. + * + * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to + * avoid splitting a match, or to avoid splitting a match such that it would produce a match + * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block. + */ +static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize) { + U32 idx = seqPos->idx; + U32 startPosInSequence = seqPos->posInSequence; + U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize; + size_t dictSize; + BYTE const* ip = (BYTE const*)(src); + BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */ + repcodes_t updatedRepcodes; + U32 bytesAdjustment = 0; + U32 finalMatchSplit = 0; + U32 litLength; + U32 matchLength; + U32 rawOffset; + U32 offCode; + + if (cctx->cdict) { + dictSize = cctx->cdict->dictContentSize; + } else if (cctx->prefixDict.dict) { + dictSize = cctx->prefixDict.dictSize; + } else { + dictSize = 0; + } + DEBUGLOG(5, "ZSTD_copySequencesToSeqStore: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); + DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { + const ZSTD_Sequence currSeq = inSeqs[idx]; + litLength = currSeq.litLength; + matchLength = currSeq.matchLength; + rawOffset = currSeq.offset; + + /* Modify the sequence depending on where endPosInSequence lies */ + if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { + if (startPosInSequence >= litLength) { + startPosInSequence -= litLength; + litLength = 0; + matchLength -= startPosInSequence; + } else { + litLength -= startPosInSequence; + } + /* Move to the next sequence */ + endPosInSequence -= currSeq.litLength + currSeq.matchLength; + startPosInSequence = 0; + idx++; + } else { + /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence + does not reach the end of the match. So, we have to split the sequence */ + DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u", + currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence); + if (endPosInSequence > litLength) { + U32 firstHalfMatchLength; + litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence; + firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength; + if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) { + /* Only ever split the match if it is larger than the block size */ + U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence; + if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) { + /* Move the endPosInSequence backward so that it creates match of minMatch length */ + endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; + bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; + firstHalfMatchLength -= bytesAdjustment; + } + matchLength = firstHalfMatchLength; + /* Flag that we split the last match - after storing the sequence, exit the loop, + but keep the value of endPosInSequence */ + finalMatchSplit = 1; + } else { + /* Move the position in sequence backwards so that we don't split match, and break to store + * the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence + * should go. We prefer to do this whenever it is not necessary to split the match, or if doing so + * would cause the first half of the match to be too small + */ + bytesAdjustment = endPosInSequence - currSeq.litLength; + endPosInSequence = currSeq.litLength; + break; + } + } else { + /* This sequence ends inside the literals, break to store the last literals */ + break; + } + } + /* Check if this offset can be represented with a repcode */ + { U32 ll0 = (litLength == 0); + offCode = ZSTD_finalizeOffCode(rawOffset, updatedRepcodes.rep, ll0); + updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); + } + + if (cctx->appliedParams.validateSequences) { + seqPos->posInSrc += litLength + matchLength; + FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, + cctx->appliedParams.cParams.minMatch), + "Sequence validation failed"); + } + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); + RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH); + ip += matchLength + litLength; + } + DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); + assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); + seqPos->idx = idx; + seqPos->posInSequence = endPosInSequence; + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + + iend -= bytesAdjustment; + if (ip != iend) { + /* Store any last literals */ + U32 lastLLSize = (U32)(iend - ip); + assert(ip <= iend); + DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize); + ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize); + seqPos->posInSrc += lastLLSize; + } + + return bytesAdjustment; +} + +typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize); +static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode) { + ZSTD_sequenceCopier sequenceCopier = NULL; + assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode)); + if (mode == ZSTD_sf_explicitBlockDelimiters) { + return ZSTD_copySequencesToSeqStoreExplicitBlockDelim; + } else if (mode == ZSTD_sf_noBlockDelimiters) { + return ZSTD_copySequencesToSeqStoreNoBlockDelim; + } + assert(sequenceCopier != NULL); + return sequenceCopier; +} + +/* Compress, block-by-block, all of the sequences given. + * + * Returns the cumulative size of all compressed blocks (including their headers), otherwise a ZSTD error. + */ +static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) { + size_t cSize = 0; + U32 lastBlock; + size_t blockSize; + size_t compressedSeqsSize; + size_t remaining = srcSize; + ZSTD_sequencePosition seqPos = {0, 0, 0}; + + BYTE const* ip = (BYTE const*)src; + BYTE* op = (BYTE*)dst; + ZSTD_sequenceCopier sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters); + + DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); + /* Special case: empty frame */ + if (remaining == 0) { + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header"); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + cSize += ZSTD_blockHeaderSize; + } + + while (remaining) { + size_t cBlockSize; + size_t additionalByteAdjustment; + lastBlock = remaining <= cctx->blockSize; + blockSize = lastBlock ? (U32)remaining : (U32)cctx->blockSize; + ZSTD_resetSeqStore(&cctx->seqStore); + DEBUGLOG(4, "Working on new block. Blocksize: %zu", blockSize); + + additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize); + FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy"); + blockSize -= additionalByteAdjustment; + + /* If blocks are too small, emit as a nocompress block */ + if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); + DEBUGLOG(4, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize); + cSize += cBlockSize; + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + continue; + } + + compressedSeqsSize = ZSTD_entropyCompressSequences(&cctx->seqStore, + &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, + &cctx->appliedParams, + op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, + blockSize, + cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + cctx->bmi2); + FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); + DEBUGLOG(4, "Compressed sequences size: %zu", compressedSeqsSize); + + if (!cctx->isFirstBlock && + ZSTD_maybeRLE(&cctx->seqStore) && + ZSTD_isRLE((BYTE const*)src, srcSize)) { + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + compressedSeqsSize = 1; + } + + if (compressedSeqsSize == 0) { + /* ZSTD_noCompressBlock writes the block header as well */ + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); + DEBUGLOG(4, "Writing out nocompress block, size: %zu", cBlockSize); + } else if (compressedSeqsSize == 1) { + cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "RLE compress block failed"); + DEBUGLOG(4, "Writing out RLE block, size: %zu", cBlockSize); + } else { + U32 cBlockHeader; + /* Error checking and repcodes update */ + ZSTD_confirmRepcodesAndEntropyTables(cctx); + if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + /* Write block header into beginning of block*/ + cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); + MEM_writeLE24(op, cBlockHeader); + cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; + DEBUGLOG(4, "Writing out compressed block, size: %zu", cBlockSize); + } + + cSize += cBlockSize; + DEBUGLOG(4, "cSize running total: %zu", cSize); + + if (lastBlock) { + break; + } else { + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + cctx->isFirstBlock = 0; + } + } + + return cSize; +} + +size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) { + BYTE* op = (BYTE*)dst; + size_t cSize = 0; + size_t compressedBlocksSize = 0; + size_t frameHeaderSize = 0; + + /* Transparent initialization stage, same as compressStream2() */ + DEBUGLOG(3, "ZSTD_compressSequences()"); + assert(cctx != NULL); + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); + /* Begin writing output, starting with frame header */ + frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID); + op += frameHeaderSize; + dstCapacity -= frameHeaderSize; + cSize += frameHeaderSize; + if (cctx->appliedParams.fParams.checksumFlag && srcSize) { + XXH64_update(&cctx->xxhState, src, srcSize); + } + /* cSize includes block header size and compressed sequences size */ + compressedBlocksSize = ZSTD_compressSequences_internal(cctx, + op, dstCapacity, + inSeqs, inSeqsSize, + src, srcSize); + FORWARD_IF_ERROR(compressedBlocksSize, "Compressing blocks failed!"); + cSize += compressedBlocksSize; + dstCapacity -= compressedBlocksSize; + + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); + DEBUGLOG(4, "Write checksum : %08X", (unsigned)checksum); + MEM_writeLE32((char*)dst + cSize, checksum); + cSize += 4; + } + + DEBUGLOG(3, "Final compressed size: %zu", cSize); + return cSize; +} + /*====== Finalize ======*/ /*! ZSTD_flushStream() : @@ -4223,25 +5075,103 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }, }; +static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict); + switch (cParams.strategy) { + case ZSTD_fast: + case ZSTD_dfast: + break; + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + cParams.hashLog += ZSTD_LAZY_DDSS_BUCKET_LOG; + break; + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + } + return cParams; +} + +static int ZSTD_dedicatedDictSearch_isSupported( + ZSTD_compressionParameters const* cParams) +{ + return (cParams->strategy >= ZSTD_greedy) && (cParams->strategy <= ZSTD_lazy2); +} + +/** + * Reverses the adjustment applied to cparams when enabling dedicated dict + * search. This is used to recover the params set to be used in the working + * context. (Otherwise, those tables would also grow.) + */ +static void ZSTD_dedicatedDictSearch_revertCParams( + ZSTD_compressionParameters* cParams) { + switch (cParams->strategy) { + case ZSTD_fast: + case ZSTD_dfast: + break; + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG; + break; + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + } +} + +static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + switch (mode) { + case ZSTD_cpm_unknown: + case ZSTD_cpm_noAttachDict: + case ZSTD_cpm_createCDict: + break; + case ZSTD_cpm_attachDict: + dictSize = 0; + break; + default: + assert(0); + break; + } + { int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN; + size_t const addedSize = unknown && dictSize > 0 ? 500 : 0; + return unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize; + } +} + /*! ZSTD_getCParams_internal() : * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. - * Use dictSize == 0 for unknown or unused. */ -static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) + * Use dictSize == 0 for unknown or unused. + * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */ +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) { - int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN; - size_t const addedSize = unknown && dictSize > 0 ? 500 : 0; - U64 const rSize = unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize; + U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode); U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); - int row = compressionLevel; + int row; DEBUGLOG(5, "ZSTD_getCParams_internal (cLevel=%i)", compressionLevel); + + /* row */ if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ - if (compressionLevel < 0) row = 0; /* entry 0 is baseline for fast mode */ - if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; + else if (compressionLevel < 0) row = 0; /* entry 0 is baseline for fast mode */ + else if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; + else row = compressionLevel; + { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; - if (compressionLevel < 0) cp.targetLength = (unsigned)(-compressionLevel); /* acceleration factor */ + /* acceleration factor */ + if (compressionLevel < 0) { + int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel); + cp.targetLength = (unsigned)(-clampedCompressionLevel); + } /* refine parameters based on srcSize & dictSize */ - return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode); } } @@ -4251,18 +5181,18 @@ static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; - return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize); + return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); } /*! ZSTD_getParams() : * same idea as ZSTD_getCParams() * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). * Fields of `ZSTD_frameParameters` are set to default values */ -static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) { ZSTD_parameters params; - ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize); + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode); DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel); - memset(¶ms, 0, sizeof(params)); + ZSTD_memset(¶ms, 0, sizeof(params)); params.cParams = cParams; params.fParams.contentSizeFlag = 1; return params; @@ -4274,5 +5204,5 @@ static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned lo * Fields of `ZSTD_frameParameters` are set to default values */ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; - return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize); + return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index db73f6ce..c04998b8 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -28,7 +28,6 @@ extern "C" { #endif - /*-************************************* * Constants ***************************************/ @@ -64,7 +63,7 @@ typedef struct { } ZSTD_localDict; typedef struct { - U32 CTable[HUF_CTABLE_SIZE_U32(255)]; + HUF_CElt CTable[HUF_CTABLE_SIZE_U32(255)]; HUF_repeat repeatMode; } ZSTD_hufCTables_t; @@ -83,11 +82,28 @@ typedef struct { } ZSTD_entropyCTables_t; typedef struct { - U32 off; - U32 len; + U32 off; /* Offset code (offset + ZSTD_REP_MOVE) for the match */ + U32 len; /* Raw length of match */ } ZSTD_match_t; typedef struct { + U32 offset; /* Offset of sequence */ + U32 litLength; /* Length of literals prior to match */ + U32 matchLength; /* Raw length of match */ +} rawSeq; + +typedef struct { + rawSeq* seq; /* The start of the sequences */ + size_t pos; /* The index in seq where reading stopped. pos <= size. */ + size_t posInSequence; /* The position within the sequence at seq[pos] where reading + stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */ + size_t size; /* The number of sequences. <= capacity. */ + size_t capacity; /* The capacity starting from `seq` pointer */ +} rawSeqStore_t; + +UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0}; + +typedef struct { int price; U32 off; U32 mlen; @@ -147,9 +163,13 @@ struct ZSTD_matchState_t { U32* hashTable; U32* hashTable3; U32* chainTable; + int dedicatedDictSearch; /* Indicates whether this matchState is using the + * dedicated dictionary search structure. + */ optState_t opt; /* optimal parser state */ const ZSTD_matchState_t* dictMatchState; ZSTD_compressionParameters cParams; + const rawSeqStore_t* ldmSeqStore; }; typedef struct { @@ -182,19 +202,6 @@ typedef struct { } ldmParams_t; typedef struct { - U32 offset; - U32 litLength; - U32 matchLength; -} rawSeq; - -typedef struct { - rawSeq* seq; /* The start of the sequences */ - size_t pos; /* The position where reading stopped. <= size. */ - size_t size; /* The number of sequences. <= capacity. */ - size_t capacity; /* The capacity starting from `seq` pointer */ -} rawSeqStore_t; - -typedef struct { int collectSequences; ZSTD_Sequence* seqStart; size_t seqIndex; @@ -228,10 +235,34 @@ struct ZSTD_CCtx_params_s { /* Long distance matching parameters */ ldmParams_t ldmParams; + /* Dedicated dict search algorithm trigger */ + int enableDedicatedDictSearch; + + /* Input/output buffer modes */ + ZSTD_bufferMode_e inBufferMode; + ZSTD_bufferMode_e outBufferMode; + + /* Sequence compression API */ + ZSTD_sequenceFormat_e blockDelimiters; + int validateSequences; + /* Internal use, for createCCtxParams() and freeCCtxParams() only */ ZSTD_customMem customMem; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ +#define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2)) +#define ENTROPY_WORKSPACE_SIZE (HUF_WORKSPACE_SIZE + COMPRESS_SEQUENCES_WORKSPACE_SIZE) + +/** + * Indicates whether this compression proceeds directly from user-provided + * source buffer to user-provided destination buffer (ZSTDb_not_buffered), or + * whether the context needs to buffer the input/output (ZSTDb_buffered). + */ +typedef enum { + ZSTDb_not_buffered, + ZSTDb_buffered +} ZSTD_buffered_policy_e; + struct ZSTD_CCtx_s { ZSTD_compressionStage_e stage; int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */ @@ -247,6 +278,7 @@ struct ZSTD_CCtx_s { unsigned long long producedCSize; XXH64_state_t xxhState; ZSTD_customMem customMem; + ZSTD_threadPool* pool; size_t staticSize; SeqCollector seqCollector; int isFirstBlock; @@ -258,7 +290,10 @@ struct ZSTD_CCtx_s { size_t maxNbLdmSequences; rawSeqStore_t externSeqStore; /* Mutable reference to external sequences */ ZSTD_blockState_t blockState; - U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ + U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */ + + /* Wether we are streaming or not */ + ZSTD_buffered_policy_e bufferedPolicy; /* streaming */ char* inBuff; @@ -273,6 +308,10 @@ struct ZSTD_CCtx_s { ZSTD_cStreamStage streamStage; U32 frameEnded; + /* Stable in/out buffer verification */ + ZSTD_inBuffer expectedInBuffer; + size_t expectedOutBufferSize; + /* Dictionary */ ZSTD_localDict localDict; const ZSTD_CDict* cdict; @@ -286,8 +325,32 @@ struct ZSTD_CCtx_s { typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e; -typedef enum { ZSTD_noDict = 0, ZSTD_extDict = 1, ZSTD_dictMatchState = 2 } ZSTD_dictMode_e; - +typedef enum { + ZSTD_noDict = 0, + ZSTD_extDict = 1, + ZSTD_dictMatchState = 2, + ZSTD_dedicatedDictSearch = 3 +} ZSTD_dictMode_e; + +typedef enum { + ZSTD_cpm_noAttachDict = 0, /* Compression with ZSTD_noDict or ZSTD_extDict. + * In this mode we use both the srcSize and the dictSize + * when selecting and adjusting parameters. + */ + ZSTD_cpm_attachDict = 1, /* Compression with ZSTD_dictMatchState or ZSTD_dedicatedDictSearch. + * In this mode we only take the srcSize into account when selecting + * and adjusting parameters. + */ + ZSTD_cpm_createCDict = 2, /* Creating a CDict. + * In this mode we take both the source size and the dictionary size + * into account when selecting and adjusting the parameters. + */ + ZSTD_cpm_unknown = 3, /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams. + * We don't know what these parameters are for. We default to the legacy + * behavior of taking both the source size and the dict size into account + * when selecting and adjusting parameters. + */ +} ZSTD_cParamMode_e; typedef size_t (*ZSTD_blockCompressor) ( ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], @@ -345,7 +408,7 @@ MEM_STATIC repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 con newReps.rep[1] = rep[0]; newReps.rep[0] = currentOffset; } else { /* repCode == 0 */ - memcpy(&newReps, rep, sizeof(newReps)); + ZSTD_memcpy(&newReps, rep, sizeof(newReps)); } } return newReps; @@ -372,7 +435,7 @@ MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const voi RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, dstSize_tooSmall, "dst buf too small for uncompressed block"); MEM_writeLE24(dst, cBlockHeader24); - memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); + ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); return ZSTD_blockHeaderSize + srcSize; } @@ -498,8 +561,12 @@ static unsigned ZSTD_NbCommonBytes (size_t val) if (MEM_isLittleEndian()) { if (MEM_64bits()) { # if defined(_MSC_VER) && defined(_WIN64) - unsigned long r = 0; - return _BitScanForward64( &r, (U64)val ) ? (unsigned)(r >> 3) : 0; +# if STATIC_BMI2 + return _tzcnt_u64(val) >> 3; +# else + unsigned long r = 0; + return _BitScanForward64( &r, (U64)val ) ? (unsigned)(r >> 3) : 0; +# endif # elif defined(__GNUC__) && (__GNUC__ >= 4) return (__builtin_ctzll((U64)val) >> 3); # else @@ -530,8 +597,12 @@ static unsigned ZSTD_NbCommonBytes (size_t val) } else { /* Big Endian CPU */ if (MEM_64bits()) { # if defined(_MSC_VER) && defined(_WIN64) - unsigned long r = 0; - return _BitScanReverse64( &r, val ) ? (unsigned)(r >> 3) : 0; +# if STATIC_BMI2 + return _lzcnt_u64(val) >> 3; +# else + unsigned long r = 0; + return _BitScanReverse64(&r, (U64)val) ? (unsigned)(r >> 3) : 0; +# endif # elif defined(__GNUC__) && (__GNUC__ >= 4) return (__builtin_clzll(val) >> 3); # else @@ -626,7 +697,8 @@ static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } -MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) { switch(mls) { @@ -742,7 +814,7 @@ MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms) return ZSTD_window_hasExtDict(ms->window) ? ZSTD_extDict : ms->dictMatchState != NULL ? - ZSTD_dictMatchState : + (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) : ZSTD_noDict; } @@ -754,8 +826,8 @@ MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms) MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, void const* srcEnd) { - U32 const current = (U32)((BYTE const*)srcEnd - window.base); - return current > ZSTD_CURRENT_MAX; + U32 const curr = (U32)((BYTE const*)srcEnd - window.base); + return curr > ZSTD_CURRENT_MAX; } /** @@ -791,14 +863,14 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, * windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32. */ U32 const cycleMask = (1U << cycleLog) - 1; - U32 const current = (U32)((BYTE const*)src - window->base); - U32 const currentCycle0 = current & cycleMask; + U32 const curr = (U32)((BYTE const*)src - window->base); + U32 const currentCycle0 = curr & cycleMask; /* Exclude zero so that newCurrent - maxDist >= 1. */ U32 const currentCycle1 = currentCycle0 == 0 ? (1U << cycleLog) : currentCycle0; U32 const newCurrent = currentCycle1 + maxDist; - U32 const correction = current - newCurrent; + U32 const correction = curr - newCurrent; assert((maxDist & cycleMask) == 0); - assert(current > newCurrent); + assert(curr > newCurrent); /* Loose bound, should be around 1<<29 (see above) */ assert(correction > 1<<28); @@ -919,7 +991,7 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window, } MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { - memset(window, 0, sizeof(*window)); + ZSTD_memset(window, 0, sizeof(*window)); window->base = (BYTE const*)""; window->dictBase = (BYTE const*)""; window->dictLimit = 1; /* start from 1, so that 1st position is valid */ @@ -973,12 +1045,16 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, /** * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix. */ -MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog) +MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) { U32 const maxDistance = 1U << windowLog; U32 const lowestValid = ms->window.lowLimit; - U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary + * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't + * valid for the entire block. So this check is sufficient to find the lowest valid match index. + */ U32 const matchLowest = isDictionary ? lowestValid : withinWindow; return matchLowest; } @@ -986,12 +1062,15 @@ MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current /** * Returns the lowest allowed match index in the prefix. */ -MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog) +MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) { U32 const maxDistance = 1U << windowLog; U32 const lowestValid = ms->window.dictLimit; - U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When computing the lowest prefix index we need to take the dictionary into account to handle + * the edge case where the dictionary and the source are contiguous in memory. + */ U32 const matchLowest = isDictionary ? lowestValid : withinWindow; return matchLowest; } @@ -1045,7 +1124,6 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) * assumptions : magic number supposed already checked * and dictSize >= 8 */ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, - short* offcodeNCount, unsigned* offcodeMaxValue, const void* const dict, size_t dictSize); void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs); @@ -1061,7 +1139,7 @@ void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs); * Note: srcSizeHint == 0 means 0! */ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( - const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize); + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); /*! ZSTD_initCStream_internal() : * Private use only. Init streaming operation. diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c index 17e7168d..6dd1c144 100644 --- a/lib/compress/zstd_compress_literals.c +++ b/lib/compress/zstd_compress_literals.c @@ -35,7 +35,7 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, assert(0); } - memcpy(ostart + flSize, src, srcSize); + ZSTD_memcpy(ostart + flSize, src, srcSize); DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); return srcSize + flSize; } @@ -86,7 +86,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, disableLiteralCompression, (U32)srcSize); /* Prepare nextEntropy assuming reusing the existing table */ - memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); if (disableLiteralCompression) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); @@ -118,11 +118,11 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, } if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) { - memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); } if (cLitSize==1) { - memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); } diff --git a/lib/compress/zstd_compress_sequences.c b/lib/compress/zstd_compress_sequences.c index f9f8097c..be30c08c 100644 --- a/lib/compress/zstd_compress_sequences.c +++ b/lib/compress/zstd_compress_sequences.c @@ -51,6 +51,19 @@ static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) { } /** + * Returns true if we should use ncount=-1 else we should + * use ncount=1 for low probability symbols instead. + */ +static unsigned ZSTD_useLowProbCount(size_t const nbSeq) +{ + /* Heuristic: This should cover most blocks <= 16K and + * start to fade out after 16K to about 32K depending on + * comprssibility. + */ + return nbSeq >= 2048; +} + +/** * Returns the cost in bytes of encoding the normalized count header. * Returns an error if any of the helper functions return an error. */ @@ -60,7 +73,7 @@ static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, BYTE wksp[FSE_NCOUNTBOUND]; S16 norm[MaxSeq + 1]; const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); - FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max), ""); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), ""); return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); } @@ -239,7 +252,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, *op = codeTable[0]; return 1; case set_repeat: - memcpy(nextCTable, prevCTable, prevCTableSize); + ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize); return 0; case set_basic: FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */ @@ -253,7 +266,8 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, nbSeq_1--; } assert(nbSeq_1 > 1); - FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max), ""); + assert(entropyWorkspaceSize >= FSE_BUILD_CTABLE_WORKSPACE_SIZE(MaxSeq, MaxFSELog)); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), ""); { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed"); FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize), ""); diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c index b693866c..e23e619e 100644 --- a/lib/compress/zstd_compress_superblock.c +++ b/lib/compress/zstd_compress_superblock.c @@ -29,7 +29,7 @@ * This metadata is populated in ZSTD_buildSuperBlockEntropy_literal() */ typedef struct { symbolEncodingType_e hType; - BYTE hufDesBuffer[500]; /* TODO give name to this value */ + BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE]; size_t hufDesSize; } ZSTD_hufCTablesMetadata_t; @@ -42,7 +42,7 @@ typedef struct { symbolEncodingType_e llType; symbolEncodingType_e ofType; symbolEncodingType_e mlType; - BYTE fseTablesBuffer[500]; /* TODO give name to this value */ + BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; size_t fseTablesSize; size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_compressSubBlock_sequences() */ } ZSTD_fseCTablesMetadata_t; @@ -79,7 +79,7 @@ static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSiz DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_literal (srcSize=%zu)", srcSize); /* Prepare nextEntropy assuming reusing the existing table */ - memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); if (disableLiteralsCompression) { DEBUGLOG(5, "set_basic - disabled"); @@ -118,7 +118,7 @@ static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSiz } /* Build Huffman Tree */ - memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); + ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue, huffLog, @@ -137,14 +137,14 @@ static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSiz (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { DEBUGLOG(5, "set_repeat - smaller"); - memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); hufMetadata->hType = set_repeat; return 0; } } if (newCSize + hSize >= srcSize) { DEBUGLOG(5, "set_basic - no gains"); - memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); hufMetadata->hType = set_basic; return 0; } @@ -188,7 +188,7 @@ static size_t ZSTD_buildSuperBlockEntropy_sequences(seqStore_t* seqStorePtr, assert(cTableWkspSize >= (1 << MaxFSELog) * sizeof(FSE_FUNCTION_TYPE)); DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_sequences (nbSeq=%zu)", nbSeq); - memset(workspace, 0, wkspSize); + ZSTD_memset(workspace, 0, wkspSize); fseMetadata->lastCountSize = 0; /* convert length/distances into codes */ @@ -348,7 +348,7 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat); if (writeEntropy && hufMetadata->hType == set_compressed) { - memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize); + ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize); op += hufMetadata->hufDesSize; cLitSize += hufMetadata->hufDesSize; DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); @@ -474,7 +474,7 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables const U32 MLtype = fseMetadata->mlType; DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize); *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); - memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize); + ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize); op += fseMetadata->fseTablesSize; } else { const U32 repeat = set_repeat; @@ -603,7 +603,7 @@ static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type, const BYTE* codeTable, unsigned maxCode, size_t nbSeq, const FSE_CTable* fseCTable, const U32* additionalBits, - short const* defaultNorm, U32 defaultNormLog, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, void* workspace, size_t wkspSize) { unsigned* const countWksp = (unsigned*)workspace; @@ -615,7 +615,11 @@ static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type, HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ if (type == set_basic) { - cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max); + /* We selected this encoding type, so it must be valid. */ + assert(max <= defaultMax); + cSymbolTypeSizeEstimateInBits = max <= defaultMax + ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max) + : ERROR(GENERIC); } else if (type == set_rle) { cSymbolTypeSizeEstimateInBits = 0; } else if (type == set_compressed || type == set_repeat) { @@ -643,15 +647,15 @@ static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable, size_t cSeqSizeEstimate = 0; cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff, nbSeq, fseTables->offcodeCTable, NULL, - OF_defaultNorm, OF_defaultNormLog, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, workspace, wkspSize); cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL, nbSeq, fseTables->litlengthCTable, LL_bits, - LL_defaultNorm, LL_defaultNormLog, + LL_defaultNorm, LL_defaultNormLog, MaxLL, workspace, wkspSize); cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML, nbSeq, fseTables->matchlengthCTable, ML_bits, - ML_defaultNorm, ML_defaultNormLog, + ML_defaultNorm, ML_defaultNormLog, MaxML, workspace, wkspSize); if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; return cSeqSizeEstimate + sequencesSectionHeaderSize; @@ -790,7 +794,7 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr, } while (!lastSequence); if (writeLitEntropy) { DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten"); - memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); + ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); } if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) { /* If we haven't written our entropy tables, then we've violated our contract and @@ -809,11 +813,11 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr, if (sp < send) { seqDef const* seq; repcodes_t rep; - memcpy(&rep, prevCBlock->rep, sizeof(rep)); + ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep)); for (seq = sstart; seq < sp; ++seq) { rep = ZSTD_updateRep(rep.rep, seq->offset - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); } - memcpy(nextCBlock->rep, &rep, sizeof(rep)); + ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep)); } } DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed"); @@ -831,7 +835,7 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, &zc->blockState.nextCBlock->entropy, &zc->appliedParams, &entropyMetadata, - zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); return ZSTD_compressSubBlock_multi(&zc->seqStore, zc->blockState.prevCBlock, @@ -841,5 +845,5 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, dst, dstCapacity, src, srcSize, zc->bmi2, lastBlock, - zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */); + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */); } diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index a25c9263..d65170b3 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -45,6 +45,16 @@ typedef enum { } ZSTD_cwksp_alloc_phase_e; /** + * Used to describe whether the workspace is statically allocated (and will not + * necessarily ever be freed), or if it's dynamically allocated and we can + * expect a well-formed caller to free this. + */ +typedef enum { + ZSTD_cwksp_dynamic_alloc, + ZSTD_cwksp_static_alloc +} ZSTD_cwksp_static_alloc_e; + +/** * Zstd fits all its internal datastructures into a single continuous buffer, * so that it only needs to perform a single OS allocation (or so that a buffer * can be provided to it and it can perform no allocations at all). This buffer @@ -92,7 +102,7 @@ typedef enum { * * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, * so that literally everything fits in a single buffer. Note: if present, - * this must be the first object in the workspace, since ZSTD_free{CCtx, + * this must be the first object in the workspace, since ZSTD_customFree{CCtx, * CDict}() rely on a pointer comparison to see whether one or two frees are * required. * @@ -137,9 +147,10 @@ typedef struct { void* tableValidEnd; void* allocStart; - int allocFailed; + BYTE allocFailed; int workspaceOversizedDuration; ZSTD_cwksp_alloc_phase_e phase; + ZSTD_cwksp_static_alloc_e isStatic; } ZSTD_cwksp; /*-************************************* @@ -178,7 +189,9 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { * else is though. */ MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { -#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + if (size == 0) + return 0; +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; #else return size; @@ -228,7 +241,10 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal( ZSTD_cwksp_internal_advance_phase(ws, phase); alloc = (BYTE *)ws->allocStart - bytes; -#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + if (bytes == 0) + return NULL; + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) /* over-reserve space */ alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; #endif @@ -247,11 +263,13 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal( } ws->allocStart = alloc; -#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on * either size. */ alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; - __asan_unpoison_memory_region(alloc, bytes); + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); + } #endif return alloc; @@ -296,8 +314,10 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { } ws->tableEnd = end; -#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - __asan_unpoison_memory_region(alloc, bytes); +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); + } #endif return alloc; @@ -311,7 +331,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { void* alloc = ws->objectEnd; void* end = (BYTE*)alloc + roundedBytes; -#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) /* over-reserve space */ end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; #endif @@ -332,11 +352,13 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { ws->tableEnd = end; ws->tableValidEnd = end; -#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on * either size. */ alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; - __asan_unpoison_memory_region(alloc, bytes); + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); + } #endif return alloc; @@ -345,7 +367,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) { DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty"); -#if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) /* To validate that the table re-use logic is sound, and that we don't * access table space that we haven't cleaned, we re-"poison" the table * space every time we mark it dirty. */ @@ -380,7 +402,7 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { assert(ws->tableValidEnd >= ws->objectEnd); assert(ws->tableValidEnd <= ws->allocStart); if (ws->tableValidEnd < ws->tableEnd) { - memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd); + ZSTD_memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd); } ZSTD_cwksp_mark_tables_clean(ws); } @@ -392,8 +414,12 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { DEBUGLOG(4, "cwksp: clearing tables!"); -#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - { +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* We don't do this when the workspace is statically allocated, because + * when that is the case, we have no capability to hook into the end of the + * workspace's lifecycle to unpoison the memory. + */ + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; __asan_poison_memory_region(ws->objectEnd, size); } @@ -410,7 +436,7 @@ MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { DEBUGLOG(4, "cwksp: clearing!"); -#if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) /* To validate that the context re-use logic is sound, and that we don't * access stuff that this compression hasn't initialized, we re-"poison" * the workspace (or at least the non-static, non-table parts of it) @@ -421,8 +447,12 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { } #endif -#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - { +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* We don't do this when the workspace is statically allocated, because + * when that is the case, we have no capability to hook into the end of the + * workspace's lifecycle to unpoison the memory. + */ + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd; __asan_poison_memory_region(ws->objectEnd, size); } @@ -442,7 +472,7 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { * Any existing values in the workspace are ignored (the previously managed * buffer, if present, must be separately freed). */ -MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { +MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_cwksp_static_alloc_e isStatic) { DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ ws->workspace = start; @@ -450,24 +480,25 @@ MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { ws->objectEnd = ws->workspace; ws->tableValidEnd = ws->objectEnd; ws->phase = ZSTD_cwksp_alloc_objects; + ws->isStatic = isStatic; ZSTD_cwksp_clear(ws); ws->workspaceOversizedDuration = 0; ZSTD_cwksp_assert_internal_consistency(ws); } MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { - void* workspace = ZSTD_malloc(size, customMem); + void* workspace = ZSTD_customMalloc(size, customMem); DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); RETURN_ERROR_IF(workspace == NULL, memory_allocation, "NULL pointer!"); - ZSTD_cwksp_init(ws, workspace, size); + ZSTD_cwksp_init(ws, workspace, size, ZSTD_cwksp_dynamic_alloc); return 0; } MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { void *ptr = ws->workspace; DEBUGLOG(4, "cwksp: freeing workspace"); - memset(ws, 0, sizeof(ZSTD_cwksp)); - ZSTD_free(ptr, customMem); + ZSTD_memset(ws, 0, sizeof(ZSTD_cwksp)); + ZSTD_customFree(ptr, customMem); } /** @@ -476,13 +507,18 @@ MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { */ MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { *dst = *src; - memset(src, 0, sizeof(ZSTD_cwksp)); + ZSTD_memset(src, 0, sizeof(ZSTD_cwksp)); } MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace); } +MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace) + + (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart); +} + MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { return ws->allocFailed; } diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c index 27eed66c..ef12a524 100644 --- a/lib/compress/zstd_double_fast.c +++ b/lib/compress/zstd_double_fast.c @@ -31,15 +31,15 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, * is empty. */ for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { - U32 const current = (U32)(ip - base); + U32 const curr = (U32)(ip - base); U32 i; for (i = 0; i < fastHashFillStep; ++i) { size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls); size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8); if (i == 0) - hashSmall[smHash] = current + i; + hashSmall[smHash] = curr + i; if (i == 0 || hashLarge[lgHash] == 0) - hashLarge[lgHash] = current + i; + hashLarge[lgHash] = curr + i; /* Only load extra positions for ZSTD_dtlm_full */ if (dtlm == ZSTD_dtlm_fast) break; @@ -108,9 +108,9 @@ size_t ZSTD_compressBlock_doubleFast_generic( /* init */ ip += (dictAndPrefixLength == 0); if (dictMode == ZSTD_noDict) { - U32 const current = (U32)(ip - base); - U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog); - U32 const maxRep = current - windowLow; + U32 const curr = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); + U32 const maxRep = curr - windowLow; if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; } @@ -129,17 +129,17 @@ size_t ZSTD_compressBlock_doubleFast_generic( size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); size_t const dictHL = ZSTD_hashPtr(ip, dictHBitsL, 8); size_t const dictHS = ZSTD_hashPtr(ip, dictHBitsS, mls); - U32 const current = (U32)(ip-base); + U32 const curr = (U32)(ip-base); U32 const matchIndexL = hashLong[h2]; U32 matchIndexS = hashSmall[h]; const BYTE* matchLong = base + matchIndexL; const BYTE* match = base + matchIndexS; - const U32 repIndex = current + 1 - offset_1; + const U32 repIndex = curr + 1 - offset_1; const BYTE* repMatch = (dictMode == ZSTD_dictMatchState && repIndex < prefixLowestIndex) ? dictBase + (repIndex - dictIndexDelta) : base + repIndex; - hashLong[h2] = hashSmall[h] = current; /* update hash tables */ + hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ /* check dictMatchState repcode */ if (dictMode == ZSTD_dictMatchState @@ -177,7 +177,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) { mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8; - offset = (U32)(current - dictMatchIndexL - dictIndexDelta); + offset = (U32)(curr - dictMatchIndexL - dictIndexDelta); while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ goto _match_found; } } @@ -209,7 +209,7 @@ _search_next_long: size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8); U32 const matchIndexL3 = hashLong[hl3]; const BYTE* matchL3 = base + matchIndexL3; - hashLong[hl3] = current + 1; + hashLong[hl3] = curr + 1; /* check prefix long +1 match */ if (matchIndexL3 > prefixLowestIndex) { @@ -228,7 +228,7 @@ _search_next_long: if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) { mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8; ip++; - offset = (U32)(current + 1 - dictMatchIndexL3 - dictIndexDelta); + offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta); while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ goto _match_found; } } } @@ -236,7 +236,7 @@ _search_next_long: /* if no long +1 match, explore the short match we found */ if (dictMode == ZSTD_dictMatchState && matchIndexS < prefixLowestIndex) { mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; - offset = (U32)(current - matchIndexS); + offset = (U32)(curr - matchIndexS); while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ } else { mLength = ZSTD_count(ip+4, match+4, iend) + 4; @@ -260,7 +260,7 @@ _match_stored: if (ip <= ilimit) { /* Complementary insertion */ /* done after iLimit test, as candidates could be > iend-8 */ - { U32 const indexToInsert = current+2; + { U32 const indexToInsert = curr+2; hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; @@ -401,12 +401,12 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base; const BYTE* matchLong = matchLongBase + matchLongIndex; - const U32 current = (U32)(ip-base); - const U32 repIndex = current + 1 - offset_1; /* offset_1 expected <= current +1 */ + const U32 curr = (U32)(ip-base); + const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; size_t mLength; - hashSmall[hSmall] = hashLong[hLong] = current; /* update hash table */ + hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */ & (repIndex > dictStartIndex)) @@ -421,7 +421,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart; U32 offset; mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8; - offset = current - matchLongIndex; + offset = curr - matchLongIndex; while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; @@ -433,19 +433,19 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base; const BYTE* match3 = match3Base + matchIndex3; U32 offset; - hashLong[h3] = current + 1; + hashLong[h3] = curr + 1; if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend; const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart; mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8; ip++; - offset = current+1 - matchIndex3; + offset = curr+1 - matchIndex3; while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ } else { const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; - offset = current - matchIndex; + offset = curr - matchIndex; while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ } offset_2 = offset_1; @@ -464,7 +464,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( if (ip <= ilimit) { /* Complementary insertion */ /* done after iLimit test, as candidates could be > iend-8 */ - { U32 const indexToInsert = current+2; + { U32 const indexToInsert = curr+2; hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c index 85a3a7a9..db7ce83d 100644 --- a/lib/compress/zstd_fast.c +++ b/lib/compress/zstd_fast.c @@ -29,16 +29,16 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, * Insert the other positions if their hash entry is empty. */ for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { - U32 const current = (U32)(ip - base); + U32 const curr = (U32)(ip - base); size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls); - hashTable[hash0] = current; + hashTable[hash0] = curr; if (dtlm == ZSTD_dtlm_fast) continue; /* Only load extra positions for ZSTD_dtlm_full */ { U32 p; for (p = 1; p < fastHashFillStep; ++p) { size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls); if (hashTable[hash] == 0) { /* not yet filled */ - hashTable[hash] = current + p; + hashTable[hash] = curr + p; } } } } } @@ -72,9 +72,9 @@ ZSTD_compressBlock_fast_generic( DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); ip0 += (ip0 == prefixStart); ip1 = ip0 + 1; - { U32 const current = (U32)(ip0 - base); - U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog); - U32 const maxRep = current - windowLow; + { U32 const curr = (U32)(ip0 - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); + U32 const maxRep = curr - windowLow; if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; } @@ -258,14 +258,14 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ size_t mLength; size_t const h = ZSTD_hashPtr(ip, hlog, mls); - U32 const current = (U32)(ip-base); + U32 const curr = (U32)(ip-base); U32 const matchIndex = hashTable[h]; const BYTE* match = base + matchIndex; - const U32 repIndex = current + 1 - offset_1; + const U32 repIndex = curr + 1 - offset_1; const BYTE* repMatch = (repIndex < prefixStartIndex) ? dictBase + (repIndex - dictIndexDelta) : base + repIndex; - hashTable[h] = current; /* update hash table */ + hashTable[h] = curr; /* update hash table */ if ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */ && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { @@ -284,7 +284,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( continue; } else { /* found a dict match */ - U32 const offset = (U32)(current-dictMatchIndex-dictIndexDelta); + U32 const offset = (U32)(curr-dictMatchIndex-dictIndexDelta); mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4; while (((ip>anchor) & (dictMatch>dictStart)) && (ip[-1] == dictMatch[-1])) { @@ -316,8 +316,8 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( if (ip <= ilimit) { /* Fill Table */ - assert(base+current+2 > istart); /* check base overflow */ - hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2; /* here because current+2 could be > iend-8 */ + assert(base+curr+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */ hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); /* check immediate repcode */ @@ -410,13 +410,13 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const U32 matchIndex = hashTable[h]; const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; const BYTE* match = matchBase + matchIndex; - const U32 current = (U32)(ip-base); - const U32 repIndex = current + 1 - offset_1; + const U32 curr = (U32)(ip-base); + const U32 repIndex = curr + 1 - offset_1; const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - hashTable[h] = current; /* update hash table */ - DEBUGLOG(7, "offset_1 = %u , current = %u", offset_1, current); - assert(offset_1 <= current +1); /* check repIndex */ + hashTable[h] = curr; /* update hash table */ + DEBUGLOG(7, "offset_1 = %u , curr = %u", offset_1, curr); + assert(offset_1 <= curr +1); /* check repIndex */ if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex)) && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { @@ -435,7 +435,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( } { const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; - U32 const offset = current - matchIndex; + U32 const offset = curr - matchIndex; size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; /* update offset history */ @@ -446,7 +446,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( if (ip <= ilimit) { /* Fill Table */ - hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2; + hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); /* check immediate repcode */ while (ip <= ilimit) { diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 4cf5c88b..49ec1b09 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -58,11 +58,11 @@ ZSTD_updateDUBT(ZSTD_matchState_t* ms, /** ZSTD_insertDUBT1() : * sort one already inserted but unsorted position - * assumption : current >= btlow == (current - btmask) + * assumption : curr >= btlow == (curr - btmask) * doesn't fail */ static void ZSTD_insertDUBT1(ZSTD_matchState_t* ms, - U32 current, const BYTE* inputEnd, + U32 curr, const BYTE* inputEnd, U32 nbCompares, U32 btLow, const ZSTD_dictMode_e dictMode) { @@ -74,41 +74,41 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms, const BYTE* const base = ms->window.base; const BYTE* const dictBase = ms->window.dictBase; const U32 dictLimit = ms->window.dictLimit; - const BYTE* const ip = (current>=dictLimit) ? base + current : dictBase + current; - const BYTE* const iend = (current>=dictLimit) ? inputEnd : dictBase + dictLimit; + const BYTE* const ip = (curr>=dictLimit) ? base + curr : dictBase + curr; + const BYTE* const iend = (curr>=dictLimit) ? inputEnd : dictBase + dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; const BYTE* match; - U32* smallerPtr = bt + 2*(current&btMask); + U32* smallerPtr = bt + 2*(curr&btMask); U32* largerPtr = smallerPtr + 1; U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ U32 dummy32; /* to be nullified at the end */ U32 const windowValid = ms->window.lowLimit; U32 const maxDistance = 1U << cParams->windowLog; - U32 const windowLow = (current - windowValid > maxDistance) ? current - maxDistance : windowValid; + U32 const windowLow = (curr - windowValid > maxDistance) ? curr - maxDistance : windowValid; DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", - current, dictLimit, windowLow); - assert(current >= btLow); + curr, dictLimit, windowLow); + assert(curr >= btLow); assert(ip < iend); /* condition for ZSTD_count */ while (nbCompares-- && (matchIndex > windowLow)) { U32* const nextPtr = bt + 2*(matchIndex & btMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - assert(matchIndex < current); + assert(matchIndex < curr); /* note : all candidates are now supposed sorted, * but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK * when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */ if ( (dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit) /* both in current segment*/ - || (current < dictLimit) /* both in extDict */) { + || (curr < dictLimit) /* both in extDict */) { const BYTE* const mBase = ( (dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) ? base : dictBase; assert( (matchIndex+matchLength >= dictLimit) /* might be wrong if extDict is incorrectly set to 0 */ - || (current < dictLimit) ); + || (curr < dictLimit) ); match = mBase + matchIndex; matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); } else { @@ -119,7 +119,7 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms, } DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ", - current, matchIndex, (U32)matchLength); + curr, matchIndex, (U32)matchLength); if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ @@ -168,7 +168,7 @@ ZSTD_DUBT_findBetterDictMatch ( const BYTE* const base = ms->window.base; const BYTE* const prefixStart = base + ms->window.dictLimit; - U32 const current = (U32)(ip-base); + U32 const curr = (U32)(ip-base); const BYTE* const dictBase = dms->window.base; const BYTE* const dictEnd = dms->window.nextSrc; U32 const dictHighLimit = (U32)(dms->window.nextSrc - dms->window.base); @@ -195,10 +195,10 @@ ZSTD_DUBT_findBetterDictMatch ( if (matchLength > bestLength) { U32 matchIndex = dictMatchIndex + dictIndexDelta; - if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) { + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) { DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)", - current, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, ZSTD_REP_MOVE + current - matchIndex, dictMatchIndex, matchIndex); - bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; + curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, ZSTD_REP_MOVE + curr - matchIndex, dictMatchIndex, matchIndex); + bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; } if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */ break; /* drop, to guarantee consistency (miss a little bit of compression) */ @@ -218,9 +218,9 @@ ZSTD_DUBT_findBetterDictMatch ( } if (bestLength >= MINMATCH) { - U32 const mIndex = current - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex; + U32 const mIndex = curr - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex; DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)", - current, (U32)bestLength, (U32)*offsetPtr, mIndex); + curr, (U32)bestLength, (U32)*offsetPtr, mIndex); } return bestLength; @@ -241,13 +241,13 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, U32 matchIndex = hashTable[h]; const BYTE* const base = ms->window.base; - U32 const current = (U32)(ip-base); - U32 const windowLow = ZSTD_getLowestMatchIndex(ms, current, cParams->windowLog); + U32 const curr = (U32)(ip-base); + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); U32* const bt = ms->chainTable; U32 const btLog = cParams->chainLog - 1; U32 const btMask = (1 << btLog) - 1; - U32 const btLow = (btMask >= current) ? 0 : current - btMask; + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; U32 const unsortLimit = MAX(btLow, windowLow); U32* nextCandidate = bt + 2*(matchIndex&btMask); @@ -256,8 +256,9 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, U32 nbCandidates = nbCompares; U32 previousCandidate = 0; - DEBUGLOG(7, "ZSTD_DUBT_findBestMatch (%u) ", current); + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch (%u) ", curr); assert(ip <= iend-8); /* required for h calculation */ + assert(dictMode != ZSTD_dedicatedDictSearch); /* reach end of unsorted candidates list */ while ( (matchIndex > unsortLimit) @@ -299,14 +300,14 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, const U32 dictLimit = ms->window.dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; - U32* smallerPtr = bt + 2*(current&btMask); - U32* largerPtr = bt + 2*(current&btMask) + 1; - U32 matchEndIdx = current + 8 + 1; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr + 8 + 1; U32 dummy32; /* to be nullified at the end */ size_t bestLength = 0; matchIndex = hashTable[h]; - hashTable[h] = current; /* Update Hash Table */ + hashTable[h] = curr; /* Update Hash Table */ while (nbCompares-- && (matchIndex > windowLow)) { U32* const nextPtr = bt + 2*(matchIndex & btMask); @@ -326,8 +327,8 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, if (matchLength > bestLength) { if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; - if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) - bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) + bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ if (dictMode == ZSTD_dictMatchState) { nbCompares = 0; /* in addition to avoiding checking any @@ -363,12 +364,12 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, mls, dictMode); } - assert(matchEndIdx > current+8); /* ensure nextToUpdate is increased */ + assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */ ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ if (bestLength >= MINMATCH) { - U32 const mIndex = current - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex; + U32 const mIndex = curr - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex; DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)", - current, (U32)bestLength, (U32)*offsetPtr, mIndex); + curr, (U32)bestLength, (U32)*offsetPtr, mIndex); } return bestLength; } @@ -446,7 +447,7 @@ static size_t ZSTD_BtFindBestMatch_extDict_selectMLS ( /* Update chains up to ip (excluded) Assumption : always within prefix (i.e. not within extDict) */ -static U32 ZSTD_insertAndFindFirstIndex_internal( +FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal( ZSTD_matchState_t* ms, const ZSTD_compressionParameters* const cParams, const BYTE* ip, U32 const mls) @@ -475,6 +476,121 @@ U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) { return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch); } +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip) +{ + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32* const hashTable = ms->hashTable; + U32* const chainTable = ms->chainTable; + U32 const chainSize = 1 << ms->cParams.chainLog; + U32 idx = ms->nextToUpdate; + U32 const minChain = chainSize < target ? target - chainSize : idx; + U32 const bucketSize = 1 << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const cacheSize = bucketSize - 1; + U32 const chainAttempts = (1 << ms->cParams.searchLog) - cacheSize; + U32 const chainLimit = chainAttempts > 255 ? 255 : chainAttempts; + + /* We know the hashtable is oversized by a factor of `bucketSize`. + * We are going to temporarily pretend `bucketSize == 1`, keeping only a + * single entry. We will use the rest of the space to construct a temporary + * chaintable. + */ + U32 const hashLog = ms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; + U32* const tmpHashTable = hashTable; + U32* const tmpChainTable = hashTable + ((size_t)1 << hashLog); + U32 const tmpChainSize = ((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog; + U32 const tmpMinChain = tmpChainSize < target ? target - tmpChainSize : idx; + + U32 hashIdx; + + assert(ms->cParams.chainLog <= 24); + assert(ms->cParams.hashLog >= ms->cParams.chainLog); + assert(idx != 0); + assert(tmpMinChain <= minChain); + + /* fill conventional hash table and conventional chain table */ + for ( ; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch); + if (idx >= tmpMinChain) { + tmpChainTable[idx - tmpMinChain] = hashTable[h]; + } + tmpHashTable[h] = idx; + } + + /* sort chains into ddss chain table */ + { + U32 chainPos = 0; + for (hashIdx = 0; hashIdx < (1U << hashLog); hashIdx++) { + U32 count; + U32 countBeyondMinChain = 0; + U32 i = tmpHashTable[hashIdx]; + for (count = 0; i >= tmpMinChain && count < cacheSize; count++) { + /* skip through the chain to the first position that won't be + * in the hash cache bucket */ + if (i < minChain) { + countBeyondMinChain++; + } + i = tmpChainTable[i - tmpMinChain]; + } + if (count == cacheSize) { + for (count = 0; count < chainLimit;) { + if (i < minChain) { + if (!i || countBeyondMinChain++ > cacheSize) { + /* only allow pulling `cacheSize` number of entries + * into the cache or chainTable beyond `minChain`, + * to replace the entries pulled out of the + * chainTable into the cache. This lets us reach + * back further without increasing the total number + * of entries in the chainTable, guaranteeing the + * DDSS chain table will fit into the space + * allocated for the regular one. */ + break; + } + } + chainTable[chainPos++] = i; + count++; + if (i < tmpMinChain) { + break; + } + i = tmpChainTable[i - tmpMinChain]; + } + } else { + count = 0; + } + if (count) { + tmpHashTable[hashIdx] = ((chainPos - count) << 8) + count; + } else { + tmpHashTable[hashIdx] = 0; + } + } + assert(chainPos <= chainSize); /* I believe this is guaranteed... */ + } + + /* move chain pointers into the last entry of each hash bucket */ + for (hashIdx = (1 << hashLog); hashIdx; ) { + U32 const bucketIdx = --hashIdx << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const chainPackedPointer = tmpHashTable[hashIdx]; + U32 i; + for (i = 0; i < cacheSize; i++) { + hashTable[bucketIdx + i] = 0; + } + hashTable[bucketIdx + bucketSize - 1] = chainPackedPointer; + } + + /* fill the buckets of the hash table */ + for (idx = ms->nextToUpdate; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch) + << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 i; + /* Shift hash cache down 1. */ + for (i = cacheSize - 1; i; i--) + hashTable[h + i] = hashTable[h + i - 1]; + hashTable[h] = idx; + } + + ms->nextToUpdate = target; +} + /* inlining is important to hardwire a hot branch (template emulation) */ FORCE_INLINE_TEMPLATE @@ -493,20 +609,33 @@ size_t ZSTD_HcFindBestMatch_generic ( const U32 dictLimit = ms->window.dictLimit; const BYTE* const prefixStart = base + dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; - const U32 current = (U32)(ip-base); + const U32 curr = (U32)(ip-base); const U32 maxDistance = 1U << cParams->windowLog; const U32 lowestValid = ms->window.lowLimit; - const U32 withinMaxDistance = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid; + const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; const U32 isDictionary = (ms->loadedDictEnd != 0); const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; - const U32 minChain = current > chainSize ? current - chainSize : 0; + const U32 minChain = curr > chainSize ? curr - chainSize : 0; U32 nbAttempts = 1U << cParams->searchLog; size_t ml=4-1; + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch + ? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch + ? ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + + U32 matchIndex; + + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32* entry = &dms->hashTable[ddsIdx]; + PREFETCH_L1(entry); + } + /* HC4 match finder */ - U32 matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls); + matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls); - for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) { + for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) { size_t currentMl=0; if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { const BYTE* const match = base + matchIndex; @@ -523,7 +652,7 @@ size_t ZSTD_HcFindBestMatch_generic ( /* save best solution */ if (currentMl > ml) { ml = currentMl; - *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; + *offsetPtr = curr - matchIndex + ZSTD_REP_MOVE; if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ } @@ -531,8 +660,92 @@ size_t ZSTD_HcFindBestMatch_generic ( matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); } - if (dictMode == ZSTD_dictMatchState) { - const ZSTD_matchState_t* const dms = ms->dictMatchState; + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32 ddsLowestIndex = dms->window.dictLimit; + const BYTE* const ddsBase = dms->window.base; + const BYTE* const ddsEnd = dms->window.nextSrc; + const U32 ddsSize = (U32)(ddsEnd - ddsBase); + const U32 ddsIndexDelta = dictLimit - ddsSize; + const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG); + const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1; + U32 ddsAttempt; + + for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) { + PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]); + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 const chainIndex = chainPackedPointer >> 8; + + PREFETCH_L1(&dms->chainTable[chainIndex]); + } + + for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->hashTable[ddsIdx + ddsAttempt]; + match = ddsBase + matchIndex; + + if (!matchIndex) { + return ml; + } + + /* guaranteed by table construction */ + (void)ddsLowestIndex; + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE; + if (ip+currentMl == iLimit) { + /* best possible, avoids read overflow on next attempt */ + return ml; + } + } + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 chainIndex = chainPackedPointer >> 8; + U32 const chainLength = chainPackedPointer & 0xFF; + U32 const chainAttempts = nbAttempts - ddsAttempt; + U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts; + U32 chainAttempt; + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) { + PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]); + } + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->chainTable[chainIndex]; + match = ddsBase + matchIndex; + + /* guaranteed by table construction */ + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE; + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + } + } + } else if (dictMode == ZSTD_dictMatchState) { const U32* const dmsChainTable = dms->chainTable; const U32 dmsChainSize = (1 << dms->cParams.chainLog); const U32 dmsChainMask = dmsChainSize - 1; @@ -545,7 +758,7 @@ size_t ZSTD_HcFindBestMatch_generic ( matchIndex = dms->hashTable[ZSTD_hashPtr(ip, dms->cParams.hashLog, mls)]; - for ( ; (matchIndex>dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) { + for ( ; (matchIndex>=dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) { size_t currentMl=0; const BYTE* const match = dmsBase + matchIndex; assert(match+4 <= dmsEnd); @@ -555,11 +768,12 @@ size_t ZSTD_HcFindBestMatch_generic ( /* save best solution */ if (currentMl > ml) { ml = currentMl; - *offsetPtr = current - (matchIndex + dmsIndexDelta) + ZSTD_REP_MOVE; + *offsetPtr = curr - (matchIndex + dmsIndexDelta) + ZSTD_REP_MOVE; if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ } if (matchIndex <= dmsMinChain) break; + matchIndex = dmsChainTable[matchIndex & dmsChainMask]; } } @@ -600,6 +814,22 @@ static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( } +static size_t ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS ( + ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr) +{ + switch(ms->cParams.minMatch) + { + default : /* includes case 3 */ + case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dedicatedDictSearch); + case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dedicatedDictSearch); + case 7 : + case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dedicatedDictSearch); + } +} + + FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* const iLimit, @@ -641,39 +871,62 @@ ZSTD_compressBlock_lazy_generic( typedef size_t (*searchMax_f)( ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - searchMax_f const searchMax = dictMode == ZSTD_dictMatchState ? - (searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_dictMatchState_selectMLS - : ZSTD_HcFindBestMatch_dictMatchState_selectMLS) : - (searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_selectMLS - : ZSTD_HcFindBestMatch_selectMLS); + + /** + * This table is indexed first by the four ZSTD_dictMode_e values, and then + * by the two searchMethod_e values. NULLs are placed for configurations + * that should never occur (extDict modes go to the other implementation + * below and there is no DDSS for binary tree search yet). + */ + const searchMax_f searchFuncs[4][2] = { + { + ZSTD_HcFindBestMatch_selectMLS, + ZSTD_BtFindBestMatch_selectMLS + }, + { + NULL, + NULL + }, + { + ZSTD_HcFindBestMatch_dictMatchState_selectMLS, + ZSTD_BtFindBestMatch_dictMatchState_selectMLS + }, + { + ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS, + NULL + } + }; + + searchMax_f const searchMax = searchFuncs[dictMode][searchMethod == search_binaryTree]; U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; + const int isDMS = dictMode == ZSTD_dictMatchState; + const int isDDS = dictMode == ZSTD_dedicatedDictSearch; + const int isDxS = isDMS || isDDS; const ZSTD_matchState_t* const dms = ms->dictMatchState; - const U32 dictLowestIndex = dictMode == ZSTD_dictMatchState ? - dms->window.dictLimit : 0; - const BYTE* const dictBase = dictMode == ZSTD_dictMatchState ? - dms->window.base : NULL; - const BYTE* const dictLowest = dictMode == ZSTD_dictMatchState ? - dictBase + dictLowestIndex : NULL; - const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ? - dms->window.nextSrc : NULL; - const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ? + const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0; + const BYTE* const dictBase = isDxS ? dms->window.base : NULL; + const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL; + const BYTE* const dictEnd = isDxS ? dms->window.nextSrc : NULL; + const U32 dictIndexDelta = isDxS ? prefixLowestIndex - (U32)(dictEnd - dictBase) : 0; const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest)); + assert(searchMax != NULL); + DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u)", (U32)dictMode); /* init */ ip += (dictAndPrefixLength == 0); if (dictMode == ZSTD_noDict) { - U32 const current = (U32)(ip - base); - U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, ms->cParams.windowLog); - U32 const maxRep = current - windowLow; + U32 const curr = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog); + U32 const maxRep = curr - windowLow; if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; } - if (dictMode == ZSTD_dictMatchState) { + if (isDxS) { /* dictMatchState repCode checks don't currently handle repCode == 0 * disabling. */ assert(offset_1 <= dictAndPrefixLength); @@ -693,9 +946,9 @@ ZSTD_compressBlock_lazy_generic( const BYTE* start=ip+1; /* check repCode */ - if (dictMode == ZSTD_dictMatchState) { + if (isDxS) { const U32 repIndex = (U32)(ip - base) + 1 - offset_1; - const BYTE* repMatch = (dictMode == ZSTD_dictMatchState + const BYTE* repMatch = ((dictMode == ZSTD_dictMatchState || dictMode == ZSTD_dedicatedDictSearch) && repIndex < prefixLowestIndex) ? dictBase + (repIndex - dictIndexDelta) : base + repIndex; @@ -736,7 +989,7 @@ ZSTD_compressBlock_lazy_generic( if ((mlRep >= 4) && (gain2 > gain1)) matchLength = mlRep, offset = 0, start = ip; } - if (dictMode == ZSTD_dictMatchState) { + if (isDxS) { const U32 repIndex = (U32)(ip - base) - offset_1; const BYTE* repMatch = repIndex < prefixLowestIndex ? dictBase + (repIndex - dictIndexDelta) : @@ -771,7 +1024,7 @@ ZSTD_compressBlock_lazy_generic( if ((mlRep >= 4) && (gain2 > gain1)) matchLength = mlRep, offset = 0, start = ip; } - if (dictMode == ZSTD_dictMatchState) { + if (isDxS) { const U32 repIndex = (U32)(ip - base) - offset_1; const BYTE* repMatch = repIndex < prefixLowestIndex ? dictBase + (repIndex - dictIndexDelta) : @@ -809,7 +1062,7 @@ ZSTD_compressBlock_lazy_generic( && (start[-1] == (start-(offset-ZSTD_REP_MOVE))[-1]) ) /* only search for offset within prefix */ { start--; matchLength++; } } - if (dictMode == ZSTD_dictMatchState) { + if (isDxS) { U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex; const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest; @@ -825,12 +1078,11 @@ _storeSequence: } /* check immediate repcode */ - if (dictMode == ZSTD_dictMatchState) { + if (isDxS) { while (ip <= ilimit) { U32 const current2 = (U32)(ip-base); U32 const repIndex = current2 - offset_2; - const BYTE* repMatch = dictMode == ZSTD_dictMatchState - && repIndex < prefixLowestIndex ? + const BYTE* repMatch = repIndex < prefixLowestIndex ? dictBase - dictIndexDelta + repIndex : base + repIndex; if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */) @@ -925,6 +1177,28 @@ size_t ZSTD_compressBlock_greedy_dictMatchState( } +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch); +} + + FORCE_INLINE_TEMPLATE size_t ZSTD_compressBlock_lazy_extDict_generic( ZSTD_matchState_t* ms, seqStore_t* seqStore, @@ -968,11 +1242,11 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( size_t matchLength=0; size_t offset=0; const BYTE* start=ip+1; - U32 current = (U32)(ip-base); + U32 curr = (U32)(ip-base); /* check repCode */ - { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, current+1, windowLog); - const U32 repIndex = (U32)(current+1 - offset_1); + { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr+1, windowLog); + const U32 repIndex = (U32)(curr+1 - offset_1); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ @@ -999,11 +1273,11 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( if (depth>=1) while (ip<ilimit) { ip ++; - current++; + curr++; /* check repCode */ if (offset) { - const U32 windowLow = ZSTD_getLowestMatchIndex(ms, current, windowLog); - const U32 repIndex = (U32)(current - offset_1); + const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog); + const U32 repIndex = (U32)(curr - offset_1); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ @@ -1030,11 +1304,11 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( /* let's find an even better one */ if ((depth==2) && (ip<ilimit)) { ip ++; - current++; + curr++; /* check repCode */ if (offset) { - const U32 windowLow = ZSTD_getLowestMatchIndex(ms, current, windowLog); - const U32 repIndex = (U32)(current - offset_1); + const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog); + const U32 repIndex = (U32)(curr - offset_1); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ diff --git a/lib/compress/zstd_lazy.h b/lib/compress/zstd_lazy.h index 581936f0..d0214d5e 100644 --- a/lib/compress/zstd_lazy.h +++ b/lib/compress/zstd_lazy.h @@ -17,8 +17,18 @@ extern "C" { #include "zstd_compress_internal.h" +/** + * Dedicated Dictionary Search Structure bucket log. In the + * ZSTD_dedicatedDictSearch mode, the hashTable has + * 2 ** ZSTD_LAZY_DDSS_BUCKET_LOG entries in each bucket, rather than just + * one. + */ +#define ZSTD_LAZY_DDSS_BUCKET_LOG 2 + U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip); +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip); + void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */ size_t ZSTD_compressBlock_btlazy2( @@ -47,6 +57,16 @@ size_t ZSTD_compressBlock_greedy_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + size_t ZSTD_compressBlock_greedy_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 8c479483..3f3d7c46 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -27,13 +27,6 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH; - if (cParams->strategy >= ZSTD_btopt) { - /* Get out of the way of the optimal parser */ - U32 const minMatch = MAX(cParams->targetLength, params->minMatchLength); - assert(minMatch >= ZSTD_LDM_MINMATCH_MIN); - assert(minMatch <= ZSTD_LDM_MINMATCH_MAX); - params->minMatchLength = minMatch; - } if (params->hashLog == 0) { params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG); assert(params->hashLog <= ZSTD_HASHLOG_MAX); @@ -150,10 +143,10 @@ static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState, * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */ static size_t ZSTD_ldm_countBackwardsMatch( const BYTE* pIn, const BYTE* pAnchor, - const BYTE* pMatch, const BYTE* pBase) + const BYTE* pMatch, const BYTE* pMatchBase) { size_t matchLength = 0; - while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) { + while (pIn > pAnchor && pMatch > pMatchBase && pIn[-1] == pMatch[-1]) { pIn--; pMatch--; matchLength++; @@ -161,6 +154,27 @@ static size_t ZSTD_ldm_countBackwardsMatch( return matchLength; } +/** ZSTD_ldm_countBackwardsMatch_2segments() : + * Returns the number of bytes that match backwards from pMatch, + * even with the backwards match spanning 2 different segments. + * + * On reaching `pMatchBase`, start counting from mEnd */ +static size_t ZSTD_ldm_countBackwardsMatch_2segments( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase, + const BYTE* pExtDictStart, const BYTE* pExtDictEnd) +{ + size_t matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase); + if (pMatch - matchLength != pMatchBase || pMatchBase == pExtDictStart) { + /* If backwards match is entirely in the extDict or prefix, immediately return */ + return matchLength; + } + DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found 2-parts backwards match (length in prefix==%zu)", matchLength); + matchLength += ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, pExtDictEnd, pExtDictStart); + DEBUGLOG(7, "final backwards match length = %zu", matchLength); + return matchLength; +} + /** ZSTD_ldm_fillFastTables() : * * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies. @@ -246,10 +260,10 @@ void ZSTD_ldm_fillHashTable( * (after a long match, only update tables a limited amount). */ static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor) { - U32 const current = (U32)(anchor - ms->window.base); - if (current > ms->nextToUpdate + 1024) { + U32 const curr = (U32)(anchor - ms->window.base); + if (curr > ms->nextToUpdate + 1024) { ms->nextToUpdate = - current - MIN(512, current - ms->nextToUpdate - 1024); + curr - MIN(512, curr - ms->nextToUpdate - 1024); } } @@ -286,7 +300,7 @@ static size_t ZSTD_ldm_generateSequences_internal( while (ip <= ilimit) { size_t mLength; - U32 const current = (U32)(ip - base); + U32 const curr = (U32)(ip - base); size_t forwardMatchLength = 0, backwardMatchLength = 0; ldmEntry_t* bestEntry = NULL; if (ip != istart) { @@ -336,8 +350,9 @@ static size_t ZSTD_ldm_generateSequences_internal( continue; } curBackwardMatchLength = - ZSTD_ldm_countBackwardsMatch(ip, anchor, pMatch, - lowMatchPtr); + ZSTD_ldm_countBackwardsMatch_2segments(ip, anchor, + pMatch, lowMatchPtr, + dictStart, dictEnd); curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength; } else { /* !extDict */ @@ -365,7 +380,7 @@ static size_t ZSTD_ldm_generateSequences_internal( /* No match found -- continue searching */ if (bestEntry == NULL) { ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash, - hBits, current, + hBits, curr, *params); ip++; continue; @@ -377,11 +392,11 @@ static size_t ZSTD_ldm_generateSequences_internal( { /* Store the sequence: - * ip = current - backwardMatchLength + * ip = curr - backwardMatchLength * The match is at (bestEntry->offset - backwardMatchLength) */ U32 const matchIndex = bestEntry->offset; - U32 const offset = current - matchIndex; + U32 const offset = curr - matchIndex; rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size; /* Out of sequence storage */ @@ -562,6 +577,23 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, return sequence; } +void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; + } +} + size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) @@ -577,6 +609,15 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, BYTE const* ip = istart; DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize); + /* If using opt parser, use LDMs only as candidates rather than always accepting them */ + if (cParams->strategy >= ZSTD_btopt) { + size_t lastLLSize; + ms->ldmSeqStore = rawSeqStore; + lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); + ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize); + return lastLLSize; + } + assert(rawSeqStore->pos <= rawSeqStore->size); assert(rawSeqStore->size <= rawSeqStore->capacity); /* Loop through each sequence and apply the block compressor to the lits */ diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h index 229ea05a..6561024e 100644 --- a/lib/compress/zstd_ldm.h +++ b/lib/compress/zstd_ldm.h @@ -78,6 +78,12 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch); +/* ZSTD_ldm_skipRawSeqStoreBytes(): + * Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'. + * Not to be used in conjunction with ZSTD_ldm_skipSequences(). + * Must be called for data with is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes); /** ZSTD_ldm_getTableSize() : * Estimate the space needed for long distance matching tables or 0 if LDM is diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 36fff050..e55c459d 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -386,32 +386,32 @@ static U32 ZSTD_insertBt1( const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; const BYTE* match; - const U32 current = (U32)(ip-base); - const U32 btLow = btMask >= current ? 0 : current - btMask; - U32* smallerPtr = bt + 2*(current&btMask); + const U32 curr = (U32)(ip-base); + const U32 btLow = btMask >= curr ? 0 : curr - btMask; + U32* smallerPtr = bt + 2*(curr&btMask); U32* largerPtr = smallerPtr + 1; U32 dummy32; /* to be nullified at the end */ U32 const windowLow = ms->window.lowLimit; - U32 matchEndIdx = current+8+1; + U32 matchEndIdx = curr+8+1; size_t bestLength = 8; U32 nbCompares = 1U << cParams->searchLog; #ifdef ZSTD_C_PREDICT - U32 predictedSmall = *(bt + 2*((current-1)&btMask) + 0); - U32 predictedLarge = *(bt + 2*((current-1)&btMask) + 1); + U32 predictedSmall = *(bt + 2*((curr-1)&btMask) + 0); + U32 predictedLarge = *(bt + 2*((curr-1)&btMask) + 1); predictedSmall += (predictedSmall>0); predictedLarge += (predictedLarge>0); #endif /* ZSTD_C_PREDICT */ - DEBUGLOG(8, "ZSTD_insertBt1 (%u)", current); + DEBUGLOG(8, "ZSTD_insertBt1 (%u)", curr); assert(ip <= iend-8); /* required for h calculation */ - hashTable[h] = current; /* Update Hash Table */ + hashTable[h] = curr; /* Update Hash Table */ assert(windowLow > 0); while (nbCompares-- && (matchIndex >= windowLow)) { U32* const nextPtr = bt + 2*(matchIndex & btMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - assert(matchIndex < current); + assert(matchIndex < curr); #ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */ const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ @@ -474,8 +474,8 @@ static U32 ZSTD_insertBt1( *smallerPtr = *largerPtr = 0; { U32 positions = 0; if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */ - assert(matchEndIdx > current + 8); - return MAX(positions, matchEndIdx - (current + 8)); + assert(matchEndIdx > curr + 8); + return MAX(positions, matchEndIdx - (curr + 8)); } } @@ -519,7 +519,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); const BYTE* const base = ms->window.base; - U32 const current = (U32)(ip-base); + U32 const curr = (U32)(ip-base); U32 const hashLog = cParams->hashLog; U32 const minMatch = (mls==3) ? 3 : 4; U32* const hashTable = ms->hashTable; @@ -533,12 +533,12 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 const dictLimit = ms->window.dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; - U32 const btLow = (btMask >= current) ? 0 : current - btMask; - U32 const windowLow = ZSTD_getLowestMatchIndex(ms, current, cParams->windowLog); + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); U32 const matchLow = windowLow ? windowLow : 1; - U32* smallerPtr = bt + 2*(current&btMask); - U32* largerPtr = bt + 2*(current&btMask) + 1; - U32 matchEndIdx = current+8+1; /* farthest referenced position of any match => detects repetitive patterns */ + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr+8+1; /* farthest referenced position of any match => detects repetitive patterns */ U32 dummy32; /* to be nullified at the end */ U32 mnum = 0; U32 nbCompares = 1U << cParams->searchLog; @@ -557,7 +557,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit; size_t bestLength = lengthToBeat-1; - DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", current); + DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", curr); /* check repCode */ assert(ll0 <= 1); /* necessarily 1 or 0 */ @@ -565,29 +565,29 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 repCode; for (repCode = ll0; repCode < lastR; repCode++) { U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; - U32 const repIndex = current - repOffset; + U32 const repIndex = curr - repOffset; U32 repLen = 0; - assert(current >= dictLimit); - if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < current-dictLimit) { /* equivalent to `current > repIndex >= dictLimit` */ + assert(curr >= dictLimit); + if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < curr-dictLimit) { /* equivalent to `curr > repIndex >= dictLimit` */ /* We must validate the repcode offset because when we're using a dictionary the * valid offset range shrinks when the dictionary goes out of bounds. */ if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) { repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch; } - } else { /* repIndex < dictLimit || repIndex >= current */ + } else { /* repIndex < dictLimit || repIndex >= curr */ const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ? dmsBase + repIndex - dmsIndexDelta : dictBase + repIndex; - assert(current >= windowLow); + assert(curr >= windowLow); if ( dictMode == ZSTD_extDict - && ( ((repOffset-1) /*intentional overflow*/ < current - windowLow) /* equivalent to `current > repIndex >= windowLow` */ + && ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */ & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */) && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; } if (dictMode == ZSTD_dictMatchState - && ( ((repOffset-1) /*intentional overflow*/ < current - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `current > repIndex >= dmsLowLimit` */ + && ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */ & ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */ && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; @@ -609,7 +609,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( if ((mls == 3) /*static*/ && (bestLength < mls)) { U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip); if ((matchIndex3 >= matchLow) - & (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { + & (curr - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { size_t mlen; if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) { const BYTE* const match = base + matchIndex3; @@ -624,26 +624,26 @@ U32 ZSTD_insertBtAndGetAllMatches ( DEBUGLOG(8, "found small match with hlog3, of length %u", (U32)mlen); bestLength = mlen; - assert(current > matchIndex3); + assert(curr > matchIndex3); assert(mnum==0); /* no prior solution */ - matches[0].off = (current - matchIndex3) + ZSTD_REP_MOVE; + matches[0].off = (curr - matchIndex3) + ZSTD_REP_MOVE; matches[0].len = (U32)mlen; mnum = 1; if ( (mlen > sufficient_len) | (ip+mlen == iLimit) ) { /* best possible length */ - ms->nextToUpdate = current+1; /* skip insertion */ + ms->nextToUpdate = curr+1; /* skip insertion */ return 1; } } } /* no dictMatchState lookup: dicts don't have a populated HC3 table */ } - hashTable[h] = current; /* Update Hash Table */ + hashTable[h] = curr; /* Update Hash Table */ while (nbCompares-- && (matchIndex >= matchLow)) { U32* const nextPtr = bt + 2*(matchIndex & btMask); const BYTE* match; size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - assert(current > matchIndex); + assert(curr > matchIndex); if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ @@ -660,12 +660,12 @@ U32 ZSTD_insertBtAndGetAllMatches ( if (matchLength > bestLength) { DEBUGLOG(8, "found match of length %u at distance %u (offCode=%u)", - (U32)matchLength, current - matchIndex, current - matchIndex + ZSTD_REP_MOVE); + (U32)matchLength, curr - matchIndex, curr - matchIndex + ZSTD_REP_MOVE); assert(matchEndIdx > matchIndex); if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; bestLength = matchLength; - matches[mnum].off = (current - matchIndex) + ZSTD_REP_MOVE; + matches[mnum].off = (curr - matchIndex) + ZSTD_REP_MOVE; matches[mnum].len = (U32)matchLength; mnum++; if ( (matchLength > ZSTD_OPT_NUM) @@ -708,11 +708,11 @@ U32 ZSTD_insertBtAndGetAllMatches ( if (matchLength > bestLength) { matchIndex = dictMatchIndex + dmsIndexDelta; DEBUGLOG(8, "found dms match of length %u at distance %u (offCode=%u)", - (U32)matchLength, current - matchIndex, current - matchIndex + ZSTD_REP_MOVE); + (U32)matchLength, curr - matchIndex, curr - matchIndex + ZSTD_REP_MOVE); if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; bestLength = matchLength; - matches[mnum].off = (current - matchIndex) + ZSTD_REP_MOVE; + matches[mnum].off = (curr - matchIndex) + ZSTD_REP_MOVE; matches[mnum].len = (U32)matchLength; mnum++; if ( (matchLength > ZSTD_OPT_NUM) @@ -733,7 +733,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( } } - assert(matchEndIdx > current+8); + assert(matchEndIdx > curr+8); ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ return mnum; } @@ -764,6 +764,140 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( } } +/************************* +* LDM helper functions * +*************************/ + +/* Struct containing info needed to make decision about ldm inclusion */ +typedef struct { + rawSeqStore_t seqStore; /* External match candidates store for this block */ + U32 startPosInBlock; /* Start position of the current match candidate */ + U32 endPosInBlock; /* End position of the current match candidate */ + U32 offset; /* Offset of the match candidate */ +} ZSTD_optLdm_t; + +/* ZSTD_optLdm_skipRawSeqStoreBytes(): + * Moves forward in rawSeqStore by nbBytes, which will update the fields 'pos' and 'posInSequence'. + */ +static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; + } +} + +/* ZSTD_opt_getNextMatchAndUpdateSeqStore(): + * Calculates the beginning and end of the next match in the current block. + * Updates 'pos' and 'posInSequence' of the ldmSeqStore. + */ +static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock, + U32 blockBytesRemaining) { + rawSeq currSeq; + U32 currBlockEndPos; + U32 literalsBytesRemaining; + U32 matchBytesRemaining; + + /* Setting match end position to MAX to ensure we never use an LDM during this block */ + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + return; + } + /* Calculate appropriate bytes left in matchLength and litLength after adjusting + based on ldmSeqStore->posInSequence */ + currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos]; + assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength); + currBlockEndPos = currPosInBlock + blockBytesRemaining; + literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ? + currSeq.litLength - (U32)optLdm->seqStore.posInSequence : + 0; + matchBytesRemaining = (literalsBytesRemaining == 0) ? + currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) : + currSeq.matchLength; + + /* If there are more literal bytes than bytes remaining in block, no ldm is possible */ + if (literalsBytesRemaining >= blockBytesRemaining) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining); + return; + } + + /* Matches may be < MINMATCH by this process. In that case, we will reject them + when we are deciding whether or not to add the ldm */ + optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining; + optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining; + optLdm->offset = currSeq.offset; + + if (optLdm->endPosInBlock > currBlockEndPos) { + /* Match ends after the block ends, we can't use the whole match */ + optLdm->endPosInBlock = currBlockEndPos; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock); + } else { + /* Consume nb of bytes equal to size of sequence left */ + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining); + } +} + +/* ZSTD_optLdm_maybeAddMatch(): + * Adds a match if it's long enough, based on it's 'matchStartPosInBlock' + * and 'matchEndPosInBlock', into 'matches'. Maintains the correct ordering of 'matches' + */ +static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, + ZSTD_optLdm_t* optLdm, U32 currPosInBlock) { + U32 posDiff = currPosInBlock - optLdm->startPosInBlock; + /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */ + U32 candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; + U32 candidateOffCode = optLdm->offset + ZSTD_REP_MOVE; + + /* Ensure that current block position is not outside of the match */ + if (currPosInBlock < optLdm->startPosInBlock + || currPosInBlock >= optLdm->endPosInBlock + || candidateMatchLength < MINMATCH) { + return; + } + + if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) { + DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u", + candidateOffCode, candidateMatchLength, currPosInBlock); + matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffCode; + (*nbMatches)++; + } +} + +/* ZSTD_optLdm_processMatchCandidate(): + * Wrapper function to update ldm seq store and call ldm functions as necessary. + */ +static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_t* matches, U32* nbMatches, + U32 currPosInBlock, U32 remainingBytes) { + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + return; + } + + if (currPosInBlock >= optLdm->endPosInBlock) { + if (currPosInBlock > optLdm->endPosInBlock) { + /* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily + * at the end of a match from the ldm seq store, and will often be some bytes + * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" + */ + U32 posOvershoot = currPosInBlock - optLdm->endPosInBlock; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot); + } + ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes); + } + ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); +} /*-******************************* * Optimal parser @@ -817,6 +951,11 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, ZSTD_optimal_t* const opt = optStatePtr->priceTable; ZSTD_match_t* const matches = optStatePtr->matchTable; ZSTD_optimal_t lastSequence; + ZSTD_optLdm_t optLdm; + + optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore; + optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0; + ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); /* init */ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", @@ -832,7 +971,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* find first match */ { U32 const litlen = (U32)(ip - anchor); U32 const ll0 = !litlen; - U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); + U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(ip-istart), (U32)(iend - ip)); if (!nbMatches) { ip++; continue; } /* initialize opt[0] */ @@ -925,9 +1066,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, if (opt[cur].mlen != 0) { U32 const prev = cur - opt[cur].mlen; repcodes_t newReps = ZSTD_updateRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0); - memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t)); + ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t)); } else { - memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t)); + ZSTD_memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t)); } /* last match must start at a minimum distance of 8 from oend */ @@ -945,8 +1086,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; U32 const previousPrice = opt[cur].price; U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); - U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); + U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); U32 matchNb; + + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(inr-istart), (U32)(iend-inr)); + if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); continue; @@ -1010,9 +1155,9 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ */ if (lastSequence.mlen != 0) { repcodes_t reps = ZSTD_updateRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0); - memcpy(rep, &reps, sizeof(reps)); + ZSTD_memcpy(rep, &reps, sizeof(reps)); } else { - memcpy(rep, opt[cur].rep, sizeof(repcodes_t)); + ZSTD_memcpy(rep, opt[cur].rep, sizeof(repcodes_t)); } { U32 const storeEnd = cur + 1; @@ -1110,7 +1255,7 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms, const void* src, size_t srcSize) { U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */ - memcpy(tmpRep, rep, sizeof(tmpRep)); + ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep)); DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize); assert(ms->opt.litLengthSum == 0); /* first block */ @@ -1143,7 +1288,7 @@ size_t ZSTD_compressBlock_btultra2( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) { - U32 const current = (U32)((const BYTE*)src - ms->window.base); + U32 const curr = (U32)((const BYTE*)src - ms->window.base); DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize); /* 2-pass strategy: @@ -1158,7 +1303,7 @@ size_t ZSTD_compressBlock_btultra2( if ( (ms->opt.litLengthSum==0) /* first block */ && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ - && (current == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ + && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ && (srcSize > ZSTD_PREDEF_THRESHOLD) ) { ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 1e3c8fdb..50454a50 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -20,8 +20,7 @@ /* ====== Dependencies ====== */ -#include <string.h> /* memcpy, memset */ -#include <limits.h> /* INT_MAX, UINT_MAX */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset, INT_MAX, UINT_MAX */ #include "../common/mem.h" /* MEM_STATIC */ #include "../common/pool.h" /* threadpool */ #include "../common/threading.h" /* mutex */ @@ -106,11 +105,11 @@ typedef struct ZSTDMT_bufferPool_s { static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbWorkers, ZSTD_customMem cMem) { unsigned const maxNbBuffers = 2*nbWorkers + 3; - ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_calloc( + ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_customCalloc( sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem); if (bufPool==NULL) return NULL; if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) { - ZSTD_free(bufPool, cMem); + ZSTD_customFree(bufPool, cMem); return NULL; } bufPool->bufferSize = 64 KB; @@ -127,10 +126,10 @@ static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool) if (!bufPool) return; /* compatibility with free on NULL */ for (u=0; u<bufPool->totalBuffers; u++) { DEBUGLOG(4, "free buffer %2u (address:%08X)", u, (U32)(size_t)bufPool->bTable[u].start); - ZSTD_free(bufPool->bTable[u].start, bufPool->cMem); + ZSTD_customFree(bufPool->bTable[u].start, bufPool->cMem); } ZSTD_pthread_mutex_destroy(&bufPool->poolMutex); - ZSTD_free(bufPool, bufPool->cMem); + ZSTD_customFree(bufPool, bufPool->cMem); } /* only works at initialization, not during compression */ @@ -201,13 +200,13 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) } /* size conditions not respected : scratch this buffer, create new one */ DEBUGLOG(5, "ZSTDMT_getBuffer: existing buffer does not meet size conditions => freeing"); - ZSTD_free(buf.start, bufPool->cMem); + ZSTD_customFree(buf.start, bufPool->cMem); } ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); /* create new buffer */ DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer"); { buffer_t buffer; - void* const start = ZSTD_malloc(bSize, bufPool->cMem); + void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); buffer.start = start; /* note : start can be NULL if malloc fails ! */ buffer.capacity = (start==NULL) ? 0 : bSize; if (start==NULL) { @@ -229,13 +228,13 @@ static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer) { size_t const bSize = bufPool->bufferSize; if (buffer.capacity < bSize) { - void* const start = ZSTD_malloc(bSize, bufPool->cMem); + void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); buffer_t newBuffer; newBuffer.start = start; newBuffer.capacity = start == NULL ? 0 : bSize; if (start != NULL) { assert(newBuffer.capacity >= buffer.capacity); - memcpy(newBuffer.start, buffer.start, buffer.capacity); + ZSTD_memcpy(newBuffer.start, buffer.start, buffer.capacity); DEBUGLOG(5, "ZSTDMT_resizeBuffer: created buffer of size %u", (U32)bSize); return newBuffer; } @@ -261,14 +260,12 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); /* Reached bufferPool capacity (should not happen) */ DEBUGLOG(5, "ZSTDMT_releaseBuffer: pool capacity reached => freeing "); - ZSTD_free(buf.start, bufPool->cMem); + ZSTD_customFree(buf.start, bufPool->cMem); } /* ===== Seq Pool Wrapper ====== */ -static rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0}; - typedef ZSTDMT_bufferPool ZSTDMT_seqPool; static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) @@ -278,7 +275,7 @@ static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) static rawSeqStore_t bufferToSeq(buffer_t buffer) { - rawSeqStore_t seq = {NULL, 0, 0, 0}; + rawSeqStore_t seq = kNullRawSeqStore; seq.seq = (rawSeq*)buffer.start; seq.capacity = buffer.capacity / sizeof(rawSeq); return seq; @@ -354,7 +351,7 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) for (cid=0; cid<pool->totalCCtx; cid++) ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */ ZSTD_pthread_mutex_destroy(&pool->poolMutex); - ZSTD_free(pool, pool->cMem); + ZSTD_customFree(pool, pool->cMem); } /* ZSTDMT_createCCtxPool() : @@ -362,12 +359,12 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers, ZSTD_customMem cMem) { - ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc( + ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_customCalloc( sizeof(ZSTDMT_CCtxPool) + (nbWorkers-1)*sizeof(ZSTD_CCtx*), cMem); assert(nbWorkers > 0); if (!cctxPool) return NULL; if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) { - ZSTD_free(cctxPool, cMem); + ZSTD_customFree(cctxPool, cMem); return NULL; } cctxPool->cMem = cMem; @@ -478,7 +475,7 @@ ZSTDMT_serialState_reset(serialState_t* serialState, serialState->ldmState.hashPower = ZSTD_rollingHash_primePower(params.ldmParams.minMatchLength); } else { - memset(¶ms.ldmParams, 0, sizeof(params.ldmParams)); + ZSTD_memset(¶ms.ldmParams, 0, sizeof(params.ldmParams)); } serialState->nextJobID = 0; if (params.fParams.checksumFlag) @@ -499,18 +496,18 @@ ZSTDMT_serialState_reset(serialState_t* serialState, ZSTD_window_init(&serialState->ldmState.window); /* Resize tables and output space if necessary. */ if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) { - ZSTD_free(serialState->ldmState.hashTable, cMem); - serialState->ldmState.hashTable = (ldmEntry_t*)ZSTD_malloc(hashSize, cMem); + ZSTD_customFree(serialState->ldmState.hashTable, cMem); + serialState->ldmState.hashTable = (ldmEntry_t*)ZSTD_customMalloc(hashSize, cMem); } if (serialState->ldmState.bucketOffsets == NULL || prevBucketLog < bucketLog) { - ZSTD_free(serialState->ldmState.bucketOffsets, cMem); - serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_malloc(bucketSize, cMem); + ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); + serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_customMalloc(bucketSize, cMem); } if (!serialState->ldmState.hashTable || !serialState->ldmState.bucketOffsets) return 1; /* Zero the tables */ - memset(serialState->ldmState.hashTable, 0, hashSize); - memset(serialState->ldmState.bucketOffsets, 0, bucketSize); + ZSTD_memset(serialState->ldmState.hashTable, 0, hashSize); + ZSTD_memset(serialState->ldmState.bucketOffsets, 0, bucketSize); /* Update window state and fill hash table with dict */ serialState->ldmState.loadedDictEnd = 0; @@ -537,7 +534,7 @@ ZSTDMT_serialState_reset(serialState_t* serialState, static int ZSTDMT_serialState_init(serialState_t* serialState) { int initError = 0; - memset(serialState, 0, sizeof(*serialState)); + ZSTD_memset(serialState, 0, sizeof(*serialState)); initError |= ZSTD_pthread_mutex_init(&serialState->mutex, NULL); initError |= ZSTD_pthread_cond_init(&serialState->cond, NULL); initError |= ZSTD_pthread_mutex_init(&serialState->ldmWindowMutex, NULL); @@ -552,8 +549,8 @@ static void ZSTDMT_serialState_free(serialState_t* serialState) ZSTD_pthread_cond_destroy(&serialState->cond); ZSTD_pthread_mutex_destroy(&serialState->ldmWindowMutex); ZSTD_pthread_cond_destroy(&serialState->ldmWindowCond); - ZSTD_free(serialState->ldmState.hashTable, cMem); - ZSTD_free(serialState->ldmState.bucketOffsets, cMem); + ZSTD_customFree(serialState->ldmState.hashTable, cMem); + ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); } static void ZSTDMT_serialState_update(serialState_t* serialState, @@ -820,7 +817,6 @@ struct ZSTDMT_CCtx_s { roundBuff_t roundBuff; serialState_t serial; rsyncState_t rsync; - unsigned singleBlockingThread; unsigned jobIDMask; unsigned doneJobID; unsigned nextJobID; @@ -832,6 +828,7 @@ struct ZSTDMT_CCtx_s { ZSTD_customMem cMem; ZSTD_CDict* cdictLocal; const ZSTD_CDict* cdict; + unsigned providedFactory: 1; }; static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZSTD_customMem cMem) @@ -842,7 +839,7 @@ static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZS ZSTD_pthread_mutex_destroy(&jobTable[jobNb].job_mutex); ZSTD_pthread_cond_destroy(&jobTable[jobNb].job_cond); } - ZSTD_free(jobTable, cMem); + ZSTD_customFree(jobTable, cMem); } /* ZSTDMT_allocJobsTable() @@ -854,7 +851,7 @@ static ZSTDMT_jobDescription* ZSTDMT_createJobsTable(U32* nbJobsPtr, ZSTD_custom U32 const nbJobs = 1 << nbJobsLog2; U32 jobNb; ZSTDMT_jobDescription* const jobTable = (ZSTDMT_jobDescription*) - ZSTD_calloc(nbJobs * sizeof(ZSTDMT_jobDescription), cMem); + ZSTD_customCalloc(nbJobs * sizeof(ZSTDMT_jobDescription), cMem); int initError = 0; if (jobTable==NULL) return NULL; *nbJobsPtr = nbJobs; @@ -885,12 +882,12 @@ static size_t ZSTDMT_expandJobsTable (ZSTDMT_CCtx* mtctx, U32 nbWorkers) { /* ZSTDMT_CCtxParam_setNbWorkers(): * Internal use only */ -size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers) +static size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers) { return ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers); } -MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem) +MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) { ZSTDMT_CCtx* mtctx; U32 nbJobs = nbWorkers + 2; @@ -903,12 +900,19 @@ MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, /* invalid custom allocator */ return NULL; - mtctx = (ZSTDMT_CCtx*) ZSTD_calloc(sizeof(ZSTDMT_CCtx), cMem); + mtctx = (ZSTDMT_CCtx*) ZSTD_customCalloc(sizeof(ZSTDMT_CCtx), cMem); if (!mtctx) return NULL; ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); mtctx->cMem = cMem; mtctx->allJobsCompleted = 1; - mtctx->factory = POOL_create_advanced(nbWorkers, 0, cMem); + if (pool != NULL) { + mtctx->factory = pool; + mtctx->providedFactory = 1; + } + else { + mtctx->factory = POOL_create_advanced(nbWorkers, 0, cMem); + mtctx->providedFactory = 0; + } mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, cMem); assert(nbJobs > 0); assert((nbJobs & (nbJobs - 1)) == 0); /* ensure nbJobs is a power of 2 */ mtctx->jobIDMask = nbJobs - 1; @@ -925,22 +929,18 @@ MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, return mtctx; } -ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem) +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) { #ifdef ZSTD_MULTITHREAD - return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem); + return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem, pool); #else (void)nbWorkers; (void)cMem; + (void)pool; return NULL; #endif } -ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers) -{ - return ZSTDMT_createCCtx_advanced(nbWorkers, ZSTD_defaultCMem); -} - /* ZSTDMT_releaseAllJobResources() : * note : ensure all workers are killed first ! */ @@ -957,7 +957,7 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx) ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); /* Clear the job description, but keep the mutex/cond */ - memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID])); + ZSTD_memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID])); mtctx->jobs[jobID].job_mutex = mutex; mtctx->jobs[jobID].job_cond = cond; } @@ -984,7 +984,8 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* mtctx) size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) { if (mtctx==NULL) return 0; /* compatible with free on NULL */ - POOL_free(mtctx->factory); /* stop and free worker threads */ + if (!mtctx->providedFactory) + POOL_free(mtctx->factory); /* stop and free worker threads */ ZSTDMT_releaseAllJobResources(mtctx); /* release job resources into pools first */ ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); ZSTDMT_freeBufferPool(mtctx->bufPool); @@ -993,8 +994,8 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) ZSTDMT_serialState_free(&mtctx->serial); ZSTD_freeCDict(mtctx->cdictLocal); if (mtctx->roundBuff.buffer) - ZSTD_free(mtctx->roundBuff.buffer, mtctx->cMem); - ZSTD_free(mtctx, mtctx->cMem); + ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); + ZSTD_customFree(mtctx, mtctx->cMem); return 0; } @@ -1011,65 +1012,6 @@ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) + mtctx->roundBuff.capacity; } -/* Internal only */ -size_t -ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, - ZSTDMT_parameter parameter, - int value) -{ - DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter"); - switch(parameter) - { - case ZSTDMT_p_jobSize : - DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value); - return ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, value); - case ZSTDMT_p_overlapLog : - DEBUGLOG(4, "ZSTDMT_p_overlapLog : %i", value); - return ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, value); - case ZSTDMT_p_rsyncable : - DEBUGLOG(4, "ZSTD_p_rsyncable : %i", value); - return ZSTD_CCtxParams_setParameter(params, ZSTD_c_rsyncable, value); - default : - return ERROR(parameter_unsupported); - } -} - -size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value) -{ - DEBUGLOG(4, "ZSTDMT_setMTCtxParameter"); - return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value); -} - -size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value) -{ - switch (parameter) { - case ZSTDMT_p_jobSize: - return ZSTD_CCtxParams_getParameter(&mtctx->params, ZSTD_c_jobSize, value); - case ZSTDMT_p_overlapLog: - return ZSTD_CCtxParams_getParameter(&mtctx->params, ZSTD_c_overlapLog, value); - case ZSTDMT_p_rsyncable: - return ZSTD_CCtxParams_getParameter(&mtctx->params, ZSTD_c_rsyncable, value); - default: - return ERROR(parameter_unsupported); - } -} - -/* Sets parameters relevant to the compression job, - * initializing others to default values. */ -static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(const ZSTD_CCtx_params* params) -{ - ZSTD_CCtx_params jobParams = *params; - /* Clear parameters related to multithreading */ - jobParams.forceWindow = 0; - jobParams.nbWorkers = 0; - jobParams.jobSize = 0; - jobParams.overlapLog = 0; - jobParams.rsyncable = 0; - memset(&jobParams.ldmParams, 0, sizeof(ldmParams_t)); - memset(&jobParams.customMem, 0, sizeof(ZSTD_customMem)); - return jobParams; -} - /* ZSTDMT_resize() : * @return : error code if fails, 0 on success */ @@ -1098,7 +1040,7 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)", compressionLevel); mtctx->params.compressionLevel = compressionLevel; - { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0); + { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); cParams.windowLog = saved_wlog; mtctx->params.cParams = cParams; } @@ -1185,8 +1127,8 @@ static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params) if (params->ldmParams.enableLdm) { /* In Long Range Mode, the windowLog is typically oversized. * In which case, it's preferable to determine the jobSize - * based on chainLog instead. */ - jobLog = MAX(21, params->cParams.chainLog + 4); + * based on cycleLog instead. */ + jobLog = MAX(21, ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy) + 3); } else { jobLog = MAX(20, params->cParams.windowLog + 2); } @@ -1240,174 +1182,6 @@ static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params) return (ovLog==0) ? 0 : (size_t)1 << ovLog; } -static unsigned -ZSTDMT_computeNbJobs(const ZSTD_CCtx_params* params, size_t srcSize, unsigned nbWorkers) -{ - assert(nbWorkers>0); - { size_t const jobSizeTarget = (size_t)1 << ZSTDMT_computeTargetJobLog(params); - size_t const jobMaxSize = jobSizeTarget << 2; - size_t const passSizeMax = jobMaxSize * nbWorkers; - unsigned const multiplier = (unsigned)(srcSize / passSizeMax) + 1; - unsigned const nbJobsLarge = multiplier * nbWorkers; - unsigned const nbJobsMax = (unsigned)(srcSize / jobSizeTarget) + 1; - unsigned const nbJobsSmall = MIN(nbJobsMax, nbWorkers); - return (multiplier>1) ? nbJobsLarge : nbJobsSmall; -} } - -/* ZSTDMT_compress_advanced_internal() : - * This is a blocking function : it will only give back control to caller after finishing its compression job. - */ -static size_t -ZSTDMT_compress_advanced_internal( - ZSTDMT_CCtx* mtctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict, - ZSTD_CCtx_params params) -{ - ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(¶ms); - size_t const overlapSize = ZSTDMT_computeOverlapSize(¶ms); - unsigned const nbJobs = ZSTDMT_computeNbJobs(¶ms, srcSize, params.nbWorkers); - size_t const proposedJobSize = (srcSize + (nbJobs-1)) / nbJobs; - size_t const avgJobSize = (((proposedJobSize-1) & 0x1FFFF) < 0x7FFF) ? proposedJobSize + 0xFFFF : proposedJobSize; /* avoid too small last block */ - const char* const srcStart = (const char*)src; - size_t remainingSrcSize = srcSize; - unsigned const compressWithinDst = (dstCapacity >= ZSTD_compressBound(srcSize)) ? nbJobs : (unsigned)(dstCapacity / ZSTD_compressBound(avgJobSize)); /* presumes avgJobSize >= 256 KB, which should be the case */ - size_t frameStartPos = 0, dstBufferPos = 0; - assert(jobParams.nbWorkers == 0); - assert(mtctx->cctxPool->totalCCtx == params.nbWorkers); - - params.jobSize = (U32)avgJobSize; - DEBUGLOG(4, "ZSTDMT_compress_advanced_internal: nbJobs=%2u (rawSize=%u bytes; fixedSize=%u) ", - nbJobs, (U32)proposedJobSize, (U32)avgJobSize); - - if ((nbJobs==1) | (params.nbWorkers<=1)) { /* fallback to single-thread mode : this is a blocking invocation anyway */ - ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0]; - DEBUGLOG(4, "ZSTDMT_compress_advanced_internal: fallback to single-thread mode"); - if (cdict) return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, jobParams.fParams); - return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, NULL, 0, &jobParams); - } - - assert(avgJobSize >= 256 KB); /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), required to compress directly into Dst (no additional buffer) */ - ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(avgJobSize) ); - /* LDM doesn't even try to load the dictionary in single-ingestion mode */ - if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize, NULL, 0, ZSTD_dct_auto)) - return ERROR(memory_allocation); - - FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbJobs) , ""); /* only expands if necessary */ - - { unsigned u; - for (u=0; u<nbJobs; u++) { - size_t const jobSize = MIN(remainingSrcSize, avgJobSize); - size_t const dstBufferCapacity = ZSTD_compressBound(jobSize); - buffer_t const dstAsBuffer = { (char*)dst + dstBufferPos, dstBufferCapacity }; - buffer_t const dstBuffer = u < compressWithinDst ? dstAsBuffer : g_nullBuffer; - size_t dictSize = u ? overlapSize : 0; - - mtctx->jobs[u].prefix.start = srcStart + frameStartPos - dictSize; - mtctx->jobs[u].prefix.size = dictSize; - mtctx->jobs[u].src.start = srcStart + frameStartPos; - mtctx->jobs[u].src.size = jobSize; assert(jobSize > 0); /* avoid job.src.size == 0 */ - mtctx->jobs[u].consumed = 0; - mtctx->jobs[u].cSize = 0; - mtctx->jobs[u].cdict = (u==0) ? cdict : NULL; - mtctx->jobs[u].fullFrameSize = srcSize; - mtctx->jobs[u].params = jobParams; - /* do not calculate checksum within sections, but write it in header for first section */ - mtctx->jobs[u].dstBuff = dstBuffer; - mtctx->jobs[u].cctxPool = mtctx->cctxPool; - mtctx->jobs[u].bufPool = mtctx->bufPool; - mtctx->jobs[u].seqPool = mtctx->seqPool; - mtctx->jobs[u].serial = &mtctx->serial; - mtctx->jobs[u].jobID = u; - mtctx->jobs[u].firstJob = (u==0); - mtctx->jobs[u].lastJob = (u==nbJobs-1); - - DEBUGLOG(5, "ZSTDMT_compress_advanced_internal: posting job %u (%u bytes)", u, (U32)jobSize); - DEBUG_PRINTHEX(6, mtctx->jobs[u].prefix.start, 12); - POOL_add(mtctx->factory, ZSTDMT_compressionJob, &mtctx->jobs[u]); - - frameStartPos += jobSize; - dstBufferPos += dstBufferCapacity; - remainingSrcSize -= jobSize; - } } - - /* collect result */ - { size_t error = 0, dstPos = 0; - unsigned jobID; - for (jobID=0; jobID<nbJobs; jobID++) { - DEBUGLOG(5, "waiting for job %u ", jobID); - ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[jobID].job_mutex); - while (mtctx->jobs[jobID].consumed < mtctx->jobs[jobID].src.size) { - DEBUGLOG(5, "waiting for jobCompleted signal from job %u", jobID); - ZSTD_pthread_cond_wait(&mtctx->jobs[jobID].job_cond, &mtctx->jobs[jobID].job_mutex); - } - ZSTD_pthread_mutex_unlock(&mtctx->jobs[jobID].job_mutex); - DEBUGLOG(5, "ready to write job %u ", jobID); - - { size_t const cSize = mtctx->jobs[jobID].cSize; - if (ZSTD_isError(cSize)) error = cSize; - if ((!error) && (dstPos + cSize > dstCapacity)) error = ERROR(dstSize_tooSmall); - if (jobID) { /* note : job 0 is written directly at dst, which is correct position */ - if (!error) - memmove((char*)dst + dstPos, mtctx->jobs[jobID].dstBuff.start, cSize); /* may overlap when job compressed within dst */ - if (jobID >= compressWithinDst) { /* job compressed into its own buffer, which must be released */ - DEBUGLOG(5, "releasing buffer %u>=%u", jobID, compressWithinDst); - ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); - } } - mtctx->jobs[jobID].dstBuff = g_nullBuffer; - mtctx->jobs[jobID].cSize = 0; - dstPos += cSize ; - } - } /* for (jobID=0; jobID<nbJobs; jobID++) */ - - DEBUGLOG(4, "checksumFlag : %u ", params.fParams.checksumFlag); - if (params.fParams.checksumFlag) { - U32 const checksum = (U32)XXH64_digest(&mtctx->serial.xxhState); - if (dstPos + 4 > dstCapacity) { - error = ERROR(dstSize_tooSmall); - } else { - DEBUGLOG(4, "writing checksum : %08X \n", checksum); - MEM_writeLE32((char*)dst + dstPos, checksum); - dstPos += 4; - } } - - if (!error) DEBUGLOG(4, "compressed size : %u ", (U32)dstPos); - return error ? error : dstPos; - } -} - -size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict, - ZSTD_parameters params, - int overlapLog) -{ - ZSTD_CCtx_params cctxParams = mtctx->params; - cctxParams.cParams = params.cParams; - cctxParams.fParams = params.fParams; - assert(ZSTD_OVERLAPLOG_MIN <= overlapLog && overlapLog <= ZSTD_OVERLAPLOG_MAX); - cctxParams.overlapLog = overlapLog; - return ZSTDMT_compress_advanced_internal(mtctx, - dst, dstCapacity, - src, srcSize, - cdict, cctxParams); -} - - -size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - int compressionLevel) -{ - ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0); - int const overlapLog = ZSTDMT_overlapLog_default(params.cParams.strategy); - params.fParams.contentSizeFlag = 1; - return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapLog); -} - - /* ====================================== */ /* ======= Streaming API ======= */ /* ====================================== */ @@ -1432,16 +1206,6 @@ size_t ZSTDMT_initCStream_internal( if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX; - mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */ - if (mtctx->singleBlockingThread) { - ZSTD_CCtx_params const singleThreadParams = ZSTDMT_initJobCCtxParams(¶ms); - DEBUGLOG(5, "ZSTDMT_initCStream_internal: switch to single blocking thread mode"); - assert(singleThreadParams.nbWorkers == 0); - return ZSTD_initCStream_internal(mtctx->cctxPool->cctx[0], - dict, dictSize, cdict, - &singleThreadParams, pledgedSrcSize); - } - DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers); if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */ @@ -1504,8 +1268,8 @@ size_t ZSTDMT_initCStream_internal( size_t const capacity = MAX(windowSize, sectionsSize) + slackSize; if (mtctx->roundBuff.capacity < capacity) { if (mtctx->roundBuff.buffer) - ZSTD_free(mtctx->roundBuff.buffer, mtctx->cMem); - mtctx->roundBuff.buffer = (BYTE*)ZSTD_malloc(capacity, mtctx->cMem); + ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); + mtctx->roundBuff.buffer = (BYTE*)ZSTD_customMalloc(capacity, mtctx->cMem); if (mtctx->roundBuff.buffer == NULL) { mtctx->roundBuff.capacity = 0; return ERROR(memory_allocation); @@ -1530,53 +1294,6 @@ size_t ZSTDMT_initCStream_internal( return 0; } -size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, - const void* dict, size_t dictSize, - ZSTD_parameters params, - unsigned long long pledgedSrcSize) -{ - ZSTD_CCtx_params cctxParams = mtctx->params; /* retrieve sticky params */ - DEBUGLOG(4, "ZSTDMT_initCStream_advanced (pledgedSrcSize=%u)", (U32)pledgedSrcSize); - cctxParams.cParams = params.cParams; - cctxParams.fParams = params.fParams; - return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, ZSTD_dct_auto, NULL, - cctxParams, pledgedSrcSize); -} - -size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, - const ZSTD_CDict* cdict, - ZSTD_frameParameters fParams, - unsigned long long pledgedSrcSize) -{ - ZSTD_CCtx_params cctxParams = mtctx->params; - if (cdict==NULL) return ERROR(dictionary_wrong); /* method incompatible with NULL cdict */ - cctxParams.cParams = ZSTD_getCParamsFromCDict(cdict); - cctxParams.fParams = fParams; - return ZSTDMT_initCStream_internal(mtctx, NULL, 0 /*dictSize*/, ZSTD_dct_auto, cdict, - cctxParams, pledgedSrcSize); -} - - -/* ZSTDMT_resetCStream() : - * pledgedSrcSize can be zero == unknown (for the time being) - * prefer using ZSTD_CONTENTSIZE_UNKNOWN, - * as `0` might mean "empty" in the future */ -size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize) -{ - if (!pledgedSrcSize) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; - return ZSTDMT_initCStream_internal(mtctx, NULL, 0, ZSTD_dct_auto, 0, mtctx->params, - pledgedSrcSize); -} - -size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel) { - ZSTD_parameters const params = ZSTD_getParams(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0); - ZSTD_CCtx_params cctxParams = mtctx->params; /* retrieve sticky params */ - DEBUGLOG(4, "ZSTDMT_initCStream (cLevel=%i)", compressionLevel); - cctxParams.cParams = params.cParams; - cctxParams.fParams = params.fParams; - return ZSTDMT_initCStream_internal(mtctx, NULL, 0, ZSTD_dct_auto, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN); -} - /* ZSTDMT_writeLastEmptyBlock() * Write a single empty block with an end-of-frame to finish a frame. @@ -1740,7 +1457,7 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u assert(cSize >= mtctx->jobs[wJobID].dstFlushed); assert(mtctx->jobs[wJobID].dstBuff.start != NULL); if (toFlush > 0) { - memcpy((char*)output->dst + output->pos, + ZSTD_memcpy((char*)output->dst + output->pos, (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed, toFlush); } @@ -1894,7 +1611,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) return 0; } ZSTDMT_waitForLdmComplete(mtctx, buffer); - memmove(start, mtctx->inBuff.prefix.start, prefixSize); + ZSTD_memmove(start, mtctx->inBuff.prefix.start, prefixSize); mtctx->inBuff.prefix.start = start; mtctx->roundBuff.pos = prefixSize; } @@ -1968,6 +1685,16 @@ findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) pos = 0; prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + if ((hash & hitMask) == hitMask) { + /* We're already at a sync point so don't load any more until + * we're able to flush this sync point. + * This likely happened because the job table was full so we + * couldn't add our job. + */ + syncPoint.toLoad = 0; + syncPoint.flush = 1; + return syncPoint; + } } else { /* We don't have enough bytes buffered to initialize the hash, but * we know we have at least RSYNC_LENGTH bytes total. @@ -2022,34 +1749,11 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, assert(output->pos <= output->size); assert(input->pos <= input->size); - if (mtctx->singleBlockingThread) { /* delegate to single-thread (synchronous) */ - return ZSTD_compressStream2(mtctx->cctxPool->cctx[0], output, input, endOp); - } - if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) { /* current frame being ended. Only flush/end are allowed */ return ERROR(stage_wrong); } - /* single-pass shortcut (note : synchronous-mode) */ - if ( (!mtctx->params.rsyncable) /* rsyncable mode is disabled */ - && (mtctx->nextJobID == 0) /* just started */ - && (mtctx->inBuff.filled == 0) /* nothing buffered */ - && (!mtctx->jobReady) /* no job already created */ - && (endOp == ZSTD_e_end) /* end order */ - && (output->size - output->pos >= ZSTD_compressBound(input->size - input->pos)) ) { /* enough space in dst */ - size_t const cSize = ZSTDMT_compress_advanced_internal(mtctx, - (char*)output->dst + output->pos, output->size - output->pos, - (const char*)input->src + input->pos, input->size - input->pos, - mtctx->cdict, mtctx->params); - if (ZSTD_isError(cSize)) return cSize; - input->pos = input->size; - output->pos += cSize; - mtctx->allJobsCompleted = 1; - mtctx->frameEnded = 1; - return 0; - } - /* fill input buffer */ if ( (!mtctx->jobReady) && (input->size > input->pos) ) { /* support NULL input */ @@ -2072,13 +1776,21 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize); DEBUGLOG(5, "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u", (U32)syncPoint.toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize); - memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad); + ZSTD_memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad); input->pos += syncPoint.toLoad; mtctx->inBuff.filled += syncPoint.toLoad; forwardInputProgress = syncPoint.toLoad>0; } - if ((input->pos < input->size) && (endOp == ZSTD_e_end)) - endOp = ZSTD_e_flush; /* can't end now : not all input consumed */ + } + if ((input->pos < input->size) && (endOp == ZSTD_e_end)) { + /* Can't end yet because the input is not fully consumed. + * We are in one of these cases: + * - mtctx->inBuff is NULL & empty: we couldn't get an input buffer so don't create a new job. + * - We filled the input buffer: flush this job but don't end the frame. + * - We hit a synchronization point: flush this job but don't end the frame. + */ + assert(mtctx->inBuff.filled == 0 || mtctx->inBuff.filled == mtctx->targetSectionSize || mtctx->params.rsyncable); + endOp = ZSTD_e_flush; } if ( (mtctx->jobReady) @@ -2097,47 +1809,3 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, return remainingToFlush; } } - - -size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input) -{ - FORWARD_IF_ERROR( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) , ""); - - /* recommended next input size : fill current input buffer */ - return mtctx->targetSectionSize - mtctx->inBuff.filled; /* note : could be zero when input buffer is fully filled and no more availability to create new job */ -} - - -static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_EndDirective endFrame) -{ - size_t const srcSize = mtctx->inBuff.filled; - DEBUGLOG(5, "ZSTDMT_flushStream_internal"); - - if ( mtctx->jobReady /* one job ready for a worker to pick up */ - || (srcSize > 0) /* still some data within input buffer */ - || ((endFrame==ZSTD_e_end) && !mtctx->frameEnded)) { /* need a last 0-size block to end frame */ - DEBUGLOG(5, "ZSTDMT_flushStream_internal : create a new job (%u bytes, end:%u)", - (U32)srcSize, (U32)endFrame); - FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) , ""); - } - - /* check if there is any data available to flush */ - return ZSTDMT_flushProduced(mtctx, output, 1 /* blockToFlush */, endFrame); -} - - -size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output) -{ - DEBUGLOG(5, "ZSTDMT_flushStream"); - if (mtctx->singleBlockingThread) - return ZSTD_flushStream(mtctx->cctxPool->cctx[0], output); - return ZSTDMT_flushStream_internal(mtctx, output, ZSTD_e_flush); -} - -size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output) -{ - DEBUGLOG(4, "ZSTDMT_endStream"); - if (mtctx->singleBlockingThread) - return ZSTD_endStream(mtctx->cctxPool->cctx[0], output); - return ZSTDMT_flushStream_internal(mtctx, output, ZSTD_e_end); -} diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h index 89914eb7..0a9e551c 100644 --- a/lib/compress/zstdmt_compress.h +++ b/lib/compress/zstdmt_compress.h @@ -19,26 +19,14 @@ /* Note : This is an internal API. * These APIs used to be exposed with ZSTDLIB_API, * because it used to be the only way to invoke MT compression. - * Now, it's recommended to use ZSTD_compress2 and ZSTD_compressStream2() - * instead. - * - * If you depend on these APIs and can't switch, then define - * ZSTD_LEGACY_MULTITHREADED_API when making the dynamic library. - * However, we may completely remove these functions in a future - * release, so please switch soon. + * Now, you must use ZSTD_compress2 and ZSTD_compressStream2() instead. * * This API requires ZSTD_MULTITHREAD to be defined during compilation, * otherwise ZSTDMT_createCCtx*() will fail. */ -#ifdef ZSTD_LEGACY_MULTITHREADED_API -# define ZSTDMT_API ZSTDLIB_API -#else -# define ZSTDMT_API -#endif - /* === Dependencies === */ -#include <stddef.h> /* size_t */ +#include "../common/zstd_deps.h" /* size_t */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ #include "../zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ @@ -54,78 +42,34 @@ #define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) +/* ======================================================== + * === Private interface, for use by ZSTD_compress.c === + * === Not exposed in libzstd. Never invoke directly === + * ======================================================== */ + /* === Memory management === */ typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; /* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */ -ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers); -/* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */ -ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, - ZSTD_customMem cMem); -ZSTDMT_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); - -ZSTDMT_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); - - -/* === Simple one-pass compression function === */ - -ZSTDMT_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - int compressionLevel); - +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, + ZSTD_customMem cMem, + ZSTD_threadPool *pool); +size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); /* === Streaming functions === */ -ZSTDMT_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel); -ZSTDMT_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */ - -ZSTDMT_API size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); -ZSTDMT_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input); - -ZSTDMT_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */ -ZSTDMT_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */ - - -/* === Advanced functions and parameters === */ - -ZSTDMT_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict, - ZSTD_parameters params, - int overlapLog); - -ZSTDMT_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, - const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */ - ZSTD_parameters params, - unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */ - -ZSTDMT_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, - const ZSTD_CDict* cdict, - ZSTD_frameParameters fparams, - unsigned long long pledgedSrcSize); /* note : zero means empty */ - -/* ZSTDMT_parameter : - * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */ -typedef enum { - ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */ - ZSTDMT_p_overlapLog, /* Each job may reload a part of previous job to enhance compression ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */ - ZSTDMT_p_rsyncable /* Enables rsyncable mode. */ -} ZSTDMT_parameter; - -/* ZSTDMT_setMTCtxParameter() : - * allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter. - * The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__ - * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions. - * @return : 0, or an error code (which can be tested using ZSTD_isError()) */ -ZSTDMT_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value); - -/* ZSTDMT_getMTCtxParameter() : - * Query the ZSTDMT_CCtx for a parameter value. - * @return : 0, or an error code (which can be tested using ZSTD_isError()) */ -ZSTDMT_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value); +size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); +/*! ZSTDMT_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * @return : 0, or an error code */ +size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, + const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); /*! ZSTDMT_compressStream_generic() : * Combines ZSTDMT_compressStream() with optional ZSTDMT_flushStream() or ZSTDMT_endStream() @@ -134,16 +78,10 @@ ZSTDMT_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter * 0 if fully flushed * or an error code * note : needs to be init using any ZSTD_initCStream*() variant */ -ZSTDMT_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective endOp); - - -/* ======================================================== - * === Private interface, for use by ZSTD_compress.c === - * === Not exposed in libzstd. Never invoke directly === - * ======================================================== */ +size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); /*! ZSTDMT_toFlushNow() * Tell how many bytes are ready to be flushed immediately. @@ -153,15 +91,6 @@ ZSTDMT_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, * therefore flushing is limited by speed of oldest job. */ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx); -/*! ZSTDMT_CCtxParam_setMTCtxParameter() - * like ZSTDMT_setMTCtxParameter(), but into a ZSTD_CCtx_Params */ -size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, int value); - -/*! ZSTDMT_CCtxParam_setNbWorkers() - * Set nbWorkers, and clamp it. - * Also reset jobSize and overlapLog */ -size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers); - /*! ZSTDMT_updateCParams_whileCompressing() : * Updates only a selected set of compression parameters, to remain compatible with current frame. * New parameters will be applied to next compression job. */ @@ -174,17 +103,6 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx); -/*! ZSTDMT_initCStream_internal() : - * Private use only. Init streaming operation. - * expects params to be valid. - * must receive dict, or cdict, or none, but not both. - * @return : 0, or an error code */ -size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, - const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, - const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); - - #if defined (__cplusplus) } #endif diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index 68293a13..14182067 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -15,7 +15,7 @@ /* ************************************************************** * Dependencies ****************************************************************/ -#include <string.h> /* memcpy, memset */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ #include "../common/compiler.h" #include "../common/bitstream.h" /* BIT_* */ #include "../common/fse.h" /* to compress headers */ @@ -103,7 +103,7 @@ typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) { DTableDesc dtd; - memcpy(&dtd, table, sizeof(dtd)); + ZSTD_memcpy(&dtd, table, sizeof(dtd)); return dtd; } @@ -115,29 +115,51 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) /*-***************************/ typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX1; /* single-symbol decoding */ +/** + * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at + * a time. + */ +static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { + U64 D4; + if (MEM_isLittleEndian()) { + D4 = symbol + (nbBits << 8); + } else { + D4 = (symbol << 8) + nbBits; + } + D4 *= 0x0001000100010001ULL; + return D4; +} + +typedef struct { + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; +} HUF_ReadDTableX1_Workspace; + + size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) { + return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); +} + +size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2) +{ U32 tableLog = 0; U32 nbSymbols = 0; size_t iSize; void* const dtPtr = DTable + 1; HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; + HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; - U32* rankVal; - BYTE* huffWeight; - size_t spaceUsed32 = 0; - - rankVal = (U32 *)workSpace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1; - huffWeight = (BYTE *)((U32 *)workSpace + spaceUsed32); - spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > wkspSize) return ERROR(tableLog_tooLarge); + DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); + if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); - /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ - iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2); if (HUF_isError(iSize)) return iSize; /* Table header */ @@ -145,52 +167,117 @@ size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ dtd.tableType = 0; dtd.tableLog = (BYTE)tableLog; - memcpy(DTable, &dtd, sizeof(dtd)); + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); } - /* Calculate starting value for each rank */ - { U32 n, nextRankStart = 0; - for (n=1; n<tableLog+1; n++) { - U32 const current = nextRankStart; - nextRankStart += (rankVal[n] << (n-1)); - rankVal[n] = current; - } } + /* Compute symbols and rankStart given rankVal: + * + * rankVal already contains the number of values of each weight. + * + * symbols contains the symbols ordered by weight. First are the rankVal[0] + * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. + * symbols[0] is filled (but unused) to avoid a branch. + * + * rankStart contains the offset where each rank belongs in the DTable. + * rankStart[0] is not filled because there are no entries in the table for + * weight 0. + */ + { + int n; + int nextRankStart = 0; + int const unroll = 4; + int const nLimit = (int)nbSymbols - unroll + 1; + for (n=0; n<(int)tableLog+1; n++) { + U32 const curr = nextRankStart; + nextRankStart += wksp->rankVal[n]; + wksp->rankStart[n] = curr; + } + for (n=0; n < nLimit; n += unroll) { + int u; + for (u=0; u < unroll; ++u) { + size_t const w = wksp->huffWeight[n+u]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); + } + } + for (; n < (int)nbSymbols; ++n) { + size_t const w = wksp->huffWeight[n]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; + } + } - /* fill DTable */ - { U32 n; - size_t const nEnd = nbSymbols; - for (n=0; n<nEnd; n++) { - size_t const w = huffWeight[n]; - size_t const length = (1 << w) >> 1; - size_t const uStart = rankVal[w]; - size_t const uEnd = uStart + length; - size_t u; - HUF_DEltX1 D; - D.byte = (BYTE)n; - D.nbBits = (BYTE)(tableLog + 1 - w); - rankVal[w] = (U32)uEnd; - if (length < 4) { - /* Use length in the loop bound so the compiler knows it is short. */ - for (u = 0; u < length; ++u) - dt[uStart + u] = D; - } else { - /* Unroll the loop 4 times, we know it is a power of 2. */ - for (u = uStart; u < uEnd; u += 4) { - dt[u + 0] = D; - dt[u + 1] = D; - dt[u + 2] = D; - dt[u + 3] = D; - } } } } + /* fill DTable + * We fill all entries of each weight in order. + * That way length is a constant for each iteration of the outter loop. + * We can switch based on the length to a different inner loop which is + * optimized for that particular case. + */ + { + U32 w; + int symbol=wksp->rankVal[0]; + int rankStart=0; + for (w=1; w<tableLog+1; ++w) { + int const symbolCount = wksp->rankVal[w]; + int const length = (1 << w) >> 1; + int uStart = rankStart; + BYTE const nbBits = (BYTE)(tableLog + 1 - w); + int s; + int u; + switch (length) { + case 1: + for (s=0; s<symbolCount; ++s) { + HUF_DEltX1 D; + D.byte = wksp->symbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart] = D; + uStart += 1; + } + break; + case 2: + for (s=0; s<symbolCount; ++s) { + HUF_DEltX1 D; + D.byte = wksp->symbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart+0] = D; + dt[uStart+1] = D; + uStart += 2; + } + break; + case 4: + for (s=0; s<symbolCount; ++s) { + U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + uStart += 4; + } + break; + case 8: + for (s=0; s<symbolCount; ++s) { + U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + MEM_write64(dt + uStart + 4, D4); + uStart += 8; + } + break; + default: + for (s=0; s<symbolCount; ++s) { + U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits); + for (u=0; u < length; u += 16) { + MEM_write64(dt + uStart + u + 0, D4); + MEM_write64(dt + uStart + u + 4, D4); + MEM_write64(dt + uStart + u + 8, D4); + MEM_write64(dt + uStart + u + 12, D4); + } + assert(u == length); + uStart += length; + } + break; + } + symbol += symbolCount; + rankStart += symbolCount * length; + } + } return iSize; } -size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_readDTableX1_wksp(DTable, src, srcSize, - workSpace, sizeof(workSpace)); -} - FORCE_INLINE_TEMPLATE BYTE HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) { @@ -389,20 +476,6 @@ size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, } -size_t HUF_decompress1X1_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); -} - size_t HUF_decompress4X1_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, @@ -419,8 +492,7 @@ static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size { const BYTE* ip = (const BYTE*) cSrc; - size_t const hSize = HUF_readDTableX1_wksp (dctx, cSrc, cSrcSize, - workSpace, wkspSize); + size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; @@ -436,18 +508,6 @@ size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, } -size_t HUF_decompress4X1_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} -size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - #endif /* HUF_FORCE_DECOMPRESS_X2 */ @@ -474,7 +534,7 @@ static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 sizeLog, const U32 co U32 rankVal[HUF_TABLELOG_MAX + 1]; /* get pre-calculated rankVal */ - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + ZSTD_memcpy(rankVal, rankValOrigin, sizeof(rankVal)); /* fill skipped values */ if (minWeight>1) { @@ -516,7 +576,7 @@ static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, const U32 minBits = nbBitsBaseline - maxWeight; U32 s; - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + ZSTD_memcpy(rankVal, rankValOrigin, sizeof(rankVal)); /* fill DTable */ for (s=0; s<sortedListSize; s++) { @@ -581,11 +641,11 @@ size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, if ((spaceUsed32 << 2) > wkspSize) return ERROR(tableLog_tooLarge); rankStart = rankStart0 + 1; - memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1)); + ZSTD_memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1)); DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); - /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; @@ -599,9 +659,9 @@ size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, /* Get start index of each weight */ { U32 w, nextRankStart = 0; for (w=1; w<maxW+1; w++) { - U32 current = nextRankStart; + U32 curr = nextRankStart; nextRankStart += rankStats[w]; - rankStart[w] = current; + rankStart[w] = curr; } rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ sizeOfSort = nextRankStart; @@ -624,9 +684,9 @@ size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, U32 nextRankVal = 0; U32 w; for (w=1; w<maxW+1; w++) { - U32 current = nextRankVal; + U32 curr = nextRankVal; nextRankVal += rankStats[w] << (w+rescale); - rankVal0[w] = current; + rankVal0[w] = curr; } } { U32 const minBits = tableLog+1 - maxW; U32 consumed; @@ -644,23 +704,16 @@ size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, dtd.tableLog = (BYTE)maxTableLog; dtd.tableType = 1; - memcpy(DTable, &dtd, sizeof(dtd)); + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); return iSize; } -size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_readDTableX2_wksp(DTable, src, srcSize, - workSpace, sizeof(workSpace)); -} - FORCE_INLINE_TEMPLATE U32 HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) { size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt+val, 2); + ZSTD_memcpy(op, dt+val, 2); BIT_skipBits(DStream, dt[val].nbBits); return dt[val].length; } @@ -669,7 +722,7 @@ FORCE_INLINE_TEMPLATE U32 HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) { size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt+val, 1); + ZSTD_memcpy(op, dt+val, 1); if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); else { if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { @@ -890,20 +943,6 @@ size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, } -size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - size_t HUF_decompress4X2_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, @@ -937,20 +976,6 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, } -size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - #endif /* HUF_FORCE_DECOMPRESS_X1 */ @@ -1051,67 +1076,6 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) } -typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); - -size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ -#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) - static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 }; -#endif - - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); -#else - return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); -#endif - } -} - -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); -#else - return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; -#endif - } -} - -size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - - size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, @@ -1145,8 +1109,8 @@ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, /* validation checks */ if (dstSize == 0) return ERROR(dstSize_tooSmall); if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); #if defined(HUF_FORCE_DECOMPRESS_X1) @@ -1168,14 +1132,6 @@ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, } } -size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) { @@ -1199,7 +1155,7 @@ size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstS { const BYTE* ip = (const BYTE*) cSrc; - size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize); + size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; @@ -1246,3 +1202,149 @@ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t ds #endif } } + +#ifndef ZSTD_NO_UNUSED_FUNCTIONS +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_readDTableX1_wksp(DTable, src, srcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X1_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); +} +#endif + +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_readDTableX2_wksp(DTable, src, srcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} +#endif + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress4X1_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} +size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} +#endif + +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} +#endif + +typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); + +size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) + static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 }; +#endif + + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); +#else + return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); +#endif + } +} + +size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); +#else + return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; +#endif + } +} + +size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} +#endif diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c index c8cb8ecc..f5cc23b3 100644 --- a/lib/decompress/zstd_ddict.c +++ b/lib/decompress/zstd_ddict.c @@ -14,7 +14,7 @@ /*-******************************************************* * Dependencies *********************************************************/ -#include <string.h> /* memcpy, memmove, memset */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ #include "../common/cpu.h" /* bmi2 */ #include "../common/mem.h" /* low level memory routines */ #define FSE_STATIC_LINKING_ONLY @@ -127,11 +127,11 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, ddict->dictContent = dict; if (!dict) dictSize = 0; } else { - void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem); + void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem); ddict->dictBuffer = internalBuffer; ddict->dictContent = internalBuffer; if (!internalBuffer) return ERROR(memory_allocation); - memcpy(internalBuffer, dict, dictSize); + ZSTD_memcpy(internalBuffer, dict, dictSize); } ddict->dictSize = dictSize; ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ @@ -147,9 +147,9 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_customMem customMem) { - if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; - { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem); if (ddict == NULL) return NULL; ddict->cMem = customMem; { size_t const initResult = ZSTD_initDDict_internal(ddict, @@ -198,7 +198,7 @@ const ZSTD_DDict* ZSTD_initStaticDDict( if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ if (sBufferSize < neededSpace) return NULL; if (dictLoadMethod == ZSTD_dlm_byCopy) { - memcpy(ddict+1, dict, dictSize); /* local copy */ + ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */ dict = ddict+1; } if (ZSTD_isError( ZSTD_initDDict_internal(ddict, @@ -213,8 +213,8 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict) { if (ddict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = ddict->cMem; - ZSTD_free(ddict->dictBuffer, cMem); - ZSTD_free(ddict, cMem); + ZSTD_customFree(ddict->dictBuffer, cMem); + ZSTD_customFree(ddict, cMem); return 0; } } diff --git a/lib/decompress/zstd_ddict.h b/lib/decompress/zstd_ddict.h index af307efd..8906a71c 100644 --- a/lib/decompress/zstd_ddict.h +++ b/lib/decompress/zstd_ddict.h @@ -15,7 +15,7 @@ /*-******************************************************* * Dependencies *********************************************************/ -#include <stddef.h> /* size_t */ +#include "../common/zstd_deps.h" /* size_t */ #include "../zstd.h" /* ZSTD_DDict, and several public functions */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index be5c7cfc..21f846bc 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -55,7 +55,7 @@ /*-******************************************************* * Dependencies *********************************************************/ -#include <string.h> /* memcpy, memmove, memset */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ #include "../common/cpu.h" /* bmi2 */ #include "../common/mem.h" /* low level memory routines */ #define FSE_STATIC_LINKING_ONLY @@ -94,11 +94,18 @@ static size_t ZSTD_startingInputLength(ZSTD_format_e format) return startingInputLength; } +static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) +{ + assert(dctx->streamStage == zdss_init); + dctx->format = ZSTD_f_zstd1; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + dctx->outBufferMode = ZSTD_bm_buffered; + dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; +} + static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) { - dctx->format = ZSTD_f_zstd1; /* ZSTD_decompressBegin() invokes ZSTD_startingInputLength() with argument dctx->format */ dctx->staticSize = 0; - dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; dctx->ddict = NULL; dctx->ddictLocal = NULL; dctx->dictEnd = NULL; @@ -113,7 +120,8 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) dctx->noForwardProgress = 0; dctx->oversizedDuration = 0; dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); - dctx->outBufferMode = ZSTD_obm_buffered; + ZSTD_DCtx_resetParameters(dctx); + dctx->validateChecksum = 1; #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION dctx->dictContentEndForFuzzing = NULL; #endif @@ -134,9 +142,9 @@ ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) { - if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; - { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(*dctx), customMem); + { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); if (!dctx) return NULL; dctx->customMem = customMem; ZSTD_initDCtx_internal(dctx); @@ -164,13 +172,13 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx"); { ZSTD_customMem const cMem = dctx->customMem; ZSTD_clearDict(dctx); - ZSTD_free(dctx->inBuff, cMem); + ZSTD_customFree(dctx->inBuff, cMem); dctx->inBuff = NULL; #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (dctx->legacyContext) ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion); #endif - ZSTD_free(dctx, cMem); + ZSTD_customFree(dctx, cMem); return 0; } } @@ -179,7 +187,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) { size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); - memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ + ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ } @@ -246,7 +254,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s const BYTE* ip = (const BYTE*)src; size_t const minInputSize = ZSTD_startingInputLength(format); - memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ if (srcSize < minInputSize) return minInputSize; RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter"); @@ -256,7 +264,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s /* skippable frame */ if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ - memset(zfhPtr, 0, sizeof(*zfhPtr)); + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); zfhPtr->frameType = ZSTD_skippableFrame; return 0; @@ -446,7 +454,8 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID), dictionary_wrong, ""); #endif - if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); + dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0; + if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0); return 0; } @@ -461,7 +470,7 @@ static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize) { ZSTD_frameSizeInfo frameSizeInfo; - memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); + ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (ZSTD_isLegacy(src, srcSize)) @@ -516,7 +525,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize ip += 4; } - frameSizeInfo.compressedSize = ip - ipstart; + frameSizeInfo.compressedSize = (size_t)(ip - ipstart); frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) ? zfh.frameContentSize : nbBlocks * zfh.blockSizeMax; @@ -579,12 +588,12 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_copyRawBlock"); + RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, ""); if (dst == NULL) { if (srcSize == 0) return 0; RETURN_ERROR(dstBuffer_null, ""); } - RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, ""); - memcpy(dst, src, srcSize); + ZSTD_memcpy(dst, src, srcSize); return srcSize; } @@ -592,12 +601,12 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, BYTE b, size_t regenSize) { + RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, ""); if (dst == NULL) { if (regenSize == 0) return 0; RETURN_ERROR(dstBuffer_null, ""); } - RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, ""); - memset(dst, b, regenSize); + ZSTD_memset(dst, b, regenSize); return regenSize; } @@ -647,13 +656,13 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, switch(blockProperties.blockType) { case bt_compressed: - decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize, /* frame */ 1); + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1); break; case bt_raw : - decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize); + decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); break; case bt_rle : - decodedSize = ZSTD_setRleBlock(op, oend-op, *ip, blockProperties.origSize); + decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize); break; case bt_reserved : default: @@ -661,7 +670,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } if (ZSTD_isError(decodedSize)) return decodedSize; - if (dctx->fParams.checksumFlag) + if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, op, decodedSize); if (decodedSize != 0) op += decodedSize; @@ -676,11 +685,13 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, corruption_detected, ""); } if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ - U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); - U32 checkRead; RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, ""); - checkRead = MEM_readLE32(ip); - RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, ""); + if (!dctx->forceIgnoreChecksum) { + U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); + U32 checkRead; + checkRead = MEM_readLE32(ip); + RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, ""); + } ip += 4; remainingSrcSize -= 4; } @@ -688,7 +699,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, /* Allow caller to get size read */ *srcPtr = ip; *srcSizePtr = remainingSrcSize; - return op-ostart; + return (size_t)(op-ostart); } static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, @@ -721,7 +732,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); if (ZSTD_isError(decodedSize)) return decodedSize; - assert(decodedSize <=- dstCapacity); + assert(decodedSize <= dstCapacity); dst = (BYTE*)dst + decodedSize; dstCapacity -= decodedSize; @@ -761,15 +772,13 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) && (moreThan1Frame==1), srcSize_wrong, - "at least one frame successfully completed, but following " - "bytes are garbage: it's more likely to be a srcSize error, " - "specifying more bytes than compressed size of frame(s). This " - "error message replaces ERROR(prefix_unknown), which would be " - "confusing, as the first header is actually correct. Note that " - "one could be unlucky, it might be a corruption error instead, " - "happening right at the place where we expect zstd magic " - "bytes. But this is _much_ less likely than a srcSize field " - "error."); + "At least one frame successfully completed, " + "but following bytes are garbage: " + "it's more likely to be a srcSize error, " + "specifying more input bytes than size of frame(s). " + "Note: one could be unlucky, it might be a corruption error instead, " + "happening right at the place where we expect zstd magic bytes. " + "But this is _much_ less likely than a srcSize field error."); if (ZSTD_isError(res)) return res; assert(res <= dstCapacity); if (res != 0) @@ -781,7 +790,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed"); - return (BYTE*)dst - (BYTE*)dststart; + return (size_t)((BYTE*)dst - (BYTE*)dststart); } size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, @@ -899,21 +908,21 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c if (dctx->format == ZSTD_f_zstd1) { /* allows header */ assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ - memcpy(dctx->headerBuffer, src, srcSize); + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ dctx->stage = ZSTDds_decodeSkippableHeader; return 0; } } dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format); if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; - memcpy(dctx->headerBuffer, src, srcSize); + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); dctx->expected = dctx->headerSize - srcSize; dctx->stage = ZSTDds_decodeFrameHeader; return 0; case ZSTDds_decodeFrameHeader: assert(src != NULL); - memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); + ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), ""); dctx->expected = ZSTD_blockHeaderSize; dctx->stage = ZSTDds_decodeBlockHeader; @@ -977,7 +986,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); dctx->decodedSize += rSize; - if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); + if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize); dctx->previousDstEnd = (char*)dst + rSize; /* Stay on the same stage until we are finished streaming the block. */ @@ -1007,10 +1016,13 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c case ZSTDds_checkChecksum: assert(srcSize == 4); /* guaranteed by dctx->expected */ - { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); - U32 const check32 = MEM_readLE32(src); - DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); - RETURN_ERROR_IF(check32 != h32, checksum_wrong, ""); + { + if (dctx->validateChecksum) { + U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); + DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); + RETURN_ERROR_IF(check32 != h32, checksum_wrong, ""); + } dctx->expected = 0; dctx->stage = ZSTDds_getFrameHeaderSize; return 0; @@ -1019,7 +1031,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c case ZSTDds_decodeSkippableHeader: assert(src != NULL); assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); - memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ + ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ dctx->stage = ZSTDds_skipFrame; return 0; @@ -1075,7 +1087,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, workspace, workspaceSize); #else size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, - dictPtr, dictEnd - dictPtr, + dictPtr, (size_t)(dictEnd - dictPtr), workspace, workspaceSize); #endif RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); @@ -1084,40 +1096,46 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, { short offcodeNCount[MaxOff+1]; unsigned offcodeMaxValue = MaxOff, offcodeLog; - size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr)); RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, ""); RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); ZSTD_buildFSETable( entropy->OFTable, offcodeNCount, offcodeMaxValue, OF_base, OF_bits, - offcodeLog); + offcodeLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */0); dictPtr += offcodeHeaderSize; } { short matchlengthNCount[MaxML+1]; unsigned matchlengthMaxValue = MaxML, matchlengthLog; - size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, ""); RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); ZSTD_buildFSETable( entropy->MLTable, matchlengthNCount, matchlengthMaxValue, ML_base, ML_bits, - matchlengthLog); + matchlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); dictPtr += matchlengthHeaderSize; } { short litlengthNCount[MaxLL+1]; unsigned litlengthMaxValue = MaxLL, litlengthLog; - size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, ""); RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); ZSTD_buildFSETable( entropy->LLTable, litlengthNCount, litlengthMaxValue, LL_base, LL_bits, - litlengthLog); + litlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); dictPtr += litlengthHeaderSize; } @@ -1131,7 +1149,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, entropy->rep[i] = rep; } } - return dictPtr - (const BYTE*)dict; + return (size_t)(dictPtr - (const BYTE*)dict); } static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) @@ -1170,7 +1188,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->dictID = 0; dctx->bType = bt_reserved; ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); - memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ dctx->LLTptr = dctx->entropy.LLTable; dctx->MLTptr = dctx->entropy.MLTable; dctx->OFTptr = dctx->entropy.OFTable; @@ -1394,7 +1412,7 @@ size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) { - return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, format); + return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format); } ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) @@ -1411,8 +1429,12 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); return bounds; case ZSTD_d_stableOutBuffer: - bounds.lowerBound = (int)ZSTD_obm_buffered; - bounds.upperBound = (int)ZSTD_obm_stable; + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + case ZSTD_d_forceIgnoreChecksum: + bounds.lowerBound = (int)ZSTD_d_validateChecksum; + bounds.upperBound = (int)ZSTD_d_ignoreChecksum; return bounds; default:; } @@ -1436,6 +1458,26 @@ static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \ } +size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value) +{ + switch (param) { + case ZSTD_d_windowLogMax: + *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize); + return 0; + case ZSTD_d_format: + *value = (int)dctx->format; + return 0; + case ZSTD_d_stableOutBuffer: + *value = (int)dctx->outBufferMode; + return 0; + case ZSTD_d_forceIgnoreChecksum: + *value = (int)dctx->forceIgnoreChecksum; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) { RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); @@ -1451,7 +1493,11 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value return 0; case ZSTD_d_stableOutBuffer: CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value); - dctx->outBufferMode = (ZSTD_outBufferMode_e)value; + dctx->outBufferMode = (ZSTD_bufferMode_e)value; + return 0; + case ZSTD_d_forceIgnoreChecksum: + CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value); + dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value; return 0; default:; } @@ -1469,8 +1515,7 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) || (reset == ZSTD_reset_session_and_parameters) ) { RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); ZSTD_clearDict(dctx); - dctx->format = ZSTD_f_zstd1; - dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + ZSTD_DCtx_resetParameters(dctx); } return 0; } @@ -1524,7 +1569,7 @@ static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const ne { if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize)) zds->oversizedDuration++; - else + else zds->oversizedDuration = 0; } @@ -1538,7 +1583,7 @@ static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* { ZSTD_outBuffer const expect = zds->expectedOutBuffer; /* No requirement when ZSTD_obm_stable is not enabled. */ - if (zds->outBufferMode != ZSTD_obm_stable) + if (zds->outBufferMode != ZSTD_bm_stable) return 0; /* Any buffer is allowed in zdss_init, this must be the same for every other call until * the context is reset. @@ -1548,7 +1593,7 @@ static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* /* The buffer must match our expectation exactly. */ if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size) return 0; - RETURN_ERROR(dstBuffer_wrong, "ZSTD_obm_stable enabled but output differs!"); + RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!"); } /* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream() @@ -1560,7 +1605,7 @@ static size_t ZSTD_decompressContinueStream( ZSTD_DStream* zds, char** op, char* oend, void const* src, size_t srcSize) { int const isSkipFrame = ZSTD_isSkipFrame(zds); - if (zds->outBufferMode == ZSTD_obm_buffered) { + if (zds->outBufferMode == ZSTD_bm_buffered) { size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart; size_t const decodedSize = ZSTD_decompressContinue(zds, zds->outBuff + zds->outStart, dstSize, src, srcSize); @@ -1573,14 +1618,14 @@ static size_t ZSTD_decompressContinueStream( } } else { /* Write directly into the output buffer */ - size_t const dstSize = isSkipFrame ? 0 : oend - *op; + size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op); size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize); FORWARD_IF_ERROR(decodedSize, ""); *op += decodedSize; /* Flushing is not needed. */ zds->streamStage = zdss_read; assert(*op <= oend); - assert(zds->outBufferMode == ZSTD_obm_stable); + assert(zds->outBufferMode == ZSTD_bm_stable); } return 0; } @@ -1663,14 +1708,14 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB assert(iend >= ip); if (toLoad > remainingInput) { /* not enough input to load full header */ if (remainingInput > 0) { - memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput); + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput); zds->lhSize += remainingInput; } input->pos = input->size; return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ } assert(ip != NULL); - memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; break; } } @@ -1678,10 +1723,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN && zds->fParams.frameType != ZSTD_skippableFrame && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { - size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart); + size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart)); if (cSize <= (size_t)(iend-istart)) { /* shortcut : using single-pass mode */ - size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, ZSTD_getDDict(zds)); + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); if (ZSTD_isError(decompressedSize)) return decompressedSize; DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") ip = istart + cSize; @@ -1693,7 +1738,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } /* Check output buffer is large enough for ZSTD_odm_stable. */ - if (zds->outBufferMode == ZSTD_obm_stable + if (zds->outBufferMode == ZSTD_bm_stable && zds->fParams.frameType != ZSTD_skippableFrame && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) { @@ -1723,7 +1768,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB /* Adapt buffer sizes to frame header instructions */ { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); - size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_obm_buffered + size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) : 0; @@ -1731,7 +1776,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds); - + if (tooSmall || tooLarge) { size_t const bufferSize = neededInBuffSize + neededOutBuffSize; DEBUGLOG(4, "inBuff : from %u to %u", @@ -1745,10 +1790,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), memory_allocation, ""); } else { - ZSTD_free(zds->inBuff, zds->customMem); + ZSTD_customFree(zds->inBuff, zds->customMem); zds->inBuffSize = 0; zds->outBuffSize = 0; - zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem); + zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem); RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, ""); } zds->inBuffSize = neededInBuffSize; @@ -1760,7 +1805,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB case zdss_read: DEBUGLOG(5, "stage zdss_read"); - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)); DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); if (neededInSize==0) { /* end of frame */ zds->streamStage = zdss_init; @@ -1790,7 +1835,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos, corruption_detected, "should never happen"); - loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip); + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); } ip += loadedSize; zds->inPos += loadedSize; @@ -1804,7 +1849,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } case zdss_flush: { size_t const toFlushSize = zds->outEnd - zds->outStart; - size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize); + size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); op += flushedSize; zds->outStart += flushedSize; if (flushedSize == toFlushSize) { /* flush completed */ diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index ad3b3d8d..bec82e85 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -14,7 +14,7 @@ /*-******************************************************* * Dependencies *********************************************************/ -#include <string.h> /* memcpy, memmove, memset */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ #include "../common/compiler.h" /* prefetch */ #include "../common/cpu.h" /* bmi2 */ #include "../common/mem.h" /* low level memory routines */ @@ -44,7 +44,7 @@ /*_******************************************************* * Memory operations **********************************************************/ -static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } +static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } /*-************************************************************* @@ -166,7 +166,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, dctx->litSize = litSize; dctx->litEntropy = 1; if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; - memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return litCSize + lhSize; } @@ -191,10 +191,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); - memcpy(dctx->litBuffer, istart+lhSize, litSize); + ZSTD_memcpy(dctx->litBuffer, istart+lhSize, litSize); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; - memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+litSize; } /* direct reference into compressed stream */ @@ -223,7 +223,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; } RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); - memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; return lhSize+1; @@ -364,23 +364,26 @@ static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U32 nbAddB * generate FSE decoding table for one symbol (ll, ml or off) * cannot fail if input is valid => * all inputs are presumed validated at this stage */ -void -ZSTD_buildFSETable(ZSTD_seqSymbol* dt, +FORCE_INLINE_TEMPLATE +void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, const U32* baseValue, const U32* nbAdditionalBits, - unsigned tableLog) + unsigned tableLog, void* wksp, size_t wkspSize) { ZSTD_seqSymbol* const tableDecode = dt+1; - U16 symbolNext[MaxSeq+1]; - U32 const maxSV1 = maxSymbolValue + 1; U32 const tableSize = 1 << tableLog; - U32 highThreshold = tableSize-1; + + U16* symbolNext = (U16*)wksp; + BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1); + U32 highThreshold = tableSize - 1; + /* Sanity Checks */ assert(maxSymbolValue <= MaxSeq); assert(tableLog <= MaxFSELog); - + assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE); + (void)wkspSize; /* Init, lay down lowprob symbols */ { ZSTD_seqSymbol_header DTableH; DTableH.tableLog = tableLog; @@ -396,16 +399,69 @@ ZSTD_buildFSETable(ZSTD_seqSymbol* dt, assert(normalizedCounter[s]>=0); symbolNext[s] = (U16)normalizedCounter[s]; } } } - memcpy(dt, &DTableH, sizeof(DTableH)); + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); } /* Spread symbols */ - { U32 const tableMask = tableSize-1; + assert(tableSize <= 512); + /* Specialized symbol spreading for the case when there are + * no low probability (-1 count) symbols. When compressing + * small blocks we avoid low probability symbols to hit this + * case, since header decoding speed matters more. + */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s<maxSV1; ++s, sv += add) { + int i; + int const n = normalizedCounter[s]; + MEM_write64(spread + pos, sv); + for (i = 8; i < n; i += 8) { + MEM_write64(spread + pos + i, sv); + } + pos += n; + } + } + /* Now we spread those positions across the table. + * The benefit of doing it in two stages is that we avoid the the + * variable size inner loop, which caused lots of branch misses. + * Now we can run through all the positions without any branch misses. + * We unroll the loop twice, since that is what emperically worked best. + */ + { + size_t position = 0; + size_t s; + size_t const unroll = 2; + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableDecode[uPosition].baseValue = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); + } + } else { + U32 const tableMask = tableSize-1; U32 const step = FSE_TABLESTEP(tableSize); U32 s, position = 0; for (s=0; s<maxSV1; s++) { int i; - for (i=0; i<normalizedCounter[s]; i++) { + int const n = normalizedCounter[s]; + for (i=0; i<n; i++) { tableDecode[position].baseValue = s; position = (position + step) & tableMask; while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ @@ -414,7 +470,8 @@ ZSTD_buildFSETable(ZSTD_seqSymbol* dt, } /* Build Decoding table */ - { U32 u; + { + U32 u; for (u=0; u<tableSize; u++) { U32 const symbol = tableDecode[u].baseValue; U32 const nextState = symbolNext[symbol]++; @@ -423,7 +480,46 @@ ZSTD_buildFSETable(ZSTD_seqSymbol* dt, assert(nbAdditionalBits[symbol] < 255); tableDecode[u].nbAdditionalBits = (BYTE)nbAdditionalBits[symbol]; tableDecode[u].baseValue = baseValue[symbol]; - } } + } + } +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static void ZSTD_buildFSETable_body_default(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize) +{ + ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue, + baseValue, nbAdditionalBits, tableLog, wksp, wkspSize); +} + +#if DYNAMIC_BMI2 +TARGET_ATTRIBUTE("bmi2") static void ZSTD_buildFSETable_body_bmi2(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize) +{ + ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue, + baseValue, nbAdditionalBits, tableLog, wksp, wkspSize); +} +#endif + +void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + ZSTD_buildFSETable_body_bmi2(dt, normalizedCounter, maxSymbolValue, + baseValue, nbAdditionalBits, tableLog, wksp, wkspSize); + return; + } +#endif + (void)bmi2; + ZSTD_buildFSETable_body_default(dt, normalizedCounter, maxSymbolValue, + baseValue, nbAdditionalBits, tableLog, wksp, wkspSize); } @@ -435,7 +531,8 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb const void* src, size_t srcSize, const U32* baseValue, const U32* nbAdditionalBits, const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable, - int ddictIsCold, int nbSeq) + int ddictIsCold, int nbSeq, U32* wksp, size_t wkspSize, + int bmi2) { switch(type) { @@ -467,7 +564,7 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, ""); RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, ""); - ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog); + ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2); *DTablePtr = DTableSpace; return headerSize; } @@ -499,7 +596,8 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, if (nbSeq > 0x7F) { if (nbSeq == 0xFF) { RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); - nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; + nbSeq = MEM_readLE16(ip) + LONGNBSEQ; + ip+=2; } else { RETURN_ERROR_IF(ip >= iend, srcSize_wrong, ""); nbSeq = ((nbSeq-0x80)<<8) + *ip++; @@ -520,7 +618,9 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, ip, iend-ip, LL_base, LL_bits, LL_defaultDTable, dctx->fseEntropy, - dctx->ddictIsCold, nbSeq); + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + dctx->bmi2); RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += llhSize; } @@ -530,7 +630,9 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, ip, iend-ip, OF_base, OF_bits, OF_defaultDTable, dctx->fseEntropy, - dctx->ddictIsCold, nbSeq); + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + dctx->bmi2); RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += ofhSize; } @@ -540,7 +642,9 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, ip, iend-ip, ML_base, ML_bits, ML_defaultDTable, dctx->fseEntropy, - dctx->ddictIsCold, nbSeq); + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + dctx->bmi2); RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += mlhSize; } @@ -686,12 +790,12 @@ size_t ZSTD_execSequenceEnd(BYTE* op, RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); match = dictEnd - (prefixStart-match); if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); + ZSTD_memmove(oLitEnd, match, sequence.matchLength); return sequenceLength; } /* span extDict & currentPrefixSegment */ { size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); + ZSTD_memmove(oLitEnd, match, length1); op = oLitEnd + length1; sequence.matchLength -= length1; match = prefixStart; @@ -752,12 +856,12 @@ size_t ZSTD_execSequence(BYTE* op, RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); match = dictEnd + (match - prefixStart); if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); + ZSTD_memmove(oLitEnd, match, sequence.matchLength); return sequenceLength; } /* span extDict & currentPrefixSegment */ { size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); + ZSTD_memmove(oLitEnd, match, length1); op = oLitEnd + length1; sequence.matchLength -= length1; match = prefixStart; @@ -948,7 +1052,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c } #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -static int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) { size_t const windowSize = dctx->fParams.windowSize; /* No dictionary used. */ @@ -969,6 +1073,7 @@ MEM_STATIC void ZSTD_assertValidSequence( seq_t const seq, BYTE const* prefixStart, BYTE const* virtualStart) { +#if DEBUGLEVEL >= 1 size_t const windowSize = dctx->fParams.windowSize; size_t const sequenceSize = seq.litLength + seq.matchLength; BYTE const* const oLitEnd = op + seq.litLength; @@ -986,6 +1091,9 @@ MEM_STATIC void ZSTD_assertValidSequence( /* Offset must be within our window. */ assert(seq.offset <= windowSize); } +#else + (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; +#endif } #endif @@ -1080,14 +1188,14 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, #endif DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); BIT_reloadDStream(&(seqState.DStream)); + op += oneSeqSize; /* gcc and clang both don't like early returns in this loop. - * gcc doesn't like early breaks either. - * Instead save an error and report it at the end. - * When there is an error, don't increment op, so we don't - * overwrite. + * Instead break and check for an error at the end of the loop. */ - if (UNLIKELY(ZSTD_isError(oneSeqSize))) error = oneSeqSize; - else op += oneSeqSize; + if (UNLIKELY(ZSTD_isError(oneSeqSize))) { + error = oneSeqSize; + break; + } if (UNLIKELY(!--nbSeq)) break; } @@ -1104,7 +1212,7 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, { size_t const lastLLSize = litEnd - litPtr; RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); if (op != NULL) { - memcpy(op, litPtr, lastLLSize); + ZSTD_memcpy(op, litPtr, lastLLSize); op += lastLLSize; } } @@ -1209,7 +1317,7 @@ ZSTD_decompressSequencesLong_body( { size_t const lastLLSize = litEnd - litPtr; RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); if (op != NULL) { - memcpy(op, litPtr, lastLLSize); + ZSTD_memcpy(op, litPtr, lastLLSize); op += lastLLSize; } } diff --git a/lib/decompress/zstd_decompress_block.h b/lib/decompress/zstd_decompress_block.h index bf39b735..b5715c16 100644 --- a/lib/decompress/zstd_decompress_block.h +++ b/lib/decompress/zstd_decompress_block.h @@ -15,7 +15,7 @@ /*-******************************************************* * Dependencies *********************************************************/ -#include <stddef.h> /* size_t */ +#include "../common/zstd_deps.h" /* size_t */ #include "../zstd.h" /* DCtx, and some public functions */ #include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */ #include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */ @@ -48,12 +48,15 @@ size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, * this function must be called with valid parameters only * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.) * in which case it cannot fail. + * The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is + * defined in zstd_decompress_internal.h. * Internal use only. */ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, const U32* baseValue, const U32* nbAdditionalBits, - unsigned tableLog); + unsigned tableLog, void* wksp, size_t wkspSize, + int bmi2); #endif /* ZSTD_DEC_BLOCK_H */ diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h index 9ad96c55..f80b471e 100644 --- a/lib/decompress/zstd_decompress_internal.h +++ b/lib/decompress/zstd_decompress_internal.h @@ -27,26 +27,26 @@ /*-******************************************************* * Constants *********************************************************/ -static const U32 LL_base[MaxLL+1] = { +static UNUSED_ATTR const U32 LL_base[MaxLL+1] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000 }; -static const U32 OF_base[MaxOff+1] = { +static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; -static const U32 OF_bits[MaxOff+1] = { +static UNUSED_ATTR const U32 OF_bits[MaxOff+1] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 }; -static const U32 ML_base[MaxML+1] = { +static UNUSED_ATTR const U32 ML_base[MaxML+1] = { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, @@ -73,12 +73,16 @@ static const U32 ML_base[MaxML+1] = { #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) + typedef struct { ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ U32 rep[ZSTD_REP_NUM]; + U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; } ZSTD_entropyDTables_t; typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, @@ -95,11 +99,6 @@ typedef enum { ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ } ZSTD_dictUses_e; -typedef enum { - ZSTD_obm_buffered = 0, /* Buffer the output */ - ZSTD_obm_stable = 1 /* ZSTD_outBuffer is stable */ -} ZSTD_outBufferMode_e; - struct ZSTD_DCtx_s { const ZSTD_seqSymbol* LLTptr; @@ -122,6 +121,8 @@ struct ZSTD_DCtx_s XXH64_state_t xxhState; size_t headerSize; ZSTD_format_e format; + ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ + U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ const BYTE* litPtr; ZSTD_customMem customMem; size_t litSize; @@ -152,7 +153,7 @@ struct ZSTD_DCtx_s U32 legacyVersion; U32 hostageByte; int noForwardProgress; - ZSTD_outBufferMode_e outBufferMode; + ZSTD_bufferMode_e outBufferMode; ZSTD_outBuffer expectedOutBuffer; /* workspace */ diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index da54ef15..c78af133 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -40,33 +40,42 @@ * Constants ***************************************/ #define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB)) -#define DEFAULT_SPLITPOINT 1.0 +#define COVER_DEFAULT_SPLITPOINT 1.0 /*-************************************* * Console display ***************************************/ +#ifndef LOCALDISPLAYLEVEL static int g_displayLevel = 2; +#endif +#undef DISPLAY #define DISPLAY(...) \ { \ fprintf(stderr, __VA_ARGS__); \ fflush(stderr); \ } +#undef LOCALDISPLAYLEVEL #define LOCALDISPLAYLEVEL(displayLevel, l, ...) \ if (displayLevel >= l) { \ DISPLAY(__VA_ARGS__); \ } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ +#undef DISPLAYLEVEL #define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__) +#ifndef LOCALDISPLAYUPDATE +static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; +#endif +#undef LOCALDISPLAYUPDATE #define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ if (displayLevel >= l) { \ - if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \ + if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \ g_time = clock(); \ DISPLAY(__VA_ARGS__); \ } \ } +#undef DISPLAYUPDATE #define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__) -static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; -static clock_t g_time = 0; /*-************************************* * Hash table @@ -120,9 +129,9 @@ static int COVER_map_init(COVER_map_t *map, U32 size) { /** * Internal hash function */ -static const U32 prime4bytes = 2654435761U; +static const U32 COVER_prime4bytes = 2654435761U; static U32 COVER_map_hash(COVER_map_t *map, U32 key) { - return (key * prime4bytes) >> (32 - map->sizeLog); + return (key * COVER_prime4bytes) >> (32 - map->sizeLog); } /** @@ -215,7 +224,7 @@ typedef struct { } COVER_ctx_t; /* We need a global context for qsort... */ -static COVER_ctx_t *g_ctx = NULL; +static COVER_ctx_t *g_coverCtx = NULL; /*-************************************* * Helper functions @@ -258,11 +267,11 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) { /** * Same as COVER_cmp() except ties are broken by pointer value - * NOTE: g_ctx must be set to call this function. A global is required because + * NOTE: g_coverCtx must be set to call this function. A global is required because * qsort doesn't take an opaque pointer. */ -static int COVER_strict_cmp(const void *lp, const void *rp) { - int result = COVER_cmp(g_ctx, lp, rp); +static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) { + int result = COVER_cmp(g_coverCtx, lp, rp); if (result == 0) { result = lp < rp ? -1 : 1; } @@ -271,8 +280,8 @@ static int COVER_strict_cmp(const void *lp, const void *rp) { /** * Faster version for d <= 8. */ -static int COVER_strict_cmp8(const void *lp, const void *rp) { - int result = COVER_cmp8(g_ctx, lp, rp); +static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) { + int result = COVER_cmp8(g_coverCtx, lp, rp); if (result == 0) { result = lp < rp ? -1 : 1; } @@ -603,7 +612,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, /* qsort doesn't take an opaque pointer, so pass as a global. * On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is. */ - g_ctx = ctx; + g_coverCtx = ctx; #if defined(__OpenBSD__) mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32), (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); @@ -946,7 +955,7 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection){ free(selection.dictContent); } -COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, +COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity, size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples, size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize) { @@ -954,8 +963,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t largestCompressed = 0; BYTE* customDictContentEnd = customDictContent + dictContentSize; - BYTE * largestDictbuffer = (BYTE *)malloc(dictContentSize); - BYTE * candidateDictBuffer = (BYTE *)malloc(dictContentSize); + BYTE * largestDictbuffer = (BYTE *)malloc(dictBufferCapacity); + BYTE * candidateDictBuffer = (BYTE *)malloc(dictBufferCapacity); double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00; if (!largestDictbuffer || !candidateDictBuffer) { @@ -967,7 +976,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, /* Initial dictionary size and compressed size */ memcpy(largestDictbuffer, customDictContent, dictContentSize); dictContentSize = ZDICT_finalizeDictionary( - largestDictbuffer, dictContentSize, customDictContent, dictContentSize, + largestDictbuffer, dictBufferCapacity, customDictContent, dictContentSize, samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams); if (ZDICT_isError(dictContentSize)) { @@ -1001,7 +1010,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, while (dictContentSize < largestDict) { memcpy(candidateDictBuffer, largestDictbuffer, largestDict); dictContentSize = ZDICT_finalizeDictionary( - candidateDictBuffer, dictContentSize, customDictContentEnd - dictContentSize, dictContentSize, + candidateDictBuffer, dictBufferCapacity, customDictContentEnd - dictContentSize, dictContentSize, samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams); if (ZDICT_isError(dictContentSize)) { @@ -1079,7 +1088,7 @@ static void COVER_tryParameters(void *opaque) { { const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict, dictBufferCapacity, parameters); - selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail, + selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail, ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets, totalCompressedSize); @@ -1106,7 +1115,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( /* constants */ const unsigned nbThreads = parameters->nbThreads; const double splitPoint = - parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; + parameters->splitPoint <= 0.0 ? COVER_DEFAULT_SPLITPOINT : parameters->splitPoint; const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; diff --git a/lib/dictBuilder/cover.h b/lib/dictBuilder/cover.h index f2aa0e35..9f1cb5fb 100644 --- a/lib/dictBuilder/cover.h +++ b/lib/dictBuilder/cover.h @@ -152,6 +152,6 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection); * smallest dictionary within a specified regression of the compressed size * from the largest dictionary. */ - COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, + COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity, size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples, size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize); diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c index 485c333b..5e60f24c 100644 --- a/lib/dictBuilder/fastcover.c +++ b/lib/dictBuilder/fastcover.c @@ -21,6 +21,7 @@ #include "../common/threading.h" #include "cover.h" #include "../common/zstd_internal.h" /* includes zstd.h */ +#include "../compress/zstd_compress_internal.h" /* ZSTD_hash*() */ #ifndef ZDICT_STATIC_LINKING_ONLY #define ZDICT_STATIC_LINKING_ONLY #endif @@ -33,7 +34,7 @@ #define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB)) #define FASTCOVER_MAX_F 31 #define FASTCOVER_MAX_ACCEL 10 -#define DEFAULT_SPLITPOINT 0.75 +#define FASTCOVER_DEFAULT_SPLITPOINT 0.75 #define DEFAULT_F 20 #define DEFAULT_ACCEL 1 @@ -41,50 +42,50 @@ /*-************************************* * Console display ***************************************/ +#ifndef LOCALDISPLAYLEVEL static int g_displayLevel = 2; +#endif +#undef DISPLAY #define DISPLAY(...) \ { \ fprintf(stderr, __VA_ARGS__); \ fflush(stderr); \ } +#undef LOCALDISPLAYLEVEL #define LOCALDISPLAYLEVEL(displayLevel, l, ...) \ if (displayLevel >= l) { \ DISPLAY(__VA_ARGS__); \ } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ +#undef DISPLAYLEVEL #define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__) +#ifndef LOCALDISPLAYUPDATE +static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; +#endif +#undef LOCALDISPLAYUPDATE #define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ if (displayLevel >= l) { \ - if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \ + if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \ g_time = clock(); \ DISPLAY(__VA_ARGS__); \ } \ } +#undef DISPLAYUPDATE #define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__) -static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; -static clock_t g_time = 0; /*-************************************* * Hash Functions ***************************************/ -static const U64 prime6bytes = 227718039650203ULL; -static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } -static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); } - -static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; -static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } -static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } - - /** - * Hash the d-byte value pointed to by p and mod 2^f + * Hash the d-byte value pointed to by p and mod 2^f into the frequency vector */ -static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 h, unsigned d) { +static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 f, unsigned d) { if (d == 6) { - return ZSTD_hash6Ptr(p, h) & ((1 << h) - 1); + return ZSTD_hash6Ptr(p, f); } - return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1); + return ZSTD_hash8Ptr(p, f); } @@ -486,7 +487,7 @@ static void FASTCOVER_tryParameters(void *opaque) parameters, segmentFreqs); const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100); - selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail, + selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail, ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets, totalCompressedSize); @@ -617,7 +618,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover( /* constants */ const unsigned nbThreads = parameters->nbThreads; const double splitPoint = - parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; + parameters->splitPoint <= 0.0 ? FASTCOVER_DEFAULT_SPLITPOINT : parameters->splitPoint; const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 6d0b0423..79c522ef 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -62,14 +62,15 @@ #define NOISELENGTH 32 -static const int g_compressionLevel_default = 3; static const U32 g_selectivity_default = 9; /*-************************************* * Console display ***************************************/ +#undef DISPLAY #define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } +#undef DISPLAYLEVEL #define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } @@ -105,20 +106,17 @@ size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize) size_t headerSize; if (dictSize <= 8 || MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return ERROR(dictionary_corrupted); - { unsigned offcodeMaxValue = MaxOff; - ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t)); + { ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t)); U32* wksp = (U32*)malloc(HUF_WORKSPACE_SIZE); - short* offcodeNCount = (short*)malloc((MaxOff+1)*sizeof(short)); - if (!bs || !wksp || !offcodeNCount) { + if (!bs || !wksp) { headerSize = ERROR(memory_allocation); } else { ZSTD_reset_compressedBlockState(bs); - headerSize = ZSTD_loadCEntropy(bs, wksp, offcodeNCount, &offcodeMaxValue, dictBuffer, dictSize); + headerSize = ZSTD_loadCEntropy(bs, wksp, dictBuffer, dictSize); } free(bs); free(wksp); - free(offcodeNCount); } return headerSize; @@ -532,6 +530,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize, clock_t displayClock = 0; clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10; +# undef DISPLAYUPDATE # define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \ if (ZDICT_clockSpan(displayClock) > refreshRate) \ { displayClock = clock(); DISPLAY(__VA_ARGS__); \ @@ -706,7 +705,7 @@ static void ZDICT_flatLit(unsigned* countLit) #define OFFCODE_MAX 30 /* only applicable to first block */ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, - unsigned compressionLevel, + int compressionLevel, const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize, unsigned notificationLevel) @@ -741,7 +740,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, memset(repOffset, 0, sizeof(repOffset)); repOffset[1] = repOffset[4] = repOffset[8] = 1; memset(bestRepOffset, 0, sizeof(bestRepOffset)); - if (compressionLevel==0) compressionLevel = g_compressionLevel_default; + if (compressionLevel==0) compressionLevel = ZSTD_CLEVEL_DEFAULT; params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); esr.dict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, params.cParams, ZSTD_defaultCMem); @@ -786,7 +785,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, /* note : the result of this phase should be used to better appreciate the impact on statistics */ total=0; for (u=0; u<=offcodeMax; u++) total+=offcodeCount[u]; - errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax); + errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax, /* useLowProbCount */ 1); if (FSE_isError(errorCode)) { eSize = errorCode; DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount \n"); @@ -795,7 +794,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, Offlog = (U32)errorCode; total=0; for (u=0; u<=MaxML; u++) total+=matchLengthCount[u]; - errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML); + errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML, /* useLowProbCount */ 1); if (FSE_isError(errorCode)) { eSize = errorCode; DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount \n"); @@ -804,7 +803,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, mlLog = (U32)errorCode; total=0; for (u=0; u<=MaxLL; u++) total+=litLengthCount[u]; - errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL); + errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL, /* useLowProbCount */ 1); if (FSE_isError(errorCode)) { eSize = errorCode; DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount \n"); @@ -893,7 +892,7 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity, size_t hSize; #define HBUFFSIZE 256 /* should prove large enough for all entropy headers */ BYTE header[HBUFFSIZE]; - int const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel; + int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel; U32 const notificationLevel = params.notificationLevel; /* check conditions */ @@ -939,7 +938,7 @@ static size_t ZDICT_addEntropyTablesFromBuffer_advanced( const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_params_t params) { - int const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel; + int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel; U32 const notificationLevel = params.notificationLevel; size_t hSize = 8; @@ -1114,8 +1113,8 @@ size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, memset(¶ms, 0, sizeof(params)); params.d = 8; params.steps = 4; - /* Default to level 6 since no compression level information is available */ - params.zParams.compressionLevel = 3; + /* Use default level since no compression level information is available */ + params.zParams.compressionLevel = ZSTD_CLEVEL_DEFAULT; #if defined(DEBUGLEVEL) && (DEBUGLEVEL>=1) params.zParams.notificationLevel = DEBUGLEVEL; #endif diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index ff2e77fa..b782993f 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -279,7 +279,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy( # define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ # define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API -# elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__) +# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405) # define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message))) # elif (ZDICT_GCC_VERSION >= 301) # define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated)) diff --git a/lib/dll/example/README.md b/lib/dll/example/README.md index e231f59c..9e30fd59 100644 --- a/lib/dll/example/README.md +++ b/lib/dll/example/README.md @@ -1,23 +1,21 @@ -ZSTD Windows binary package -==================================== +# ZSTD Windows binary package -#### The package contents +## The package contents -- `zstd.exe` : Command Line Utility, supporting gzip-like arguments -- `dll\libzstd.dll` : The ZSTD dynamic library (DLL) -- `dll\libzstd.lib` : The import library of the ZSTD dynamic library (DLL) for Visual C++ -- `example\` : The example of usage of the ZSTD library -- `include\` : Header files required by the ZSTD library +- `zstd.exe` : Command Line Utility, supporting gzip-like arguments +- `dll\libzstd.dll` : The ZSTD dynamic library (DLL) +- `dll\libzstd.lib` : The import library of the ZSTD dynamic library (DLL) for Visual C++ +- `example\` : The example of usage of the ZSTD library +- `include\` : Header files required by the ZSTD library - `static\libzstd_static.lib` : The static ZSTD library (LIB) - -#### Usage of Command Line Interface +## Usage of Command Line Interface Command Line Interface (CLI) supports gzip-like arguments. By default CLI takes an input file and compresses it to an output file: -``` + Usage: zstd [arg] [input] [output] -``` + The full list of commands for CLI can be obtained with `-h` or `-H`. The ratio can be improved with commands from `-3` to `-16` but higher levels also have slower compression. CLI includes in-memory compression benchmark module with compression @@ -25,36 +23,32 @@ levels starting from `-b` and ending with `-e` with iteration time of `-i` secon CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`. - -#### The example of usage of static and dynamic ZSTD libraries with gcc/MinGW +## The example of usage of static and dynamic ZSTD libraries with gcc/MinGW Use `cd example` and `make` to build `fullbench-dll` and `fullbench-lib`. `fullbench-dll` uses a dynamic ZSTD library from the `dll` directory. `fullbench-lib` uses a static ZSTD library from the `lib` directory. - -#### Using ZSTD DLL with gcc/MinGW +## Using ZSTD DLL with gcc/MinGW The header files from `include\` and the dynamic library `dll\libzstd.dll` are required to compile a project using gcc/MinGW. The dynamic library has to be added to linking options. It means that if a project that uses ZSTD consists of a single `test-dll.c` file it should be linked with `dll\libzstd.dll`. For example: -``` + gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\libzstd.dll -``` -The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`. +The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`. -#### The example of usage of static and dynamic ZSTD libraries with Visual C++ +## The example of usage of static and dynamic ZSTD libraries with Visual C++ Open `example\fullbench-dll.sln` to compile `fullbench-dll` that uses a dynamic ZSTD library from the `dll` directory. The solution works with Visual C++ 2010 or newer. When one will open the solution with Visual C++ newer than 2010 then the solution will upgraded to the current version. - -#### Using ZSTD DLL with Visual C++ +## Using ZSTD DLL with Visual C++ The header files from `include\` and the import library `dll\libzstd.lib` are required to compile a project using Visual C++. diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index eb236283..13115bec 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -1280,7 +1280,11 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si * Basic Types *********************************************************/ #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# include <stdint.h> +# if defined(_AIX) +# include <inttypes.h> +# else +# include <stdint.h> /* intptr_t */ +# endif typedef uint8_t BYTE; typedef uint16_t U16; typedef int16_t S16; diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index 32d45a6d..9abb6d03 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -89,7 +89,11 @@ extern "C" { * Basic Types *****************************************************************/ #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# include <stdint.h> +# if defined(_AIX) +# include <inttypes.h> +# else +# include <stdint.h> /* intptr_t */ +# endif typedef uint8_t BYTE; typedef uint16_t U16; typedef int16_t S16; diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index b541eae2..a19cb205 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -90,7 +90,11 @@ extern "C" { * Basic Types *****************************************************************/ #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# include <stdint.h> +# if defined(_AIX) +# include <inttypes.h> +# else +# include <stdint.h> /* intptr_t */ +# endif typedef uint8_t BYTE; typedef uint16_t U16; typedef int16_t S16; diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 56bf4522..77d52555 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -52,7 +52,11 @@ extern "C" { * Basic Types *****************************************************************/ #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# include <stdint.h> +# if defined(_AIX) +# include <inttypes.h> +# else +# include <stdint.h> /* intptr_t */ +# endif typedef uint8_t BYTE; typedef uint16_t U16; typedef int16_t S16; @@ -74,7 +78,7 @@ extern "C" { /*-************************************* * Debug ***************************************/ -#include "debug.h" +#include "../common/debug.h" #ifndef assert # define assert(condition) ((void)0) #endif diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 243d2225..ca8d5c9b 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -80,7 +80,11 @@ extern "C" { * Basic Types *****************************************************************/ #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# include <stdint.h> +# if defined(_AIX) +# include <inttypes.h> +# else +# include <stdint.h> /* intptr_t */ +# endif typedef uint8_t BYTE; typedef uint16_t U16; typedef int16_t S16; diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index c56f5827..c4ac7dba 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -82,7 +82,11 @@ extern "C" { * Basic Types *****************************************************************/ #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include <stdint.h> +# if defined(_AIX) +# include <inttypes.h> +# else +# include <stdint.h> /* intptr_t */ +# endif typedef uint8_t BYTE; typedef uint16_t U16; typedef int16_t S16; diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 9f3a597f..049ba474 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -242,7 +242,11 @@ extern "C" { * Basic Types *****************************************************************/ #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include <stdint.h> +# if defined(_AIX) +# include <inttypes.h> +# else +# include <stdint.h> /* intptr_t */ +# endif typedef uint8_t BYTE; typedef uint16_t U16; typedef int16_t S16; diff --git a/lib/libzstd.pc.in b/lib/libzstd.pc.in index 8ec0235a..8465c977 100644 --- a/lib/libzstd.pc.in +++ b/lib/libzstd.pc.in @@ -3,9 +3,9 @@ # BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) prefix=@PREFIX@ -exec_prefix=${prefix} -includedir=${prefix}/@INCLUDEDIR@ -libdir=${exec_prefix}/@LIBDIR@ +exec_prefix=@EXEC_PREFIX@ +includedir=@INCLUDEDIR@ +libdir=@LIBDIR@ Name: zstd Description: fast lossless compression algorithm library @@ -72,16 +72,21 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 5 - +#define ZSTD_VERSION_RELEASE 7 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) -ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ + +/*! ZSTD_versionNumber() : + * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ +ZSTDLIB_API unsigned ZSTD_versionNumber(void); #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) -ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */ + +/*! ZSTD_versionString() : + * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ +ZSTDLIB_API const char* ZSTD_versionString(void); /* ************************************* * Default constant @@ -334,7 +339,9 @@ typedef enum { * for large inputs, by finding large matches at long distance. * It increases memory usage and window size. * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB - * except when expressly set to a different value. */ + * except when expressly set to a different value. + * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and + * compression strategy >= ZSTD_btopt (== compression level 16+) */ ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. * Larger values increase memory usage and compression ratio, * but decrease compression speed. @@ -365,16 +372,20 @@ typedef enum { ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ /* multi-threading parameters */ - /* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). - * They return an error otherwise. */ + /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. + * In a situation where it's unknown if the linked library supports multi-threading or not, + * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. + */ ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. - * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() : + * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, - * while compression work is performed in parallel, within worker threads. + * while compression is performed in parallel, within worker thread(s). * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). * More workers improve speed, but also increase memory usage. - * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */ + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, + * compression is performed inside Caller's thread, and all invocations are blocking */ ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. * 0 means default, which is dynamically determined based on compression parameters. @@ -403,6 +414,11 @@ typedef enum { * ZSTD_c_literalCompressionMode * ZSTD_c_targetCBlockSize * ZSTD_c_srcSizeHint + * ZSTD_c_enableDedicatedDictSearch + * ZSTD_c_stableInBuffer + * ZSTD_c_stableOutBuffer + * ZSTD_c_blockDelimiters + * ZSTD_c_validateSequences * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -413,7 +429,12 @@ typedef enum { ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam5=1002, ZSTD_c_experimentalParam6=1003, - ZSTD_c_experimentalParam7=1004 + ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam8=1005, + ZSTD_c_experimentalParam9=1006, + ZSTD_c_experimentalParam10=1007, + ZSTD_c_experimentalParam11=1008, + ZSTD_c_experimentalParam12=1009 } ZSTD_cParameter; typedef struct { @@ -524,11 +545,13 @@ typedef enum { * At the time of this writing, they include : * ZSTD_d_format * ZSTD_d_stableOutBuffer + * ZSTD_d_forceIgnoreChecksum * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly */ ZSTD_d_experimentalParam1=1000, - ZSTD_d_experimentalParam2=1001 + ZSTD_d_experimentalParam2=1001, + ZSTD_d_experimentalParam3=1002 } ZSTD_dParameter; @@ -664,8 +687,9 @@ typedef enum { * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) * - output->pos must be <= dstCapacity, input->pos must be <= srcSize * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - endOp must be a valid directive * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. - * - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available, + * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, * and then immediately returns, just indicating that there is some data remaining to be flushed. * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. @@ -1100,21 +1124,40 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; typedef struct { - unsigned int matchPos; /* Match pos in dst */ - /* If seqDef.offset > 3, then this is seqDef.offset - 3 - * If seqDef.offset < 3, then this is the corresponding repeat offset - * But if seqDef.offset < 3 and litLength == 0, this is the - * repeat offset before the corresponding repeat offset - * And if seqDef.offset == 3 and litLength == 0, this is the - * most recent repeat offset - 1 - */ - unsigned int offset; - unsigned int litLength; /* Literal length */ - unsigned int matchLength; /* Match length */ - /* 0 when seq not rep and seqDef.offset otherwise - * when litLength == 0 this will be <= 4, otherwise <= 3 like normal - */ - unsigned int rep; + unsigned int offset; /* The offset of the match. (NOT the same as the offset code) + * If offset == 0 and matchLength == 0, this sequence represents the last + * literals in the block of litLength size. + */ + + unsigned int litLength; /* Literal length of the sequence. */ + unsigned int matchLength; /* Match length of the sequence. */ + + /* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0. + * In this case, we will treat the sequence as a marker for a block boundary. + */ + + unsigned int rep; /* Represents which repeat offset is represented by the field 'offset'. + * Ranges from [0, 3]. + * + * Repeat offsets are essentially previous offsets from previous sequences sorted in + * recency order. For more detail, see doc/zstd_compression_format.md + * + * If rep == 0, then 'offset' does not contain a repeat offset. + * If rep > 0: + * If litLength != 0: + * rep == 1 --> offset == repeat_offset_1 + * rep == 2 --> offset == repeat_offset_2 + * rep == 3 --> offset == repeat_offset_3 + * If litLength == 0: + * rep == 1 --> offset == repeat_offset_2 + * rep == 2 --> offset == repeat_offset_3 + * rep == 3 --> offset == repeat_offset_1 - 1 + * + * Note: This field is optional. ZSTD_generateSequences() will calculate the value of + * 'rep', but repeat offsets do not necessarily need to be calculated from an external + * sequence provider's perspective. For example, ZSTD_compressSequences() does not + * use this 'rep' field at all (as of now). + */ } ZSTD_Sequence; typedef struct { @@ -1157,6 +1200,12 @@ typedef enum { } ZSTD_format_e; typedef enum { + /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1 +} ZSTD_forceIgnoreChecksum_e; + +typedef enum { /* Note: this enum and the behavior it controls are effectively internal * implementation details of the compressor. They are expected to continue * to evolve and should be considered only in the context of extremely @@ -1253,14 +1302,74 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS * or an error code (if srcSize is too small) */ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); -/*! ZSTD_getSequences() : - * Extract sequences from the sequence store +typedef enum { + ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ + ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ +} ZSTD_sequenceFormat_e; + +/*! ZSTD_generateSequences() : + * Generate sequences using ZSTD_compress2, given a source buffer. + * + * Each block will end with a dummy sequence + * with offset == 0, matchLength == 0, and litLength == length of last literals. + * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) + * simply acts as a block delimiter. + * * zc can be used to insert custom compression params. * This function invokes ZSTD_compress2 - * @return : number of sequences extracted + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters + * @return : number of sequences generated + */ + +ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize); + +/*! ZSTD_mergeBlockDelimiters() : + * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals + * by merging them into into the literals of the next sequence. + * + * As such, the final generated result has no explicit representation of block boundaries, + * and the final last literals segment is not represented in the sequences. + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters + * @return : number of sequences left after merging + */ +ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); + +/*! ZSTD_compressSequences() : + * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. + * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) + * The entire source is compressed into a single frame. + * + * The compression behavior changes based on cctx params. In particular: + * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on + * the block size derived from the cctx, and sequences may be split. This is the default setting. + * + * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * + * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. + * + * In addition to the two adjustable experimental params, there are other important cctx params. + * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. + * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. + * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset + * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + * + * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. + * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history + * @return : final compressed size or a ZSTD error. */ -ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, - size_t outSeqsSize, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); /*************************************** @@ -1372,7 +1481,11 @@ ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); typedef void (*ZSTD_freeFunction) (void* opaque, void* address); typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; -static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); @@ -1385,13 +1498,36 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS ZSTD_compressionParameters cParams, ZSTD_customMem customMem); +/* ! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool. + */ +typedef struct POOL_ctx_s ZSTD_threadPool; +ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); +ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); + +/* + * This API is temporary and is expected to change or disappear in the future! + */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* cctxParams, + ZSTD_customMem customMem); + ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, ZSTD_customMem customMem); - - /*************************************** * Advanced compression functions ***************************************/ @@ -1404,6 +1540,12 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); +/*! ZSTD_getDictID_fromCDict() : + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + /*! ZSTD_getCParams() : * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. * `estimatedSrcSize` value is optional, select 0 if not known */ @@ -1518,6 +1660,143 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre * but compression ratio may regress significantly if guess considerably underestimates */ #define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 +/* Controls whether the new and experimental "dedicated dictionary search + * structure" can be used. This feature is still rough around the edges, be + * prepared for surprising behavior! + * + * How to use it: + * + * When using a CDict, whether to use this feature or not is controlled at + * CDict creation, and it must be set in a CCtxParams set passed into that + * construction (via ZSTD_createCDict_advanced2()). A compression will then + * use the feature or not based on how the CDict was constructed; the value of + * this param, set in the CCtx, will have no effect. + * + * However, when a dictionary buffer is passed into a CCtx, such as via + * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control + * whether the CDict that is created internally can use the feature or not. + * + * What it does: + * + * Normally, the internal data structures of the CDict are analogous to what + * would be stored in a CCtx after compressing the contents of a dictionary. + * To an approximation, a compression using a dictionary can then use those + * data structures to simply continue what is effectively a streaming + * compression where the simulated compression of the dictionary left off. + * Which is to say, the search structures in the CDict are normally the same + * format as in the CCtx. + * + * It is possible to do better, since the CDict is not like a CCtx: the search + * structures are written once during CDict creation, and then are only read + * after that, while the search structures in the CCtx are both read and + * written as the compression goes along. This means we can choose a search + * structure for the dictionary that is read-optimized. + * + * This feature enables the use of that different structure. + * + * Note that some of the members of the ZSTD_compressionParameters struct have + * different semantics and constraints in the dedicated search structure. It is + * highly recommended that you simply set a compression level in the CCtxParams + * you pass into the CDict creation call, and avoid messing with the cParams + * directly. + * + * Effects: + * + * This will only have any effect when the selected ZSTD_strategy + * implementation supports this feature. Currently, that's limited to + * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. + * + * Note that this means that the CDict tables can no longer be copied into the + * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be + * useable. The dictionary can only be attached or reloaded. + * + * In general, you should expect compression to be faster--sometimes very much + * so--and CDict creation to be slightly slower. Eventually, we will probably + * make this mode the default. + */ +#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 + +/* ZSTD_c_stableInBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the compressor, and + * compression will fail if it ever changes. This means the only flush + * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end + * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos) + * MUST not be modified during compression or you will get data corruption. + * + * When this flag is enabled zstd won't allocate an input window buffer, + * because the user guarantees it can reference the ZSTD_inBuffer until + * the frame is complete. But, it will still allocate an output buffer + * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also + * avoid the memcpy() from the input buffer to the input window buffer. + * + * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used. + * That means this flag cannot be used with ZSTD_compressStream(). + * + * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, compression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST + * not be modified during compression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_inBuffer to find + * matches. Normally zstd maintains its own window buffer for this purpose, + * but passing this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 + +/* ZSTD_c_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells he compressor that the ZSTD_outBuffer will not be resized between + * calls. Specifically: (out.size - out.pos) will never grow. This gives the + * compressor the freedom to say: If the compressed data doesn't fit in the + * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to + * always decompress directly into the output buffer, instead of decompressing + * into an internal buffer and copying to the output buffer. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer. It will still allocate the + * input window buffer (see ZSTD_c_stableInBuffer). + * + * Zstd will check that (out.size - out.pos) never grows and return an error + * if it does. While not strictly necessary, this should prevent surprises. + */ +#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 + +/* ZSTD_c_blockDelimiters + * Default is 0 == ZSTD_sf_noBlockDelimiters. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * + * Designates whether or not the given array of ZSTD_Sequence contains block delimiters + * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. + * See the definition of ZSTD_Sequence for more specifics. + */ +#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 + +/* ZSTD_c_validateSequences + * Default is 0 == disabled. Set to 1 to enable sequence validation. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * Designates whether or not we validate sequences provided to ZSTD_compressSequences() + * during function execution. + * + * Without validation, providing a sequence that does not conform to the zstd spec will cause + * undefined behavior, and may produce a corrupted block. + * + * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and + * return an error. + * + */ +#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 + /*! ZSTD_CCtx_getParameter() : * Get the requested compression parameter value, selected by enum ZSTD_cParameter, * and store it into int* value. @@ -1566,8 +1845,10 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, Z /*! ZSTD_CCtxParams_setParameter() : * Similar to ZSTD_CCtx_setParameter. * Set one compression parameter, selected by enum ZSTD_cParameter. - * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams(). - * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Parameters must be applied to a ZSTD_CCtx using + * ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : a code representing success or failure (which can be tested with + * ZSTD_isError()). */ ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); @@ -1647,6 +1928,13 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* pre */ ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); +/*! ZSTD_DCtx_getParameter() : + * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); + /* ZSTD_d_format * experimental parameter, * allowing selection between ZSTD_format_e input compression formats @@ -1684,6 +1972,17 @@ ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowS */ #define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 +/* ZSTD_d_forceIgnoreChecksum + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * Tells the decompressor to skip checksum validation during decompression, regardless + * of whether checksumming was specified during compression. This offers some + * slight performance benefits, and may be useful for debugging. + * Param has values of type ZSTD_forceIgnoreChecksum_e + */ +#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 + /*! ZSTD_DCtx_setFormat() : * Instruct the decoder context about what kind of data to decode next. * This instruction is mandatory to decode data without a fully-formed header, @@ -1711,7 +2010,8 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( ********************************************************************/ /*===== Advanced Streaming compression functions =====*/ -/**! ZSTD_initCStream_srcSize() : + +/*! ZSTD_initCStream_srcSize() : * This function is deprecated, and equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) @@ -1728,7 +2028,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); -/**! ZSTD_initCStream_usingDict() : +/*! ZSTD_initCStream_usingDict() : * This function is deprecated, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); @@ -1745,7 +2045,7 @@ ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); -/**! ZSTD_initCStream_advanced() : +/*! ZSTD_initCStream_advanced() : * This function is deprecated, and is approximately equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * // Pseudocode: Set each zstd parameter and leave the rest as-is. @@ -1766,7 +2066,7 @@ ZSTD_initCStream_advanced(ZSTD_CStream* zcs, ZSTD_parameters params, unsigned long long pledgedSrcSize); -/**! ZSTD_initCStream_usingCDict() : +/*! ZSTD_initCStream_usingCDict() : * This function is deprecated, and equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, cdict); @@ -1776,7 +2076,7 @@ ZSTD_initCStream_advanced(ZSTD_CStream* zcs, */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); -/**! ZSTD_initCStream_usingCDict_advanced() : +/*! ZSTD_initCStream_usingCDict_advanced() : * This function is DEPRECATED, and is approximately equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. @@ -1849,7 +2149,8 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); /*===== Advanced Streaming decompression functions =====*/ -/** + +/*! * This function is deprecated, and is equivalent to: * * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); @@ -1860,7 +2161,7 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); */ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); -/** +/*! * This function is deprecated, and is equivalent to: * * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); @@ -1871,7 +2172,7 @@ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dic */ ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); -/** +/*! * This function is deprecated, and is equivalent to: * * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); @@ -1933,7 +2234,7 @@ ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstC ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -/*- +/** Buffer-less streaming decompression (synchronous mode) A ZSTD_DCtx object is required to track streaming operations. diff --git a/programs/Makefile b/programs/Makefile index 418ad4e6..8641d0ee 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -15,7 +15,14 @@ # zstd-decompress : decompressor-only version of zstd # ########################################################################## -ZSTDDIR = ../lib +.PHONY: default +default: zstd-release + +# silent mode by default; verbose can be triggered by V=1 or VERBOSE=1 +$(V)$(VERBOSE).SILENT: + + +ZSTDDIR := ../lib # Version numbers LIBVER_SRC := $(ZSTDDIR)/zstd.h @@ -33,64 +40,94 @@ ZSTD_VERSION = $(LIBVER) HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) GREP_OPTIONS ?= ifeq ($HAVE_COLORNEVER, 1) -GREP_OPTIONS += --color=never + GREP_OPTIONS += --color=never endif GREP = grep $(GREP_OPTIONS) ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1) -ALIGN_LOOP = -falign-loops=32 + ALIGN_LOOP = -falign-loops=32 else -ALIGN_LOOP = + ALIGN_LOOP = endif -CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_ +DEBUGLEVEL ?= 0 +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL) ifeq ($(OS),Windows_NT) # MinGW assumed -CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting + CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif -CFLAGS ?= -O3 +CFLAGS ?= -O3 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef -Wpointer-arith \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wredundant-decls -Wmissing-prototypes -Wc++-compat -CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +ZSTDLIB_COMMON := $(ZSTDDIR)/common +ZSTDLIB_COMPRESS := $(ZSTDDIR)/compress +ZSTDLIB_DECOMPRESS := $(ZSTDDIR)/decompress +ZDICT_DIR := $(ZSTDDIR)/dictBuilder +ZSTDLEGACY_DIR := $(ZSTDDIR)/legacy -ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c -ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c -ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c -ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) -ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c -ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o +vpath %.c $(ZSTDLIB_COMMON) $(ZSTDLIB_COMPRESS) $(ZSTDLIB_DECOMPRESS) $(ZDICT_DIR) $(ZSTDLEGACY_DIR) + +ZSTDLIB_COMMON_C := $(wildcard $(ZSTDLIB_COMMON)/*.c) +ZSTDLIB_COMPRESS_C := $(wildcard $(ZSTDLIB_COMPRESS)/*.c) +ZSTDLIB_DECOMPRESS_C := $(wildcard $(ZSTDLIB_DECOMPRESS)/*.c) +ZSTDLIB_CORE_SRC := $(ZSTDLIB_DECOMPRESS_C) $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) +ZDICT_SRC := $(wildcard $(ZDICT_DIR)/*.c) ZSTD_LEGACY_SUPPORT ?= 5 -ZSTDLEGACY_FILES := +ZSTDLEGACY_SRC := ifneq ($(ZSTD_LEGACY_SUPPORT), 0) ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) - ZSTDLEGACY_FILES += $(shell ls $(ZSTDDIR)/legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') + ZSTDLEGACY_SRC += $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') endif -else endif # Sort files in alphabetical order for reproducible builds -ZSTDLIB_FILES := $(sort $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES))) - -ZSTD_CLI_FILES := $(wildcard *.c) -ZSTD_CLI_OBJ := $(patsubst %.c,%.o,$(ZSTD_CLI_FILES)) +ZSTDLIB_FULL_SRC = $(sort $(ZSTDLIB_CORE_SRC) $(ZSTDLEGACY_SRC) $(ZDICT_SRC)) +ZSTDLIB_LOCAL_SRC := $(notdir $(ZSTDLIB_FULL_SRC)) +ZSTDLIB_LOCAL_OBJ := $(ZSTDLIB_LOCAL_SRC:.c=.o) + +ZSTD_CLI_SRC := $(wildcard *.c) +ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o) + +ZSTD_ALL_SRC := $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC) +ZSTD_ALL_OBJ := $(ZSTD_ALL_SRC:.c=.o) + +UNAME := $(shell uname) +ifeq ($(UNAME), Darwin) + HASH ?= md5 +else ifeq ($(UNAME), FreeBSD) + HASH ?= gmd5sum +else ifeq ($(UNAME), OpenBSD) + HASH ?= md5 +endif +HASH ?= md5sum +HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0) + +ifndef BUILD_DIR +HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " ") +ifeq ($(HAVE_HASH),0) + $(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags) + BUILD_DIR := obj/generic_noconf +endif +endif # BUILD_DIR # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -RES64_FILE = windres/zstd64.res -RES32_FILE = windres/zstd32.res + EXT =.exe + RES64_FILE = windres/zstd64.res + RES32_FILE = windres/zstd32.res ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine))) RES_FILE = $(RES64_FILE) else RES_FILE = $(RES32_FILE) endif else -EXT = + EXT = endif VOID = /dev/null @@ -103,60 +140,64 @@ NO_THREAD_MSG := ==> no threads, building without multithreading support HAVE_PTHREAD := $(shell printf '$(NUM_SYMBOL)include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c) HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0) ifeq ($(HAVE_THREAD), 1) -THREAD_MSG := ==> building with threading support -THREAD_CPP := -DZSTD_MULTITHREAD -THREAD_LD := -pthread + THREAD_MSG := ==> building with threading support + THREAD_CPP := -DZSTD_MULTITHREAD + THREAD_LD := -pthread else -THREAD_MSG := $(NO_THREAD_MSG) + THREAD_MSG := $(NO_THREAD_MSG) endif # zlib detection NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support HAVE_ZLIB := $(shell printf '$(NUM_SYMBOL)include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c) ifeq ($(HAVE_ZLIB), 1) -ZLIB_MSG := ==> building zstd with .gz compression support -ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS -ZLIBLD = -lz + ZLIB_MSG := ==> building zstd with .gz compression support + ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS + ZLIBLD = -lz else -ZLIB_MSG := $(NO_ZLIB_MSG) + ZLIB_MSG := $(NO_ZLIB_MSG) endif # lzma detection NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support HAVE_LZMA := $(shell printf '$(NUM_SYMBOL)include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c) ifeq ($(HAVE_LZMA), 1) -LZMA_MSG := ==> building zstd with .xz/.lzma compression support -LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS -LZMALD = -llzma + LZMA_MSG := ==> building zstd with .xz/.lzma compression support + LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS + LZMALD = -llzma else -LZMA_MSG := $(NO_LZMA_MSG) + LZMA_MSG := $(NO_LZMA_MSG) endif # lz4 detection NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support HAVE_LZ4 := $(shell printf '$(NUM_SYMBOL)include <lz4frame.h>\n$(NUM_SYMBOL)include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c) ifeq ($(HAVE_LZ4), 1) -LZ4_MSG := ==> building zstd with .lz4 compression support -LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS -LZ4LD = -llz4 + LZ4_MSG := ==> building zstd with .lz4 compression support + LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS + LZ4LD = -llz4 else -LZ4_MSG := $(NO_LZ4_MSG) + LZ4_MSG := $(NO_LZ4_MSG) endif # explicit backtrace enable/disable for Linux & Darwin ifeq ($(BACKTRACE), 0) -DEBUGFLAGS += -DBACKTRACE_ENABLE=0 + DEBUGFLAGS += -DBACKTRACE_ENABLE=0 endif ifeq (,$(filter Windows%, $(OS))) ifeq ($(BACKTRACE), 1) -DEBUGFLAGS += -DBACKTRACE_ENABLE=1 -DEBUGFLAGS_LD += -rdynamic + DEBUGFLAGS += -DBACKTRACE_ENABLE=1 + DEBUGFLAGS_LD += -rdynamic endif endif +SET_CACHE_DIRECTORY = \ + $(MAKE) --no-print-directory $@ \ + BUILD_DIR=obj/$(HASH_DIR) \ + CPPFLAGS="$(CPPFLAGS)" \ + CFLAGS="$(CFLAGS)" \ + LDFLAGS="$(LDFLAGS)" -.PHONY: default -default: zstd-release .PHONY: all all: zstd @@ -164,21 +205,51 @@ all: zstd .PHONY: allVariants allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy zstd-dictBuilder -$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) - +.PHONY: zstd # must always be run zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP) zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD) zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) ifneq (,$(filter Windows%,$(OS))) zstd : $(RES_FILE) endif -zstd : $(ZSTDLIB_FILES) $(ZSTD_CLI_OBJ) + +ifndef BUILD_DIR +# generate BUILD_DIR from flags + +zstd: + $(SET_CACHE_DIRECTORY) + +else +# BUILD_DIR is defined + +ZSTD_OBJ := $(addprefix $(BUILD_DIR)/, $(ZSTD_ALL_OBJ)) +$(BUILD_DIR)/zstd : $(ZSTD_OBJ) @echo "$(THREAD_MSG)" @echo "$(ZLIB_MSG)" @echo "$(LZMA_MSG)" @echo "$(LZ4_MSG)" + @echo LINK $@ $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) +ifeq ($(HAVE_HASH),1) +SRCBIN_HASH = $(shell cat $(BUILD_DIR)/zstd 2> $(VOID) | $(HASH) | cut -f 1 -d " ") +DSTBIN_HASH = $(shell cat zstd 2> $(VOID) | $(HASH) | cut -f 1 -d " ") +BIN_ISDIFFERENT = $(if $(filter $(SRCBIN_HASH),$(DSTBIN_HASH)),0,1) +else +BIN_ISDIFFERENT = 1 +endif + +zstd : $(BUILD_DIR)/zstd + if [ $(BIN_ISDIFFERENT) -eq 1 ]; then \ + cp -f $< $@; \ + echo zstd build completed; \ + else \ + echo zstd already built; \ + fi + +endif # BUILD_DIR + + .PHONY: zstd-release zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0 zstd-release: DEBUGFLAGS_LD := @@ -190,12 +261,12 @@ zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) ifneq (,$(filter Windows%,$(OS))) zstd32 : $(RES32_FILE) endif -zstd32 : $(ZSTDLIB_FILES) $(ZSTD_CLI_FILES) +zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_SRC) $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) ## zstd-nolegacy: same scope as zstd, with just support of legacy formats removed zstd-nolegacy : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD) -zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) $(ZSTD_CLI_OBJ) +zstd-nolegacy : $(ZSTDLIB_CORE_SRC) $(ZDICT_SRC) $(ZSTD_CLI_OBJ) $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) zstd-nomt : THREAD_CPP := @@ -222,12 +293,12 @@ zstd-noxz : zstd # It's unclear at this stage if this is a scenario that must be supported .PHONY: zstd-dll zstd-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd -zstd-dll : ZSTDLIB_FILES = +zstd-dll : ZSTDLIB_FULL_SRC = zstd-dll : $(ZSTD_CLI_OBJ) $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) -## zstd-pgo: zstd executable optimized with pgo. `gcc` only. +## zstd-pgo: zstd executable optimized with PGO. zstd-pgo : $(MAKE) clean $(MAKE) zstd MOREFLAGS=-fprofile-generate @@ -237,24 +308,24 @@ zstd-pgo : ./zstd -b $(PROFILE_WITH) ./zstd -b7i2 $(PROFILE_WITH) ./zstd -b5 $(PROFILE_WITH) - $(RM) zstd *.o $(ZSTDDECOMP_O) $(ZSTDDIR)/compress/*.o + $(RM) zstd *.o case $(CC) in *clang*) if ! [ -e default.profdata ]; then llvm-profdata merge -output=default.profdata default*.profraw; fi ;; esac $(MAKE) zstd MOREFLAGS=-fprofile-use ## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format. zstd-small: CFLAGS = -Os -s -zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c util.c timefn.c fileio.c +zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o $@$(EXT) -zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c util.c timefn.c fileio.c +zstd-decompress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_DECOMPRESS_C) zstdcli.c util.c timefn.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT) -zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c util.c timefn.c fileio.c +zstd-compress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) zstdcli.c util.c timefn.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) ## zstd-dictBuilder: executable supporting dictionary creation and compression (only) zstd-dictBuilder: CPPFLAGS += -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -zstd-dictBuilder: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) $(ZDICT_FILES) zstdcli.c util.c timefn.c fileio.c dibio.c +zstd-dictBuilder: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c dibio.c $(CC) $(FLAGS) $^ -o $@$(EXT) zstdmt: zstd @@ -274,12 +345,11 @@ endif .PHONY: clean clean: - $(MAKE) -C $(ZSTDDIR) clean - @$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda - @$(RM) core *.o tmp* result* *.gcda dictionary *.zst \ + $(RM) core *.o tmp* result* *.gcda dictionary *.zst \ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \ zstd-dictBuilder$(EXT) *.gcda default*.profraw default.profdata have_zlib$(EXT) + $(RM) -r obj/* @echo Cleaning completed MD2ROFF = ronn @@ -309,15 +379,34 @@ preview-man: clean-man man man ./zstdgrep.1 man ./zstdless.1 + +# Generate .h dependencies automatically + +DEPFLAGS = -MT $@ -MMD -MP -MF + +$(BUILD_DIR)/%.o : %.c $(BUILD_DIR)/%.d | $(BUILD_DIR) + @echo CC $@ + $(COMPILE.c) $(DEPFLAGS) $(BUILD_DIR)/$*.d $(OUTPUT_OPTION) $< + +MKDIR ?= mkdir +$(BUILD_DIR): ; $(MKDIR) -p $@ + +DEPFILES := $(ZSTD_OBJ:.o=.d) +$(DEPFILES): + +include $(wildcard $(DEPFILES)) + + + #----------------------------------------------------------------------------- # make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets #----------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) +ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) EGREP_OPTIONS ?= ifeq ($HAVE_COLORNEVER, 1) -EGREP_OPTIONS += --color=never + EGREP_OPTIONS += --color=never endif EGREP = egrep $(EGREP_OPTIONS) AWK = awk @@ -328,7 +417,7 @@ AWK = awk ## list: Print all targets and their descriptions (if provided) .PHONY: list list: - @TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \ + TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \ | $(AWK) -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \ | $(EGREP) -v -e '^[^[:alnum:]]' | sort); \ { \ @@ -355,17 +444,17 @@ datarootdir ?= $(PREFIX)/share mandir ?= $(datarootdir)/man man1dir ?= $(mandir)/man1 -ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) -MANDIR ?= $(PREFIX)/man -MAN1DIR ?= $(MANDIR)/man1 +ifneq (,$(filter $(UNAME),OpenBSD FreeBSD NetBSD DragonFly SunOS)) + MANDIR ?= $(PREFIX)/man + MAN1DIR ?= $(MANDIR)/man1 else -MAN1DIR ?= $(man1dir) + MAN1DIR ?= $(man1dir) endif -ifneq (,$(filter $(shell uname),SunOS)) -INSTALL ?= ginstall +ifneq (,$(filter $(UNAME),SunOS)) + INSTALL ?= ginstall else -INSTALL ?= install + INSTALL ?= install endif INSTALL_PROGRAM ?= $(INSTALL) @@ -374,36 +463,39 @@ INSTALL_DATA ?= $(INSTALL) -m 644 INSTALL_MAN ?= $(INSTALL_DATA) .PHONY: install -install: zstd +install: + # generate zstd only if not already present + [ -e zstd ] || $(MAKE) zstd-release + [ -e $(DESTDIR)$(BINDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ + [ -e $(DESTDIR)$(MAN1DIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(MAN1DIR)/ @echo Installing binaries - @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MAN1DIR)/ - @$(INSTALL_PROGRAM) zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT) - @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat$(EXT) - @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd$(EXT) - @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdmt$(EXT) - @$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless - @$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep + $(INSTALL_PROGRAM) zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT) + ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat$(EXT) + ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd$(EXT) + ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdmt$(EXT) + $(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless + $(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep @echo Installing man pages - @$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1 - @ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1 - @ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1 - @$(INSTALL_MAN) zstdgrep.1 $(DESTDIR)$(MAN1DIR)/zstdgrep.1 - @$(INSTALL_MAN) zstdless.1 $(DESTDIR)$(MAN1DIR)/zstdless.1 + $(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1 + ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1 + ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1 + $(INSTALL_MAN) zstdgrep.1 $(DESTDIR)$(MAN1DIR)/zstdgrep.1 + $(INSTALL_MAN) zstdless.1 $(DESTDIR)$(MAN1DIR)/zstdless.1 @echo zstd installation completed .PHONY: uninstall uninstall: - @$(RM) $(DESTDIR)$(BINDIR)/zstdgrep - @$(RM) $(DESTDIR)$(BINDIR)/zstdless - @$(RM) $(DESTDIR)$(BINDIR)/zstdcat - @$(RM) $(DESTDIR)$(BINDIR)/unzstd - @$(RM) $(DESTDIR)$(BINDIR)/zstdmt - @$(RM) $(DESTDIR)$(BINDIR)/zstd - @$(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1 - @$(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1 - @$(RM) $(DESTDIR)$(MAN1DIR)/zstdcat.1 - @$(RM) $(DESTDIR)$(MAN1DIR)/unzstd.1 - @$(RM) $(DESTDIR)$(MAN1DIR)/zstd.1 + $(RM) $(DESTDIR)$(BINDIR)/zstdgrep + $(RM) $(DESTDIR)$(BINDIR)/zstdless + $(RM) $(DESTDIR)$(BINDIR)/zstdcat + $(RM) $(DESTDIR)$(BINDIR)/unzstd + $(RM) $(DESTDIR)$(BINDIR)/zstdmt + $(RM) $(DESTDIR)$(BINDIR)/zstd + $(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1 + $(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1 + $(RM) $(DESTDIR)$(MAN1DIR)/zstdcat.1 + $(RM) $(DESTDIR)$(MAN1DIR)/unzstd.1 + $(RM) $(DESTDIR)$(MAN1DIR)/zstd.1 @echo zstd programs successfully uninstalled endif diff --git a/programs/README.md b/programs/README.md index 53706de7..cf7f5ba4 100644 --- a/programs/README.md +++ b/programs/README.md @@ -3,7 +3,7 @@ Command Line Interface for Zstandard library Command Line Interface (CLI) can be created using the `make` command without any additional parameters. There are however other Makefile targets that create different variations of CLI: -- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and support for decompression of legacy zstd formats +- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and supports decompression of legacy zstd formats - `zstd_nolegacy` : Same as `zstd` but without support for legacy zstd formats - `zstd-small` : CLI optimized for minimal size; no dictionary builder, no benchmark, and no support for legacy zstd formats - `zstd-compress` : version of CLI which can only compress into zstd format @@ -147,71 +147,91 @@ FILE : a filename Arguments : -# : # compression level (1-19, default: 3) -d : decompression - -D file: use `file` as Dictionary - -o file: result stored into `file` (only if 1 input file) - -f : overwrite output without prompting and (de)compress links + -D DICT: use DICT as Dictionary for compression or decompression + -o file: result stored into `file` (only 1 output file) + -f : overwrite output without prompting, also (de)compress links --rm : remove source file(s) after successful de/compression -k : preserve source file(s) (default) -h/-H : display help/long help and exit Advanced arguments : -V : display Version number and exit + -c : force write to standard output, even if it is the console -v : verbose mode; specify multiple times to increase verbosity -q : suppress warnings; specify twice to suppress errors too - -c : force write to standard output, even if it is the console - -l : print information about zstd compressed files ---exclude-compressed: only compress files that are not previously compressed +--no-progress : do not display the progress counter + -r : operate recursively on directories +--filelist FILE : read list of files to operate upon from FILE +--output-dir-flat DIR : processed files are stored into DIR +--output-dir-mirror DIR : processed files are stored into DIR respecting original directory structure +--[no-]check : during compression, add XXH64 integrity checksum to frame (default: enabled). If specified with -d, decompressor will ignore/validate checksums in compressed frame (default: validate). +-- : All arguments after "--" are treated as files + +Advanced compression arguments : --ultra : enable levels beyond 19, up to 22 (requires more memory) --long[=#]: enable long distance matching with given window log (default: 27) --fast[=#]: switch to very fast compression levels (default: 1) --adapt : dynamically adapt compression level to I/O conditions ---stream-size=# : optimize compression parameters for streaming input of given number of bytes ---size-hint=# optimize compression parameters for streaming input of approximately this size ---target-compressed-block-size=# : make compressed block near targeted size -T# : spawns # compression threads (default: 1, 0==# cores) -B# : select size of each job (default: 0==automatic) +--single-thread : use a single thread for both I/O and compression (result slightly different than -T1) --rsyncable : compress using a rsync-friendly method (-B sets block size) ---no-dictID : don't write dictID into header (dictionary compression) ---[no-]check : integrity check (default: enabled) +--exclude-compressed: only compress files that are not already compressed +--stream-size=# : specify size of streaming input from `stdin` +--size-hint=# optimize compression parameters for streaming input of approximately this size +--target-compressed-block-size=# : generate compressed block of approximately targeted size +--no-dictID : don't write dictID into header (dictionary compression only) --[no-]compress-literals : force (un)compressed literals - -r : operate recursively on directories ---output-dir-flat[=directory]: all resulting files stored into `directory`. --format=zstd : compress files to the .zst format (default) --format=gzip : compress files to the .gz format +--format=xz : compress files to the .xz format +--format=lzma : compress files to the .lzma format +--format=lz4 : compress files to the .lz4 format + +Advanced decompression arguments : + -l : print information about zstd compressed files --test : test compressed file integrity ---[no-]sparse : sparse mode (default: disabled) -M# : Set a memory usage limit for decompression ---no-progress : do not display the progress bar --- : All arguments after "--" are treated as files +--[no-]sparse : sparse mode (default: disabled) Dictionary builder : --train ## : create a dictionary from a training set of files --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]] : use the cover algorithm with optional args --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]] : use the fast cover algorithm with optional args --train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9) - -o file : `file` is dictionary name (default: dictionary) + -o DICT : DICT is dictionary name (default: dictionary) --maxdict=# : limit dictionary to specified size (default: 112640) --dictID=# : force dictionary ID to specified value (default: random) Benchmark arguments : -b# : benchmark file(s), using # compression level (default: 3) - -e# : test all compression levels from -bX to # (default: 1) + -e# : test all compression levels successively from -b# to -e# (default: 1) -i# : minimum evaluation time in seconds (default: 3s) -B# : cut file into independent blocks of size # (default: no block) + -S : output one benchmark result per input file (default: consolidated result) --priority=rt : set process priority to real-time ``` ### Passing parameters through Environment Variables +There is no "generic" way to pass "any kind of parameter" to `zstd` in a pass-through manner. +Using environment variables for this purpose has security implications. +Therefore, this avenue is intentionally restricted and only supports `ZSTD_CLEVEL` and `ZSTD_NBTHREADS`. + `ZSTD_CLEVEL` can be used to modify the default compression level of `zstd` (usually set to `3`) to another value between 1 and 19 (the "normal" range). -This can be useful when `zstd` CLI is invoked in a way that doesn't allow passing arguments. + +`ZSTD_NBTHREADS` can be used to specify a number of threads +that `zstd` will use for compression, which by default is `1`. +This functionality only exists when `zstd` is compiled with multithread support. +`0` means "use as many threads as detected cpu cores on local system". +The max # of threads is capped at: `ZSTDMT_NBWORKERS_MAX==200`. + +This functionality can be useful when `zstd` CLI is invoked in a way that doesn't allow passing arguments. One such scenario is `tar --zstd`. -As `ZSTD_CLEVEL` only replaces the default compression level, -it can then be overridden by corresponding command line arguments. +As `ZSTD_CLEVEL` and `ZSTD_NBTHREADS` only replace the default compression level +and number of threads respectively, they can both be overridden by corresponding command line arguments: +`-#` for compression level and `-T#` for number of threads. -There is no "generic" way to pass "any kind of parameter" to `zstd` in a pass-through manner. -Using environment variables for this purpose has security implications. -Therefore, this avenue is intentionally restricted and only supports `ZSTD_CLEVEL`. ### Long distance matching mode The long distance matching mode, enabled with `--long`, is designed to improve diff --git a/programs/dibio.c b/programs/dibio.c index 463095a8..cb3829e3 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -301,7 +301,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n"); EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */ } - if (fs.totalSizeToLoad < (unsigned long long)(8 * maxDictSize)) { + if (fs.totalSizeToLoad < (unsigned long long)maxDictSize * 8) { DISPLAYLEVEL(2, "! Warning : data size of samples too small for target dictionary size \n"); DISPLAYLEVEL(2, "! Samples should be about 100x larger than target dictionary size \n"); } diff --git a/programs/fileio.c b/programs/fileio.c index d72879d6..65f2d531 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -321,6 +321,25 @@ struct FIO_prefs_s { int contentSize; }; +/*-************************************* +* Parameters: FIO_ctx_t +***************************************/ + +/* typedef'd to FIO_ctx_t within fileio.h */ +struct FIO_ctx_s { + + /* file i/o info */ + int nbFilesTotal; + int hasStdinInput; + int hasStdoutOutput; + + /* file i/o state */ + int currFileIdx; + int nbFilesProcessed; + size_t totalBytesInput; + size_t totalBytesOutput; +}; + /*-************************************* * Parameters: Initialization @@ -363,11 +382,31 @@ FIO_prefs_t* FIO_createPreferences(void) return ret; } +FIO_ctx_t* FIO_createContext(void) +{ + FIO_ctx_t* const ret = (FIO_ctx_t*)malloc(sizeof(FIO_ctx_t)); + if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); + + ret->currFileIdx = 0; + ret->hasStdinInput = 0; + ret->hasStdoutOutput = 0; + ret->nbFilesTotal = 1; + ret->nbFilesProcessed = 0; + ret->totalBytesInput = 0; + ret->totalBytesOutput = 0; + return ret; +} + void FIO_freePreferences(FIO_prefs_t* const prefs) { free(prefs); } +void FIO_freeContext(FIO_ctx_t* const fCtx) +{ + free(fCtx); +} + /*-************************************* * Parameters: Display Options @@ -382,6 +421,8 @@ void FIO_setNoProgress(unsigned noProgress) { g_display_prefs.noProgress = noPro * Parameters: Setters ***************************************/ +/* FIO_prefs_t functions */ + void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = compressionType; } void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; } @@ -495,21 +536,49 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value) prefs->contentSize = value != 0; } +/* FIO_ctx_t functions */ + +void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) { + fCtx->hasStdoutOutput = value; +} + +void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value) +{ + fCtx->nbFilesTotal = value; +} + +void FIO_determineHasStdinInput(FIO_ctx_t* const fCtx, const FileNamesTable* const filenames) { + size_t i = 0; + for ( ; i < filenames->tableSize; ++i) { + if (!strcmp(stdinmark, filenames->fileNames[i])) { + fCtx->hasStdinInput = 1; + return; + } + } +} + /*-************************************* * Functions ***************************************/ -/** FIO_remove() : +/** FIO_removeFile() : * @result : Unlink `fileName`, even if it's read-only */ -static int FIO_remove(const char* path) +static int FIO_removeFile(const char* path) { - if (!UTIL_isRegularFile(path)) { - DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s \n", path); + stat_t statbuf; + if (!UTIL_stat(path, &statbuf)) { + DISPLAYLEVEL(2, "zstd: Failed to stat %s while trying to remove it\n", path); + return 0; + } + if (!UTIL_isRegularFileStat(&statbuf)) { + DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path); return 0; } #if defined(_WIN32) || defined(WIN32) /* windows doesn't allow remove read-only files, * so try to make it writable first */ - UTIL_chmod(path, _S_IWRITE); + if (!(statbuf.st_mode & _S_IWRITE)) { + UTIL_chmod(path, &statbuf, _S_IWRITE); + } #endif return remove(path); } @@ -519,6 +588,7 @@ static int FIO_remove(const char* path) * @result : FILE* to `srcFileName`, or NULL if it fails */ static FILE* FIO_openSrcFile(const char* srcFileName) { + stat_t statbuf; assert(srcFileName != NULL); if (!strcmp (srcFileName, stdinmark)) { DISPLAYLEVEL(4,"Using stdin for input \n"); @@ -526,14 +596,14 @@ static FILE* FIO_openSrcFile(const char* srcFileName) return stdin; } - if (!UTIL_fileExist(srcFileName)) { + if (!UTIL_stat(srcFileName, &statbuf)) { DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n", srcFileName, strerror(errno)); return NULL; } - if (!UTIL_isRegularFile(srcFileName) - && !UTIL_isFIFO(srcFileName) + if (!UTIL_isRegularFileStat(&statbuf) + && !UTIL_isFIFOStat(&statbuf) ) { DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); @@ -551,8 +621,8 @@ static FILE* FIO_openSrcFile(const char* srcFileName) * condition : `dstFileName` must be non-NULL. * @result : FILE* to `dstFileName`, or NULL if it fails */ static FILE* -FIO_openDstFile(FIO_prefs_t* const prefs, - const char* srcFileName, const char* dstFileName) +FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, + const char* srcFileName, const char* dstFileName) { if (prefs->testMode) return NULL; /* do not open file in test mode */ @@ -597,18 +667,12 @@ FIO_openDstFile(FIO_prefs_t* const prefs, dstFileName); return NULL; } - DISPLAY("zstd: %s already exists; overwrite (y/N) ? ", - dstFileName); - { int ch = getchar(); - if ((ch!='Y') && (ch!='y')) { - DISPLAY(" not overwritten \n"); - return NULL; - } - /* flush rest of input line */ - while ((ch!=EOF) && (ch!='\n')) ch = getchar(); - } } + DISPLAY("zstd: %s already exists; ", dstFileName); + if (UTIL_requireUserConfirmation("overwrite (y/n) ? ", "Not overwritten \n", "yY", fCtx->hasStdinInput)) + return NULL; + } /* need to unlink */ - FIO_remove(dstFileName); + FIO_removeFile(dstFileName); } } { FILE* const f = fopen( dstFileName, "wb" ); @@ -618,13 +682,12 @@ FIO_openDstFile(FIO_prefs_t* const prefs, && strcmp (srcFileName, stdinmark) && strcmp(dstFileName, nulmark) ) { /* reduce rights on newly created dst file while compression is ongoing */ - UTIL_chmod(dstFileName, 00600); + UTIL_chmod(dstFileName, NULL, 00600); } return f; } } - /*! FIO_createDictBuffer() : * creates a buffer, pointed by `*bufferPtr`, * loads `filename` content into it, up to DICTSIZE_MAX bytes. @@ -669,15 +732,9 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_p * Checks for and warns if there are any files that would have the same output path */ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { - const char **filenameTableSorted, *c, *prevElem, *filename; + const char **filenameTableSorted, *prevElem, *filename; unsigned u; - #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ - c = "\\"; - #else - c = "/"; - #endif - filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles); if (!filenameTableSorted) { DISPLAY("Unable to malloc new str array, not checking for name collisions\n"); @@ -685,7 +742,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { } for (u = 0; u < nbFiles; ++u) { - filename = strrchr(filenameTable[u], c[0]); + filename = strrchr(filenameTable[u], PATH_SEP); if (filename == NULL) { filenameTableSorted[u] = filenameTable[u]; } else { @@ -771,12 +828,57 @@ static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs, unsigned long long const maxSrcFileSize) { unsigned long long maxSize = MAX(prefs->memLimit, MAX(dictSize, maxSrcFileSize)); + unsigned const maxWindowSize = (1U << ZSTD_WINDOWLOG_MAX); + if (maxSize == UTIL_FILESIZE_UNKNOWN) + EXM_THROW(42, "Using --patch-from with stdin requires --stream-size"); assert(maxSize != UTIL_FILESIZE_UNKNOWN); - if (maxSize > UINT_MAX) - EXM_THROW(42, "Can't handle files larger than %u GB\n", UINT_MAX/(1 GB) + 1); + if (maxSize > maxWindowSize) + EXM_THROW(42, "Can't handle files larger than %u GB\n", maxWindowSize/(1 GB)); FIO_setMemLimit(prefs, (unsigned)maxSize); } +/* FIO_removeMultiFilesWarning() : + * Returns 1 if the console should abort, 0 if console should proceed. + * This function handles logic when processing multiple files with -o, displaying the appropriate warnings/prompts. + * + * If -f is specified, or there is just 1 file, zstd will always proceed as usual. + * If --rm is specified, there will be a prompt asking for user confirmation. + * If -f is specified with --rm, zstd will proceed as usual + * If -q is specified with --rm, zstd will abort pre-emptively + * If neither flag is specified, zstd will prompt the user for confirmation to proceed. + * If --rm is not specified, then zstd will print a warning to the user (which can be silenced with -q). + * However, if the output is stdout, we will always abort rather than displaying the warning prompt. + */ +static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* const prefs, const char* outFileName, int displayLevelCutoff) +{ + int error = 0; + if (fCtx->nbFilesTotal > 1 && !prefs->overwrite) { + if (g_display_prefs.displayLevel <= displayLevelCutoff) { + if (prefs->removeSrcFile) { + DISPLAYLEVEL(1, "zstd: Aborting... not deleting files and processing into dst: %s", outFileName); + error = 1; + } + } else { + if (!strcmp(outFileName, stdoutmark)) { + DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into stdout. "); + } else { + DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into a single output file: %s ", outFileName); + } + DISPLAYLEVEL(2, "\nThe concatenated output CANNOT regenerate the original directory tree. ") + if (prefs->removeSrcFile) { + if (fCtx->hasStdoutOutput) { + DISPLAYLEVEL(1, "\nAborting. Use -f if you really want to delete the files and output to stdout"); + error = 1; + } else { + error = g_display_prefs.displayLevel > displayLevelCutoff && UTIL_requireUserConfirmation("This is a destructive operation. Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput); + } + } + } + DISPLAY("\n"); + } + return error; +} + #ifndef ZSTD_NOCOMPRESS /* ********************************************************************** @@ -807,9 +909,9 @@ static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs, if (fileWindowLog > ZSTD_WINDOWLOG_MAX) DISPLAYLEVEL(1, "Max window log exceeded by file (compression ratio will suffer)\n"); comprParams->windowLog = MIN(ZSTD_WINDOWLOG_MAX, fileWindowLog); - if (fileWindowLog > ZSTD_cycleLog(cParams.hashLog, cParams.strategy)) { + if (fileWindowLog > ZSTD_cycleLog(cParams.chainLog, cParams.strategy)) { if (!prefs->ldmFlag) - DISPLAYLEVEL(1, "long mode automaticaly triggered\n"); + DISPLAYLEVEL(1, "long mode automatically triggered\n"); FIO_setLdmFlag(prefs, 1); } if (cParams.strategy >= ZSTD_btopt) { @@ -838,8 +940,10 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, /* need to update memLimit before calling createDictBuffer * because of memLimit check inside it */ - if (prefs->patchFromMode) - FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), maxSrcFileSize, cLevel); + if (prefs->patchFromMode) { + unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize; + FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), ssSize > 0 ? ssSize : maxSrcFileSize, cLevel); + } ress.dstBuffer = malloc(ress.dstBufferSize); ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs); /* works with dictFileName==NULL */ if (!ress.srcBuffer || !ress.dstBuffer) @@ -881,6 +985,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableDedicatedDictSearch, 1) ); /* multi-threading */ #ifdef ZSTD_MULTITHREAD DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers); @@ -902,12 +1007,12 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, return ress; } -static void FIO_freeCResources(cRess_t ress) +static void FIO_freeCResources(const cRess_t* const ress) { - free(ress.srcBuffer); - free(ress.dstBuffer); - free(ress.dictBuffer); - ZSTD_freeCStream(ress.cctx); /* never fails */ + free(ress->srcBuffer); + free(ress->dstBuffer); + free(ress->dictBuffer); + ZSTD_freeCStream(ress->cctx); /* never fails */ } @@ -1172,7 +1277,8 @@ FIO_compressLz4Frame(cRess_t* ress, static unsigned long long -FIO_compressZstdFrame(FIO_prefs_t* const prefs, +FIO_compressZstdFrame(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, const cRess_t* ressPtr, const char* srcFileName, U64 fileSize, int compressionLevel, U64* readsize) @@ -1254,11 +1360,24 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs, (unsigned)(zfp.consumed >> 20), (unsigned)(zfp.produced >> 20), cShare ); - } else { /* summarized notifications if == 2; */ - DISPLAYLEVEL(2, "\rRead : %u ", (unsigned)(zfp.consumed >> 20)); + } else { /* summarized notifications if == 2 */ + DISPLAYLEVEL(2, "\r%79s\r", ""); /* Clear out the current displayed line */ + if (fCtx->nbFilesTotal > 1) { + size_t srcFileNameSize = strlen(srcFileName); + /* Ensure that the string we print is roughly the same size each time */ + if (srcFileNameSize > 18) { + const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; + DISPLAYLEVEL(2, "Compress: %u/%u files. Current: ...%s ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName); + } else { + DISPLAYLEVEL(2, "Compress: %u/%u files. Current: %*s ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName); + } + } + DISPLAYLEVEL(2, "Read : %2u ", (unsigned)(zfp.consumed >> 20)); if (fileSize != UTIL_FILESIZE_UNKNOWN) - DISPLAYLEVEL(2, "/ %u ", (unsigned)(fileSize >> 20)); - DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare); + DISPLAYLEVEL(2, "/ %2u ", (unsigned)(fileSize >> 20)); + DISPLAYLEVEL(2, "MB ==> %2.f%%", cShare); DELAY_NEXT_UPDATE(); } @@ -1369,7 +1488,8 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs, * 1 : missing or pb opening srcFileName */ static int -FIO_compressFilename_internal(FIO_prefs_t* const prefs, +FIO_compressFilename_internal(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, cRess_t ress, const char* dstFileName, const char* srcFileName, int compressionLevel) @@ -1385,7 +1505,7 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, switch (prefs->compressionType) { default: case FIO_zstdCompression: - compressedfilesize = FIO_compressZstdFrame(prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize); + compressedfilesize = FIO_compressZstdFrame(fCtx, prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize); break; case FIO_gzipCompression: @@ -1421,18 +1541,24 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, } /* Status */ + fCtx->totalBytesInput += (size_t)readsize; + fCtx->totalBytesOutput += (size_t)compressedfilesize; DISPLAYLEVEL(2, "\r%79s\r", ""); - if (readsize == 0) { - DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", - srcFileName, - (unsigned long long)readsize, (unsigned long long) compressedfilesize, - dstFileName); - } else { - DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", - srcFileName, - (double)compressedfilesize / readsize * 100, - (unsigned long long)readsize, (unsigned long long) compressedfilesize, - dstFileName); + if (g_display_prefs.displayLevel >= 2 && + !fCtx->hasStdoutOutput && + (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) { + if (readsize == 0) { + DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", + srcFileName, + (unsigned long long)readsize, (unsigned long long) compressedfilesize, + dstFileName); + } else { + DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", + srcFileName, + (double)compressedfilesize / readsize * 100, + (unsigned long long)readsize, (unsigned long long) compressedfilesize, + dstFileName); + } } /* Elapsed Time and CPU Load */ @@ -1457,7 +1583,8 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, * @return : 0 : compression completed correctly, * 1 : pb */ -static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, +static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, cRess_t ress, const char* dstFileName, const char* srcFileName, @@ -1471,7 +1598,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, if (ress.dstFile == NULL) { closeDstFile = 1; DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName); - ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName); + ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName); if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, @@ -1480,11 +1607,12 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, addHandler(dstFileName); if ( strcmp (srcFileName, stdinmark) - && UTIL_getFileStat(srcFileName, &statbuf)) + && UTIL_stat(srcFileName, &statbuf) + && UTIL_isRegularFileStat(&statbuf) ) transfer_permissions = 1; } - result = FIO_compressFilename_internal(prefs, ress, dstFileName, srcFileName, compressionLevel); + result = FIO_compressFilename_internal(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); if (closeDstFile) { FILE* const dstFile = ress.dstFile; @@ -1498,14 +1626,11 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, result=1; } if ( (result != 0) /* operation failure */ - && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null */ && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ ) { - FIO_remove(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */ - } else if ( strcmp(dstFileName, stdoutmark) - && strcmp(dstFileName, nulmark) - && transfer_permissions) { - DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: transfering permissions into dst: %s \n", dstFileName); + FIO_removeFile(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */ + } else if (transfer_permissions) { + DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: transferring permissions into dst: %s \n", dstFileName); UTIL_setFileStat(dstFileName, &statbuf); } else { DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: do not transfer permissions into dst: %s \n", dstFileName); @@ -1536,7 +1661,8 @@ static const char *compressedFileExtensions[] = { * 1 : missing or pb opening srcFileName */ static int -FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, +FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, cRess_t ress, const char* dstFileName, const char* srcFileName, @@ -1569,7 +1695,7 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, ress.srcFile = FIO_openSrcFile(srcFileName); if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ - result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel); + result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); fclose(ress.srcFile); ress.srcFile = NULL; @@ -1581,21 +1707,22 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, * delete both the source and destination files. */ clearHandler(); - if (FIO_remove(srcFileName)) + if (FIO_removeFile(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } return result; } -int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName, +int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* dstFileName, const char* srcFileName, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams); - int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); + int const result = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); +#define DISPLAY_LEVEL_DEFAULT 2 - FIO_freeCResources(ress); + FIO_freeCResources(&ress); return result; } @@ -1656,45 +1783,79 @@ static unsigned long long FIO_getLargestFileSize(const char** inFileNames, unsig * or into one file each (outFileName == NULL, but suffix != NULL), * or into a destination folder (specified with -O) */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - const char** inFileNamesTable, unsigned nbFiles, +int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, + const char** inFileNamesTable, + const char* outMirroredRootDirName, const char* outDirName, const char* outFileName, const char* suffix, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { + int status; int error = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, - FIO_getLargestFileSize(inFileNamesTable, nbFiles), + FIO_getLargestFileSize(inFileNamesTable, fCtx->nbFilesTotal), compressionLevel, comprParams); /* init */ assert(outFileName != NULL || suffix != NULL); if (outFileName != NULL) { /* output into a single destination (stdout typically) */ - ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); + if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) { + FIO_freeCResources(&ress); + return 1; + } + ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName); if (ress.dstFile == NULL) { /* could not open outFileName */ error = 1; } else { - unsigned u; - for (u=0; u<nbFiles; u++) - error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[u], compressionLevel); + for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { + status = FIO_compressFilename_srcFile(fCtx, prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); + if (!status) fCtx->nbFilesProcessed++; + error |= status; + } if (fclose(ress.dstFile)) EXM_THROW(29, "Write error (%s) : cannot properly close %s", strerror(errno), outFileName); ress.dstFile = NULL; } } else { - unsigned u; - for (u=0; u<nbFiles; u++) { - const char* const srcFileName = inFileNamesTable[u]; - const char* const dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ - error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); + if (outMirroredRootDirName) + UTIL_mirrorSourceFilesDirectories(inFileNamesTable, fCtx->nbFilesTotal, outMirroredRootDirName); + + for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { + const char* const srcFileName = inFileNamesTable[fCtx->currFileIdx]; + const char* dstFileName = NULL; + if (outMirroredRootDirName) { + char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); + if (validMirroredDirName) { + dstFileName = FIO_determineCompressedName(srcFileName, validMirroredDirName, suffix); + free(validMirroredDirName); + } else { + DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot compress '%s' into '%s' \n", srcFileName, outMirroredRootDirName); + error=1; + continue; + } + } else { + dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ + } + status = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); + if (!status) fCtx->nbFilesProcessed++; + error |= status; } + if (outDirName) - FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles); + FIO_checkFilenameCollisions(inFileNamesTable , fCtx->nbFilesTotal); } - FIO_freeCResources(ress); + if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) { + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "%d files compressed : %.2f%% (%6zu => %6zu bytes)\n", fCtx->nbFilesProcessed, + (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, + fCtx->totalBytesInput, fCtx->totalBytesOutput); + } + + FIO_freeCResources(&ress); return error; } @@ -1730,6 +1891,8 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi if (ress.dctx==NULL) EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno)); CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) ); + CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag)); + ress.srcBufferSize = ZSTD_DStreamInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); ress.dstBufferSize = ZSTD_DStreamOutSize(); @@ -1923,7 +2086,7 @@ FIO_zstdErrorHelp(const FIO_prefs_t* const prefs, */ #define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) static unsigned long long -FIO_decompressZstdFrame(dRess_t* ress, FILE* finput, +FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, const FIO_prefs_t* const prefs, const char* srcFileName, U64 alreadyDecoded) /* for multi-frames streams */ @@ -1961,8 +2124,22 @@ FIO_decompressZstdFrame(dRess_t* ress, FILE* finput, /* Write block */ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips); frameSize += outBuff.pos; - DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", - srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + if (!fCtx->hasStdoutOutput) { + if (fCtx->nbFilesTotal > 1) { + size_t srcFileNameSize = strlen(srcFileName); + if (srcFileNameSize > 18) { + const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; + DISPLAYUPDATE(2, "\rDecompress: %2u/%2u files. Current: ...%s : %u MB... ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + } else { + DISPLAYUPDATE(2, "\rDecompress: %2u/%2u files. Current: %s : %u MB... ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + } + } else { + DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", + srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + } + } if (inBuff.pos > 0) { memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos); @@ -2220,7 +2397,8 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, * @return : 0 : OK * 1 : error */ -static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, +static int FIO_decompressFrames(FIO_ctx_t* const fCtx, + dRess_t ress, FILE* srcFile, const FIO_prefs_t* const prefs, const char* dstFileName, const char* srcFileName) { @@ -2249,7 +2427,7 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, return 1; } if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) { - unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, prefs, srcFileName, filesize); + unsigned long long const frameSize = FIO_decompressZstdFrame(fCtx, &ress, srcFile, prefs, srcFileName, filesize); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ @@ -2291,8 +2469,14 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, } } /* for each frame */ /* Final Status */ + fCtx->totalBytesOutput += (size_t)filesize; DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); + /* No status message in pipe mode (stdin - stdout) or multi-files mode */ + if (g_display_prefs.displayLevel >= 2) { + if (fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3) { + DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); + } + } return 0; } @@ -2304,7 +2488,8 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, @return : 0 : OK 1 : operation aborted */ -static int FIO_decompressDstFile(FIO_prefs_t* const prefs, +static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, dRess_t ress, FILE* srcFile, const char* dstFileName, const char* srcFileName) { @@ -2316,7 +2501,7 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs, if ((ress.dstFile == NULL) && (prefs->testMode==0)) { releaseDstFile = 1; - ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName); + ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName); if (ress.dstFile==NULL) return 1; /* Must only be added after FIO_openDstFile() succeeds. @@ -2326,11 +2511,12 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs, addHandler(dstFileName); if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ - && UTIL_getFileStat(srcFileName, &statbuf) ) + && UTIL_stat(srcFileName, &statbuf) + && UTIL_isRegularFileStat(&statbuf) ) transfer_permissions = 1; } - result = FIO_decompressFrames(ress, srcFile, prefs, dstFileName, srcFileName); + result = FIO_decompressFrames(fCtx, ress, srcFile, prefs, dstFileName, srcFileName); if (releaseDstFile) { FILE* const dstFile = ress.dstFile; @@ -2342,15 +2528,11 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs, } if ( (result != 0) /* operation failure */ - && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null (#316) */ && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ ) { - FIO_remove(dstFileName); /* remove decompression artefact; note: don't do anything special if remove() fails */ - } else { /* operation success */ - if ( strcmp(dstFileName, stdoutmark) /* special case : don't chmod stdout */ - && strcmp(dstFileName, nulmark) /* special case : don't chmod /dev/null */ - && transfer_permissions ) /* file permissions correctly extracted from src */ - UTIL_setFileStat(dstFileName, &statbuf); /* transfer file permissions from src into dst */ + FIO_removeFile(dstFileName); /* remove decompression artefact; note: don't do anything special if remove() fails */ + } else if ( transfer_permissions /* file permissions correctly extracted from src */ ) { + UTIL_setFileStat(dstFileName, &statbuf); /* transfer file permissions from src into dst */ } } @@ -2363,7 +2545,7 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs, @return : 0 : OK 1 : error */ -static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName) +static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName) { FILE* srcFile; int result; @@ -2377,7 +2559,7 @@ static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const c if (srcFile==NULL) return 1; ress.srcBufferLoaded = 0; - result = FIO_decompressDstFile(prefs, ress, srcFile, dstFileName, srcFileName); + result = FIO_decompressDstFile(fCtx, prefs, ress, srcFile, dstFileName, srcFileName); /* Close file */ if (fclose(srcFile)) { @@ -2391,7 +2573,7 @@ static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const c * delete both the source and destination files. */ clearHandler(); - if (FIO_remove(srcFileName)) { + if (FIO_removeFile(srcFileName)) { /* failed to remove src file */ DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); return 1; @@ -2401,13 +2583,13 @@ static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const c -int FIO_decompressFilename(FIO_prefs_t* const prefs, +int FIO_decompressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* dstFileName, const char* srcFileName, const char* dictFileName) { dRess_t const ress = FIO_createDResources(prefs, dictFileName); - int const decodingError = FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); + int const decodingError = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName); FIO_freeDResources(ress); return decodingError; @@ -2416,6 +2598,9 @@ int FIO_decompressFilename(FIO_prefs_t* const prefs, static const char *suffixList[] = { ZSTD_EXTENSION, TZSTD_EXTENSION, +#ifndef ZSTD_NODECOMPRESS + ZSTD_ALT_EXTENSION, +#endif #ifdef ZSTD_GZDECOMPRESS GZ_EXTENSION, TGZ_EXTENSION, @@ -2531,39 +2716,64 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) /* note : dstFileNameBuffer memory is not going to be free */ } - int -FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, - const char** srcNamesTable, unsigned nbFiles, +FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, + const char** srcNamesTable, + const char* outMirroredRootDirName, const char* outDirName, const char* outFileName, const char* dictFileName) { + int status; int error = 0; dRess_t ress = FIO_createDResources(prefs, dictFileName); if (outFileName) { - unsigned u; + if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) { + FIO_freeDResources(ress); + return 1; + } if (!prefs->testMode) { - ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); + ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName); if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); } - for (u=0; u<nbFiles; u++) - error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[u]); + for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { + status = FIO_decompressSrcFile(fCtx, prefs, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); + if (!status) fCtx->nbFilesProcessed++; + error |= status; + } if ((!prefs->testMode) && (fclose(ress.dstFile))) EXM_THROW(72, "Write error : %s : cannot properly close output file", strerror(errno)); } else { - unsigned u; - for (u=0; u<nbFiles; u++) { /* create dstFileName */ - const char* const srcFileName = srcNamesTable[u]; - const char* const dstFileName = FIO_determineDstName(srcFileName, outDirName); + if (outMirroredRootDirName) + UTIL_mirrorSourceFilesDirectories(srcNamesTable, fCtx->nbFilesTotal, outMirroredRootDirName); + + for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { /* create dstFileName */ + const char* const srcFileName = srcNamesTable[fCtx->currFileIdx]; + const char* dstFileName = NULL; + if (outMirroredRootDirName) { + char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); + if (validMirroredDirName) { + dstFileName = FIO_determineDstName(srcFileName, validMirroredDirName); + free(validMirroredDirName); + } else { + DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot decompress '%s' into '%s'\n", srcFileName, outMirroredRootDirName); + } + } else { + dstFileName = FIO_determineDstName(srcFileName, outDirName); + } if (dstFileName == NULL) { error=1; continue; } - - error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); + status = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName); + if (!status) fCtx->nbFilesProcessed++; + error |= status; } if (outDirName) - FIO_checkFilenameCollisions(srcNamesTable ,nbFiles); + FIO_checkFilenameCollisions(srcNamesTable , fCtx->nbFilesTotal); } + + if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesOutput != 0) + DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput); FIO_freeDResources(ress); return error; diff --git a/programs/fileio.h b/programs/fileio.h index 2fbf01f8..05e6d068 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -44,6 +44,7 @@ extern "C" { #define ZSTD_EXTENSION ".zst" #define TZSTD_EXTENSION ".tzst" +#define ZSTD_ALT_EXTENSION ".zstd" /* allow decompression of .zstd files */ #define LZ4_EXTENSION ".lz4" #define TLZ4_EXTENSION ".tlz4" @@ -59,11 +60,18 @@ typedef struct FIO_prefs_s FIO_prefs_t; FIO_prefs_t* FIO_createPreferences(void); void FIO_freePreferences(FIO_prefs_t* const prefs); +/* Mutable struct containing relevant context and state regarding (de)compression with respect to file I/O */ +typedef struct FIO_ctx_s FIO_ctx_t; + +FIO_ctx_t* FIO_createContext(void); +void FIO_freeContext(FIO_ctx_t* const fCtx); + typedef struct FIO_display_prefs_s FIO_display_prefs_t; /*-************************************* * Parameters ***************************************/ +/* FIO_prefs_t functions */ void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType); void FIO_overwriteMode(FIO_prefs_t* const prefs); void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt); @@ -97,19 +105,24 @@ void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompresse void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value); void FIO_setContentSize(FIO_prefs_t* const prefs, int value); +/* FIO_ctx_t functions */ +void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value); +void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value); +void FIO_determineHasStdinInput(FIO_ctx_t* const fCtx, const FileNamesTable* const filenames); + /*-************************************* * Single File functions ***************************************/ /** FIO_compressFilename() : * @return : 0 == ok; 1 == pb with src file. */ -int FIO_compressFilename (FIO_prefs_t* const prefs, +int FIO_compressFilename (FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams); /** FIO_decompressFilename() : * @return : 0 == ok; 1 == pb with src file. */ -int FIO_decompressFilename (FIO_prefs_t* const prefs, +int FIO_decompressFilename (FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* outfilename, const char* infilename, const char* dictFileName); int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel); @@ -120,8 +133,10 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis ***************************************/ /** FIO_compressMultipleFilenames() : * @return : nb of missing files */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - const char** inFileNamesTable, unsigned nbFiles, +int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, + const char** inFileNamesTable, + const char* outMirroredDirName, const char* outDirName, const char* outFileName, const char* suffix, const char* dictFileName, int compressionLevel, @@ -129,8 +144,10 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, /** FIO_decompressMultipleFilenames() : * @return : nb of missing or skipped files */ -int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, - const char** srcNamesTable, unsigned nbFiles, +int FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, + const char** srcNamesTable, + const char* outMirroredDirName, const char* outDirName, const char* outFileName, const char* dictFileName); diff --git a/programs/platform.h b/programs/platform.h index 2b4b9f2d..68be70bb 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -102,6 +102,12 @@ extern "C" { # define PLATFORM_POSIX_VERSION 1 # endif +# ifdef __UCLIBC__ +# ifndef __USE_MISC +# define __USE_MISC /* enable st_mtim on uclibc */ +# endif +# endif + # else /* non-unix target platform (like Windows) */ # define PLATFORM_POSIX_VERSION 0 # endif diff --git a/programs/timefn.h b/programs/timefn.h index eb3c1309..5d2818e8 100644 --- a/programs/timefn.h +++ b/programs/timefn.h @@ -28,7 +28,11 @@ extern "C" { ******************************************/ #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include <stdint.h> +# if defined(_AIX) +# include <inttypes.h> +# else +# include <stdint.h> /* intptr_t */ +# endif typedef uint64_t PTime; /* Precise Time */ #else typedef unsigned long long PTime; /* does not support compilers without long long support */ diff --git a/programs/util.c b/programs/util.c index ab1abd3b..5386d005 100644 --- a/programs/util.c +++ b/programs/util.c @@ -28,7 +28,7 @@ extern "C" { # include <io.h> /* _chmod */ #else # include <unistd.h> /* chown, stat */ -# if PLATFORM_POSIX_VERSION < 200809L +# if PLATFORM_POSIX_VERSION < 200809L || !defined(st_mtime) # include <utime.h> /* utime */ # else # include <fcntl.h> /* AT_FDCWD */ @@ -45,7 +45,6 @@ extern "C" { # include <string.h> /* strerror, memcpy */ #endif /* #ifdef _WIN32 */ - /*-**************************************** * Internal Macros ******************************************/ @@ -88,6 +87,28 @@ UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size) ******************************************/ int g_utilDisplayLevel; +int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, + const char* acceptableLetters, int hasStdinInput) { + int ch, result; + + if (hasStdinInput) { + UTIL_DISPLAY("stdin is an input - not proceeding.\n"); + return 1; + } + + UTIL_DISPLAY("%s", prompt); + ch = getchar(); + result = 0; + if (strchr(acceptableLetters, ch) == NULL) { + UTIL_DISPLAY("%s", abortMsg); + result = 1; + } + /* flush the rest */ + while ((ch!=EOF) && (ch!='\n')) + ch = getchar(); + return result; +} + /*-************************************* * Constants @@ -100,48 +121,50 @@ int g_utilDisplayLevel; * Functions ***************************************/ -int UTIL_fileExist(const char* filename) +int UTIL_stat(const char* filename, stat_t* statbuf) { - stat_t statbuf; #if defined(_MSC_VER) - int const stat_error = _stat64(filename, &statbuf); + return !_stat64(filename, statbuf); +#elif defined(__MINGW32__) && defined (__MSVCRT__) + return !_stati64(filename, statbuf); #else - int const stat_error = stat(filename, &statbuf); + return !stat(filename, statbuf); #endif - return !stat_error; } int UTIL_isRegularFile(const char* infilename) { stat_t statbuf; - return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */ + return UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf); } -int UTIL_getFileStat(const char* infilename, stat_t *statbuf) +int UTIL_isRegularFileStat(const stat_t* statbuf) { - int r; #if defined(_MSC_VER) - r = _stat64(infilename, statbuf); - if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ + return (statbuf->st_mode & S_IFREG) != 0; #else - r = stat(infilename, statbuf); - if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */ + return S_ISREG(statbuf->st_mode) != 0; #endif - return 1; } /* like chmod, but avoid changing permission of /dev/null */ -int UTIL_chmod(char const* filename, mode_t permissions) +int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions) { - if (!strcmp(filename, "/dev/null")) return 0; /* pretend success, but don't change anything */ + stat_t localStatBuf; + if (statbuf == NULL) { + if (!UTIL_stat(filename, &localStatBuf)) return 0; + statbuf = &localStatBuf; + } + if (!UTIL_isRegularFileStat(statbuf)) return 0; /* pretend success, but don't change anything */ return chmod(filename, permissions); } -int UTIL_setFileStat(const char *filename, stat_t *statbuf) +int UTIL_setFileStat(const char *filename, const stat_t *statbuf) { int res = 0; - if (!UTIL_isRegularFile(filename)) + stat_t curStatBuf; + if (!UTIL_stat(filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) return -1; /* set access and modification times */ @@ -169,7 +192,7 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf) res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */ #endif - res += UTIL_chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */ + res += UTIL_chmod(filename, &curStatBuf, statbuf->st_mode & 07777); /* Copy file permissions */ errno = 0; return -res; /* number of errors is returned */ @@ -178,14 +201,16 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf) int UTIL_isDirectory(const char* infilename) { stat_t statbuf; + return UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf); +} + +int UTIL_isDirectoryStat(const stat_t* statbuf) +{ #if defined(_MSC_VER) - int const r = _stat64(infilename, &statbuf); - if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; + return (statbuf->st_mode & _S_IFDIR) != 0; #else - int const r = stat(infilename, &statbuf); - if (!r && S_ISDIR(statbuf.st_mode)) return 1; + return S_ISDIR(statbuf->st_mode) != 0; #endif - return 0; } int UTIL_compareStr(const void *p1, const void *p2) { @@ -204,8 +229,8 @@ int UTIL_isSameFile(const char* fName1, const char* fName2) #else { stat_t file1Stat; stat_t file2Stat; - return UTIL_getFileStat(fName1, &file1Stat) - && UTIL_getFileStat(fName2, &file2Stat) + return UTIL_stat(fName1, &file1Stat) + && UTIL_stat(fName2, &file2Stat) && (file1Stat.st_dev == file2Stat.st_dev) && (file1Stat.st_ino == file2Stat.st_ino); } @@ -218,13 +243,23 @@ int UTIL_isFIFO(const char* infilename) /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ #if PLATFORM_POSIX_VERSION >= 200112L stat_t statbuf; - int const r = UTIL_getFileStat(infilename, &statbuf); - if (!r && S_ISFIFO(statbuf.st_mode)) return 1; + if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) return 1; #endif (void)infilename; return 0; } +/* UTIL_isFIFO : distinguish named pipes */ +int UTIL_isFIFOStat(const stat_t* statbuf) +{ +/* macro guards, as defined in : https://linux.die.net/man/2/lstat */ +#if PLATFORM_POSIX_VERSION >= 200112L + if (S_ISFIFO(statbuf->st_mode)) return 1; +#endif + (void)statbuf; + return 0; +} + int UTIL_isLink(const char* infilename) { /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ @@ -239,23 +274,22 @@ int UTIL_isLink(const char* infilename) U64 UTIL_getFileSize(const char* infilename) { - if (!UTIL_isRegularFile(infilename)) return UTIL_FILESIZE_UNKNOWN; - { int r; + stat_t statbuf; + if (!UTIL_stat(infilename, &statbuf)) return UTIL_FILESIZE_UNKNOWN; + return UTIL_getFileSizeStat(&statbuf); +} + +U64 UTIL_getFileSizeStat(const stat_t* statbuf) +{ + if (!UTIL_isRegularFileStat(statbuf)) return UTIL_FILESIZE_UNKNOWN; #if defined(_MSC_VER) - struct __stat64 statbuf; - r = _stat64(infilename, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; + if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; #elif defined(__MINGW32__) && defined (__MSVCRT__) - struct _stati64 statbuf; - r = _stati64(infilename, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; + if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; #else - struct stat statbuf; - r = stat(infilename, &statbuf); - if (r || !S_ISREG(statbuf.st_mode)) return UTIL_FILESIZE_UNKNOWN; + if (!S_ISREG(statbuf->st_mode)) return UTIL_FILESIZE_UNKNOWN; #endif - return (U64)statbuf.st_size; - } + return (U64)statbuf->st_size; } @@ -332,11 +366,12 @@ UTIL_createFileNamesTable_fromFileName(const char* inputFileName) char* buf; size_t bufSize; size_t pos = 0; + stat_t statbuf; - if (!UTIL_fileExist(inputFileName) || !UTIL_isRegularFile(inputFileName)) + if (!UTIL_stat(inputFileName, &statbuf) || !UTIL_isRegularFileStat(&statbuf)) return NULL; - { U64 const inputFileSize = UTIL_getFileSize(inputFileName); + { U64 const inputFileSize = UTIL_getFileSizeStat(&statbuf); if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE) return NULL; bufSize = (size_t)(inputFileSize + 1); /* (+1) to add '\0' at the end of last filename */ @@ -633,6 +668,290 @@ const char* UTIL_getFileExtension(const char* infilename) return extension; } +static int pathnameHas2Dots(const char *pathname) +{ + return NULL != strstr(pathname, ".."); +} + +static int isFileNameValidForMirroredOutput(const char *filename) +{ + return !pathnameHas2Dots(filename); +} + + +#define DIR_DEFAULT_MODE 0755 +static mode_t getDirMode(const char *dirName) +{ + stat_t st; + if (!UTIL_stat(dirName, &st)) { + UTIL_DISPLAY("zstd: failed to get DIR stats %s: %s\n", dirName, strerror(errno)); + return DIR_DEFAULT_MODE; + } + if (!UTIL_isDirectoryStat(&st)) { + UTIL_DISPLAY("zstd: expected directory: %s\n", dirName); + return DIR_DEFAULT_MODE; + } + return st.st_mode; +} + +static int makeDir(const char *dir, mode_t mode) +{ +#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) + int ret = _mkdir(dir); + (void) mode; +#else + int ret = mkdir(dir, mode); +#endif + if (ret != 0) { + if (errno == EEXIST) + return 0; + UTIL_DISPLAY("zstd: failed to create DIR %s: %s\n", dir, strerror(errno)); + } + return ret; +} + +/* this function requires a mutable input string */ +static void convertPathnameToDirName(char *pathname) +{ + size_t len = 0; + char* pos = NULL; + /* get dir name from pathname similar to 'dirname()' */ + assert(pathname != NULL); + + /* remove trailing '/' chars */ + len = strlen(pathname); + assert(len > 0); + while (pathname[len] == PATH_SEP) { + pathname[len] = '\0'; + len--; + } + if (len == 0) return; + + /* if input is a single file, return '.' instead. i.e. + * "xyz/abc/file.txt" => "xyz/abc" + "./file.txt" => "." + "file.txt" => "." + */ + pos = strrchr(pathname, PATH_SEP); + if (pos == NULL) { + pathname[0] = '.'; + pathname[1] = '\0'; + } else { + *pos = '\0'; + } +} + +/* pathname must be valid */ +static const char* trimLeadingRootChar(const char *pathname) +{ + assert(pathname != NULL); + if (pathname[0] == PATH_SEP) + return pathname + 1; + return pathname; +} + +/* pathname must be valid */ +static const char* trimLeadingCurrentDirConst(const char *pathname) +{ + assert(pathname != NULL); + if ((pathname[0] == '.') && (pathname[1] == PATH_SEP)) + return pathname + 2; + return pathname; +} + +static char* +trimLeadingCurrentDir(char *pathname) +{ + /* 'union charunion' can do const-cast without compiler warning */ + union charunion { + char *chr; + const char* cchr; + } ptr; + ptr.cchr = trimLeadingCurrentDirConst(pathname); + return ptr.chr; +} + +/* remove leading './' or '/' chars here */ +static const char * trimPath(const char *pathname) +{ + return trimLeadingRootChar( + trimLeadingCurrentDirConst(pathname)); +} + +static char* mallocAndJoin2Dir(const char *dir1, const char *dir2) +{ + const size_t dir1Size = strlen(dir1); + const size_t dir2Size = strlen(dir2); + char *outDirBuffer, *buffer, trailingChar; + + assert(dir1 != NULL && dir2 != NULL); + outDirBuffer = (char *) malloc(dir1Size + dir2Size + 2); + CONTROL(outDirBuffer != NULL); + + memcpy(outDirBuffer, dir1, dir1Size); + outDirBuffer[dir1Size] = '\0'; + + if (dir2[0] == '.') + return outDirBuffer; + + buffer = outDirBuffer + dir1Size; + trailingChar = *(buffer - 1); + if (trailingChar != PATH_SEP) { + *buffer = PATH_SEP; + buffer++; + } + memcpy(buffer, dir2, dir2Size); + buffer[dir2Size] = '\0'; + + return outDirBuffer; +} + +/* this function will return NULL if input srcFileName is not valid name for mirrored output path */ +char* UTIL_createMirroredDestDirName(const char* srcFileName, const char* outDirRootName) +{ + char* pathname = NULL; + if (!isFileNameValidForMirroredOutput(srcFileName)) + return NULL; + + pathname = mallocAndJoin2Dir(outDirRootName, trimPath(srcFileName)); + + convertPathnameToDirName(pathname); + return pathname; +} + +static int +mirrorSrcDir(char* srcDirName, const char* outDirName) +{ + mode_t srcMode; + int status = 0; + char* newDir = mallocAndJoin2Dir(outDirName, trimPath(srcDirName)); + if (!newDir) + return -ENOMEM; + + srcMode = getDirMode(srcDirName); + status = makeDir(newDir, srcMode); + free(newDir); + return status; +} + +static int +mirrorSrcDirRecursive(char* srcDirName, const char* outDirName) +{ + int status = 0; + char* pp = trimLeadingCurrentDir(srcDirName); + char* sp = NULL; + + while ((sp = strchr(pp, PATH_SEP)) != NULL) { + if (sp != pp) { + *sp = '\0'; + status = mirrorSrcDir(srcDirName, outDirName); + if (status != 0) + return status; + *sp = PATH_SEP; + } + pp = sp + 1; + } + status = mirrorSrcDir(srcDirName, outDirName); + return status; +} + +static void +makeMirroredDestDirsWithSameSrcDirMode(char** srcDirNames, unsigned nbFile, const char* outDirName) +{ + unsigned int i = 0; + for (i = 0; i < nbFile; i++) + mirrorSrcDirRecursive(srcDirNames[i], outDirName); +} + +static int +firstIsParentOrSameDirOfSecond(const char* firstDir, const char* secondDir) +{ + size_t firstDirLen = strlen(firstDir), + secondDirLen = strlen(secondDir); + return firstDirLen <= secondDirLen && + (secondDir[firstDirLen] == PATH_SEP || secondDir[firstDirLen] == '\0') && + 0 == strncmp(firstDir, secondDir, firstDirLen); +} + +static int compareDir(const void* pathname1, const void* pathname2) { + /* sort it after remove the leading '/' or './'*/ + const char* s1 = trimPath(*(char * const *) pathname1); + const char* s2 = trimPath(*(char * const *) pathname2); + return strcmp(s1, s2); +} + +static void +makeUniqueMirroredDestDirs(char** srcDirNames, unsigned nbFile, const char* outDirName) +{ + unsigned int i = 0, uniqueDirNr = 0; + char** uniqueDirNames = NULL; + + if (nbFile == 0) + return; + + uniqueDirNames = (char** ) malloc(nbFile * sizeof (char *)); + CONTROL(uniqueDirNames != NULL); + + /* if dirs is "a/b/c" and "a/b/c/d", we only need call: + * we just need "a/b/c/d" */ + qsort((void *)srcDirNames, nbFile, sizeof(char*), compareDir); + + uniqueDirNr = 1; + uniqueDirNames[uniqueDirNr - 1] = srcDirNames[0]; + for (i = 1; i < nbFile; i++) { + char* prevDirName = srcDirNames[i - 1]; + char* currDirName = srcDirNames[i]; + + /* note: we alwasy compare trimmed path, i.e.: + * src dir of "./foo" and "/foo" will be both saved into: + * "outDirName/foo/" */ + if (!firstIsParentOrSameDirOfSecond(trimPath(prevDirName), + trimPath(currDirName))) + uniqueDirNr++; + + /* we need maintain original src dir name instead of trimmed + * dir, so we can retrive the original src dir's mode_t */ + uniqueDirNames[uniqueDirNr - 1] = currDirName; + } + + makeMirroredDestDirsWithSameSrcDirMode(uniqueDirNames, uniqueDirNr, outDirName); + + free(uniqueDirNames); +} + +static void +makeMirroredDestDirs(char** srcFileNames, unsigned nbFile, const char* outDirName) +{ + unsigned int i = 0; + for (i = 0; i < nbFile; ++i) + convertPathnameToDirName(srcFileNames[i]); + makeUniqueMirroredDestDirs(srcFileNames, nbFile, outDirName); +} + +void UTIL_mirrorSourceFilesDirectories(const char** inFileNames, unsigned int nbFile, const char* outDirName) +{ + unsigned int i = 0, validFilenamesNr = 0; + char** srcFileNames = (char **) malloc(nbFile * sizeof (char *)); + CONTROL(srcFileNames != NULL); + + /* check input filenames is valid */ + for (i = 0; i < nbFile; ++i) { + if (isFileNameValidForMirroredOutput(inFileNames[i])) { + char* fname = STRDUP(inFileNames[i]); + CONTROL(fname != NULL); + srcFileNames[validFilenamesNr++] = fname; + } + } + + if (validFilenamesNr > 0) { + makeDir(outDirName, DIR_DEFAULT_MODE); + makeMirroredDestDirs(srcFileNames, validFilenamesNr, outDirName); + } + + for (i = 0; i < validFilenamesNr; i++) + free(srcFileNames[i]); + free(srcFileNames); +} FileNamesTable* UTIL_createExpandedFNT(const char** inputNames, size_t nbIfns, int followLinks) diff --git a/programs/util.h b/programs/util.h index 8e187e4f..25fa3f53 100644 --- a/programs/util.h +++ b/programs/util.h @@ -93,6 +93,13 @@ extern "C" { ******************************************/ extern int g_utilDisplayLevel; +/** + * Displays a message prompt and returns success (0) if first character from stdin + * matches any from acceptableLetters. Otherwise, returns failure (1) and displays abortMsg. + * If any of the inputs are stdin itself, then automatically return failure (1). + */ +int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const char* acceptableLetters, int hasStdinInput); + /*-**************************************** * File functions @@ -100,12 +107,57 @@ extern int g_utilDisplayLevel; #if defined(_MSC_VER) typedef struct __stat64 stat_t; typedef int mode_t; +#elif defined(__MINGW32__) && defined (__MSVCRT__) + typedef struct _stati64 stat_t; #else typedef struct stat stat_t; #endif +#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ +#define PATH_SEP '\\' +#define STRDUP(s) _strdup(s) +#else +#define PATH_SEP '/' +#include <libgen.h> +#define STRDUP(s) strdup(s) +#endif + +/** + * Calls platform's equivalent of stat() on filename and writes info to statbuf. + * Returns success (1) or failure (0). + */ +int UTIL_stat(const char* filename, stat_t* statbuf); + +/** + * Instead of getting a file's stats, this updates them with the info in the + * provided stat_t. Currently sets owner, group, atime, and mtime. Will only + * update this info for regular files. + */ +int UTIL_setFileStat(const char* filename, const stat_t* statbuf); + +/* + * These helpers operate on a pre-populated stat_t, i.e., the result of + * calling one of the above functions. + */ + +int UTIL_isRegularFileStat(const stat_t* statbuf); +int UTIL_isDirectoryStat(const stat_t* statbuf); +int UTIL_isFIFOStat(const stat_t* statbuf); +U64 UTIL_getFileSizeStat(const stat_t* statbuf); + +/** + * Like chmod(), but only modifies regular files. Provided statbuf may be NULL, + * in which case this function will stat() the file internally, in order to + * check whether it should be modified. + */ +int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions); + +/* + * In the absence of a pre-existing stat result on the file in question, these + * functions will do a stat() call internally and then use that result to + * compute the needed information. + */ -int UTIL_fileExist(const char* filename); int UTIL_isRegularFile(const char* infilename); int UTIL_isDirectory(const char* infilename); int UTIL_isSameFile(const char* file1, const char* file2); @@ -116,11 +168,12 @@ int UTIL_isFIFO(const char* infilename); #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) U64 UTIL_getFileSize(const char* infilename); U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles); -int UTIL_getFileStat(const char* infilename, stat_t* statbuf); -int UTIL_setFileStat(const char* filename, stat_t* statbuf); -int UTIL_chmod(char const* filename, mode_t permissions); /*< like chmod, but avoid changing permission of /dev/null */ + int UTIL_compareStr(const void *p1, const void *p2); const char* UTIL_getFileExtension(const char* infilename); +void UTIL_mirrorSourceFilesDirectories(const char** fileNamesTable, unsigned int nbFiles, const char *outDirName); +char* UTIL_createMirroredDestDirName(const char* srcFileName, const char* outDirRootName); + /*-**************************************** @@ -207,6 +260,7 @@ void UTIL_refFilename(FileNamesTable* fnt, const char* filename); # define UTIL_HAS_CREATEFILELIST #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ # define UTIL_HAS_CREATEFILELIST +# define UTIL_HAS_MIRRORFILELIST #else /* do not define UTIL_HAS_CREATEFILELIST */ #endif diff --git a/programs/zstd.1 b/programs/zstd.1 index 9ba0b4fa..0335b175 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -1,5 +1,5 @@ . -.TH "ZSTD" "1" "May 2020" "zstd 1.4.5" "User Commands" +.TH "ZSTD" "1" "December 2020" "zstd 1.4.7" "User Commands" . .SH "NAME" \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files @@ -99,10 +99,19 @@ Display information related to a zstd compressed file, such as size, ratio, and \fB\-#\fR: \fB#\fR compression level [1\-19] (default: 3) . .IP "\(bu" 4 +\fB\-\-ultra\fR: unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\. +. +.IP "\(bu" 4 \fB\-\-fast[=#]\fR: switch to ultra\-fast compression levels\. If \fB=#\fR is not present, it defaults to \fB1\fR\. The higher the value, the faster the compression speed, at the cost of some compression ratio\. This setting overwrites compression level if one was set previously\. Similarly, if a compression level is set after \fB\-\-fast\fR, it overrides it\. . .IP "\(bu" 4 -\fB\-\-ultra\fR: unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\. +\fB\-T#\fR, \fB\-\-threads=#\fR: Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to ZSTDMT_NBWORKERS_MAX==200\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\. +. +.IP "\(bu" 4 +\fB\-\-single\-thread\fR: Does not spawn a thread for compression, use a single thread for both I/O and compression\. In this mode, compression is serialized with I/O, which is slightly slower\. (This is different from \fB\-T1\fR, which spawns 1 compression thread in parallel of I/O)\. This mode is the only one available when multithread support is disabled\. Single\-thread mode features lower memory usage\. Final compressed result is slightly different from \fB\-T1\fR\. +. +.IP "\(bu" 4 +\fB\-\-adapt[=min=#,max=#]\fR : \fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\. . .IP "\(bu" 4 \fB\-\-long[=#]\fR: enables long distance matching with \fB#\fR \fBwindowLog\fR, if not \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\. @@ -111,43 +120,40 @@ Display information related to a zstd compressed file, such as size, ratio, and Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\. . .IP "\(bu" 4 -\fB\-\-patch\-from=FILE\fR: Specify the file to be used as a reference point for zstd\'s diff engine\. This is effectively dictionary compression with some convenient parameter selection, namely that windowSize > srcSize\. -. -.IP -Note: cannot use both this and \-D together Note: \fB\-\-long\fR mode will be automatically activated if chainLog < fileLog (fileLog being the windowLog requried to cover the whole file)\. You can also manually force it\. Node: for all levels, you can use \-\-patch\-from in \-\-single\-thread mode to improve compression ratio at the cost of speed Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e 4096), and by setting a large \fB\-\-zstd=chainLog=\fR +\fB\-D DICT\fR: use \fBDICT\fR as Dictionary to compress or decompress FILE(s) . .IP "\(bu" 4 -\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, Zstandard uses 128 MB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (ie\. you can increase or decrease it)\. +\fB\-\-patch\-from FILE\fR: Specify the file to be used as a reference point for zstd\'s diff engine\. This is effectively dictionary compression with some convenient parameter selection, namely that windowSize > srcSize\. . .IP -This is also used during compression when using with \-\-patch\-from=\. In this case, this parameter overrides that maximum size allowed for a dictionary\. (128 MB)\. +Note: cannot use both this and \-D together Note: \fB\-\-long\fR mode will be automatically activated if chainLog < fileLog (fileLog being the windowLog required to cover the whole file)\. You can also manually force it\. Node: for all levels, you can use \-\-patch\-from in \-\-single\-thread mode to improve compression ratio at the cost of speed Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e 4096), and by setting a large \fB\-\-zstd=chainLog=\fR . .IP "\(bu" 4 -\fB\-T#\fR, \fB\-\-threads=#\fR: Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to ZSTDMT_NBTHREADS_MAX==200\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\. +\fB\-\-rsyncable\fR : \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your milage may vary\. . .IP "\(bu" 4 -\fB\-\-single\-thread\fR: Does not spawn a thread for compression, use a single thread for both I/O and compression\. In this mode, compression is serialized with I/O, which is slightly slower\. (This is different from \fB\-T1\fR, which spawns 1 compression thread in parallel of I/O)\. This mode is the only one available when multithread support is disabled\. Single\-thread mode features lower memory usage\. Final compressed result is slightly different from \fB\-T1\fR\. +\fB\-C\fR, \fB\-\-[no\-]check\fR: add integrity check computed from uncompressed data (default: enabled) . .IP "\(bu" 4 -\fB\-\-adapt[=min=#,max=#]\fR : \fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\. +\fB\-\-[no\-]content\-size\fR: enable / disable whether or not the original size of the file is placed in the header of the compressed file\. The default option is \-\-content\-size (meaning that the original size will be placed in the header)\. . .IP "\(bu" 4 -\fB\-\-stream\-size=#\fR : Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\. +\fB\-\-no\-dictID\fR: do not store dictionary ID within frame header (dictionary compression)\. The decoder will have to rely on implicit knowledge about which dictionary to use, it won\'t be able to check if it\'s correct\. . .IP "\(bu" 4 -\fB\-\-size\-hint=#\fR: When handling input from a stream, \fBzstd\fR must guess how large the source size will be when optimizing compression parameters\. If the stream size is relatively small, this guess may be a poor one, resulting in a higher compression ratio than expected\. This feature allows for controlling the guess when needed\. Exact guesses result in better compression ratios\. Overestimates result in slightly degraded compression ratios, while underestimates may result in significant degradation\. +\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, Zstandard uses 128 MB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (ie\. you can increase or decrease it)\. . -.IP "\(bu" 4 -\fB\-\-rsyncable\fR : \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your milage may vary\. +.IP +This is also used during compression when using with \-\-patch\-from=\. In this case, this parameter overrides that maximum size allowed for a dictionary\. (128 MB)\. . .IP "\(bu" 4 -\fB\-D file\fR: use \fBfile\fR as Dictionary to compress or decompress FILE(s) +\fB\-\-stream\-size=#\fR : Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\. . .IP "\(bu" 4 -\fB\-\-no\-dictID\fR: do not store dictionary ID within frame header (dictionary compression)\. The decoder will have to rely on implicit knowledge about which dictionary to use, it won\'t be able to check if it\'s correct\. +\fB\-\-size\-hint=#\fR: When handling input from a stream, \fBzstd\fR must guess how large the source size will be when optimizing compression parameters\. If the stream size is relatively small, this guess may be a poor one, resulting in a higher compression ratio than expected\. This feature allows for controlling the guess when needed\. Exact guesses result in better compression ratios\. Overestimates result in slightly degraded compression ratios, while underestimates may result in significant degradation\. . .IP "\(bu" 4 -\fB\-o file\fR: save result into \fBfile\fR (only possible with a single \fIINPUT\-FILE\fR) +\fB\-o FILE\fR: save result into \fBFILE\fR . .IP "\(bu" 4 \fB\-f\fR, \fB\-\-force\fR: overwrite output without prompting, and (de)compress symbolic links @@ -159,10 +165,7 @@ This is also used during compression when using with \-\-patch\-from=\. In this \fB\-\-[no\-]sparse\fR: enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default: enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\. . .IP "\(bu" 4 -\fB\-\-[no\-]content\-size\fR: enable / disable whether or not the original size of the file is placed in the header of the compressed file\. The default option is \-\-content\-size (meaning that the original size will be placed in the header)\. -. -.IP "\(bu" 4 -\fB\-\-rm\fR: remove source file(s) after successful compression or decompression +\fB\-\-rm\fR: remove source file(s) after successful compression or decompression\. If used in combination with \-o, will trigger a confirmation prompt (which can be silenced with \-f), as this is a destructive operation\. . .IP "\(bu" 4 \fB\-k\fR, \fB\-\-keep\fR: keep source file(s) after successful compression or decompression\. This is the default behavior\. @@ -171,25 +174,28 @@ This is also used during compression when using with \-\-patch\-from=\. In this \fB\-r\fR: operate recursively on directories . .IP "\(bu" 4 -\fB\-\-filelist=FILE\fR read a list of files to process as content from \fBFILE\fR\. Format is compatible with \fBls\fR output, with one file per line\. +\fB\-\-filelist FILE\fR read a list of files to process as content from \fBFILE\fR\. Format is compatible with \fBls\fR output, with one file per line\. . .IP "\(bu" 4 -\fB\-\-output\-dir\-flat[=dir]\fR: resulting files are stored into target \fBdir\fR directory, instead of same directory as origin file\. Be aware that this command can introduce name collision issues, if multiple files, from different directories, end up having the same name\. Collision resolution ensures first file with a given name will be present in \fBdir\fR, while in combination with \fB\-f\fR, the last file will be present instead\. +\fB\-\-output\-dir\-flat DIR\fR: resulting files are stored into target \fBDIR\fR directory, instead of same directory as origin file\. Be aware that this command can introduce name collision issues, if multiple files, from different directories, end up having the same name\. Collision resolution ensures first file with a given name will be present in \fBDIR\fR, while in combination with \fB\-f\fR, the last file will be present instead\. . .IP "\(bu" 4 -\fB\-\-format=FORMAT\fR: compress and decompress in other formats\. If compiled with support, zstd can compress to or decompress from other compression algorithm formats\. Possibly available options are \fBzstd\fR, \fBgzip\fR, \fBxz\fR, \fBlzma\fR, and \fBlz4\fR\. If no such format is provided, \fBzstd\fR is the default\. +\fB\-\-output\-dir\-mirror DIR\fR: similar to \fB\-\-output\-dir\-flat\fR, the output files are stored underneath target \fBDIR\fR directory, but this option will replicate input directory hierarchy into output \fBDIR\fR\. +. +.IP +If input directory contains "\.\.", the files in this directory will be ignored\. If input directory is an absolute directory (i\.e\. "/var/tmp/abc"), it will be stored into the "output\-dir/var/tmp/abc"\. If there are multiple input files or directories, name collision resolution will follow the same rules as \fB\-\-output\-dir\-flat\fR\. . .IP "\(bu" 4 -\fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR: display help/long help and exit +\fB\-\-format=FORMAT\fR: compress and decompress in other formats\. If compiled with support, zstd can compress to or decompress from other compression algorithm formats\. Possibly available options are \fBzstd\fR, \fBgzip\fR, \fBxz\fR, \fBlzma\fR, and \fBlz4\fR\. If no such format is provided, \fBzstd\fR is the default\. . .IP "\(bu" 4 -\fB\-V\fR, \fB\-\-version\fR: display version number and exit\. Advanced : \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\. +\fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR: display help/long help and exit . .IP "\(bu" 4 -\fB\-v\fR, \fB\-\-verbose\fR: verbose mode +\fB\-V\fR, \fB\-\-version\fR: display version number and exit\. Advanced : \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\. \fB\-q\fR will only display the version number, suitable for machine reading\. . .IP "\(bu" 4 -\fB\-\-show\-default\-cparams\fR: Shows the default compresssion parameters that will be used for a particular src file\. If the provided src file is not a regular file (eg\. named pipe), the cli will just output the default paramters\. That is, the parameters that are used when the src size is unknown\. +\fB\-v\fR, \fB\-\-verbose\fR: verbose mode, display more information . .IP "\(bu" 4 \fB\-q\fR, \fB\-\-quiet\fR: suppress warnings, interactivity, and notifications\. specify twice to suppress errors too\. @@ -198,7 +204,7 @@ This is also used during compression when using with \-\-patch\-from=\. In this \fB\-\-no\-progress\fR: do not display the progress bar, but keep all other messages\. . .IP "\(bu" 4 -\fB\-C\fR, \fB\-\-[no\-]check\fR: add integrity check computed from uncompressed data (default: enabled) +\fB\-\-show\-default\-cparams\fR: Shows the default compression parameters that will be used for a particular src file\. If the provided src file is not a regular file (eg\. named pipe), the cli will just output the default parameters\. That is, the parameters that are used when the src size is unknown\. . .IP "\(bu" 4 \fB\-\-\fR: All arguments after \fB\-\-\fR are treated as files @@ -206,7 +212,16 @@ This is also used during compression when using with \-\-patch\-from=\. In this .IP "" 0 . .SS "Restricted usage of Environment Variables" -Using environment variables to set parameters has security implications\. Therefore, this avenue is intentionally restricted\. Only \fBZSTD_CLEVEL\fR is supported currently, for setting compression level\. \fBZSTD_CLEVEL\fR can be used to set the level between 1 and 19 (the "normal" range)\. If the value of \fBZSTD_CLEVEL\fR is not a valid integer, it will be ignored with a warning message\. \fBZSTD_CLEVEL\fR just replaces the default compression level (\fB3\fR)\. It can be overridden by corresponding command line arguments\. +Using environment variables to set parameters has security implications\. Therefore, this avenue is intentionally restricted\. Only \fBZSTD_CLEVEL\fR and \fBZSTD_NBTHREADS\fR are currently supported\. They set the compression level and number of threads to use during compression, respectively\. +. +.P +\fBZSTD_CLEVEL\fR can be used to set the level between 1 and 19 (the "normal" range)\. If the value of \fBZSTD_CLEVEL\fR is not a valid integer, it will be ignored with a warning message\. \fBZSTD_CLEVEL\fR just replaces the default compression level (\fB3\fR)\. +. +.P +\fBZSTD_NBTHREADS\fR can be used to set the number of threads \fBzstd\fR will attempt to use during compression\. If the value of \fBZSTD_NBTHREADS\fR is not a valid unsigned integer, it will be ignored with a warning message\. \'ZSTD_NBTHREADS\fBhas a default value of (\fR1\fB), and is capped at ZSTDMT_NBWORKERS_MAX==200\.\fRzstd` must be compiled with multithread support for this to have any effect\. +. +.P +They can both be overridden by corresponding command line arguments: \fB\-#\fR for compression level and \fB\-T#\fR for number of compression threads\. . .SH "DICTIONARY BUILDER" \fBzstd\fR offers \fIdictionary\fR compression, which greatly improves efficiency on small files and messages\. It\'s possible to train \fBzstd\fR with a set of samples, the result of which is saved into a file called a \fBdictionary\fR\. Then during compression and decompression, reference the same dictionary, using command \fB\-D dictionaryFileName\fR\. Compression of small files similar to the sample set will be greatly improved\. diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 550e2e53..73670daf 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -102,6 +102,9 @@ the last one takes effect. * `-#`: `#` compression level \[1-19] (default: 3) +* `--ultra`: + unlocks high compression levels 20+ (maximum 22), using a lot more memory. + Note that decompression will also require more memory when using these levels. * `--fast[=#]`: switch to ultra-fast compression levels. If `=#` is not present, it defaults to `1`. @@ -109,9 +112,28 @@ the last one takes effect. at the cost of some compression ratio. This setting overwrites compression level if one was set previously. Similarly, if a compression level is set after `--fast`, it overrides it. -* `--ultra`: - unlocks high compression levels 20+ (maximum 22), using a lot more memory. - Note that decompression will also require more memory when using these levels. +* `-T#`, `--threads=#`: + Compress using `#` working threads (default: 1). + If `#` is 0, attempt to detect and use the number of physical CPU cores. + In all cases, the nb of threads is capped to ZSTDMT_NBWORKERS_MAX==200. + This modifier does nothing if `zstd` is compiled without multithread support. +* `--single-thread`: + Does not spawn a thread for compression, use a single thread for both I/O and compression. + In this mode, compression is serialized with I/O, which is slightly slower. + (This is different from `-T1`, which spawns 1 compression thread in parallel of I/O). + This mode is the only one available when multithread support is disabled. + Single-thread mode features lower memory usage. + Final compressed result is slightly different from `-T1`. +* `--adapt[=min=#,max=#]` : + `zstd` will dynamically adapt compression level to perceived I/O conditions. + Compression level adaptation can be observed live by using command `-v`. + Adaptation can be constrained between supplied `min` and `max` levels. + The feature works when combined with multi-threading and `--long` mode. + It does not work with `--single-thread`. + It sets window size to 8 MB by default (can be changed manually, see `wlog`). + Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible. + _note_ : at the time of this writing, `--adapt` can remain stuck at low speed + when combined with multiple worker threads (>=2). * `--long[=#]`: enables long distance matching with `#` `windowLog`, if not `#` is not present it defaults to `27`. @@ -122,20 +144,40 @@ the last one takes effect. Note: If `windowLog` is set to larger than 27, `--long=windowLog` or `--memory=windowSize` needs to be passed to the decompressor. -* `--patch-from=FILE`: +* `-D DICT`: + use `DICT` as Dictionary to compress or decompress FILE(s) +* `--patch-from FILE`: Specify the file to be used as a reference point for zstd's diff engine. This is effectively dictionary compression with some convenient parameter selection, namely that windowSize > srcSize. Note: cannot use both this and -D together Note: `--long` mode will be automatically activated if chainLog < fileLog - (fileLog being the windowLog requried to cover the whole file). You + (fileLog being the windowLog required to cover the whole file). You can also manually force it. Node: for all levels, you can use --patch-from in --single-thread mode to improve compression ratio at the cost of speed - Note: for level 19, you can get increased compression ratio at the cost - of speed by specifying `--zstd=targetLength=` to be something large + Note: for level 19, you can get increased compression ratio at the cost + of speed by specifying `--zstd=targetLength=` to be something large (i.e 4096), and by setting a large `--zstd=chainLog=` +* `--rsyncable` : + `zstd` will periodically synchronize the compression state to make the + compressed file more rsync-friendly. There is a negligible impact to + compression ratio, and the faster compression levels will see a small + compression speed hit. + This feature does not work with `--single-thread`. You probably don't want + to use it with long range mode, since it will decrease the effectiveness of + the synchronization points, but your milage may vary. +* `-C`, `--[no-]check`: + add integrity check computed from uncompressed data (default: enabled) +* `--[no-]content-size`: + enable / disable whether or not the original size of the file is placed in + the header of the compressed file. The default option is + --content-size (meaning that the original size will be placed in the header). +* `--no-dictID`: + do not store dictionary ID within frame header (dictionary compression). + The decoder will have to rely on implicit knowledge about which dictionary to use, + it won't be able to check if it's correct. * `-M#`, `--memory=#`: Set a memory usage limit. By default, Zstandard uses 128 MB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can @@ -144,28 +186,6 @@ the last one takes effect. This is also used during compression when using with --patch-from=. In this case, this parameter overrides that maximum size allowed for a dictionary. (128 MB). -* `-T#`, `--threads=#`: - Compress using `#` working threads (default: 1). - If `#` is 0, attempt to detect and use the number of physical CPU cores. - In all cases, the nb of threads is capped to ZSTDMT_NBTHREADS_MAX==200. - This modifier does nothing if `zstd` is compiled without multithread support. -* `--single-thread`: - Does not spawn a thread for compression, use a single thread for both I/O and compression. - In this mode, compression is serialized with I/O, which is slightly slower. - (This is different from `-T1`, which spawns 1 compression thread in parallel of I/O). - This mode is the only one available when multithread support is disabled. - Single-thread mode features lower memory usage. - Final compressed result is slightly different from `-T1`. -* `--adapt[=min=#,max=#]` : - `zstd` will dynamically adapt compression level to perceived I/O conditions. - Compression level adaptation can be observed live by using command `-v`. - Adaptation can be constrained between supplied `min` and `max` levels. - The feature works when combined with multi-threading and `--long` mode. - It does not work with `--single-thread`. - It sets window size to 8 MB by default (can be changed manually, see `wlog`). - Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible. - _note_ : at the time of this writing, `--adapt` can remain stuck at low speed - when combined with multiple worker threads (>=2). * `--stream-size=#` : Sets the pledged source size of input coming from a stream. This value must be exact, as it will be included in the produced frame header. Incorrect stream sizes will cause an error. @@ -178,22 +198,8 @@ the last one takes effect. expected. This feature allows for controlling the guess when needed. Exact guesses result in better compression ratios. Overestimates result in slightly degraded compression ratios, while underestimates may result in significant degradation. -* `--rsyncable` : - `zstd` will periodically synchronize the compression state to make the - compressed file more rsync-friendly. There is a negligible impact to - compression ratio, and the faster compression levels will see a small - compression speed hit. - This feature does not work with `--single-thread`. You probably don't want - to use it with long range mode, since it will decrease the effectiveness of - the synchronization points, but your milage may vary. -* `-D file`: - use `file` as Dictionary to compress or decompress FILE(s) -* `--no-dictID`: - do not store dictionary ID within frame header (dictionary compression). - The decoder will have to rely on implicit knowledge about which dictionary to use, - it won't be able to check if it's correct. -* `-o file`: - save result into `file` (only possible with a single _INPUT-FILE_) +* `-o FILE`: + save result into `FILE` * `-f`, `--force`: overwrite output without prompting, and (de)compress symbolic links * `-c`, `--stdout`: @@ -206,27 +212,34 @@ the last one takes effect. default: enabled when output is into a file, and disabled when output is stdout. This setting overrides default and can force sparse mode over stdout. -* `--[no-]content-size`: - enable / disable whether or not the original size of the file is placed in - the header of the compressed file. The default option is - --content-size (meaning that the original size will be placed in the header). * `--rm`: - remove source file(s) after successful compression or decompression + remove source file(s) after successful compression or decompression. If used in combination with + -o, will trigger a confirmation prompt (which can be silenced with -f), as this is a destructive operation. * `-k`, `--keep`: keep source file(s) after successful compression or decompression. This is the default behavior. * `-r`: operate recursively on directories -* `--filelist=FILE` +* `--filelist FILE` read a list of files to process as content from `FILE`. Format is compatible with `ls` output, with one file per line. -* `--output-dir-flat[=dir]`: - resulting files are stored into target `dir` directory, +* `--output-dir-flat DIR`: + resulting files are stored into target `DIR` directory, instead of same directory as origin file. Be aware that this command can introduce name collision issues, if multiple files, from different directories, end up having the same name. - Collision resolution ensures first file with a given name will be present in `dir`, + Collision resolution ensures first file with a given name will be present in `DIR`, while in combination with `-f`, the last file will be present instead. +* `--output-dir-mirror DIR`: + similar to `--output-dir-flat`, + the output files are stored underneath target `DIR` directory, + but this option will replicate input directory hierarchy into output `DIR`. + + If input directory contains "..", the files in this directory will be ignored. + If input directory is an absolute directory (i.e. "/var/tmp/abc"), + it will be stored into the "output-dir/var/tmp/abc". + If there are multiple input files or directories, + name collision resolution will follow the same rules as `--output-dir-flat`. * `--format=FORMAT`: compress and decompress in other formats. If compiled with support, zstd can compress to or decompress from other compression algorithm @@ -238,21 +251,19 @@ the last one takes effect. display version number and exit. Advanced : `-vV` also displays supported formats. `-vvV` also displays POSIX support. + `-q` will only display the version number, suitable for machine reading. * `-v`, `--verbose`: - verbose mode -* `--show-default-cparams`: - Shows the default compresssion parameters that will be used for a - particular src file. If the provided src file is not a regular file - (eg. named pipe), the cli will just output the default paramters. - That is, the parameters that are used when the src size is - unknown. + verbose mode, display more information * `-q`, `--quiet`: suppress warnings, interactivity, and notifications. specify twice to suppress errors too. * `--no-progress`: do not display the progress bar, but keep all other messages. -* `-C`, `--[no-]check`: - add integrity check computed from uncompressed data (default: enabled) +* `--show-default-cparams`: + Shows the default compression parameters that will be used for a + particular src file. If the provided src file is not a regular file + (eg. named pipe), the cli will just output the default parameters. + That is, the parameters that are used when the src size is unknown. * `--`: All arguments after `--` are treated as files @@ -260,11 +271,20 @@ the last one takes effect. Using environment variables to set parameters has security implications. Therefore, this avenue is intentionally restricted. -Only `ZSTD_CLEVEL` is supported currently, for setting compression level. +Only `ZSTD_CLEVEL` and `ZSTD_NBTHREADS` are currently supported. +They set the compression level and number of threads to use during compression, respectively. + `ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range). If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message. `ZSTD_CLEVEL` just replaces the default compression level (`3`). -It can be overridden by corresponding command line arguments. + +`ZSTD_NBTHREADS` can be used to set the number of threads `zstd` will attempt to use during compression. +If the value of `ZSTD_NBTHREADS` is not a valid unsigned integer, it will be ignored with a warning message. +'ZSTD_NBTHREADS` has a default value of (`1`), and is capped at ZSTDMT_NBWORKERS_MAX==200. `zstd` must be +compiled with multithread support for this to have any effect. + +They can both be overridden by corresponding command line arguments: +`-#` for compression level and `-T#` for number of compression threads. DICTIONARY BUILDER diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 70e2b706..9b6f9153 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -20,7 +20,9 @@ # define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */ #endif - +#ifndef ZSTDCLI_NBTHREADS_DEFAULT +# define ZSTDCLI_NBTHREADS_DEFAULT 1 +#endif /*-************************************ * Dependencies @@ -146,9 +148,26 @@ static void usage_advanced(const char* programName) #ifdef UTIL_HAS_CREATEFILELIST DISPLAYOUT( " -r : operate recursively on directories \n"); - DISPLAYOUT( "--filelist=FILE : read list of files to operate upon from FILE \n"); - DISPLAYOUT( "--output-dir-flat=DIR : all resulting files are stored into DIR \n"); + DISPLAYOUT( "--filelist FILE : read list of files to operate upon from FILE \n"); + DISPLAYOUT( "--output-dir-flat DIR : processed files are stored into DIR \n"); +#endif + +#ifdef UTIL_HAS_MIRRORFILELIST + DISPLAYOUT( "--output-dir-mirror DIR : processed files are stored into DIR respecting original directory structure \n"); +#endif + + +#ifndef ZSTD_NOCOMPRESS + DISPLAYOUT( "--[no-]check : during compression, add XXH64 integrity checksum to frame (default: enabled)"); +#ifndef ZSTD_NODECOMPRESS + DISPLAYOUT( ". If specified with -d, decompressor will ignore/validate checksums in compressed frame (default: validate)."); #endif +#else +#ifdef ZSTD_NOCOMPRESS + DISPLAYOUT( "--[no-]check : during decompression, ignore/validate checksums in compressed frame (default: validate)."); +#endif +#endif /* ZSTD_NOCOMPRESS */ + DISPLAYOUT( "\n"); DISPLAYOUT( "-- : All arguments after \"--\" are treated as files \n"); @@ -170,7 +189,6 @@ static void usage_advanced(const char* programName) DISPLAYOUT( "--size-hint=# optimize compression parameters for streaming input of approximately this size \n"); DISPLAYOUT( "--target-compressed-block-size=# : generate compressed block of approximately targeted size \n"); DISPLAYOUT( "--no-dictID : don't write dictID into header (dictionary compression only) \n"); - DISPLAYOUT( "--[no-]check : add XXH64 integrity checksum to frame (default: enabled) \n"); DISPLAYOUT( "--[no-]compress-literals : force (un)compressed literals \n"); DISPLAYOUT( "--format=zstd : compress files to the .zst format (default) \n"); @@ -544,6 +562,11 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi static void printVersion(void) { + if (g_displayLevel < DISPLAY_LEVEL_DEFAULT) { + DISPLAYOUT("%s\n", ZSTD_VERSION_STRING); + return; + } + DISPLAYOUT(WELCOME_MESSAGE); if (g_displayLevel >= 3) { /* format support */ @@ -577,6 +600,7 @@ static void printVersion(void) /* Environment variables for parameter setting */ #define ENV_CLEVEL "ZSTD_CLEVEL" +#define ENV_NBTHREADS "ZSTD_NBTHREADS" /* takes lower precedence than directly specifying -T# in the CLI */ /* pick up environment variable */ static int init_cLevel(void) { @@ -606,8 +630,51 @@ static int init_cLevel(void) { return ZSTDCLI_CLEVEL_DEFAULT; } -#define ZSTD_NB_STRATEGIES 9 +#ifdef ZSTD_MULTITHREAD +static unsigned init_nbThreads(void) { + const char* const env = getenv(ENV_NBTHREADS); + if (env != NULL) { + const char* ptr = env; + if ((*ptr>='0') && (*ptr<='9')) { + unsigned nbThreads; + if (readU32FromCharChecked(&ptr, &nbThreads)) { + DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBTHREADS, env); + return ZSTDCLI_NBTHREADS_DEFAULT; + } else if (*ptr == 0) { + return nbThreads; + } + } + DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBTHREADS, env); + } + return ZSTDCLI_NBTHREADS_DEFAULT; +} +#endif + +#define NEXT_FIELD(ptr) { \ + if (*argument == '=') { \ + ptr = ++argument; \ + argument += strlen(ptr); \ + } else { \ + argNb++; \ + if (argNb >= argCount) { \ + DISPLAY("error: missing command argument \n"); \ + CLEAN_RETURN(1); \ + } \ + ptr = argv[argNb]; \ + assert(ptr != NULL); \ + if (ptr[0]=='-') { \ + DISPLAY("error: command cannot be separated from its argument by another command \n"); \ + CLEAN_RETURN(1); \ +} } } + +#define NEXT_UINT32(val32) { \ + const char* __nb; \ + NEXT_FIELD(__nb); \ + val32 = readU32FromChar(&__nb); \ +} + +#define ZSTD_NB_STRATEGIES 9 static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast", "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2", "ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"}; @@ -630,7 +697,7 @@ int main(int const argCount, const char* argv[]) int argNb, followLinks = 0, forceStdout = 0, - lastCommand = 0, + hasStdout = 0, ldmFlag = 0, main_pause = 0, nbWorkers = 0, @@ -638,12 +705,7 @@ int main(int const argCount, const char* argv[]) adaptMin = MINCLEVEL, adaptMax = MAXCLEVEL, rsyncable = 0, - nextArgumentIsOutFileName = 0, - nextArgumentIsOutDirName = 0, - nextArgumentIsMaxDict = 0, - nextArgumentIsDictID = 0, nextArgumentsAreFiles = 0, - nextEntryIsDictionary = 0, operationResult = 0, separateFiles = 0, setRealTimePrio = 0, @@ -656,6 +718,7 @@ int main(int const argCount, const char* argv[]) size_t blockSize = 0; FIO_prefs_t* const prefs = FIO_createPreferences(); + FIO_ctx_t* const fCtx = FIO_createContext(); zstd_operation_mode operation = zom_compress; ZSTD_compressionParameters compressionParams; int cLevel = init_cLevel(); @@ -667,6 +730,7 @@ int main(int const argCount, const char* argv[]) const char* programName = argv[0]; const char* outFileName = NULL; const char* outDirName = NULL; + const char* outMirroredDirName = NULL; const char* dictFileName = NULL; const char* patchFromDictFileName = NULL; const char* suffix = ZSTD_EXTENSION; @@ -695,7 +759,7 @@ int main(int const argCount, const char* argv[]) if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAY("zstd: allocation error \n"); exit(1); } programName = lastNameFromPath(programName); #ifdef ZSTD_MULTITHREAD - nbWorkers = 1; + nbWorkers = init_nbThreads(); #endif /* preset behaviors */ @@ -756,13 +820,10 @@ int main(int const argCount, const char* argv[]) if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; } if (!strcmp(argument, "--test")) { operation=zom_test; continue; } if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; } - if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ - if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(prefs, 0); continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } - if (!strcmp(argument, "--output-dir-flat")) {nextArgumentIsOutDirName=1; lastCommand=1; continue; } if (!strcmp(argument, "--show-default-cparams")) { showDefaultCParams = 1; continue; } if (!strcmp(argument, "--content-size")) { contentSize = 1; continue; } if (!strcmp(argument, "--no-content-size")) { contentSize = 0; continue; } @@ -785,6 +846,7 @@ int main(int const argCount, const char* argv[]) if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; } if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; } if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; } + /* long commands with arguments */ #ifndef ZSTD_NODICT if (longCommandWArg(&argument, "--train-cover")) { @@ -821,19 +883,22 @@ int main(int const argCount, const char* argv[]) continue; } #endif - if (longCommandWArg(&argument, "--threads=")) { nbWorkers = (int)readU32FromChar(&argument); continue; } - if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; } - if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; } - if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--threads")) { NEXT_UINT32(nbWorkers); continue; } + if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; } + if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; } + if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; } if (longCommandWArg(&argument, "--block-size=")) { blockSize = readSizeTFromChar(&argument); continue; } - if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } - if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; } + if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; } if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badusage(programName); CLEAN_RETURN(1); } continue; } if (longCommandWArg(&argument, "--stream-size=")) { streamSrcSize = readSizeTFromChar(&argument); continue; } if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readSizeTFromChar(&argument); continue; } if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readSizeTFromChar(&argument); continue; } - if (longCommandWArg(&argument, "--output-dir-flat=")) { outDirName = argument; continue; } - if (longCommandWArg(&argument, "--patch-from=")) { patchFromDictFileName = argument; continue; } + if (longCommandWArg(&argument, "--output-dir-flat")) { NEXT_FIELD(outDirName); continue; } +#ifdef UTIL_HAS_MIRRORFILELIST + if (longCommandWArg(&argument, "--output-dir-mirror")) { NEXT_FIELD(outMirroredDirName); continue; } +#endif + if (longCommandWArg(&argument, "--patch-from")) { NEXT_FIELD(patchFromDictFileName); continue; } if (longCommandWArg(&argument, "--long")) { unsigned ldmWindowLog = 0; ldmFlag = 1; @@ -877,8 +942,10 @@ int main(int const argCount, const char* argv[]) } #endif - if (longCommandWArg(&argument, "--filelist=")) { - UTIL_refFilename(file_of_names, argument); + if (longCommandWArg(&argument, "--filelist")) { + const char* listName; + NEXT_FIELD(listName); + UTIL_refFilename(file_of_names, listName); continue; } @@ -887,10 +954,7 @@ int main(int const argCount, const char* argv[]) argument++; while (argument[0]!=0) { - if (lastCommand) { - DISPLAY("error : command must be followed by argument \n"); - CLEAN_RETURN(1); - } + #ifndef ZSTD_NOCOMPRESS /* compression Level */ if ((*argument>='0') && (*argument<='9')) { @@ -921,7 +985,7 @@ int main(int const argCount, const char* argv[]) case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break; /* Use file content as dictionary */ - case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break; + case 'D': argument++; NEXT_FIELD(dictFileName); break; /* Overwrite */ case 'f': FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; argument++; break; @@ -942,7 +1006,7 @@ int main(int const argCount, const char* argv[]) case 't': operation=zom_test; argument++; break; /* destination file name */ - case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; + case 'o': argument++; NEXT_FIELD(outFileName); break; /* limit memory */ case 'M': @@ -1013,10 +1077,9 @@ int main(int const argCount, const char* argv[]) /* Select compressibility of synthetic sample */ case 'P': - { argument++; + argument++; compressibility = (double)readU32FromChar(&argument) / 100; - } - break; + break; /* unknown command */ default : badusage(programName); CLEAN_RETURN(1); @@ -1025,51 +1088,10 @@ int main(int const argCount, const char* argv[]) continue; } /* if (argument[0]=='-') */ - if (nextArgumentIsMaxDict) { /* kept available for compatibility with old syntax ; will be removed one day */ - nextArgumentIsMaxDict = 0; - lastCommand = 0; - maxDictSize = readU32FromChar(&argument); - continue; - } - - if (nextArgumentIsDictID) { /* kept available for compatibility with old syntax ; will be removed one day */ - nextArgumentIsDictID = 0; - lastCommand = 0; - dictID = readU32FromChar(&argument); - continue; - } - - if (nextEntryIsDictionary) { - nextEntryIsDictionary = 0; - lastCommand = 0; - dictFileName = argument; - continue; - } - - if (nextArgumentIsOutFileName) { - nextArgumentIsOutFileName = 0; - lastCommand = 0; - outFileName = argument; - if (!strcmp(outFileName, "-")) outFileName = stdoutmark; - continue; - } - - if (nextArgumentIsOutDirName) { - nextArgumentIsOutDirName = 0; - lastCommand = 0; - outDirName = argument; - continue; - } - /* none of the above : add filename to list */ UTIL_refFilename(filenames, argument); } - if (lastCommand) { /* forgotten argument */ - DISPLAY("error : command must be followed by argument \n"); - CLEAN_RETURN(1); - } - /* Welcome message (if verbose) */ DISPLAYLEVEL(3, WELCOME_MESSAGE); @@ -1222,7 +1244,7 @@ int main(int const argCount, const char* argv[]) /* Check if input/output defined as console; trigger an error in this case */ if (!strcmp(filenames->fileNames[0], stdinmark) && IS_CONSOLE(stdin) ) { - badusage(programName); + DISPLAYLEVEL(1, "stdin is a console, aborting\n"); CLEAN_RETURN(1); } if ( outFileName && !strcmp(outFileName, stdoutmark) @@ -1230,7 +1252,7 @@ int main(int const argCount, const char* argv[]) && !strcmp(filenames->fileNames[0], stdinmark) && !forceStdout && operation!=zom_decompress ) { - badusage(programName); + DISPLAYLEVEL(1, "stdout is a console, aborting\n"); CLEAN_RETURN(1); } @@ -1259,12 +1281,16 @@ int main(int const argCount, const char* argv[]) DISPLAY("error : can't use --patch-from=# on multiple files \n"); CLEAN_RETURN(1); } + + /* No status message in pipe mode (stdin - stdout) */ + hasStdout = outFileName && !strcmp(outFileName,stdoutmark); - /* No status message in pipe mode (stdin - stdout) or multi-files mode */ - if (!strcmp(filenames->fileNames[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1; - if ((filenames->tableSize > 1) & (g_displayLevel==2)) g_displayLevel=1; + if (hasStdout && (g_displayLevel==2)) g_displayLevel=1; /* IO Stream/File */ + FIO_setHasStdoutOutput(fCtx, hasStdout); + FIO_setNbFilesTotal(fCtx, (int)filenames->tableSize); + FIO_determineHasStdinInput(fCtx, filenames); FIO_setNotificationLevel(g_displayLevel); FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL); if (memLimit == 0) { @@ -1323,9 +1349,9 @@ int main(int const argCount, const char* argv[]) } if ((filenames->tableSize==1) && outFileName) - operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); + operationResult = FIO_compressFilename(fCtx, prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); + operationResult = FIO_compressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); #else (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); @@ -1333,9 +1359,9 @@ int main(int const argCount, const char* argv[]) } else { /* decompression or test */ #ifndef ZSTD_NODECOMPRESS if (filenames->tableSize == 1 && outFileName) { - operationResult = FIO_decompressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName); + operationResult = FIO_decompressFilename(fCtx, prefs, outFileName, filenames->fileNames[0], dictFileName); } else { - operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outDirName, outFileName, dictFileName); + operationResult = FIO_decompressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName); } #else DISPLAY("Decompression not supported \n"); @@ -1344,6 +1370,7 @@ int main(int const argCount, const char* argv[]) _end: FIO_freePreferences(prefs); + FIO_freeContext(fCtx); if (main_pause) waitEnter(); UTIL_freeFileNamesTable(filenames); UTIL_freeFileNamesTable(file_of_names); diff --git a/programs/zstdgrep.1 b/programs/zstdgrep.1 index fe1d29bb..c8af908c 100644 --- a/programs/zstdgrep.1 +++ b/programs/zstdgrep.1 @@ -1,5 +1,5 @@ . -.TH "ZSTDGREP" "1" "May 2020" "zstd 1.4.5" "User Commands" +.TH "ZSTDGREP" "1" "December 2020" "zstd 1.4.7" "User Commands" . .SH "NAME" \fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files diff --git a/programs/zstdless.1 b/programs/zstdless.1 index d54c6400..be92e351 100644 --- a/programs/zstdless.1 +++ b/programs/zstdless.1 @@ -1,5 +1,5 @@ . -.TH "ZSTDLESS" "1" "May 2020" "zstd 1.4.5" "User Commands" +.TH "ZSTDLESS" "1" "December 2020" "zstd 1.4.7" "User Commands" . .SH "NAME" \fBzstdless\fR \- view zstandard\-compressed files diff --git a/tests/Makefile b/tests/Makefile index d347a948..42bc353c 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -25,6 +25,7 @@ PYTHON ?= python3 TESTARTEFACT := versionsTest DEBUGLEVEL ?= 1 +export DEBUGLEVEL # transmit value to sub-makefiles DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL) CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) @@ -77,39 +78,37 @@ FUZZERTEST ?= -T200s ZSTDRTTEST = --test-large-data DECODECORPUS_TESTTIME ?= -T30 -.PHONY: default all all32 allnothread dll clean test test32 test-all versionsTest - +.PHONY: default default: fullbench - @echo $(ZSTDMT_OBJECTS) -all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash \ - fullbench-lib poolTests +.PHONY: all +all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash poolTests +.PHONY: all32 all32: fullbench32 fuzzer32 zstreamtest32 +.PHONY: allnothread allnothread: MULTITHREAD_CPP= allnothread: MULTITHREAD_LD= allnothread: fullbench fuzzer paramgrill datagen decodecorpus # note : broken : requires symbols unavailable from dynamic library +.PHONY: dll dll: fuzzer-dll zstreamtest-dll -PHONY: zstd zstd32 zstd-nolegacy # must be phony, only external makefile knows how to build them, or if they need an update +.PHONY: zstd zstd32 zstd-nolegacy # only external makefile knows how to build or update them zstd zstd32 zstd-nolegacy: $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" -gzstd: - $(MAKE) -C $(PRGDIR) $@ HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)" - .PHONY: libzstd libzstd : - $(MAKE) -C $(ZSTDDIR) libzstd + $(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS+="$(DEBUGFLAGS)" %-dll : libzstd -%-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd +%-dll : LDFLAGS += -L$(ZSTDDIR) -lzstd -.PHONY: zstd-staticLib -zstd-staticLib : +.PHONY: $(ZSTDDIR)/libzstd.a +$(ZSTDDIR)/libzstd.a : $(MAKE) -C $(ZSTDDIR) libzstd.a zstdm_%.o : $(ZSTDDIR)/common/%.c @@ -138,25 +137,25 @@ fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD) fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements fullbench fullbench32 : $(ZSTD_FILES) fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c - $(CC) $(FLAGS) $^ -o $@$(EXT) + $(LINK.c) $^ -o $@$(EXT) fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -fullbench-lib : zstd-staticLib -fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c - $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) $(ZSTDDIR)/libzstd.a +fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(ZSTDDIR)/libzstd.a fullbench.c + $(LINK.c) $^ -o $@$(EXT) # note : broken : requires symbols unavailable from dynamic library fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/timefn.c fullbench.c # $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) -fuzzer : LDFLAGS += $(MULTITHREAD_LD) -fuzzer32: CFLAGS += -m32 -fuzzer : $(ZSTDMT_OBJECTS) -fuzzer32: $(ZSTD_FILES) +fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) +fuzzer : LDFLAGS += $(MULTITHREAD_LD) +fuzzer : $(ZSTDMT_OBJECTS) fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c - $(CC) $(FLAGS) $^ -o $@$(EXT) + +fuzzer32 : CFLAGS += -m32 $(MULTITHREAD) +fuzzer32 : $(ZSTD_FILES) + $(LINK.c) $^ -o $@$(EXT) # note : broken : requires symbols unavailable from dynamic library fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c @@ -180,66 +179,65 @@ zstreamtest zstreamtest32 : LDFLAGS += $(MULTITHREAD_LD) zstreamtest : $(ZSTDMT_OBJECTS) $(ZSTREAM_PROPER_FILES) zstreamtest32 : $(ZSTREAMFILES) zstreamtest zstreamtest32 : - $(CC) $(FLAGS) $^ -o $@$(EXT) + $(LINK.c) $^ -o $@$(EXT) zstreamtest_asan : CFLAGS += -fsanitize=address zstreamtest_asan : $(ZSTREAMFILES) - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) + $(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT) zstreamtest_tsan : CFLAGS += -fsanitize=thread zstreamtest_tsan : $(ZSTREAMFILES) - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) + $(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT) # note : broken : requires symbols unavailable from dynamic library zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -paramgrill : DEBUGFLAGS = # turn off assert() by default for speed measurements +paramgrill : DEBUGFLAGS = # turn off debug for speed measurements +paramgrill : LDLIBS += -lm paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c - $(CC) $(FLAGS) $^ -lm -o $@$(EXT) datagen : $(PRGDIR)/datagen.c datagencli.c - $(CC) $(FLAGS) $^ -o $@$(EXT) + $(LINK.c) $^ -o $@$(EXT) +roundTripCrash: CFLAGS += $(MULTITHREAD) roundTripCrash : $(ZSTD_OBJECTS) roundTripCrash.c - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) -longmatch : $(ZSTD_OBJECTS) longmatch.c - $(CC) $(FLAGS) $^ -o $@$(EXT) +longmatch : $(ZSTD_OBJECTS) longmatch.c +bigdict: CFLAGS += $(MULTITHREAD) bigdict: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c bigdict.c - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c - $(CC) $(FLAGS) $^ -o $@$(EXT) legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -DZSTD_LEGACY_SUPPORT=4 legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c - $(CC) $(FLAGS) $^ -o $@$(EXT) +decodecorpus : LDLIBS += -lm decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c decodecorpus.c - $(CC) $(FLAGS) $^ -o $@$(EXT) -lm poolTests : $(PRGDIR)/util.c $(PRGDIR)/timefn.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) + $(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT) .PHONY: versionsTest versionsTest: clean $(PYTHON) test-zstd-versions.py +.PHONY: automated_benchmarking automated_benchmarking: clean $(PYTHON) automated_benchmarking.py -checkTag: checkTag.c $(ZSTDDIR)/zstd.h - $(CC) $(FLAGS) $< -o $@$(EXT) +# make checkTag +checkTag.o : $(ZSTDDIR)/zstd.h +.PHONY: clean clean: $(MAKE) -C $(ZSTDDIR) clean $(MAKE) -C $(PRGDIR) clean - @$(RM) -fR $(TESTARTEFACT) - @$(RM) -rf tmp* # some test directories are named tmp* - @$(RM) core *.o *.tmp result* *.gcda dictionary *.zst \ + $(RM) -fR $(TESTARTEFACT) + $(RM) -rf tmp* # some test directories are named tmp* + $(RM) core *.o *.tmp result* *.gcda dictionary *.zst \ $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fullbench-lib$(EXT) fullbench-dll$(EXT) \ @@ -255,7 +253,8 @@ clean: #---------------------------------------------------------------------------------- # valgrind tests are validated only for some posix platforms #---------------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) +UNAME := $(shell uname) +ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) HOST_OS = POSIX valgrindTest: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1 @@ -275,8 +274,8 @@ valgrindTest: zstd datagen fuzzer fullbench endif -ifneq (,$(filter MINGW% MSYS%,$(shell uname))) -HOST_OS = MSYS +ifneq (,$(filter MINGW% MSYS%,$(UNAME))) + HOST_OS = MSYS endif @@ -286,8 +285,8 @@ endif ifneq (,$(filter $(HOST_OS),MSYS POSIX)) DIFF:=diff -ifneq (,$(filter $(shell uname),SunOS)) -DIFF:=gdiff +ifneq (,$(filter $(UNAME),SunOS)) + DIFF:=gdiff endif .PHONY: list @@ -295,9 +294,12 @@ list: @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs .PHONY: shortest -shortest: ZSTDRTTEST= +shortest: ZSTDRTTEST= # remove long tests shortest: test-zstd +.PHONY: check +check: shortest + .PHONY: fuzztest fuzztest: test-fuzzer test-zstream test-decodecorpus @@ -307,11 +309,12 @@ ifeq ($(QEMU_SYS),) test: test-pool endif +.PHONY: test32 test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 +.PHONY: test-all test-all: test test32 valgrindTest test-decodecorpus-cli - .PHONY: test-zstd test-zstd32 test-zstd-nolegacy test-zstdgrep test-zstd: ZSTD = $(PRGDIR)/zstd test-zstd: zstd @@ -326,35 +329,6 @@ test-zstd test-zstd32 test-zstd-nolegacy: datagen file $(ZSTD) EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST) - -test-gzstd: gzstd - $(PRGDIR)/zstd -f README.md test-zstd-speed.py - gzip -f README.md test-zstd-speed.py - cat README.md.zst test-zstd-speed.py.gz >zstd_gz.zst - cat README.md.gz test-zstd-speed.py.zst >gz_zstd.gz - $(PRGDIR)/zstd -df README.md.gz -o README2.md - $(PRGDIR)/zstd -df README.md.gz test-zstd-speed.py.gz - $(PRGDIR)/zstd -df zstd_gz.zst gz_zstd.gz - $(DIFF) -q zstd_gz gz_zstd - echo Hello World ZSTD | $(PRGDIR)/zstd -c - >hello.zst - echo Hello World GZIP | gzip -c - >hello.gz - echo Hello World TEXT >hello.txt - cat hello.zst hello.gz hello.txt >hello_zst_gz_txt.gz - $(PRGDIR)/zstd -dcf hello.* - $(PRGDIR)/zstd -dcf - <hello_zst_gz_txt.gz - $(RM) *.gz *.zst README2.md gz_zstd zstd_gz hello.txt - -test-zstdgrep: gzstd - -[ -f /tmp/zstdcat ] || ln -s $(PWD)/$(PRGDIR)/zstd /tmp/zstdcat - echo a | $(PRGDIR)/zstd | env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep a - echo a | $(PRGDIR)/zstd | env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep b && return 1 || return 0 - -echo 'hello world' > test.txt && $(PRGDIR)/zstd test.txt - env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep hello test.txt.zst - env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep weird test.txt.zst && return 1 || return 0 - -echo 'hello' > pattern.txt - env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep -f pattern.txt test.txt.zst - $(RM) test.txt test.txt.zst pattern.txt - test-fullbench: fullbench datagen $(QEMU_SYS) ./fullbench -i1 $(QEMU_SYS) ./fullbench -i1 -P0 @@ -380,7 +354,6 @@ test-zbuff32: zbufftest32 test-zstream: zstreamtest $(QEMU_SYS) ./zstreamtest -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) - $(QEMU_SYS) ./zstreamtest --mt -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) $(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) test-zstream32: zstreamtest32 diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index a46fc24d..50935d31 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -199,7 +199,7 @@ typedef struct { int hufInit; /* the distribution used in the previous block for repeat mode */ BYTE hufDist[DISTSIZE]; - U32 hufTable [256]; /* HUF_CElt is an incomplete type */ + HUF_CElt hufTable [256]; int fseInit; FSE_CTable offcodeCTable [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; @@ -535,7 +535,7 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con * actual data to avoid bugs with symbols that were in the * distribution but never showed up in the output */ hufHeaderSize = writeHufHeader( - seed, (HUF_CElt*)frame->stats.hufTable, op, opend - op, + seed, frame->stats.hufTable, op, opend - op, frame->stats.hufDist, DISTSIZE); CHECKERR(hufHeaderSize); /* repeat until a valid header is written */ @@ -558,10 +558,10 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con sizeFormat == 0 ? HUF_compress1X_usingCTable( op, opend - op, LITERAL_BUFFER, litSize, - (HUF_CElt*)frame->stats.hufTable) + frame->stats.hufTable) : HUF_compress4X_usingCTable( op, opend - op, LITERAL_BUFFER, litSize, - (HUF_CElt*)frame->stats.hufTable); + frame->stats.hufTable); CHECKERR(compressedSize); /* this only occurs when it could not compress or similar */ } while (compressedSize <= 0); @@ -815,7 +815,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, BYTE* const oend = (BYTE*)frame->dataEnd; BYTE* op = (BYTE*)frame->data; BYTE* seqHead; - BYTE scratchBuffer[1<<MAX(MLFSELog,LLFSELog)]; + BYTE scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE(MaxSeq, MaxFSELog)]; /* literals compressing block removed so that can be done separately */ @@ -852,18 +852,18 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, LLtype = set_rle; } else if (!(RAND(seed) & 3)) { /* maybe use the default distribution */ - FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); + CHECKERR(FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer))); LLtype = set_basic; } else { /* fall back on a full table */ size_t nbSeq_1 = nbSeq; const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max); if (count[llCodeTable[nbSeq-1]]>1) { count[llCodeTable[nbSeq-1]]--; nbSeq_1--; } - FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); + FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max, nbSeq >= 2048); { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ if (FSE_isError(NCountSize)) return ERROR(GENERIC); op += NCountSize; } - FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); + CHECKERR(FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer))); LLtype = set_compressed; } } @@ -887,7 +887,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, size_t nbSeq_1 = nbSeq; const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max); if (count[ofCodeTable[nbSeq-1]]>1) { count[ofCodeTable[nbSeq-1]]--; nbSeq_1--; } - FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); + FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max, nbSeq >= 2048); { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ if (FSE_isError(NCountSize)) return ERROR(GENERIC); op += NCountSize; } @@ -917,7 +917,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, size_t nbSeq_1 = nbSeq; const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max); if (count[mlCodeTable[nbSeq-1]]>1) { count[mlCodeTable[nbSeq-1]]--; nbSeq_1--; } - FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); + FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max, nbSeq >= 2048); { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ if (FSE_isError(NCountSize)) return ERROR(GENERIC); op += NCountSize; } diff --git a/tests/fullbench.c b/tests/fullbench.c index f0179a9d..37f0e242 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -21,6 +21,7 @@ #include "mem.h" /* U32 */ #ifndef ZSTD_DLL_IMPORT #include "zstd_internal.h" /* ZSTD_decodeSeqHeaders, ZSTD_blockHeaderSize, ZSTD_getcBlockSize, blockType_e, KB, MB */ + #include "decompress/zstd_decompress_internal.h" /* ZSTD_DCtx struct */ #else #define KB *(1 <<10) #define MB *(1 <<20) @@ -134,6 +135,65 @@ static size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* (void)src; (void)srcSize; (void)dst; (void)dstSize; return ZSTD_decodeSeqHeaders(g_zdc, &nbSeq, buff2, g_cSize); } + +FORCE_NOINLINE size_t ZSTD_decodeLiteralsHeader(ZSTD_DCtx* dctx, void const* src, size_t srcSize) +{ + RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); + { + BYTE const* istart = (BYTE const*)src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + if (litEncType == set_compressed) { + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3"); + { + size_t lhSize, litSize, litCSize; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); + break; + } + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); +#ifndef HUF_FORCE_DECOMPRESS_X2 + return HUF_readDTableX1_wksp_bmi2( + dctx->entropy.hufTable, + istart+lhSize, litCSize, + dctx->workspace, sizeof(dctx->workspace), + dctx->bmi2); +#else + return HUF_readDTableX2_wksp( + dctx->entropy.hufTable, + istart+lhSize, litCSize, + dctx->workspace, sizeof(dctx->workspace)); +#endif + } + } + } + return 0; +} + +static size_t local_ZSTD_decodeLiteralsHeader(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) +{ + (void)dst, (void)dstSize, (void)src, (void)srcSize; + return ZSTD_decodeLiteralsHeader(g_zdc, buff2, g_cSize); +} #endif static ZSTD_CStream* g_cstream= NULL; @@ -172,23 +232,40 @@ local_ZSTD_compressStream_freshCCtx(const void* src, size_t srcSize, r = local_ZSTD_compressStream(src, srcSize, dst, dstCapacity, payload); ZSTD_freeCCtx(cctx); - return r; } static size_t -local_ZSTD_compress_generic_end(const void* src, size_t srcSize, - void* dst, size_t dstCapacity, - void* payload) +local_ZSTD_compress2(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) { (void)payload; return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize); } static size_t -local_ZSTD_compress_generic_continue(const void* src, size_t srcSize, - void* dst, size_t dstCapacity, - void* payload) +local_ZSTD_compressStream2_end(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) +{ + ZSTD_outBuffer buffOut; + ZSTD_inBuffer buffIn; + (void)payload; + buffOut.dst = dst; + buffOut.size = dstCapacity; + buffOut.pos = 0; + buffIn.src = src; + buffIn.size = srcSize; + buffIn.pos = 0; + ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end); + return buffOut.pos; +} + +static size_t +local_ZSTD_compressStream2_continue(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) { ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; @@ -358,6 +435,9 @@ static int benchMem(unsigned benchNb, case 13: benchFunction = local_ZSTD_decompressContinue; benchName = "decompressContinue"; break; + case 30: + benchFunction = local_ZSTD_decodeLiteralsHeader; benchName = "decodeLiteralsHeader"; + break; case 31: benchFunction = local_ZSTD_decodeLiteralsBlock; benchName = "decodeLiteralsBlock"; break; @@ -374,11 +454,17 @@ static int benchMem(unsigned benchNb, case 43: benchFunction = local_ZSTD_compressStream_freshCCtx; benchName = "compressStream_freshCCtx"; break; + case 50: + benchFunction = local_ZSTD_compress2; benchName = "compress2"; + break; case 51: - benchFunction = local_ZSTD_compress_generic_continue; benchName = "compress_generic, continue"; + benchFunction = local_ZSTD_compressStream2_end; benchName = "compressStream2, end"; break; case 52: - benchFunction = local_ZSTD_compress_generic_end; benchName = "compress_generic, end"; + benchFunction = local_ZSTD_compressStream2_end; benchName = "compressStream2, end & short"; + break; + case 53: + benchFunction = local_ZSTD_compressStream2_continue; benchName = "compressStream2, continue"; break; case 61: benchFunction = local_ZSTD_compress_generic_T2_continue; benchName = "compress_generic, -T2, continue"; @@ -446,6 +532,8 @@ static int benchMem(unsigned benchNb, case 13 : g_cSize = ZSTD_compress(dstBuff2, dstBuffSize, src, srcSize, cLevel); break; + case 30: /* ZSTD_decodeLiteralsHeader */ + /* fall-through */ case 31: /* ZSTD_decodeLiteralsBlock : starts literals block in dstBuff2 */ { size_t frameHeaderSize; g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); @@ -508,6 +596,11 @@ static int benchMem(unsigned benchNb, payload = &cparams; break; + case 52 : + /* compressStream2, short dstCapacity */ + dstBuffSize--; + break; + /* test functions */ /* convention: test functions have ID > 100 */ diff --git a/tests/fuzz/.gitignore b/tests/fuzz/.gitignore index b6fc6e50..9bd280c0 100644 --- a/tests/fuzz/.gitignore +++ b/tests/fuzz/.gitignore @@ -13,7 +13,12 @@ simple_round_trip stream_decompress stream_round_trip zstd_frame_info +decompress_dstSize_tooSmall +fse_read_ncount +sequence_compression_api fuzz-*.log +rt_lib_* +d_lib_* # misc trace diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile index 1af3dc73..36232a8c 100644 --- a/tests/fuzz/Makefile +++ b/tests/fuzz/Makefile @@ -95,9 +95,12 @@ FUZZ_TARGETS := \ simple_compress \ dictionary_loader \ raw_dictionary_round_trip \ - dictionary_stream_round_trip + dictionary_stream_round_trip \ + decompress_dstSize_tooSmall \ + fse_read_ncount \ + sequence_compression_api -all: $(FUZZ_TARGETS) +all: libregression.a $(FUZZ_TARGETS) rt_lib_common_%.o: $(ZSTDDIR)/common/%.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ @@ -180,6 +183,15 @@ zstd_frame_info: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_zstd_frame_info.o dictionary_loader: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_loader.o $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_loader.o $(LIB_FUZZING_ENGINE) -o $@ +decompress_dstSize_tooSmall: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_decompress_dstSize_tooSmall.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_decompress_dstSize_tooSmall.o $(LIB_FUZZING_ENGINE) -o $@ + +fse_read_ncount: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_fse_read_ncount.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_fse_read_ncount.o $(LIB_FUZZING_ENGINE) -o $@ + +sequence_compression_api: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_sequence_compression_api.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_sequence_compression_api.o $(LIB_FUZZING_ENGINE) -o $@ + libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c d_fuzz_regression_driver.o $(AR) $(FUZZ_ARFLAGS) $@ d_fuzz_regression_driver.o @@ -201,12 +213,10 @@ regressiontest: corpora $(PYTHON) ./fuzz.py regression all clean: - @$(RM) *.a *.o - @$(RM) simple_round_trip stream_round_trip simple_decompress \ - stream_decompress block_decompress block_round_trip \ - simple_compress dictionary_round_trip dictionary_decompress \ - zstd_frame_info + @$(RM) *.a *.o $(FUZZ_TARGETS) + @echo Cleaning completed cleanall: @$(RM) -r Fuzzer @$(RM) -r corpora + @echo Cleaning completed diff --git a/tests/fuzz/decompress_dstSize_tooSmall.c b/tests/fuzz/decompress_dstSize_tooSmall.c new file mode 100644 index 00000000..e47b3d04 --- /dev/null +++ b/tests/fuzz/decompress_dstSize_tooSmall.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target attempts to decompress a valid compressed frame into + * an output buffer that is too small to ensure we always get + * ZSTD_error_dstSize_tooSmall. + */ + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include "fuzz_helpers.h" +#include "zstd.h" +#include "zstd_errors.h" +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" + +static ZSTD_CCtx *cctx = NULL; +static ZSTD_DCtx *dctx = NULL; + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t rBufSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_remainingBytes(producer); + /* Ensure the round-trip buffer is too small. */ + if (rBufSize >= size) { + rBufSize = size > 0 ? size - 1 : 0; + } + size_t const cBufSize = ZSTD_compressBound(size); + + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + void *cBuf = FUZZ_malloc(cBufSize); + void *rBuf = FUZZ_malloc(rBufSize); + size_t const cSize = ZSTD_compressCCtx(cctx, cBuf, cBufSize, src, size, 1); + FUZZ_ZASSERT(cSize); + size_t const rSize = ZSTD_decompressDCtx(dctx, rBuf, rBufSize, cBuf, cSize); + if (size == 0) { + FUZZ_ASSERT(rSize == 0); + } else { + FUZZ_ASSERT(ZSTD_isError(rSize)); + FUZZ_ASSERT(ZSTD_getErrorCode(rSize) == ZSTD_error_dstSize_tooSmall); + } + free(cBuf); + free(rBuf); + FUZZ_dataProducer_free(producer); +#ifndef STATEFUL_FUZZING + ZSTD_freeCCtx(cctx); cctx = NULL; + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/tests/fuzz/fse_read_ncount.c b/tests/fuzz/fse_read_ncount.c new file mode 100644 index 00000000..e20a9382 --- /dev/null +++ b/tests/fuzz/fse_read_ncount.c @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target round trips the FSE normalized count with FSE_writeNCount() + * and FSE_readNcount() to ensure that it can always round trip correctly. + */ + +#define FSE_STATIC_LINKING_ONLY +#define ZSTD_STATIC_LINKING_ONLY + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "fuzz_helpers.h" +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" +#include "fse.h" + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + + /* Pick a random tableLog and maxSymbolValue */ + unsigned const tableLog = FUZZ_dataProducer_uint32Range(producer, FSE_MIN_TABLELOG, FSE_MAX_TABLELOG); + unsigned const maxSymbolValue = FUZZ_dataProducer_uint32Range(producer, 0, 255); + + unsigned remainingWeight = (1u << tableLog) - 1; + size_t dataSize; + BYTE data[512]; + short ncount[256]; + + /* Randomly fill the normalized count */ + memset(ncount, 0, sizeof(ncount)); + { + unsigned s; + for (s = 0; s < maxSymbolValue && remainingWeight > 0; ++s) { + short n = (short)FUZZ_dataProducer_int32Range(producer, -1, remainingWeight); + ncount[s] = n; + if (n < 0) { + remainingWeight -= 1; + } else { + assert((unsigned)n <= remainingWeight); + remainingWeight -= n; + } + } + /* Ensure ncount[maxSymbolValue] != 0 and the sum is (1<<tableLog) */ + ncount[maxSymbolValue] = remainingWeight + 1; + if (ncount[maxSymbolValue] == 1 && FUZZ_dataProducer_uint32Range(producer, 0, 1) == 1) { + ncount[maxSymbolValue] = -1; + } + } + /* Write the normalized count */ + { + FUZZ_ASSERT(sizeof(data) >= FSE_NCountWriteBound(maxSymbolValue, tableLog)); + dataSize = FSE_writeNCount(data, sizeof(data), ncount, maxSymbolValue, tableLog); + FUZZ_ZASSERT(dataSize); + } + /* Read & validate the normalized count */ + { + short rtNcount[256]; + unsigned rtMaxSymbolValue = 255; + unsigned rtTableLog; + /* Copy into a buffer with a random amount of random data at the end */ + size_t const buffSize = (size_t)FUZZ_dataProducer_uint32Range(producer, dataSize, sizeof(data)); + BYTE* const buff = FUZZ_malloc(buffSize); + size_t rtDataSize; + memcpy(buff, data, dataSize); + { + size_t b; + for (b = dataSize; b < buffSize; ++b) { + buff[b] = (BYTE)FUZZ_dataProducer_uint32Range(producer, 0, 255); + } + } + + rtDataSize = FSE_readNCount(rtNcount, &rtMaxSymbolValue, &rtTableLog, buff, buffSize); + FUZZ_ZASSERT(rtDataSize); + FUZZ_ASSERT(rtDataSize == dataSize); + FUZZ_ASSERT(rtMaxSymbolValue == maxSymbolValue); + FUZZ_ASSERT(rtTableLog == tableLog); + { + unsigned s; + for (s = 0; s <= maxSymbolValue; ++s) { + FUZZ_ASSERT(ncount[s] == rtNcount[s]); + } + } + free(buff); + } + + FUZZ_dataProducer_free(producer); + return 0; +} diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py index 6875d1d6..ef94a53b 100755 --- a/tests/fuzz/fuzz.py +++ b/tests/fuzz/fuzz.py @@ -59,6 +59,9 @@ TARGET_INFO = { 'dictionary_loader': TargetInfo(InputType.DICTIONARY_DATA), 'raw_dictionary_round_trip': TargetInfo(InputType.RAW_DATA), 'dictionary_stream_round_trip': TargetInfo(InputType.RAW_DATA), + 'decompress_dstSize_tooSmall': TargetInfo(InputType.RAW_DATA), + 'fse_read_ncount': TargetInfo(InputType.RAW_DATA), + 'sequence_compression_api': TargetInfo(InputType.RAW_DATA), } TARGETS = list(TARGET_INFO.keys()) ALL_TARGETS = TARGETS + ['all'] diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c index 6518af30..f2d5a1b5 100644 --- a/tests/fuzz/fuzz_data_producer.c +++ b/tests/fuzz/fuzz_data_producer.c @@ -66,6 +66,10 @@ size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){ return producer->size; } +int FUZZ_dataProducer_empty(FUZZ_dataProducer_t *producer) { + return producer->size == 0; +} + size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize) { newSize = newSize > producer->size ? producer->size : newSize; diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h index 41e0b52d..25cc937f 100644 --- a/tests/fuzz/fuzz_data_producer.h +++ b/tests/fuzz/fuzz_data_producer.h @@ -49,6 +49,9 @@ int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer, /* Returns the size of the remaining bytes of data in the producer */ size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer); +/* Returns true if the data producer is out of bytes */ +int FUZZ_dataProducer_empty(FUZZ_dataProducer_t *producer); + /* Restricts the producer to only the last newSize bytes of data. If newSize > current data size, nothing happens. Returns the number of bytes the producer won't use anymore, after contracting. */ diff --git a/tests/fuzz/sequence_compression_api.c b/tests/fuzz/sequence_compression_api.c new file mode 100644 index 00000000..e838687a --- /dev/null +++ b/tests/fuzz/sequence_compression_api.c @@ -0,0 +1,303 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target performs a zstd round-trip test by generating an arbitrary + * array of sequences, generating the associated source buffer, calling + * ZSTD_compressSequences(), and then decompresses and compares the result with + * the original generated source buffer. + */ + +#define ZSTD_STATIC_LINKING_ONLY + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <time.h> +#include "fuzz_helpers.h" +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" + +static ZSTD_CCtx *cctx = NULL; +static ZSTD_DCtx *dctx = NULL; +static void* literalsBuffer = NULL; +static void* generatedSrc = NULL; +static ZSTD_Sequence* generatedSequences = NULL; + +#define ZSTD_FUZZ_GENERATED_SRC_MAXSIZE (1 << 20) /* Allow up to 1MB generated data */ +#define ZSTD_FUZZ_MATCHLENGTH_MAXSIZE (1 << 18) /* Allow up to 256KB matches */ +#define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << 18) /* Allow up to a 256KB dict */ +#define ZSTD_FUZZ_GENERATED_LITERALS_SIZE (1 << 18) /* Fixed size 256KB literals buffer */ +#define ZSTD_FUZZ_MAX_NBSEQ (1 << 17) /* Maximum of 128K sequences */ + +/* Deterministic random number generator */ +#define FUZZ_RDG_rotl32(x,r) ((x << r) | (x >> (32 - r))) +static uint32_t FUZZ_RDG_rand(uint32_t* src) +{ + static const uint32_t prime1 = 2654435761U; + static const uint32_t prime2 = 2246822519U; + uint32_t rand32 = *src; + rand32 *= prime1; + rand32 ^= prime2; + rand32 = FUZZ_RDG_rotl32(rand32, 13); + *src = rand32; + return rand32 >> 5; +} + +/* Make a pseudorandom string - this simple function exists to avoid + * taking a dependency on datagen.h to have RDG_genBuffer(). + */ +static char *generatePseudoRandomString(char *str, size_t size) { + const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_"; + uint32_t seed = 0; + if (size) { + for (size_t n = 0; n < size; n++) { + int key = FUZZ_RDG_rand(&seed) % (int) (sizeof charset - 1); + str[n] = charset[key]; + } + } + return str; +} + +/* Returns size of source buffer */ +static size_t decodeSequences(void* dst, size_t nbSequences, + size_t literalsSize, const void* dict, size_t dictSize) { + const uint8_t* litPtr = literalsBuffer; + const uint8_t* const litBegin = literalsBuffer; + const uint8_t* const litEnd = literalsBuffer + literalsSize; + const uint8_t* dictPtr = dict; + uint8_t* op = dst; + const uint8_t* const oend = dst + ZSTD_FUZZ_GENERATED_SRC_MAXSIZE; + size_t generatedSrcBufferSize = 0; + size_t bytesWritten = 0; + uint32_t lastLLSize; + + for (size_t i = 0; i < nbSequences; ++i) { + FUZZ_ASSERT(generatedSequences[i].matchLength != 0); + FUZZ_ASSERT(generatedSequences[i].offset != 0); + + if (litPtr + generatedSequences[i].litLength > litEnd) { + litPtr = litBegin; + } + ZSTD_memcpy(op, litPtr, generatedSequences[i].litLength); + bytesWritten += generatedSequences[i].litLength; + op += generatedSequences[i].litLength; + litPtr += generatedSequences[i].litLength; + + FUZZ_ASSERT(generatedSequences[i].offset != 0); + /* Copy over the match */ + { size_t matchLength = generatedSequences[i].matchLength; + size_t j = 0; + size_t k = 0; + if (dictSize != 0) { + if (generatedSequences[i].offset > bytesWritten) { + /* Offset goes into the dictionary */ + size_t offsetFromEndOfDict = generatedSequences[i].offset - bytesWritten; + for (; k < offsetFromEndOfDict && k < matchLength; ++k) { + op[k] = dictPtr[dictSize - offsetFromEndOfDict + k]; + } + matchLength -= k; + op += k; + } + } + for (; j < matchLength; ++j) { + op[j] = op[j-(int)generatedSequences[i].offset]; + } + op += j; + FUZZ_ASSERT(generatedSequences[i].matchLength == j + k); + bytesWritten += generatedSequences[i].matchLength; + } + } + generatedSrcBufferSize = bytesWritten; + FUZZ_ASSERT(litPtr <= litEnd); + lastLLSize = (uint32_t)(litEnd - litPtr); + if (lastLLSize <= oend - op) { + ZSTD_memcpy(op, litPtr, lastLLSize); + generatedSrcBufferSize += lastLLSize; + } + return generatedSrcBufferSize; +} + +/* Returns nb sequences generated + * TODO: Add repcode fuzzing once we support repcode match splits + */ +static size_t generateRandomSequences(FUZZ_dataProducer_t* producer, + size_t literalsSizeLimit, size_t dictSize, + size_t windowLog) { + uint32_t bytesGenerated = 0; + uint32_t nbSeqGenerated = 0; + uint32_t litLength; + uint32_t matchLength; + uint32_t matchBound; + uint32_t offset; + uint32_t offsetBound; + uint32_t repCode = 0; + uint32_t isFirstSequence = 1; + uint32_t windowSize = 1 << windowLog; + + while (nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ + && bytesGenerated < ZSTD_FUZZ_GENERATED_SRC_MAXSIZE + && !FUZZ_dataProducer_empty(producer)) { + matchBound = ZSTD_FUZZ_MATCHLENGTH_MAXSIZE; + litLength = isFirstSequence && dictSize == 0 ? FUZZ_dataProducer_uint32Range(producer, 1, literalsSizeLimit) + : FUZZ_dataProducer_uint32Range(producer, 0, literalsSizeLimit); + bytesGenerated += litLength; + if (bytesGenerated > ZSTD_FUZZ_GENERATED_SRC_MAXSIZE) { + break; + } + offsetBound = bytesGenerated > windowSize ? windowSize : bytesGenerated + dictSize; + offset = FUZZ_dataProducer_uint32Range(producer, 1, offsetBound); + if (dictSize > 0 && bytesGenerated <= windowSize) { + /* Prevent match length from being such that it would be associated with an offset too large + * from the decoder's perspective. If not possible (match would be too small), + * then reduce the offset if necessary. + */ + size_t bytesToReachWindowSize = windowSize - bytesGenerated; + if (bytesToReachWindowSize < ZSTD_MINMATCH_MIN) { + uint32_t newOffsetBound = offsetBound > windowSize ? windowSize : offsetBound; + offset = FUZZ_dataProducer_uint32Range(producer, 1, newOffsetBound); + } else { + matchBound = bytesToReachWindowSize > ZSTD_FUZZ_MATCHLENGTH_MAXSIZE ? + ZSTD_FUZZ_MATCHLENGTH_MAXSIZE : bytesToReachWindowSize; + } + } + matchLength = FUZZ_dataProducer_uint32Range(producer, ZSTD_MINMATCH_MIN, matchBound); + bytesGenerated += matchLength; + if (bytesGenerated > ZSTD_FUZZ_GENERATED_SRC_MAXSIZE) { + break; + } + ZSTD_Sequence seq = {offset, litLength, matchLength, repCode}; + generatedSequences[nbSeqGenerated++] = seq; + isFirstSequence = 0; + } + + return nbSeqGenerated; +} + +static size_t roundTripTest(void *result, size_t resultCapacity, + void *compressed, size_t compressedCapacity, + size_t srcSize, + const void *dict, size_t dictSize, + size_t generatedSequencesSize, + size_t wLog, unsigned cLevel, unsigned hasDict) +{ + size_t cSize; + size_t dSize; + ZSTD_CDict* cdict = NULL; + ZSTD_DDict* ddict = NULL; + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1); + /* TODO: Add block delim mode fuzzing */ + ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters); + if (hasDict) { + FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary(cctx, dict, dictSize)); + FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary(dctx, dict, dictSize)); + } + + cSize = ZSTD_compressSequences(cctx, compressed, compressedCapacity, + generatedSequences, generatedSequencesSize, + generatedSrc, srcSize); + FUZZ_ZASSERT(cSize); + dSize = ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize); + FUZZ_ZASSERT(dSize); + + if (cdict) { + ZSTD_freeCDict(cdict); + } + if (ddict) { + ZSTD_freeDDict(ddict); + } + return dSize; +} + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + void* rBuf; + size_t rBufSize; + void* cBuf; + size_t cBufSize; + size_t generatedSrcSize; + size_t nbSequences; + void* dictBuffer; + size_t dictSize = 0; + unsigned hasDict; + unsigned wLog; + int cLevel; + + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + if (literalsBuffer == NULL) { + literalsBuffer = FUZZ_malloc(ZSTD_FUZZ_GENERATED_LITERALS_SIZE); + literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE); + } + + hasDict = FUZZ_dataProducer_uint32Range(producer, 0, 1); + if (hasDict) { + dictSize = FUZZ_dataProducer_uint32Range(producer, 1, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE); + dictBuffer = FUZZ_malloc(dictSize); + dictBuffer = generatePseudoRandomString(dictBuffer, dictSize); + } + /* Generate window log first so we dont generate offsets too large */ + wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX_32); + cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22); + + if (!generatedSequences) { + generatedSequences = FUZZ_malloc(sizeof(ZSTD_Sequence)*ZSTD_FUZZ_MAX_NBSEQ); + } + if (!generatedSrc) { + generatedSrc = FUZZ_malloc(ZSTD_FUZZ_GENERATED_SRC_MAXSIZE); + } + nbSequences = generateRandomSequences(producer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictSize, wLog); + generatedSrcSize = decodeSequences(generatedSrc, nbSequences, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictBuffer, dictSize); + cBufSize = ZSTD_compressBound(generatedSrcSize); + cBuf = FUZZ_malloc(cBufSize); + + rBufSize = generatedSrcSize; + rBuf = FUZZ_malloc(rBufSize); + + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + size_t const result = roundTripTest(rBuf, rBufSize, + cBuf, cBufSize, + generatedSrcSize, + dictBuffer, dictSize, + nbSequences, + wLog, cLevel, hasDict); + FUZZ_ZASSERT(result); + FUZZ_ASSERT_MSG(result == generatedSrcSize, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!FUZZ_memcmp(generatedSrc, rBuf, generatedSrcSize), "Corruption!"); + + free(rBuf); + free(cBuf); + FUZZ_dataProducer_free(producer); + if (hasDict) { + free(dictBuffer); + } +#ifndef STATEFUL_FUZZING + ZSTD_freeCCtx(cctx); cctx = NULL; + ZSTD_freeDCtx(dctx); dctx = NULL; + free(generatedSequences); generatedSequences = NULL; + free(generatedSrc); generatedSrc = NULL; + free(literalsBuffer); literalsBuffer = NULL; +#endif + return 0; +} diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c index b64f373e..620177fb 100644 --- a/tests/fuzz/simple_compress.c +++ b/tests/fuzz/simple_compress.c @@ -19,6 +19,7 @@ #include <stdio.h> #include "fuzz_helpers.h" #include "zstd.h" +#include "zstd_errors.h" #include "zstd_helpers.h" #include "fuzz_data_producer.h" @@ -42,7 +43,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) } void *rBuf = FUZZ_malloc(bufSize); - ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, cLevel); + size_t const ret = ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, cLevel); + if (ZSTD_isError(ret)) { + FUZZ_ASSERT(ZSTD_getErrorCode(ret) == ZSTD_error_dstSize_tooSmall); + } free(rBuf); FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index 2f008d06..6e58fb1c 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -47,8 +47,12 @@ static size_t roundTripTest(void *result, size_t resultCapacity, FUZZ_ZASSERT(cSize); dSize = ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize); FUZZ_ZASSERT(dSize); - /* When superblock is enabled make sure we don't expand the block more than expected. */ - if (targetCBlockSize != 0) { + /* When superblock is enabled make sure we don't expand the block more than expected. + * NOTE: This test is currently disabled because superblock mode can arbitrarily + * expand the block in the worst case. Once superblock mode has been improved we can + * re-enable this test. + */ + if (0 && targetCBlockSize != 0) { size_t normalCSize; FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 0)); normalCSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c index 25901b1e..5d2bb2aa 100644 --- a/tests/fuzz/stream_decompress.c +++ b/tests/fuzz/stream_decompress.c @@ -22,18 +22,19 @@ #include "zstd.h" #include "fuzz_data_producer.h" -static size_t const kBufSize = ZSTD_BLOCKSIZE_MAX; - static ZSTD_DStream *dstream = NULL; -static void* buf = NULL; uint32_t seed; -static ZSTD_outBuffer makeOutBuffer(FUZZ_dataProducer_t *producer, uint32_t min) +static ZSTD_outBuffer makeOutBuffer(FUZZ_dataProducer_t *producer, void* buf, size_t bufSize) { ZSTD_outBuffer buffer = { buf, 0, 0 }; - buffer.size = (FUZZ_dataProducer_uint32Range(producer, min, kBufSize)); - FUZZ_ASSERT(buffer.size <= kBufSize); + if (FUZZ_dataProducer_empty(producer)) { + buffer.size = bufSize; + } else { + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 0, bufSize)); + } + FUZZ_ASSERT(buffer.size <= bufSize); if (buffer.size == 0) { buffer.dst = NULL; @@ -43,13 +44,16 @@ static ZSTD_outBuffer makeOutBuffer(FUZZ_dataProducer_t *producer, uint32_t min) } static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, - FUZZ_dataProducer_t *producer, - uint32_t min) + FUZZ_dataProducer_t *producer) { ZSTD_inBuffer buffer = { *src, 0, 0 }; FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_dataProducer_uint32Range(producer, min, *size)); + if (FUZZ_dataProducer_empty(producer)) { + buffer.size = *size; + } else { + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 0, *size)); + } FUZZ_ASSERT(buffer.size <= *size); *src += buffer.size; *size -= buffer.size; @@ -66,18 +70,15 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - /* Guarantee forward progress by refusing to generate 2 zero sized - * buffers in a row. */ - int prevInWasZero = 0; - int prevOutWasZero = 0; int stableOutBuffer; ZSTD_outBuffer out; + void* buf; + size_t bufSize; size = FUZZ_dataProducer_reserveDataPrefix(producer); + bufSize = MAX(10 * size, ZSTD_BLOCKSIZE_MAX); /* Allocate all buffers and contexts if not already allocated */ - if (!buf) { - buf = FUZZ_malloc(kBufSize); - } + buf = FUZZ_malloc(bufSize); if (!dstream) { dstream = ZSTD_createDStream(); @@ -90,18 +91,19 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) if (stableOutBuffer) { FUZZ_ZASSERT(ZSTD_DCtx_setParameter(dstream, ZSTD_d_stableOutBuffer, 1)); out.dst = buf; - out.size = kBufSize; + out.size = bufSize; out.pos = 0; + } else { + out = makeOutBuffer(producer, buf, bufSize); } while (size > 0) { - ZSTD_inBuffer in = makeInBuffer(&src, &size, producer, prevInWasZero ? 1 : 0); - prevInWasZero = in.size == 0; + ZSTD_inBuffer in = makeInBuffer(&src, &size, producer); while (in.pos != in.size) { - if (!stableOutBuffer || prevOutWasZero || FUZZ_dataProducer_uint32Range(producer, 0, 100) == 55) { - out = makeOutBuffer(producer, prevOutWasZero ? 1 : 0); + if (out.pos == out.size) { + if (stableOutBuffer) goto error; + out = makeOutBuffer(producer, buf, bufSize); } - prevOutWasZero = out.size == 0; size_t const rc = ZSTD_decompressStream(dstream, &out, &in); if (ZSTD_isError(rc)) goto error; } @@ -112,5 +114,6 @@ error: ZSTD_freeDStream(dstream); dstream = NULL; #endif FUZZ_dataProducer_free(producer); + free(buf); return 0; } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 8ac2864f..2e5d70ef 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -32,7 +32,6 @@ #include "fse.h" #include "zstd.h" /* ZSTD_VERSION_STRING */ #include "zstd_errors.h" /* ZSTD_getErrorCode */ -#include "zstdmt_compress.h" #define ZDICT_STATIC_LINKING_ONLY #include "zdict.h" /* ZDICT_trainFromBuffer */ #include "mem.h" @@ -306,26 +305,34 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) #endif -static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, BYTE* src, size_t size) +static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, + BYTE* src, size_t size, ZSTD_sequenceFormat_e format) { size_t i; size_t j; - for(i = 0; i < seqsSize - 1; ++i) { - assert(dst + seqs[i].litLength + seqs[i].matchLength < dst + size); - assert(src + seqs[i].litLength + seqs[i].matchLength < src + size); + for(i = 0; i < seqsSize; ++i) { + assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size); + assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size); + if (format == ZSTD_sf_noBlockDelimiters) { + assert(seqs[i].matchLength != 0 || seqs[i].offset != 0); + } memcpy(dst, src, seqs[i].litLength); dst += seqs[i].litLength; src += seqs[i].litLength; size -= seqs[i].litLength; - for (j = 0; j < seqs[i].matchLength; ++j) - dst[j] = dst[j - seqs[i].offset]; - dst += seqs[i].matchLength; - src += seqs[i].matchLength; - size -= seqs[i].matchLength; + if (seqs[i].offset != 0) { + for (j = 0; j < seqs[i].matchLength; ++j) + dst[j] = dst[j - seqs[i].offset]; + dst += seqs[i].matchLength; + src += seqs[i].matchLength; + size -= seqs[i].matchLength; + } + } + if (format == ZSTD_sf_noBlockDelimiters) { + memcpy(dst, src, size); } - memcpy(dst, src, size); } /*============================================= @@ -372,6 +379,19 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "%u (OK) \n", vn); } + DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++); + { + ZSTD_compressionParameters params; + memset(¶ms, 0, sizeof(params)); + params.windowLog = 10; + params.hashLog = 19; + params.chainLog = 19; + params = ZSTD_adjustCParams(params, 1000, 100000); + if (params.hashLog != 18) goto _output_error; + if (params.chainLog != 17) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); if (cctx==NULL) goto _output_error; @@ -439,6 +459,17 @@ static int basicUnitTests(U32 const seed, double compressibility) } } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 }; + ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 }; + if (cctx==NULL) goto _output_error; + CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) ); /* must fail */ + CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) ); /* must fail */ + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++); { ZSTD_parameters params = ZSTD_getParams(3, 0, 0); @@ -544,6 +575,45 @@ static int basicUnitTests(U32 const seed, double compressibility) if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++); + { /* create compressed buffer with checksumming enabled */ + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + if (!cctx) { + DISPLAY("Not enough memory, aborting\n"); + testResult = 1; + goto _end; + } + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) ); + CHECK_VAR(cSize, ZSTD_compress2(cctx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize) ); + ZSTD_freeCCtx(cctx); + } + { /* copy the compressed buffer and corrupt the checksum */ + size_t r; + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + if (!dctx) { + DISPLAY("Not enough memory, aborting\n"); + testResult = 1; + goto _end; + } + + ((char*)compressedBuffer)[cSize-1] += 1; + r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (!ZSTD_isError(r)) goto _output_error; + if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error; + + CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum)); + r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1); + if (!ZSTD_isError(r)) goto _output_error; /* wrong checksum size should still throw error */ + r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (ZSTD_isError(r)) goto _output_error; + + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++); { /* create compressed buffer with content size missing */ ZSTD_CCtx* const cctx = ZSTD_createCCtx(); @@ -655,6 +725,48 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++); + { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t const srcSize = 300 KB; + size_t const flushSize = 128 KB + 5; + size_t const dstSize = ZSTD_compressBound(srcSize); + char* src = (char*)CNBuffer; + char* dst = (char*)compressedBuffer; + + ZSTD_outBuffer out = { dst, dstSize, 0 }; + ZSTD_inBuffer in = { src, flushSize, 0 }; + + if (!cctx || !dctx) { + DISPLAY("Not enough memory, aborting\n"); + testResult = 1; + goto _end; + } + + RDG_genBuffer(src, srcSize, 0.5, 0.5, seed); + /* Force an LDM to exist that crosses block boundary into uncompressible block */ + memcpy(src + 125 KB, src, 3 KB + 5); + + /* Enable MT, LDM, and opt parser */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); + + /* Flushes a block of 128 KB and block of 5 bytes */ + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + + /* Compress the rest */ + in.size = 300 KB; + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos)); + + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); @@ -827,36 +939,35 @@ static int basicUnitTests(U32 const seed, double compressibility) assert(cctx != NULL); ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81); - { size_t read; - for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) { - ZSTD_inBuffer in = {src, streamCompressDelta, 0}; - ZSTD_outBuffer out = {dst, dstCapacity, 0}; - CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); - CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); - src += streamCompressDelta; srcSize -= streamCompressDelta; - dst += out.pos; dstCapacity -= out.pos;}} + { size_t read; + for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) { + ZSTD_inBuffer in = {src, streamCompressDelta, 0}; + ZSTD_outBuffer out = {dst, dstCapacity, 0}; + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + src += streamCompressDelta; srcSize -= streamCompressDelta; + dst += out.pos; dstCapacity -= out.pos; + } } /* This is trying to catch a dstSize_tooSmall error */ - { ZSTD_inBuffer in = {src, srcSize, 0}; - ZSTD_outBuffer out = {dst, dstCapacity, 0}; - CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));} + { ZSTD_inBuffer in = {src, srcSize, 0}; + ZSTD_outBuffer out = {dst, dstCapacity, 0}; + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + } ZSTD_freeCCtx(cctx); } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++); /* Generate the same data 20 times over */ - { - size_t const avgChunkSize = CNBuffSize / 20; + { size_t const avgChunkSize = CNBuffSize / 20; size_t b; for (b = 0; b < CNBuffSize; b += avgChunkSize) { size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize); RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed); - } - } - { - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + } } + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); size_t const allowedExpansion = (CNBuffSize * 3 / 1000); size_t superCSize; @@ -875,12 +986,11 @@ static int basicUnitTests(U32 const seed, double compressibility) RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed); DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++) - { - /* This tests whether or not we leave enough room for the checksum at the end - * of the dst buffer. The bug that motivated this test was found by the - * stream_round_trip fuzzer but this crashes for the same reason and is - * far more compact than re-creating the stream_round_trip fuzzer's code path */ - ZSTD_CCtx *cctx = ZSTD_createCCtx(); + /* This tests whether or not we leave enough room for the checksum at the end + * of the dst buffer. The bug that motivated this test was found by the + * stream_round_trip fuzzer but this crashes for the same reason and is + * far more compact than re-creating the stream_round_trip fuzzer's code path */ + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64); assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278))); ZSTD_freeCCtx(cctx); @@ -1097,6 +1207,26 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_outBuffer out = {NULL, 0, 0}; + ZSTD_inBuffer in = {NULL, 0, 0}; + int value; + + /* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt), + * we should not modify the actual parameter specified by the user within the CCtx + */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt)); + + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value)); + CHECK_EQ(value, 0); + + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + /* this test is really too long, and should be made faster */ DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); @@ -1304,19 +1434,20 @@ static int basicUnitTests(U32 const seed, double compressibility) /* ZSTDMT simple MT compression test */ DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++); - { ZSTDMT_CCtx* const mtctx = ZSTDMT_createCCtx(2); + { ZSTD_CCtx* const mtctx = ZSTD_createCCtx(); if (mtctx==NULL) { DISPLAY("mtctx : not enough memory, aborting \n"); testResult = 1; goto _end; } + CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) ); + CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) ); DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize); - CHECK_VAR(cSize, ZSTDMT_compressCCtx(mtctx, + CHECK_VAR(cSize, ZSTD_compress2(mtctx, compressedBuffer, compressedBufferSize, - CNBuffer, CNBuffSize, - 1) ); + CNBuffer, CNBuffSize) ); DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++); @@ -1340,14 +1471,12 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++); - { ZSTD_parameters params = ZSTD_getParams(1, CNBuffSize, 0); - params.fParams.checksumFlag = 1; - params.fParams.contentSizeFlag = 1; - CHECK_VAR(cSize, ZSTDMT_compress_advanced(mtctx, - compressedBuffer, compressedBufferSize, - CNBuffer, CNBuffSize, - NULL, params, 3 /*overlapRLog*/) ); - } + CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) ); + CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) ); + CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) ); + CHECK_VAR(cSize, ZSTD_compress2(mtctx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize) ); DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); @@ -1355,7 +1484,7 @@ static int basicUnitTests(U32 const seed, double compressibility) if (r != CNBuffSize) goto _output_error; } DISPLAYLEVEL(3, "OK \n"); - ZSTDMT_freeCCtx(mtctx); + ZSTD_freeCCtx(mtctx); } DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++); @@ -1573,11 +1702,11 @@ static int basicUnitTests(U32 const seed, double compressibility) const void* const contentStart = (const char*)dict + flatdictSize; size_t const target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770, 3770, 3770, 3770, 3750, 3750, - 3740, 3670, 3670, 3660, 3660, + 3742, 3670, 3670, 3660, 3660, 3660, 3660, 3660, 3660, 3660, 3660, 3660, 3660 }; size_t const target_wdict_cSize[22+1] = { 2830, 2890, 2890, 2820, 2940, - 2950, 2950, 2920, 2900, 2890, + 2950, 2950, 2921, 2900, 2891, 2910, 2910, 2910, 2770, 2760, 2750, 2750, 2750, 2750, 2750, 2750, 2750, 2750 }; @@ -1789,6 +1918,7 @@ static int basicUnitTests(U32 const seed, double compressibility) cParams, ZSTD_defaultCMem); assert(cdict != NULL); DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict)); + assert(ZSTD_getDictID_fromDict(dictBuffer, dictSize) == ZSTD_getDictID_fromCDict(cdict)); cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, cdict); ZSTD_freeCDict(cdict); @@ -1923,34 +2053,53 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++); - { - size_t ret; - MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); - /* Either operation is allowed to fail, but one must fail. */ - ret = ZSTD_CCtx_loadDictionary_advanced( - cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_auto); - if (!ZSTD_isError(ret)) { + { char* rawDictBuffer = (char*)malloc(dictSize); + assert(rawDictBuffer); + memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2); + memset(rawDictBuffer + dictSize - 2, 0, 2); + MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY); + + DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++); + { + size_t ret; + /* Either operation is allowed to fail, but one must fail. */ + ret = ZSTD_CCtx_loadDictionary_advanced( + cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); + if (!ZSTD_isError(ret)) { + ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); + if (!ZSTD_isError(ret)) goto _output_error; + } + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++); + { + size_t ret; + ret = ZSTD_CCtx_loadDictionary_advanced( + cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent); + if (ZSTD_isError(ret)) goto _output_error; ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); - if (!ZSTD_isError(ret)) goto _output_error; + if (ZSTD_isError(ret)) goto _output_error; } - } - DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++); - { - size_t ret; - MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); - ret = ZSTD_CCtx_loadDictionary_advanced( - cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_rawContent); - if (ZSTD_isError(ret)) goto _output_error; - ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); - if (ZSTD_isError(ret)) goto _output_error; + DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++); + { size_t const srcSize = MIN(CNBuffSize, 100); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + /* Force the dictionary to be reloaded in raw content mode */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad)); + CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize); + CHECK_Z(cSize); + } + DISPLAYLEVEL(3, "OK \n"); + + free(rawDictBuffer); } - DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++); { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); @@ -2440,6 +2589,34 @@ static int basicUnitTests(U32 const seed, double compressibility) ZSTD_freeDCtx(dctx); } + DISPLAYLEVEL(3, "test%3i : Decompression parameter reset test : ", testNb++); + { + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + /* Attempt to future proof this to new parameters. */ + int const maxParam = 2000; + int param; + if (ZSTD_d_experimentalParam3 > maxParam) goto _output_error; + for (param = 0; param < maxParam; ++param) { + ZSTD_dParameter dParam = (ZSTD_dParameter)param; + ZSTD_bounds bounds = ZSTD_dParam_getBounds(dParam); + int value1; + int value2; + int check; + if (ZSTD_isError(bounds.error)) + continue; + CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &value1)); + value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound; + CHECK(ZSTD_DCtx_setParameter(dctx, dParam, value2)); + CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check)); + if (check != value2) goto _output_error; + CHECK(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters)); + CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check)); + if (check != value1) goto _output_error; + } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + /* block API tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); @@ -2532,9 +2709,9 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); } - DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences decode from sequences test : ", testNb++); + DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences decode from sequences test : ", testNb++); { - size_t srcSize = 100 KB; + size_t srcSize = 150 KB; BYTE* src = (BYTE*)CNBuffer; BYTE* decoded = (BYTE*)compressedBuffer; @@ -2544,20 +2721,83 @@ static int basicUnitTests(U32 const seed, double compressibility) if (seqs == NULL) goto _output_error; assert(cctx != NULL); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19); + /* Populate src with random data */ + RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed); + + /* Test with block delimiters roundtrip */ + seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); + FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters); + assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); + + /* Test no block delimiters roundtrip */ + seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize); + FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters); + assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); + + ZSTD_freeCCtx(cctx); + free(seqs); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++); + { + size_t srcSize = 500 KB; + BYTE* src = (BYTE*)CNBuffer; + BYTE* dst = (BYTE*)compressedBuffer; + size_t dstSize = ZSTD_compressBound(srcSize); + size_t decompressSize = srcSize; + char* decompressBuffer = (char*)malloc(decompressSize); + size_t compressedSize; + size_t dSize; + + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); + size_t seqsSize; + + if (seqs == NULL) goto _output_error; + assert(cctx != NULL); /* Populate src with random data */ RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed); - /* get the sequences */ - seqsSize = ZSTD_getSequences(cctx, seqs, srcSize, src, srcSize); + /* Test with block delimiters roundtrip */ + seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters); + compressedSize = ZSTD_compressSequences(cctx, dst, dstSize, seqs, seqsSize, src, srcSize); + if (ZSTD_isError(compressedSize)) { + DISPLAY("Error in sequence compression with block delims\n"); + goto _output_error; + } + dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize); + if (ZSTD_isError(dSize)) { + DISPLAY("Error in sequence compression roundtrip with block delims\n"); + goto _output_error; + } + assert(!memcmp(decompressBuffer, src, srcSize)); - /* "decode" and compare the sequences */ - FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize); - assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); + /* Test with no block delimiters roundtrip */ + seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters); + compressedSize = ZSTD_compressSequences(cctx, dst, dstSize, seqs, seqsSize, src, srcSize); + if (ZSTD_isError(compressedSize)) { + DISPLAY("Error in sequence compression with no block delims\n"); + goto _output_error; + } + dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize); + if (ZSTD_isError(dSize)) { + DISPLAY("Error in sequence compression roundtrip with no block delims\n"); + goto _output_error; + } + assert(!memcmp(decompressBuffer, src, srcSize)); ZSTD_freeCCtx(cctx); + free(decompressBuffer); free(seqs); } + DISPLAYLEVEL(3, "OK \n"); /* Multiple blocks of zeros test */ #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */ @@ -2744,15 +2984,13 @@ static int basicUnitTests(U32 const seed, double compressibility) /* Calling FSE_normalizeCount() on a uniform distribution should not * cause a division by zero. */ - FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue); + FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue, /* useLowProbCount */ 1); } DISPLAYLEVEL(3, "OK \n"); #ifdef ZSTD_MULTITHREAD DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++); - { - ZSTD_CCtx* cctx = ZSTD_createCCtx(); - /* A little more than ZSTDMT_JOBSIZE_MIN */ - size_t const srcSize = 1 MB + 5; + { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */ size_t const dstSize = ZSTD_compressBound(srcSize); void* const src = CNBuffer; void* const dst = compressedBuffer; @@ -2769,10 +3007,8 @@ static int basicUnitTests(U32 const seed, double compressibility) /* lie and claim this is a full dict */ CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict)); - { - ZSTD_outBuffer out = {dst, dstSize, 0}; + { ZSTD_outBuffer out = {dst, dstSize, 0}; ZSTD_inBuffer in = {src, srcSize, 0}; - /* should fail because its not a full dict like we said it was */ assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush))); } @@ -2783,10 +3019,8 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++); - { - ZSTD_CCtx* cctx = ZSTD_createCCtx(); - /* A little more than ZSTDMT_JOBSIZE_MIN */ - size_t const srcSize = 1 MB + 5; + { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */ size_t const dictSize = 10; size_t const dstSize = ZSTD_compressBound(srcSize); void* const src = CNBuffer; @@ -2812,10 +3046,40 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); #endif +_end: + free(CNBuffer); + free(compressedBuffer); + free(decodedBuffer); + return testResult; + +_output_error: + testResult = 1; + DISPLAY("Error detected in Unit tests ! \n"); + goto _end; +} + +static int longUnitTests(U32 const seed, double compressibility) +{ + size_t const CNBuffSize = 5 MB; + void* const CNBuffer = malloc(CNBuffSize); + size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); + void* const compressedBuffer = malloc(compressedBufferSize); + void* const decodedBuffer = malloc(CNBuffSize); + int testResult = 0; + unsigned testNb=0; + size_t cSize; + + /* Create compressible noise */ + if (!CNBuffer || !compressedBuffer || !decodedBuffer) { + DISPLAY("Not enough memory, aborting\n"); + testResult = 1; + goto _end; + } + RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); + /* note : this test is rather long, it would be great to find a way to speed up its execution */ - DISPLAYLEVEL(3, "test%3i : table cleanliness through index reduction : ", testNb++); - { - int cLevel; + DISPLAYLEVEL(3, "longtest%3i : table cleanliness through index reduction : ", testNb++); + { int cLevel; size_t approxIndex = 0; size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */ @@ -2883,6 +3147,105 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++); + { + size_t cSizeLdm; + size_t cSizeNoLdm; + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + + RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed); + + /* Enable checksum to verify round trip. */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); + + /* Round trip once with ldm. */ + cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK_Z(cSizeLdm); + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm)); + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 0)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); + + /* Round trip once without ldm. */ + cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK_Z(cSizeNoLdm); + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm)); + + if (cSizeLdm > cSizeNoLdm) { + DISPLAY("Using long mode should not cause regressions for btopt+\n"); + testResult = 1; + goto _end; + } + + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "longtest%3i : testing cdict compression with different attachment strategies : ", testNb++); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + size_t dictSize = CNBuffSize; + void* dict = (void*)malloc(dictSize); + ZSTD_CCtx_params* cctx_params = ZSTD_createCCtxParams(); + ZSTD_dictAttachPref_e const attachPrefs[] = { + ZSTD_dictDefaultAttach, + ZSTD_dictForceAttach, + ZSTD_dictForceCopy, + ZSTD_dictForceLoad, + ZSTD_dictDefaultAttach, + ZSTD_dictForceAttach, + ZSTD_dictForceCopy, + ZSTD_dictForceLoad + }; + int const enableDedicatedDictSearch[] = {0, 0, 0, 0, 1, 1, 1, 1}; + int cLevel; + int i; + + RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); + RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed); + + CHECK(cctx_params != NULL); + + for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) { + DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize); + for (cLevel = 4; cLevel < 13; cLevel++) { + for (i = 0; i < 8; ++i) { + ZSTD_dictAttachPref_e const attachPref = attachPrefs[i]; + int const enableDDS = enableDedicatedDictSearch[i]; + ZSTD_CDict* cdict; + + DISPLAYLEVEL(5, "\n dictSize %u cLevel %d iter %d ", (U32)dictSize, cLevel, i); + + ZSTD_CCtxParams_init(cctx_params, cLevel); + CHECK_Z(ZSTD_CCtxParams_setParameter(cctx_params, ZSTD_c_enableDedicatedDictSearch, enableDDS)); + + cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctx_params, ZSTD_defaultCMem); + CHECK(cdict != NULL); + + CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, attachPref)); + + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK_Z(cSize); + CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); + + DISPLAYLEVEL(5, "compressed to %u bytes ", (U32)cSize); + + CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); + ZSTD_freeCDict(cdict); + } } } + + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + ZSTD_freeCCtxParams(cctx_params); + free(dict); + } + DISPLAYLEVEL(3, "OK \n"); + _end: free(CNBuffer); free(compressedBuffer); @@ -3053,7 +3416,7 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n", testNb, (unsigned)tooSmallSize, (unsigned)missing); { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); - CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); } + CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); } { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck)); CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); } } } @@ -3100,7 +3463,7 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const static const BYTE token = 0xA9; dstBuffer[tooSmallSize] = token; { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize); - CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); } + CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); } CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); } @@ -3312,10 +3675,12 @@ int main(int argc, const char** argv) int nbTests = nbTestsDefault; int testNb = 0; int proba = FUZ_compressibility_default; + double probfloat; int result = 0; U32 mainPause = 0; U32 maxDuration = 0; int bigTests = 1; + int longTests = 0; U32 memTestsOnly = 0; const char* const programName = argv[0]; @@ -3331,6 +3696,8 @@ int main(int argc, const char** argv) if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; } if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } + if (!strcmp(argument, "--long-tests")) { longTests=1; continue; } + if (!strcmp(argument, "--no-long-tests")) { longTests=0; continue; } argument++; while (*argument!=0) { @@ -3401,15 +3768,22 @@ int main(int argc, const char** argv) DISPLAY("Seed = %u\n", (unsigned)seed); if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba); + probfloat = ((double)proba) / 100; + if (memTestsOnly) { g_displayLevel = MAX(3, g_displayLevel); - return FUZ_mallocTests(seed, ((double)proba) / 100, memTestsOnly); + return FUZ_mallocTests(seed, probfloat, memTestsOnly); } if (nbTests < testNb) nbTests = testNb; - if (testNb==0) - result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */ + if (testNb==0) { + result = basicUnitTests(0, probfloat); /* constant seed for predictability */ + + if (!result && longTests) { + result = longUnitTests(0, probfloat); + } + } if (!result) result = fuzzerTests(seed, nbTests, testNb, maxDuration, ((double)proba) / 100, bigTests); if (mainPause) { diff --git a/tests/golden-compression/http b/tests/golden-compression/http new file mode 100644 index 00000000..a3908cbc --- /dev/null +++ b/tests/golden-compression/http @@ -0,0 +1 @@ +<?xml version="1.0" encoding="UTF-8" ?><VAST version="2.0"><Ad id="43C53990160C658B"><Wrapper><AdSystem version="1.0">ads60.vertamedia.com</AdSystem><VASTAdTagURI>http://ads60.vertamedia.com/ops/43C53990160C658B/46991</VASTAdTagURI><Impression><![CDATA[http://ads60.vertamedia.com/i/?x=1&adId=43C53990160C658B&aid=46991&cmpId=13765&fi=24929&advId=13177&pubId=28786&sid=0&width=300&height=250&domain=m.gazetaexpress.com]]></Impression><Creatives><Creative><Linear><TrackingEvents><Tracking event="start">http://ads60.vertamedia.com/ve/43C53990160C658B/53</Tracking><Tracking event="midpoint">http://ads60.vertamedia.com/ve/43C53990160C658B/54</Tracking><Tracking event="firstQuartile">http://ads60.vertamedia.com/ve/43C53990160C658B/55</Tracking><Tracking event="thirdQuartile">http://ads60.vertamedia.com/ve/43C53990160C658B/56</Tracking><Tracking event="complete">http://ads60.vertamedia.com/ve/43C53990160C658B/57</Tracking><Tracking event="skip">http://ads60.vertamedia.com/ve/43C53990160C658B/66</Tracking></TrackingEvents><VideoClicks><ClickTracking>http://ads60.vertamedia.com/ve/43C53990160C658B/71</ClickTracking></VideoClicks></Linear></Creative></Creatives></Wrapper></Ad></VAST>
\ No newline at end of file diff --git a/tests/golden-dictionaries/http-dict-missing-symbols b/tests/golden-dictionaries/http-dict-missing-symbols Binary files differnew file mode 100644 index 00000000..0fa5ca61 --- /dev/null +++ b/tests/golden-dictionaries/http-dict-missing-symbols diff --git a/tests/libzstd_partial_builds.sh b/tests/libzstd_partial_builds.sh index b1c1e3b1..bee2dbda 100755 --- a/tests/libzstd_partial_builds.sh +++ b/tests/libzstd_partial_builds.sh @@ -22,7 +22,6 @@ mustBeAbsent() { } # default compilation : all features enabled -make clean > /dev/null $ECHO "testing default library compilation" CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog diff --git a/tests/paramgrill.c b/tests/paramgrill.c index e9cc2a94..439aebed 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -462,8 +462,8 @@ static paramValues_t emptyParams(void) static winnerInfo_t initWinnerInfo(const paramValues_t p) { winnerInfo_t w1; - w1.result.cSpeed = 0.; - w1.result.dSpeed = 0.; + w1.result.cSpeed = 0; + w1.result.dSpeed = 0; w1.result.cMem = (size_t)-1; w1.result.cSize = (size_t)-1; w1.params = p; @@ -581,7 +581,7 @@ resultScore(const BMK_benchResult_t res, const size_t srcSize, const constraint_ static double resultDistLvl(const BMK_benchResult_t result1, const BMK_benchResult_t lvlRes) { - double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1; + double normalizedCSpeedGain1 = ((double)result1.cSpeed / lvlRes.cSpeed) - 1; double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1; if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) { return 0.0; @@ -1133,7 +1133,7 @@ typedef struct { } contexts_t; static void freeNonSrcBuffers(const buffers_t b) { - free(b.srcPtrs); + free((void*)b.srcPtrs); free(b.srcSizes); if(b.dstPtrs != NULL) { @@ -1240,7 +1240,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab size_t nbFiles) { size_t pos = 0; size_t n; - size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); + size_t totalSizeToLoad = (size_t)UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); size_t* fileSizes = calloc(sizeof(size_t), nbFiles); void* srcBuffer = NULL; @@ -1325,7 +1325,7 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { } { U64 const dictFileSize = UTIL_getFileSize(dictFileName); assert(dictFileSize != UTIL_FILESIZE_UNKNOWN); - ctx->dictSize = dictFileSize; + ctx->dictSize = (size_t)dictFileSize; assert((U64)ctx->dictSize == dictFileSize); /* check overflow */ } ctx->dictBuffer = malloc(ctx->dictSize); @@ -1492,7 +1492,7 @@ createMemoTableArray(const paramValues_t p, if(memoTableLog != PARAM_UNSET && mtl > (1ULL << memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */ mtAll[i].tableType = xxhashMap; - mtl = (1ULL << memoTableLog); + mtl = ((size_t)1 << memoTableLog); } mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl); @@ -1669,7 +1669,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx, } /* Bench */ - bResult.cMem = (1 << (comprParams->vals[wlog_ind])) + ZSTD_sizeof_CCtx(cctx); + bResult.cMem = ((size_t)1 << (comprParams->vals[wlog_ind])) + ZSTD_sizeof_CCtx(cctx); { BMK_benchOutcome_t bOut; bOut.tag = 0; @@ -1804,7 +1804,7 @@ static void BMK_init_level_constraints(int bytePerSec_level1) assert(NB_LEVELS_TRACKED >= ZSTD_maxCLevel()); memset(g_level_constraint, 0, sizeof(g_level_constraint)); g_level_constraint[1].cSpeed_min = bytePerSec_level1; - g_level_constraint[1].dSpeed_min = 0.; + g_level_constraint[1].dSpeed_min = 0; g_level_constraint[1].windowLog_max = 19; g_level_constraint[1].strategy_max = ZSTD_fast; @@ -1812,7 +1812,7 @@ static void BMK_init_level_constraints(int bytePerSec_level1) { int l; for (l=2; l<=NB_LEVELS_TRACKED; l++) { g_level_constraint[l].cSpeed_min = (g_level_constraint[l-1].cSpeed_min * 49) / 64; - g_level_constraint[l].dSpeed_min = 0.; + g_level_constraint[l].dSpeed_min = 0; g_level_constraint[l].windowLog_max = (l<20) ? 23 : l+5; /* only --ultra levels >= 20 can use windowlog > 23 */ g_level_constraint[l].strategy_max = ZSTD_STRATEGY_MAX; } } @@ -1837,7 +1837,7 @@ static int BMK_seed(winnerInfo_t* winners, continue; /* not fast enough for this level */ if (params.vals[wlog_ind] > g_level_constraint[cLevel].windowLog_max) continue; /* too much memory for this level */ - if (params.vals[strt_ind] > g_level_constraint[cLevel].strategy_max) + if (params.vals[strt_ind] > (U32)g_level_constraint[cLevel].strategy_max) continue; /* forbidden strategy for this level */ if (winners[cLevel].result.cSize==0) { /* first solution for this cLevel */ @@ -1859,16 +1859,16 @@ static int BMK_seed(winnerInfo_t* winners, double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - size_t W_CMemUsed = (1 << params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(params)); - size_t O_CMemUsed = (1 << winners[cLevel].params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(winners[cLevel].params)); + size_t W_CMemUsed = ((size_t)1 << params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(params)); + size_t O_CMemUsed = ((size_t)1 << winners[cLevel].params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(winners[cLevel].params)); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); - double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed); - double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed); + double W_CSpeed_note = W_ratioNote * (double)( 30 + 10*cLevel) + log(testResult.cSpeed); + double O_CSpeed_note = O_ratioNote * (double)( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed); - double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed); - double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed); + double W_DSpeed_note = W_ratioNote * (double)( 20 + 2*cLevel) + log(testResult.dSpeed); + double O_DSpeed_note = O_ratioNote * (double)( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed); if (W_DMemUsed_note < O_DMemUsed_note) { /* uses too much Decompression memory for too little benefit */ @@ -2227,7 +2227,7 @@ static winnerInfo_t climbOnce(const constraint_t target, } for (dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */ - for (i = 0; i < (1 << varLen) / varLen + 2; i++) { + for (i = 0; i < (1ULL << varLen) / varLen + 2; i++) { int res; CHECKTIME(winnerInfo); candidateInfo.params = cparam; diff --git a/tests/playTests.sh b/tests/playTests.sh index f353229b..51b42b60 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -8,22 +8,22 @@ die() { } datagen() { - "$DATAGEN_BIN" $@ + "$DATAGEN_BIN" "$@" } zstd() { if [ -z "$EXEC_PREFIX" ]; then - "$ZSTD_BIN" $@ + "$ZSTD_BIN" "$@" else - "$EXEC_PREFIX" "$ZSTD_BIN" $@ + "$EXEC_PREFIX" "$ZSTD_BIN" "$@" fi } sudoZstd() { if [ -z "$EXEC_PREFIX" ]; then - sudo "$ZSTD_BIN" $@ + sudo "$ZSTD_BIN" "$@" else - sudo "$EXEC_PREFIX" "$ZSTD_BIN" $@ + sudo "$EXEC_PREFIX" "$ZSTD_BIN" "$@" fi } @@ -78,6 +78,11 @@ println() { printf '%b\n' "${*}" } +if [ -z "${size}" ]; then + size= +else + size=${size} +fi SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) PRGDIR="$SCRIPT_DIR/../programs" @@ -123,10 +128,25 @@ case "$UNAME" in SunOS) DIFF="gdiff" ;; esac -println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN' DATAGEN_BIN='$DATAGEN_BIN'" -[ -n "$ZSTD_BIN" ] || die "\$ZSTD_BIN variable must be defined!" -[ -n "$DATAGEN_BIN" ] || die "\$DATAGEN_BIN variable must be defined!" +# check if ZSTD_BIN is defined. if not, use the default value +if [ -z "${ZSTD_BIN}" ]; then + println "\nZSTD_BIN is not set. Using the default value..." + ZSTD_BIN="$PRGDIR/zstd" +fi + +# check if DATAGEN_BIN is defined. if not, use the default value +if [ -z "${DATAGEN_BIN}" ]; then + println "\nDATAGEN_BIN is not set. Using the default value..." + DATAGEN_BIN="$TESTDIR/datagen" +fi + +ZSTD_BIN="$EXE_PREFIX$ZSTD_BIN" + +# assertions +[ -n "$ZSTD_BIN" ] || die "zstd not found at $ZSTD_BIN! \n Please define ZSTD_BIN pointing to the zstd binary. You might also consider rebuilding zstd follwing the instructions in README.md" +[ -n "$DATAGEN_BIN" ] || die "datagen not found at $DATAGEN_BIN! \n Please define DATAGEN_BIN pointing to the datagen binary. You might also consider rebuilding zstd tests following the instructions in README.md. " +println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN' DATAGEN_BIN='$DATAGEN_BIN'" if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled' then @@ -174,8 +194,6 @@ println "test : compress to named file" rm tmpCompressed zstd tmp -o tmpCompressed test -f tmpCompressed # file must be created -println "test : -o must be followed by filename (must fail)" -zstd tmp -of tmpCompressed && die "-o must be followed by filename " println "test : force write, correct order" zstd tmp -fo tmpCompressed println "test : forgotten argument" @@ -242,16 +260,23 @@ zstd tmp -c --compress-literals --fast=1 | zstd -t zstd tmp -c --compress-literals -19 | zstd -t zstd -b --fast=1 -i0e1 tmp --compress-literals zstd -b --fast=1 -i0e1 tmp --no-compress-literals +println "test: --no-check for decompression" +zstd -f tmp -o tmp_corrupt.zst --check +zstd -f tmp -o tmp.zst --no-check +printf '\xDE\xAD\xBE\xEF' | dd of=tmp_corrupt.zst bs=1 seek=$(($(wc -c < "tmp_corrupt.zst") - 4)) count=4 conv=notrunc # corrupt checksum in tmp +zstd -d -f tmp_corrupt.zst --no-check +zstd -d -f tmp_corrupt.zst --check --no-check # final flag overrides +zstd -d -f tmp.zst --no-check println "\n===> zstdgrep tests" ln -sf "$ZSTD_BIN" zstdcat rm -f tmp_grep echo "1234" > tmp_grep zstd -f tmp_grep -lines=$(ZCAT=./zstdcat $ZSTDGREP 2>&1 "1234" tmp_grep tmp_grep.zst | wc -l) +lines=$(ZCAT=./zstdcat "$ZSTDGREP" 2>&1 "1234" tmp_grep tmp_grep.zst | wc -l) test 2 -eq $lines -ZCAT=./zstdcat $ZSTDGREP 2>&1 "1234" tmp_grep_bad.zst && die "Should have failed" -ZCAT=./zstdcat $ZSTDGREP 2>&1 "1234" tmp_grep_bad.zst | grep "No such file or directory" || true +ZCAT=./zstdcat "$ZSTDGREP" 2>&1 "1234" tmp_grep_bad.zst && die "Should have failed" +ZCAT=./zstdcat "$ZSTDGREP" 2>&1 "1234" tmp_grep_bad.zst | grep "No such file or directory" || true rm -f tmp_grep* println "\n===> --exclude-compressed flag" @@ -285,6 +310,23 @@ test -f precompressedFilterTestDir/input.6.zst.zst println "Test completed" + +println "\n===> warning prompts should not occur if stdin is an input" +println "y" > tmpPrompt +println "hello world" >> tmpPrompt +zstd tmpPrompt -f +zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" +zstd < tmpPrompt -o tmpPrompt.zst -f # should successfully overwrite with -f +zstd -q -d -f tmpPrompt.zst -o tmpPromptRegenerated +$DIFF tmpPromptRegenerated tmpPrompt # the first 'y' character should not be swallowed + +echo 'yes' | zstd tmpPrompt -o tmpPrompt.zst # accept piped "y" input to force overwrite when using files +echo 'yes' | zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" +zstd tmpPrompt - < tmpPrompt -o tmpPromp.zst --rm && die "should have aborted immediately and failed to remove" + +println "Test completed" + + println "\n===> recursive mode test " # combination of -r with empty list of input file zstd -c -r < tmp > tmp.zst @@ -343,7 +385,22 @@ zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f zstd -d tmp1.zst tmp2.zst -o tmp touch tmpexists zstd tmp1 tmp2 -f -o tmpexists -zstd tmp1 tmp2 -o tmpexists && die "should have refused to overwrite" +zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite" +println gooder > tmp_rm1 +println boi > tmp_rm2 +println worldly > tmp_rm3 +echo 'y' | zstd tmp_rm1 tmp_rm2 -o tmp_rm3.zst --rm # tests the warning prompt for --rm with multiple inputs into once source +test ! -f tmp_rm1 +test ! -f tmp_rm2 +cp tmp_rm3.zst tmp_rm4.zst +echo 'Y' | zstd -d tmp_rm3.zst tmp_rm4.zst -o tmp_rm_out --rm +test ! -f tmp_rm3.zst +test ! -f tmp_rm4.zst +echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm && die "compressing multiple files to stdout with --rm should fail unless -f is specified" +echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm -v && die "compressing multiple files to stdout with --rm should fail unless -f is specified" +println gooder > tmpexists1 +zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID + # Bug: PR #972 if [ "$?" -eq 139 ]; then die "should not have segfaulted" @@ -425,6 +482,31 @@ test -f tmpOutDirDecomp/tmp2 test -f tmpOutDirDecomp/tmp1 rm -rf tmp* +if [ "$isWindows" = false ] ; then + println "\n===> compress multiple files into an output directory and mirror input folder, --output-dir-mirror" + println "test --output-dir-mirror" > tmp1 + mkdir -p tmpInputTestDir/we/must/go/deeper + println cool > tmpInputTestDir/we/must/go/deeper/tmp2 + zstd tmp1 -r tmpInputTestDir --output-dir-mirror tmpOutDir + test -f tmpOutDir/tmp1.zst + test -f tmpOutDir/tmpInputTestDir/we/must/go/deeper/tmp2.zst + + println "test: compress input dir will be ignored if it has '..'" + zstd -r tmpInputTestDir/we/must/../must --output-dir-mirror non-exist && die "input cannot contain '..'" + test ! -d non-exist + + println "test : decompress multiple files into an output directory, --output-dir-mirror" + zstd tmpOutDir -r -d --output-dir-mirror tmpOutDirDecomp + test -f tmpOutDirDecomp/tmpOutDir/tmp1 + test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/must/go/deeper/tmp2 + + println "test: decompress input dir will be ignored if it has '..'" + zstd -r tmpOutDir/tmpInputTestDir/we/must/../must --output-dir-mirror non-exist && die "input cannot contain '..'" + test ! -d non-exist + + rm -rf tmp* +fi + println "test : compress multiple files reading them from a file, --filelist=FILE" println "Hello world!, file1" > tmp1 @@ -435,6 +517,11 @@ zstd -f --filelist=tmp_fileList test -f tmp2.zst test -f tmp1.zst +println "test : alternate syntax: --filelist FILE" +zstd -f --filelist tmp_fileList +test -f tmp2.zst +test -f tmp1.zst + println "test : reading file list from a symlink, --filelist=FILE" rm -f *.zst ln -s tmp_fileList tmp_symLink @@ -641,7 +728,7 @@ if [ "$stream_size" -gt "$file_size" ]; then die "hinted compression larger than expected" fi println "test : sized streaming compression and decompression" -cat tmp | zstd -14 -f tmp -o --stream-size=11000 tmp.zst +cat tmp | zstd -14 -f tmp -o tmp.zst --stream-size=11000 zstd -df tmp.zst -o tmp_decompress cmp tmp tmp_decompress || die "difference between original and decompressed file" println "test : incorrect stream size" @@ -738,8 +825,6 @@ println "- Compress without dictID" zstd -f tmp -D tmpDict1 --no-dictID zstd -d tmp.zst -D tmpDict -fo result $DIFF "$TESTFILE" result -println "- Compress with wrong argument order (must fail)" -zstd tmp -Df tmpDict1 -c > $INTOVOID && die "-D must be followed by dictionary name " println "- Compress multiple files with dictionary" rm -rf dirTestDict mkdir dirTestDict @@ -873,8 +958,9 @@ datagen | zstd -c | zstd -t println "\n===> golden files tests " -zstd -t -r "$TESTDIR/golden-compression" +zstd -t -r "$TESTDIR/golden-decompression" zstd -c -r "$TESTDIR/golden-compression" | zstd -t +zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" "$TESTDIR/golden-compression/http" -c | zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" -t println "\n===> benchmark mode tests " @@ -1017,7 +1103,7 @@ if [ $LZ4MODE -eq 1 ]; then datagen > tmp zstd --format=lz4 -f tmp lz4 -t -v tmp.lz4 - lz4 -f tmp + lz4 -f -m tmp # ensure result is sent into tmp.lz4, not stdout zstd -d -f -v tmp.lz4 rm tmp* else @@ -1058,10 +1144,13 @@ if [ $LZ4MODE -ne 1 ]; then grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" fi +touch tmp1 +zstd tmp1 -o tmp1.zstd +zstd -d -f tmp1.zstd # support .zstd suffix even though it's not the default suffix println "\n===> tar extension tests " -rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 +rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 tmp1.zstd datagen > tmp tar cf tmp.tar tmp @@ -1117,6 +1206,7 @@ roundTripTest -g1000K "1 --single-thread --long" roundTripTest -g517K "6 --single-thread --long" roundTripTest -g516K "16 --single-thread --long" roundTripTest -g518K "19 --single-thread --long" +roundTripTest -g2M "22 --single-thread --ultra --long" fileRoundTripTest -g5M "3 --single-thread --long" @@ -1126,12 +1216,26 @@ then println "\n===> zstdmt round-trip tests " roundTripTest -g4M "1 -T0" roundTripTest -g8M "3 -T2" + roundTripTest -g8M "19 -T0 --long" roundTripTest -g8000K "2 --threads=2" fileRoundTripTest -g4M "19 -T2 -B1M" println "\n===> zstdmt long distance matching round-trip tests " roundTripTest -g8M "3 --long=24 -T2" + println "\n===> zstdmt environment variable tests " + echo "multifoo" >> mt_tmp + ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting + ZSTD_NBTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting + ZSTD_NBTHREADS=- zstd -f mt_tmp # malformed env var, warn and revert to default setting + ZSTD_NBTHREADS=a zstd -f mt_tmp # malformed env var, warn and revert to default setting + ZSTD_NBTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting + ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting + ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting= + ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage + ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread + rm mt_tmp* + println "\n===> ovLog tests " datagen -g2MB > tmp refSize=$(zstd tmp -6 -c --zstd=wlog=18 | wc -c) @@ -1231,6 +1335,28 @@ roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB" +println "\n===> zstd long distance matching with optimal parser compressed size tests " +optCSize16=$(datagen -g511K | zstd -16 -c | wc -c) +longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c) +optCSize19=$(datagen -g2M | zstd -19 -c | wc -c) +longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c) +optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c) +longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c) +optCSize22=$(datagen -g900K | zstd -22 --ultra -c | wc -c) +longCSize22=$(datagen -g900K | zstd -22 --ultra --long -c | wc -c) +if [ "$longCSize16" -gt "$optCSize16" ]; then + echo using --long on compression level 16 should not cause compressed size regression + exit 1 +elif [ "$longCSize19" -gt "$optCSize19" ]; then + echo using --long on compression level 19 should not cause compressed size regression + exit 1 +elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then + echo using --long on compression level 19 with wLog=23 should not cause compressed size regression + exit 1 +elif [ "$longCSize22" -gt "$optCSize22" ]; then + echo using --long on compression level 22 should not cause compressed size regression + exit 1 +fi if [ "$1" != "--test-large-data" ]; then @@ -1258,17 +1384,20 @@ then zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread" fi -println "\n===> patch-from tests" - +println "\n===> patch-from=origin tests" datagen -g1000 -P50 > tmp_dict datagen -g1000 -P10 > tmp_patch zstd --patch-from=tmp_dict tmp_patch -o tmp_patch_diff zstd -d --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon $DIFF -s tmp_patch_recon tmp_patch + +println "\n===> alternate syntax: patch-from origin" +zstd -f --patch-from tmp_dict tmp_patch -o tmp_patch_diff +zstd -df --patch-from tmp_dict tmp_patch_diff -o tmp_patch_recon +$DIFF -s tmp_patch_recon tmp_patch rm -rf tmp_* println "\n===> patch-from recursive tests" - mkdir tmp_dir datagen > tmp_dir/tmp1 datagen > tmp_dir/tmp2 @@ -1277,10 +1406,16 @@ zstd --patch-from=tmp_dict -r tmp_dir && die rm -rf tmp* println "\n===> patch-from long mode trigger larger file test" - datagen -g5000000 > tmp_dict datagen -g5000000 > tmp_patch -zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automaticaly triggered" +zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automatically triggered" +rm -rf tmp* + +println "\n===> patch-from --stream-size test" +datagen -g1000 -P50 > tmp_dict +datagen -g1000 -P10 > tmp_patch +cat tmp_patch | zstd -f --patch-from=tmp_dict -c -o tmp_patch_diff && die +cat tmp_patch | zstd -f --patch-from=tmp_dict --stream-size=1000 -c -o tmp_patch_diff rm -rf tmp* println "\n===> large files tests " diff --git a/tests/regression/README.md b/tests/regression/README.md new file mode 100644 index 00000000..bb36b1dd --- /dev/null +++ b/tests/regression/README.md @@ -0,0 +1,28 @@ +# Regression tests + +The regression tests run zstd in many scenarios and ensures that the size of the compressed results doesn't change. This helps us ensure that we don't accidentally regress zstd's compression ratio. + +These tests get run every night by CircleCI. If the job fails you can read the diff printed by the job to ensure the change isn't a regression. If all is well you can download the `results.csv` artifact and commit the new results. Or you can rebuild it yourself following the instructions below. + +## Rebuilding results.csv + +From the root of the zstd repo run: + +``` +# Build the zstd binary +make clean +make -j zstd + +# Build the regression test binary +cd tests/regression +make clean +make -j test + +# Run the regression test +./test --cache data-cache --zstd ../../zstd --output results.csv + +# Check results.csv to ensure the new results are okay +git diff + +# Then submit the PR +``` diff --git a/tests/regression/results.csv b/tests/regression/results.csv index 4db42a48..979b1d25 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -1,611 +1,611 @@ Data, Config, Method, Total compressed size -silesia.tar, level -5, compress simple, 6738558 -silesia.tar, level -3, compress simple, 6446362 -silesia.tar, level -1, compress simple, 6186038 -silesia.tar, level 0, compress simple, 4861374 -silesia.tar, level 1, compress simple, 5334825 -silesia.tar, level 3, compress simple, 4861374 -silesia.tar, level 4, compress simple, 4799583 -silesia.tar, level 5, compress simple, 4722271 -silesia.tar, level 6, compress simple, 4672231 -silesia.tar, level 7, compress simple, 4606657 -silesia.tar, level 9, compress simple, 4554099 -silesia.tar, level 13, compress simple, 4491706 -silesia.tar, level 16, compress simple, 4381265 -silesia.tar, level 19, compress simple, 4281551 -silesia.tar, uncompressed literals, compress simple, 4861374 -silesia.tar, uncompressed literals optimal, compress simple, 4281551 -silesia.tar, huffman literals, compress simple, 6186038 -silesia, level -5, compress cctx, 6737567 -silesia, level -3, compress cctx, 6444663 -silesia, level -1, compress cctx, 6178442 -silesia, level 0, compress cctx, 4849491 -silesia, level 1, compress cctx, 5313144 -silesia, level 3, compress cctx, 4849491 -silesia, level 4, compress cctx, 4786913 -silesia, level 5, compress cctx, 4710178 -silesia, level 6, compress cctx, 4659996 -silesia, level 7, compress cctx, 4596234 -silesia, level 9, compress cctx, 4543862 -silesia, level 13, compress cctx, 4482073 -silesia, level 16, compress cctx, 4377389 -silesia, level 19, compress cctx, 4293262 -silesia, long distance mode, compress cctx, 4849491 -silesia, multithreaded, compress cctx, 4849491 -silesia, multithreaded long distance mode, compress cctx, 4849491 -silesia, small window log, compress cctx, 7078156 -silesia, small hash log, compress cctx, 6554898 -silesia, small chain log, compress cctx, 4931093 -silesia, explicit params, compress cctx, 4794609 -silesia, uncompressed literals, compress cctx, 4849491 -silesia, uncompressed literals optimal, compress cctx, 4293262 -silesia, huffman literals, compress cctx, 6178442 -silesia, multithreaded with advanced params, compress cctx, 4849491 +silesia.tar, level -5, compress simple, 6738593 +silesia.tar, level -3, compress simple, 6446372 +silesia.tar, level -1, compress simple, 6186042 +silesia.tar, level 0, compress simple, 4861425 +silesia.tar, level 1, compress simple, 5334885 +silesia.tar, level 3, compress simple, 4861425 +silesia.tar, level 4, compress simple, 4799630 +silesia.tar, level 5, compress simple, 4722324 +silesia.tar, level 6, compress simple, 4672279 +silesia.tar, level 7, compress simple, 4606715 +silesia.tar, level 9, compress simple, 4554147 +silesia.tar, level 13, compress simple, 4491764 +silesia.tar, level 16, compress simple, 4381332 +silesia.tar, level 19, compress simple, 4281605 +silesia.tar, uncompressed literals, compress simple, 4861425 +silesia.tar, uncompressed literals optimal, compress simple, 4281605 +silesia.tar, huffman literals, compress simple, 6186042 +silesia, level -5, compress cctx, 6737607 +silesia, level -3, compress cctx, 6444677 +silesia, level -1, compress cctx, 6178460 +silesia, level 0, compress cctx, 4849552 +silesia, level 1, compress cctx, 5313204 +silesia, level 3, compress cctx, 4849552 +silesia, level 4, compress cctx, 4786970 +silesia, level 5, compress cctx, 4710236 +silesia, level 6, compress cctx, 4660056 +silesia, level 7, compress cctx, 4596296 +silesia, level 9, compress cctx, 4543925 +silesia, level 13, compress cctx, 4482135 +silesia, level 16, compress cctx, 4377465 +silesia, level 19, compress cctx, 4293330 +silesia, long distance mode, compress cctx, 4849552 +silesia, multithreaded, compress cctx, 4849552 +silesia, multithreaded long distance mode, compress cctx, 4849552 +silesia, small window log, compress cctx, 7084179 +silesia, small hash log, compress cctx, 6555021 +silesia, small chain log, compress cctx, 4931148 +silesia, explicit params, compress cctx, 4794677 +silesia, uncompressed literals, compress cctx, 4849552 +silesia, uncompressed literals optimal, compress cctx, 4293330 +silesia, huffman literals, compress cctx, 6178460 +silesia, multithreaded with advanced params, compress cctx, 4849552 github, level -5, compress cctx, 205285 github, level -5 with dict, compress cctx, 47294 github, level -3, compress cctx, 190643 github, level -3 with dict, compress cctx, 48047 github, level -1, compress cctx, 175568 github, level -1 with dict, compress cctx, 43527 -github, level 0, compress cctx, 136311 +github, level 0, compress cctx, 136335 github, level 0 with dict, compress cctx, 41534 -github, level 1, compress cctx, 142450 +github, level 1, compress cctx, 142465 github, level 1 with dict, compress cctx, 42157 -github, level 3, compress cctx, 136311 +github, level 3, compress cctx, 136335 github, level 3 with dict, compress cctx, 41534 -github, level 4, compress cctx, 136144 +github, level 4, compress cctx, 136199 github, level 4 with dict, compress cctx, 41725 -github, level 5, compress cctx, 135106 +github, level 5, compress cctx, 135121 github, level 5 with dict, compress cctx, 38934 -github, level 6, compress cctx, 135108 +github, level 6, compress cctx, 135122 github, level 6 with dict, compress cctx, 38628 -github, level 7, compress cctx, 135108 -github, level 7 with dict, compress cctx, 38741 -github, level 9, compress cctx, 135108 -github, level 9 with dict, compress cctx, 39335 -github, level 13, compress cctx, 133717 -github, level 13 with dict, compress cctx, 39923 -github, level 16, compress cctx, 133717 +github, level 7, compress cctx, 135122 +github, level 7 with dict, compress cctx, 38745 +github, level 9, compress cctx, 135122 +github, level 9 with dict, compress cctx, 39341 +github, level 13, compress cctx, 134064 +github, level 13 with dict, compress cctx, 39948 +github, level 16, compress cctx, 134064 github, level 16 with dict, compress cctx, 37568 -github, level 19, compress cctx, 133717 +github, level 19, compress cctx, 134064 github, level 19 with dict, compress cctx, 37567 -github, long distance mode, compress cctx, 141101 -github, multithreaded, compress cctx, 141101 -github, multithreaded long distance mode, compress cctx, 141101 -github, small window log, compress cctx, 141101 -github, small hash log, compress cctx, 138943 -github, small chain log, compress cctx, 139239 -github, explicit params, compress cctx, 140924 -github, uncompressed literals, compress cctx, 136311 -github, uncompressed literals optimal, compress cctx, 133717 +github, long distance mode, compress cctx, 141102 +github, multithreaded, compress cctx, 141102 +github, multithreaded long distance mode, compress cctx, 141102 +github, small window log, compress cctx, 141102 +github, small hash log, compress cctx, 138949 +github, small chain log, compress cctx, 139242 +github, explicit params, compress cctx, 140932 +github, uncompressed literals, compress cctx, 136335 +github, uncompressed literals optimal, compress cctx, 134064 github, huffman literals, compress cctx, 175568 -github, multithreaded with advanced params, compress cctx, 141101 -silesia, level -5, zstdcli, 6882514 -silesia, level -3, zstdcli, 6568406 -silesia, level -1, zstdcli, 6183433 -silesia, level 0, zstdcli, 4849539 -silesia, level 1, zstdcli, 5314157 -silesia, level 3, zstdcli, 4849539 -silesia, level 4, zstdcli, 4786961 -silesia, level 5, zstdcli, 4710226 -silesia, level 6, zstdcli, 4660044 -silesia, level 7, zstdcli, 4596282 -silesia, level 9, zstdcli, 4543910 -silesia, level 13, zstdcli, 4482121 -silesia, level 16, zstdcli, 4377437 -silesia, level 19, zstdcli, 4293310 -silesia, long distance mode, zstdcli, 4839698 -silesia, multithreaded, zstdcli, 4849539 -silesia, multithreaded long distance mode, zstdcli, 4839698 -silesia, small window log, zstdcli, 7104616 -silesia, small hash log, zstdcli, 6554946 -silesia, small chain log, zstdcli, 4931141 -silesia, explicit params, zstdcli, 4797048 -silesia, uncompressed literals, zstdcli, 5128008 -silesia, uncompressed literals optimal, zstdcli, 4325482 -silesia, huffman literals, zstdcli, 5331158 -silesia, multithreaded with advanced params, zstdcli, 5128008 -silesia.tar, level -5, zstdcli, 6738906 -silesia.tar, level -3, zstdcli, 6448409 -silesia.tar, level -1, zstdcli, 6186908 -silesia.tar, level 0, zstdcli, 4861462 -silesia.tar, level 1, zstdcli, 5336255 -silesia.tar, level 3, zstdcli, 4861462 -silesia.tar, level 4, zstdcli, 4800482 -silesia.tar, level 5, zstdcli, 4723312 -silesia.tar, level 6, zstdcli, 4673616 -silesia.tar, level 7, zstdcli, 4608346 -silesia.tar, level 9, zstdcli, 4554702 -silesia.tar, level 13, zstdcli, 4491710 -silesia.tar, level 16, zstdcli, 4381269 -silesia.tar, level 19, zstdcli, 4281555 -silesia.tar, no source size, zstdcli, 4861458 -silesia.tar, long distance mode, zstdcli, 4853140 -silesia.tar, multithreaded, zstdcli, 4861462 -silesia.tar, multithreaded long distance mode, zstdcli, 4853140 -silesia.tar, small window log, zstdcli, 7095284 -silesia.tar, small hash log, zstdcli, 6587841 -silesia.tar, small chain log, zstdcli, 4943269 -silesia.tar, explicit params, zstdcli, 4822318 -silesia.tar, uncompressed literals, zstdcli, 5129548 -silesia.tar, uncompressed literals optimal, zstdcli, 4320914 -silesia.tar, huffman literals, zstdcli, 5347560 -silesia.tar, multithreaded with advanced params, zstdcli, 5129548 +github, multithreaded with advanced params, compress cctx, 141102 +silesia, level -5, zstdcli, 6882553 +silesia, level -3, zstdcli, 6568424 +silesia, level -1, zstdcli, 6183451 +silesia, level 0, zstdcli, 4849600 +silesia, level 1, zstdcli, 5314210 +silesia, level 3, zstdcli, 4849600 +silesia, level 4, zstdcli, 4787018 +silesia, level 5, zstdcli, 4710284 +silesia, level 6, zstdcli, 4660104 +silesia, level 7, zstdcli, 4596344 +silesia, level 9, zstdcli, 4543973 +silesia, level 13, zstdcli, 4482183 +silesia, level 16, zstdcli, 4377513 +silesia, level 19, zstdcli, 4293378 +silesia, long distance mode, zstdcli, 4839756 +silesia, multithreaded, zstdcli, 4849600 +silesia, multithreaded long distance mode, zstdcli, 4839756 +silesia, small window log, zstdcli, 7111012 +silesia, small hash log, zstdcli, 6555069 +silesia, small chain log, zstdcli, 4931196 +silesia, explicit params, zstdcli, 4797112 +silesia, uncompressed literals, zstdcli, 5128030 +silesia, uncompressed literals optimal, zstdcli, 4325520 +silesia, huffman literals, zstdcli, 5331216 +silesia, multithreaded with advanced params, zstdcli, 5128030 +silesia.tar, level -5, zstdcli, 6738934 +silesia.tar, level -3, zstdcli, 6448419 +silesia.tar, level -1, zstdcli, 6186912 +silesia.tar, level 0, zstdcli, 4861512 +silesia.tar, level 1, zstdcli, 5336318 +silesia.tar, level 3, zstdcli, 4861512 +silesia.tar, level 4, zstdcli, 4800529 +silesia.tar, level 5, zstdcli, 4723364 +silesia.tar, level 6, zstdcli, 4673663 +silesia.tar, level 7, zstdcli, 4608403 +silesia.tar, level 9, zstdcli, 4554751 +silesia.tar, level 13, zstdcli, 4491768 +silesia.tar, level 16, zstdcli, 4381336 +silesia.tar, level 19, zstdcli, 4281609 +silesia.tar, no source size, zstdcli, 4861508 +silesia.tar, long distance mode, zstdcli, 4853190 +silesia.tar, multithreaded, zstdcli, 4861512 +silesia.tar, multithreaded long distance mode, zstdcli, 4853190 +silesia.tar, small window log, zstdcli, 7101576 +silesia.tar, small hash log, zstdcli, 6587959 +silesia.tar, small chain log, zstdcli, 4943310 +silesia.tar, explicit params, zstdcli, 4822362 +silesia.tar, uncompressed literals, zstdcli, 5129559 +silesia.tar, uncompressed literals optimal, zstdcli, 4320931 +silesia.tar, huffman literals, zstdcli, 5347610 +silesia.tar, multithreaded with advanced params, zstdcli, 5129559 github, level -5, zstdcli, 207285 github, level -5 with dict, zstdcli, 48718 github, level -3, zstdcli, 192643 github, level -3 with dict, zstdcli, 47395 github, level -1, zstdcli, 177568 github, level -1 with dict, zstdcli, 45170 -github, level 0, zstdcli, 138311 +github, level 0, zstdcli, 138335 github, level 0 with dict, zstdcli, 43148 -github, level 1, zstdcli, 144450 +github, level 1, zstdcli, 144465 github, level 1 with dict, zstdcli, 43682 -github, level 3, zstdcli, 138311 +github, level 3, zstdcli, 138335 github, level 3 with dict, zstdcli, 43148 -github, level 4, zstdcli, 138144 +github, level 4, zstdcli, 138199 github, level 4 with dict, zstdcli, 43251 -github, level 5, zstdcli, 137106 -github, level 5 with dict, zstdcli, 40938 -github, level 6, zstdcli, 137108 +github, level 5, zstdcli, 137121 +github, level 5 with dict, zstdcli, 40741 +github, level 6, zstdcli, 137122 github, level 6 with dict, zstdcli, 40632 -github, level 7, zstdcli, 137108 -github, level 7 with dict, zstdcli, 40766 -github, level 9, zstdcli, 137108 -github, level 9 with dict, zstdcli, 41326 -github, level 13, zstdcli, 135717 -github, level 13 with dict, zstdcli, 41716 -github, level 16, zstdcli, 135717 +github, level 7, zstdcli, 137122 +github, level 7 with dict, zstdcli, 40771 +github, level 9, zstdcli, 137122 +github, level 9 with dict, zstdcli, 41332 +github, level 13, zstdcli, 136064 +github, level 13 with dict, zstdcli, 41743 +github, level 16, zstdcli, 136064 github, level 16 with dict, zstdcli, 39577 -github, level 19, zstdcli, 135717 +github, level 19, zstdcli, 136064 github, level 19 with dict, zstdcli, 39576 -github, long distance mode, zstdcli, 138311 -github, multithreaded, zstdcli, 138311 -github, multithreaded long distance mode, zstdcli, 138311 -github, small window log, zstdcli, 138311 -github, small hash log, zstdcli, 137467 -github, small chain log, zstdcli, 138314 -github, explicit params, zstdcli, 136140 +github, long distance mode, zstdcli, 138335 +github, multithreaded, zstdcli, 138335 +github, multithreaded long distance mode, zstdcli, 138335 +github, small window log, zstdcli, 138335 +github, small hash log, zstdcli, 137590 +github, small chain log, zstdcli, 138341 +github, explicit params, zstdcli, 136197 github, uncompressed literals, zstdcli, 167915 -github, uncompressed literals optimal, zstdcli, 158824 -github, huffman literals, zstdcli, 144450 +github, uncompressed literals optimal, zstdcli, 159227 +github, huffman literals, zstdcli, 144465 github, multithreaded with advanced params, zstdcli, 167915 -silesia, level -5, advanced one pass, 6737567 -silesia, level -3, advanced one pass, 6444663 -silesia, level -1, advanced one pass, 6178442 -silesia, level 0, advanced one pass, 4849491 -silesia, level 1, advanced one pass, 5313144 -silesia, level 3, advanced one pass, 4849491 -silesia, level 4, advanced one pass, 4786913 -silesia, level 5, advanced one pass, 4710178 -silesia, level 6, advanced one pass, 4659996 -silesia, level 7, advanced one pass, 4596234 -silesia, level 9, advanced one pass, 4543862 -silesia, level 13, advanced one pass, 4482073 -silesia, level 16, advanced one pass, 4377389 -silesia, level 19, advanced one pass, 4293262 -silesia, no source size, advanced one pass, 4849491 -silesia, long distance mode, advanced one pass, 4839650 -silesia, multithreaded, advanced one pass, 4849491 -silesia, multithreaded long distance mode, advanced one pass, 4839650 -silesia, small window log, advanced one pass, 7089646 -silesia, small hash log, advanced one pass, 6554898 -silesia, small chain log, advanced one pass, 4931093 -silesia, explicit params, advanced one pass, 4797035 -silesia, uncompressed literals, advanced one pass, 5127960 -silesia, uncompressed literals optimal, advanced one pass, 4325434 -silesia, huffman literals, advanced one pass, 5326210 -silesia, multithreaded with advanced params, advanced one pass, 5127960 -silesia.tar, level -5, advanced one pass, 6738558 -silesia.tar, level -3, advanced one pass, 6446362 -silesia.tar, level -1, advanced one pass, 6186038 -silesia.tar, level 0, advanced one pass, 4861374 -silesia.tar, level 1, advanced one pass, 5334825 -silesia.tar, level 3, advanced one pass, 4861374 -silesia.tar, level 4, advanced one pass, 4799583 -silesia.tar, level 5, advanced one pass, 4722271 -silesia.tar, level 6, advanced one pass, 4672231 -silesia.tar, level 7, advanced one pass, 4606657 -silesia.tar, level 9, advanced one pass, 4554099 -silesia.tar, level 13, advanced one pass, 4491706 -silesia.tar, level 16, advanced one pass, 4381265 -silesia.tar, level 19, advanced one pass, 4281551 -silesia.tar, no source size, advanced one pass, 4861374 -silesia.tar, long distance mode, advanced one pass, 4848046 -silesia.tar, multithreaded, advanced one pass, 4860726 -silesia.tar, multithreaded long distance mode, advanced one pass, 4847343 -silesia.tar, small window log, advanced one pass, 7095237 -silesia.tar, small hash log, advanced one pass, 6587833 -silesia.tar, small chain log, advanced one pass, 4943266 -silesia.tar, explicit params, advanced one pass, 4808543 -silesia.tar, uncompressed literals, advanced one pass, 5129447 -silesia.tar, uncompressed literals optimal, advanced one pass, 4320910 -silesia.tar, huffman literals, advanced one pass, 5347283 -silesia.tar, multithreaded with advanced params, advanced one pass, 5129766 +silesia, level -5, advanced one pass, 6737607 +silesia, level -3, advanced one pass, 6444677 +silesia, level -1, advanced one pass, 6178460 +silesia, level 0, advanced one pass, 4849552 +silesia, level 1, advanced one pass, 5313204 +silesia, level 3, advanced one pass, 4849552 +silesia, level 4, advanced one pass, 4786970 +silesia, level 5, advanced one pass, 4710236 +silesia, level 6, advanced one pass, 4660056 +silesia, level 7, advanced one pass, 4596296 +silesia, level 9, advanced one pass, 4543925 +silesia, level 13, advanced one pass, 4482135 +silesia, level 16, advanced one pass, 4377465 +silesia, level 19, advanced one pass, 4293330 +silesia, no source size, advanced one pass, 4849552 +silesia, long distance mode, advanced one pass, 4839708 +silesia, multithreaded, advanced one pass, 4849552 +silesia, multithreaded long distance mode, advanced one pass, 4839708 +silesia, small window log, advanced one pass, 7095919 +silesia, small hash log, advanced one pass, 6555021 +silesia, small chain log, advanced one pass, 4931148 +silesia, explicit params, advanced one pass, 4797095 +silesia, uncompressed literals, advanced one pass, 5127982 +silesia, uncompressed literals optimal, advanced one pass, 4325472 +silesia, huffman literals, advanced one pass, 5326268 +silesia, multithreaded with advanced params, advanced one pass, 5127982 +silesia.tar, level -5, advanced one pass, 6738593 +silesia.tar, level -3, advanced one pass, 6446372 +silesia.tar, level -1, advanced one pass, 6186042 +silesia.tar, level 0, advanced one pass, 4861425 +silesia.tar, level 1, advanced one pass, 5334885 +silesia.tar, level 3, advanced one pass, 4861425 +silesia.tar, level 4, advanced one pass, 4799630 +silesia.tar, level 5, advanced one pass, 4722324 +silesia.tar, level 6, advanced one pass, 4672279 +silesia.tar, level 7, advanced one pass, 4606715 +silesia.tar, level 9, advanced one pass, 4554147 +silesia.tar, level 13, advanced one pass, 4491764 +silesia.tar, level 16, advanced one pass, 4381332 +silesia.tar, level 19, advanced one pass, 4281605 +silesia.tar, no source size, advanced one pass, 4861425 +silesia.tar, long distance mode, advanced one pass, 4848098 +silesia.tar, multithreaded, advanced one pass, 4861508 +silesia.tar, multithreaded long distance mode, advanced one pass, 4853186 +silesia.tar, small window log, advanced one pass, 7101530 +silesia.tar, small hash log, advanced one pass, 6587951 +silesia.tar, small chain log, advanced one pass, 4943307 +silesia.tar, explicit params, advanced one pass, 4808589 +silesia.tar, uncompressed literals, advanced one pass, 5129458 +silesia.tar, uncompressed literals optimal, advanced one pass, 4320927 +silesia.tar, huffman literals, advanced one pass, 5347335 +silesia.tar, multithreaded with advanced params, advanced one pass, 5129555 github, level -5, advanced one pass, 205285 github, level -5 with dict, advanced one pass, 46718 github, level -3, advanced one pass, 190643 github, level -3 with dict, advanced one pass, 45395 github, level -1, advanced one pass, 175568 github, level -1 with dict, advanced one pass, 43170 -github, level 0, advanced one pass, 136311 +github, level 0, advanced one pass, 136335 github, level 0 with dict, advanced one pass, 41148 -github, level 1, advanced one pass, 142450 +github, level 1, advanced one pass, 142465 github, level 1 with dict, advanced one pass, 41682 -github, level 3, advanced one pass, 136311 +github, level 3, advanced one pass, 136335 github, level 3 with dict, advanced one pass, 41148 -github, level 4, advanced one pass, 136144 +github, level 4, advanced one pass, 136199 github, level 4 with dict, advanced one pass, 41251 -github, level 5, advanced one pass, 135106 +github, level 5, advanced one pass, 135121 github, level 5 with dict, advanced one pass, 38938 -github, level 6, advanced one pass, 135108 +github, level 6, advanced one pass, 135122 github, level 6 with dict, advanced one pass, 38632 -github, level 7, advanced one pass, 135108 -github, level 7 with dict, advanced one pass, 38766 -github, level 9, advanced one pass, 135108 -github, level 9 with dict, advanced one pass, 39326 -github, level 13, advanced one pass, 133717 -github, level 13 with dict, advanced one pass, 39716 -github, level 16, advanced one pass, 133717 +github, level 7, advanced one pass, 135122 +github, level 7 with dict, advanced one pass, 38771 +github, level 9, advanced one pass, 135122 +github, level 9 with dict, advanced one pass, 39332 +github, level 13, advanced one pass, 134064 +github, level 13 with dict, advanced one pass, 39743 +github, level 16, advanced one pass, 134064 github, level 16 with dict, advanced one pass, 37577 -github, level 19, advanced one pass, 133717 +github, level 19, advanced one pass, 134064 github, level 19 with dict, advanced one pass, 37576 -github, no source size, advanced one pass, 136311 -github, long distance mode, advanced one pass, 136311 -github, multithreaded, advanced one pass, 136311 -github, multithreaded long distance mode, advanced one pass, 136311 -github, small window log, advanced one pass, 136311 -github, small hash log, advanced one pass, 135467 -github, small chain log, advanced one pass, 136314 -github, explicit params, advanced one pass, 137670 +github, no source size, advanced one pass, 136335 +github, long distance mode, advanced one pass, 136335 +github, multithreaded, advanced one pass, 136335 +github, multithreaded long distance mode, advanced one pass, 136335 +github, small window log, advanced one pass, 136335 +github, small hash log, advanced one pass, 135590 +github, small chain log, advanced one pass, 136341 +github, explicit params, advanced one pass, 137727 github, uncompressed literals, advanced one pass, 165915 -github, uncompressed literals optimal, advanced one pass, 156824 -github, huffman literals, advanced one pass, 142450 +github, uncompressed literals optimal, advanced one pass, 157227 +github, huffman literals, advanced one pass, 142465 github, multithreaded with advanced params, advanced one pass, 165915 -silesia, level -5, advanced one pass small out, 6737567 -silesia, level -3, advanced one pass small out, 6444663 -silesia, level -1, advanced one pass small out, 6178442 -silesia, level 0, advanced one pass small out, 4849491 -silesia, level 1, advanced one pass small out, 5313144 -silesia, level 3, advanced one pass small out, 4849491 -silesia, level 4, advanced one pass small out, 4786913 -silesia, level 5, advanced one pass small out, 4710178 -silesia, level 6, advanced one pass small out, 4659996 -silesia, level 7, advanced one pass small out, 4596234 -silesia, level 9, advanced one pass small out, 4543862 -silesia, level 13, advanced one pass small out, 4482073 -silesia, level 16, advanced one pass small out, 4377389 -silesia, level 19, advanced one pass small out, 4293262 -silesia, no source size, advanced one pass small out, 4849491 -silesia, long distance mode, advanced one pass small out, 4839650 -silesia, multithreaded, advanced one pass small out, 4849491 -silesia, multithreaded long distance mode, advanced one pass small out, 4839650 -silesia, small window log, advanced one pass small out, 7089646 -silesia, small hash log, advanced one pass small out, 6554898 -silesia, small chain log, advanced one pass small out, 4931093 -silesia, explicit params, advanced one pass small out, 4797035 -silesia, uncompressed literals, advanced one pass small out, 5127960 -silesia, uncompressed literals optimal, advanced one pass small out, 4325434 -silesia, huffman literals, advanced one pass small out, 5326210 -silesia, multithreaded with advanced params, advanced one pass small out, 5127960 -silesia.tar, level -5, advanced one pass small out, 6738558 -silesia.tar, level -3, advanced one pass small out, 6446362 -silesia.tar, level -1, advanced one pass small out, 6186038 -silesia.tar, level 0, advanced one pass small out, 4861374 -silesia.tar, level 1, advanced one pass small out, 5334825 -silesia.tar, level 3, advanced one pass small out, 4861374 -silesia.tar, level 4, advanced one pass small out, 4799583 -silesia.tar, level 5, advanced one pass small out, 4722271 -silesia.tar, level 6, advanced one pass small out, 4672231 -silesia.tar, level 7, advanced one pass small out, 4606657 -silesia.tar, level 9, advanced one pass small out, 4554099 -silesia.tar, level 13, advanced one pass small out, 4491706 -silesia.tar, level 16, advanced one pass small out, 4381265 -silesia.tar, level 19, advanced one pass small out, 4281551 -silesia.tar, no source size, advanced one pass small out, 4861374 -silesia.tar, long distance mode, advanced one pass small out, 4848046 -silesia.tar, multithreaded, advanced one pass small out, 4860726 -silesia.tar, multithreaded long distance mode, advanced one pass small out, 4847343 -silesia.tar, small window log, advanced one pass small out, 7095237 -silesia.tar, small hash log, advanced one pass small out, 6587833 -silesia.tar, small chain log, advanced one pass small out, 4943266 -silesia.tar, explicit params, advanced one pass small out, 4808543 -silesia.tar, uncompressed literals, advanced one pass small out, 5129447 -silesia.tar, uncompressed literals optimal, advanced one pass small out, 4320910 -silesia.tar, huffman literals, advanced one pass small out, 5347283 -silesia.tar, multithreaded with advanced params, advanced one pass small out, 5129766 +silesia, level -5, advanced one pass small out, 6737607 +silesia, level -3, advanced one pass small out, 6444677 +silesia, level -1, advanced one pass small out, 6178460 +silesia, level 0, advanced one pass small out, 4849552 +silesia, level 1, advanced one pass small out, 5313204 +silesia, level 3, advanced one pass small out, 4849552 +silesia, level 4, advanced one pass small out, 4786970 +silesia, level 5, advanced one pass small out, 4710236 +silesia, level 6, advanced one pass small out, 4660056 +silesia, level 7, advanced one pass small out, 4596296 +silesia, level 9, advanced one pass small out, 4543925 +silesia, level 13, advanced one pass small out, 4482135 +silesia, level 16, advanced one pass small out, 4377465 +silesia, level 19, advanced one pass small out, 4293330 +silesia, no source size, advanced one pass small out, 4849552 +silesia, long distance mode, advanced one pass small out, 4839708 +silesia, multithreaded, advanced one pass small out, 4849552 +silesia, multithreaded long distance mode, advanced one pass small out, 4839708 +silesia, small window log, advanced one pass small out, 7095919 +silesia, small hash log, advanced one pass small out, 6555021 +silesia, small chain log, advanced one pass small out, 4931148 +silesia, explicit params, advanced one pass small out, 4797095 +silesia, uncompressed literals, advanced one pass small out, 5127982 +silesia, uncompressed literals optimal, advanced one pass small out, 4325472 +silesia, huffman literals, advanced one pass small out, 5326268 +silesia, multithreaded with advanced params, advanced one pass small out, 5127982 +silesia.tar, level -5, advanced one pass small out, 6738593 +silesia.tar, level -3, advanced one pass small out, 6446372 +silesia.tar, level -1, advanced one pass small out, 6186042 +silesia.tar, level 0, advanced one pass small out, 4861425 +silesia.tar, level 1, advanced one pass small out, 5334885 +silesia.tar, level 3, advanced one pass small out, 4861425 +silesia.tar, level 4, advanced one pass small out, 4799630 +silesia.tar, level 5, advanced one pass small out, 4722324 +silesia.tar, level 6, advanced one pass small out, 4672279 +silesia.tar, level 7, advanced one pass small out, 4606715 +silesia.tar, level 9, advanced one pass small out, 4554147 +silesia.tar, level 13, advanced one pass small out, 4491764 +silesia.tar, level 16, advanced one pass small out, 4381332 +silesia.tar, level 19, advanced one pass small out, 4281605 +silesia.tar, no source size, advanced one pass small out, 4861425 +silesia.tar, long distance mode, advanced one pass small out, 4848098 +silesia.tar, multithreaded, advanced one pass small out, 4861508 +silesia.tar, multithreaded long distance mode, advanced one pass small out, 4853186 +silesia.tar, small window log, advanced one pass small out, 7101530 +silesia.tar, small hash log, advanced one pass small out, 6587951 +silesia.tar, small chain log, advanced one pass small out, 4943307 +silesia.tar, explicit params, advanced one pass small out, 4808589 +silesia.tar, uncompressed literals, advanced one pass small out, 5129458 +silesia.tar, uncompressed literals optimal, advanced one pass small out, 4320927 +silesia.tar, huffman literals, advanced one pass small out, 5347335 +silesia.tar, multithreaded with advanced params, advanced one pass small out, 5129555 github, level -5, advanced one pass small out, 205285 github, level -5 with dict, advanced one pass small out, 46718 github, level -3, advanced one pass small out, 190643 github, level -3 with dict, advanced one pass small out, 45395 github, level -1, advanced one pass small out, 175568 github, level -1 with dict, advanced one pass small out, 43170 -github, level 0, advanced one pass small out, 136311 +github, level 0, advanced one pass small out, 136335 github, level 0 with dict, advanced one pass small out, 41148 -github, level 1, advanced one pass small out, 142450 +github, level 1, advanced one pass small out, 142465 github, level 1 with dict, advanced one pass small out, 41682 -github, level 3, advanced one pass small out, 136311 +github, level 3, advanced one pass small out, 136335 github, level 3 with dict, advanced one pass small out, 41148 -github, level 4, advanced one pass small out, 136144 +github, level 4, advanced one pass small out, 136199 github, level 4 with dict, advanced one pass small out, 41251 -github, level 5, advanced one pass small out, 135106 +github, level 5, advanced one pass small out, 135121 github, level 5 with dict, advanced one pass small out, 38938 -github, level 6, advanced one pass small out, 135108 +github, level 6, advanced one pass small out, 135122 github, level 6 with dict, advanced one pass small out, 38632 -github, level 7, advanced one pass small out, 135108 -github, level 7 with dict, advanced one pass small out, 38766 -github, level 9, advanced one pass small out, 135108 -github, level 9 with dict, advanced one pass small out, 39326 -github, level 13, advanced one pass small out, 133717 -github, level 13 with dict, advanced one pass small out, 39716 -github, level 16, advanced one pass small out, 133717 +github, level 7, advanced one pass small out, 135122 +github, level 7 with dict, advanced one pass small out, 38771 +github, level 9, advanced one pass small out, 135122 +github, level 9 with dict, advanced one pass small out, 39332 +github, level 13, advanced one pass small out, 134064 +github, level 13 with dict, advanced one pass small out, 39743 +github, level 16, advanced one pass small out, 134064 github, level 16 with dict, advanced one pass small out, 37577 -github, level 19, advanced one pass small out, 133717 +github, level 19, advanced one pass small out, 134064 github, level 19 with dict, advanced one pass small out, 37576 -github, no source size, advanced one pass small out, 136311 -github, long distance mode, advanced one pass small out, 136311 -github, multithreaded, advanced one pass small out, 136311 -github, multithreaded long distance mode, advanced one pass small out, 136311 -github, small window log, advanced one pass small out, 136311 -github, small hash log, advanced one pass small out, 135467 -github, small chain log, advanced one pass small out, 136314 -github, explicit params, advanced one pass small out, 137670 +github, no source size, advanced one pass small out, 136335 +github, long distance mode, advanced one pass small out, 136335 +github, multithreaded, advanced one pass small out, 136335 +github, multithreaded long distance mode, advanced one pass small out, 136335 +github, small window log, advanced one pass small out, 136335 +github, small hash log, advanced one pass small out, 135590 +github, small chain log, advanced one pass small out, 136341 +github, explicit params, advanced one pass small out, 137727 github, uncompressed literals, advanced one pass small out, 165915 -github, uncompressed literals optimal, advanced one pass small out, 156824 -github, huffman literals, advanced one pass small out, 142450 +github, uncompressed literals optimal, advanced one pass small out, 157227 +github, huffman literals, advanced one pass small out, 142465 github, multithreaded with advanced params, advanced one pass small out, 165915 -silesia, level -5, advanced streaming, 6882466 -silesia, level -3, advanced streaming, 6568358 -silesia, level -1, advanced streaming, 6183385 -silesia, level 0, advanced streaming, 4849491 -silesia, level 1, advanced streaming, 5314109 -silesia, level 3, advanced streaming, 4849491 -silesia, level 4, advanced streaming, 4786913 -silesia, level 5, advanced streaming, 4710178 -silesia, level 6, advanced streaming, 4659996 -silesia, level 7, advanced streaming, 4596234 -silesia, level 9, advanced streaming, 4543862 -silesia, level 13, advanced streaming, 4482073 -silesia, level 16, advanced streaming, 4377389 -silesia, level 19, advanced streaming, 4293262 -silesia, no source size, advanced streaming, 4849455 -silesia, long distance mode, advanced streaming, 4839650 -silesia, multithreaded, advanced streaming, 4849491 -silesia, multithreaded long distance mode, advanced streaming, 4839650 -silesia, small window log, advanced streaming, 7105714 -silesia, small hash log, advanced streaming, 6554898 -silesia, small chain log, advanced streaming, 4931093 -silesia, explicit params, advanced streaming, 4797048 -silesia, uncompressed literals, advanced streaming, 5127960 -silesia, uncompressed literals optimal, advanced streaming, 4325434 -silesia, huffman literals, advanced streaming, 5331110 -silesia, multithreaded with advanced params, advanced streaming, 5127960 -silesia.tar, level -5, advanced streaming, 6982738 -silesia.tar, level -3, advanced streaming, 6641264 -silesia.tar, level -1, advanced streaming, 6190789 -silesia.tar, level 0, advanced streaming, 4861376 -silesia.tar, level 1, advanced streaming, 5336879 -silesia.tar, level 3, advanced streaming, 4861376 -silesia.tar, level 4, advanced streaming, 4799583 -silesia.tar, level 5, advanced streaming, 4722276 -silesia.tar, level 6, advanced streaming, 4672240 -silesia.tar, level 7, advanced streaming, 4606657 -silesia.tar, level 9, advanced streaming, 4554106 -silesia.tar, level 13, advanced streaming, 4491707 -silesia.tar, level 16, advanced streaming, 4381284 -silesia.tar, level 19, advanced streaming, 4281511 -silesia.tar, no source size, advanced streaming, 4861372 -silesia.tar, long distance mode, advanced streaming, 4848046 -silesia.tar, multithreaded, advanced streaming, 4861458 -silesia.tar, multithreaded long distance mode, advanced streaming, 4853136 -silesia.tar, small window log, advanced streaming, 7112148 -silesia.tar, small hash log, advanced streaming, 6587834 -silesia.tar, small chain log, advanced streaming, 4943271 -silesia.tar, explicit params, advanced streaming, 4808570 -silesia.tar, uncompressed literals, advanced streaming, 5129450 -silesia.tar, uncompressed literals optimal, advanced streaming, 4320841 -silesia.tar, huffman literals, advanced streaming, 5352306 -silesia.tar, multithreaded with advanced params, advanced streaming, 5129544 +silesia, level -5, advanced streaming, 6882505 +silesia, level -3, advanced streaming, 6568376 +silesia, level -1, advanced streaming, 6183403 +silesia, level 0, advanced streaming, 4849552 +silesia, level 1, advanced streaming, 5314162 +silesia, level 3, advanced streaming, 4849552 +silesia, level 4, advanced streaming, 4786970 +silesia, level 5, advanced streaming, 4710236 +silesia, level 6, advanced streaming, 4660056 +silesia, level 7, advanced streaming, 4596296 +silesia, level 9, advanced streaming, 4543925 +silesia, level 13, advanced streaming, 4482135 +silesia, level 16, advanced streaming, 4377465 +silesia, level 19, advanced streaming, 4293330 +silesia, no source size, advanced streaming, 4849516 +silesia, long distance mode, advanced streaming, 4839708 +silesia, multithreaded, advanced streaming, 4849552 +silesia, multithreaded long distance mode, advanced streaming, 4839708 +silesia, small window log, advanced streaming, 7112062 +silesia, small hash log, advanced streaming, 6555021 +silesia, small chain log, advanced streaming, 4931148 +silesia, explicit params, advanced streaming, 4797112 +silesia, uncompressed literals, advanced streaming, 5127982 +silesia, uncompressed literals optimal, advanced streaming, 4325472 +silesia, huffman literals, advanced streaming, 5331168 +silesia, multithreaded with advanced params, advanced streaming, 5127982 +silesia.tar, level -5, advanced streaming, 6982759 +silesia.tar, level -3, advanced streaming, 6641283 +silesia.tar, level -1, advanced streaming, 6190795 +silesia.tar, level 0, advanced streaming, 4861427 +silesia.tar, level 1, advanced streaming, 5336939 +silesia.tar, level 3, advanced streaming, 4861427 +silesia.tar, level 4, advanced streaming, 4799630 +silesia.tar, level 5, advanced streaming, 4722329 +silesia.tar, level 6, advanced streaming, 4672288 +silesia.tar, level 7, advanced streaming, 4606715 +silesia.tar, level 9, advanced streaming, 4554154 +silesia.tar, level 13, advanced streaming, 4491765 +silesia.tar, level 16, advanced streaming, 4381350 +silesia.tar, level 19, advanced streaming, 4281562 +silesia.tar, no source size, advanced streaming, 4861423 +silesia.tar, long distance mode, advanced streaming, 4848098 +silesia.tar, multithreaded, advanced streaming, 4861508 +silesia.tar, multithreaded long distance mode, advanced streaming, 4853186 +silesia.tar, small window log, advanced streaming, 7118769 +silesia.tar, small hash log, advanced streaming, 6587952 +silesia.tar, small chain log, advanced streaming, 4943312 +silesia.tar, explicit params, advanced streaming, 4808618 +silesia.tar, uncompressed literals, advanced streaming, 5129461 +silesia.tar, uncompressed literals optimal, advanced streaming, 4320858 +silesia.tar, huffman literals, advanced streaming, 5352360 +silesia.tar, multithreaded with advanced params, advanced streaming, 5129555 github, level -5, advanced streaming, 205285 github, level -5 with dict, advanced streaming, 46718 github, level -3, advanced streaming, 190643 github, level -3 with dict, advanced streaming, 45395 github, level -1, advanced streaming, 175568 github, level -1 with dict, advanced streaming, 43170 -github, level 0, advanced streaming, 136311 +github, level 0, advanced streaming, 136335 github, level 0 with dict, advanced streaming, 41148 -github, level 1, advanced streaming, 142450 +github, level 1, advanced streaming, 142465 github, level 1 with dict, advanced streaming, 41682 -github, level 3, advanced streaming, 136311 +github, level 3, advanced streaming, 136335 github, level 3 with dict, advanced streaming, 41148 -github, level 4, advanced streaming, 136144 +github, level 4, advanced streaming, 136199 github, level 4 with dict, advanced streaming, 41251 -github, level 5, advanced streaming, 135106 +github, level 5, advanced streaming, 135121 github, level 5 with dict, advanced streaming, 38938 -github, level 6, advanced streaming, 135108 +github, level 6, advanced streaming, 135122 github, level 6 with dict, advanced streaming, 38632 -github, level 7, advanced streaming, 135108 -github, level 7 with dict, advanced streaming, 38766 -github, level 9, advanced streaming, 135108 -github, level 9 with dict, advanced streaming, 39326 -github, level 13, advanced streaming, 133717 -github, level 13 with dict, advanced streaming, 39716 -github, level 16, advanced streaming, 133717 +github, level 7, advanced streaming, 135122 +github, level 7 with dict, advanced streaming, 38771 +github, level 9, advanced streaming, 135122 +github, level 9 with dict, advanced streaming, 39332 +github, level 13, advanced streaming, 134064 +github, level 13 with dict, advanced streaming, 39743 +github, level 16, advanced streaming, 134064 github, level 16 with dict, advanced streaming, 37577 -github, level 19, advanced streaming, 133717 +github, level 19, advanced streaming, 134064 github, level 19 with dict, advanced streaming, 37576 -github, no source size, advanced streaming, 136311 -github, long distance mode, advanced streaming, 136311 -github, multithreaded, advanced streaming, 136311 -github, multithreaded long distance mode, advanced streaming, 136311 -github, small window log, advanced streaming, 136311 -github, small hash log, advanced streaming, 135467 -github, small chain log, advanced streaming, 136314 -github, explicit params, advanced streaming, 137670 +github, no source size, advanced streaming, 136335 +github, long distance mode, advanced streaming, 136335 +github, multithreaded, advanced streaming, 136335 +github, multithreaded long distance mode, advanced streaming, 136335 +github, small window log, advanced streaming, 136335 +github, small hash log, advanced streaming, 135590 +github, small chain log, advanced streaming, 136341 +github, explicit params, advanced streaming, 137727 github, uncompressed literals, advanced streaming, 165915 -github, uncompressed literals optimal, advanced streaming, 156824 -github, huffman literals, advanced streaming, 142450 +github, uncompressed literals optimal, advanced streaming, 157227 +github, huffman literals, advanced streaming, 142465 github, multithreaded with advanced params, advanced streaming, 165915 -silesia, level -5, old streaming, 6882466 -silesia, level -3, old streaming, 6568358 -silesia, level -1, old streaming, 6183385 -silesia, level 0, old streaming, 4849491 -silesia, level 1, old streaming, 5314109 -silesia, level 3, old streaming, 4849491 -silesia, level 4, old streaming, 4786913 -silesia, level 5, old streaming, 4710178 -silesia, level 6, old streaming, 4659996 -silesia, level 7, old streaming, 4596234 -silesia, level 9, old streaming, 4543862 -silesia, level 13, old streaming, 4482073 -silesia, level 16, old streaming, 4377389 -silesia, level 19, old streaming, 4293262 -silesia, no source size, old streaming, 4849455 -silesia, uncompressed literals, old streaming, 4849491 -silesia, uncompressed literals optimal, old streaming, 4293262 -silesia, huffman literals, old streaming, 6183385 -silesia.tar, level -5, old streaming, 6982738 -silesia.tar, level -3, old streaming, 6641264 -silesia.tar, level -1, old streaming, 6190789 -silesia.tar, level 0, old streaming, 4861376 -silesia.tar, level 1, old streaming, 5336879 -silesia.tar, level 3, old streaming, 4861376 -silesia.tar, level 4, old streaming, 4799583 -silesia.tar, level 5, old streaming, 4722276 -silesia.tar, level 6, old streaming, 4672240 -silesia.tar, level 7, old streaming, 4606657 -silesia.tar, level 9, old streaming, 4554106 -silesia.tar, level 13, old streaming, 4491707 -silesia.tar, level 16, old streaming, 4381284 -silesia.tar, level 19, old streaming, 4281511 -silesia.tar, no source size, old streaming, 4861372 -silesia.tar, uncompressed literals, old streaming, 4861376 -silesia.tar, uncompressed literals optimal, old streaming, 4281511 -silesia.tar, huffman literals, old streaming, 6190789 +silesia, level -5, old streaming, 6882505 +silesia, level -3, old streaming, 6568376 +silesia, level -1, old streaming, 6183403 +silesia, level 0, old streaming, 4849552 +silesia, level 1, old streaming, 5314162 +silesia, level 3, old streaming, 4849552 +silesia, level 4, old streaming, 4786970 +silesia, level 5, old streaming, 4710236 +silesia, level 6, old streaming, 4660056 +silesia, level 7, old streaming, 4596296 +silesia, level 9, old streaming, 4543925 +silesia, level 13, old streaming, 4482135 +silesia, level 16, old streaming, 4377465 +silesia, level 19, old streaming, 4293330 +silesia, no source size, old streaming, 4849516 +silesia, uncompressed literals, old streaming, 4849552 +silesia, uncompressed literals optimal, old streaming, 4293330 +silesia, huffman literals, old streaming, 6183403 +silesia.tar, level -5, old streaming, 6982759 +silesia.tar, level -3, old streaming, 6641283 +silesia.tar, level -1, old streaming, 6190795 +silesia.tar, level 0, old streaming, 4861427 +silesia.tar, level 1, old streaming, 5336939 +silesia.tar, level 3, old streaming, 4861427 +silesia.tar, level 4, old streaming, 4799630 +silesia.tar, level 5, old streaming, 4722329 +silesia.tar, level 6, old streaming, 4672288 +silesia.tar, level 7, old streaming, 4606715 +silesia.tar, level 9, old streaming, 4554154 +silesia.tar, level 13, old streaming, 4491765 +silesia.tar, level 16, old streaming, 4381350 +silesia.tar, level 19, old streaming, 4281562 +silesia.tar, no source size, old streaming, 4861423 +silesia.tar, uncompressed literals, old streaming, 4861427 +silesia.tar, uncompressed literals optimal, old streaming, 4281562 +silesia.tar, huffman literals, old streaming, 6190795 github, level -5, old streaming, 205285 github, level -5 with dict, old streaming, 46718 github, level -3, old streaming, 190643 github, level -3 with dict, old streaming, 45395 github, level -1, old streaming, 175568 github, level -1 with dict, old streaming, 43170 -github, level 0, old streaming, 136311 +github, level 0, old streaming, 136335 github, level 0 with dict, old streaming, 41148 -github, level 1, old streaming, 142450 +github, level 1, old streaming, 142465 github, level 1 with dict, old streaming, 41682 -github, level 3, old streaming, 136311 +github, level 3, old streaming, 136335 github, level 3 with dict, old streaming, 41148 -github, level 4, old streaming, 136144 +github, level 4, old streaming, 136199 github, level 4 with dict, old streaming, 41251 -github, level 5, old streaming, 135106 +github, level 5, old streaming, 135121 github, level 5 with dict, old streaming, 38938 -github, level 6, old streaming, 135108 +github, level 6, old streaming, 135122 github, level 6 with dict, old streaming, 38632 -github, level 7, old streaming, 135108 -github, level 7 with dict, old streaming, 38766 -github, level 9, old streaming, 135108 -github, level 9 with dict, old streaming, 39326 -github, level 13, old streaming, 133717 -github, level 13 with dict, old streaming, 39716 -github, level 16, old streaming, 133717 +github, level 7, old streaming, 135122 +github, level 7 with dict, old streaming, 38771 +github, level 9, old streaming, 135122 +github, level 9 with dict, old streaming, 39332 +github, level 13, old streaming, 134064 +github, level 13 with dict, old streaming, 39743 +github, level 16, old streaming, 134064 github, level 16 with dict, old streaming, 37577 -github, level 19, old streaming, 133717 +github, level 19, old streaming, 134064 github, level 19 with dict, old streaming, 37576 -github, no source size, old streaming, 140631 -github, uncompressed literals, old streaming, 136311 -github, uncompressed literals optimal, old streaming, 133717 +github, no source size, old streaming, 140632 +github, uncompressed literals, old streaming, 136335 +github, uncompressed literals optimal, old streaming, 134064 github, huffman literals, old streaming, 175568 -silesia, level -5, old streaming advanced, 6882466 -silesia, level -3, old streaming advanced, 6568358 -silesia, level -1, old streaming advanced, 6183385 -silesia, level 0, old streaming advanced, 4849491 -silesia, level 1, old streaming advanced, 5314109 -silesia, level 3, old streaming advanced, 4849491 -silesia, level 4, old streaming advanced, 4786913 -silesia, level 5, old streaming advanced, 4710178 -silesia, level 6, old streaming advanced, 4659996 -silesia, level 7, old streaming advanced, 4596234 -silesia, level 9, old streaming advanced, 4543862 -silesia, level 13, old streaming advanced, 4482073 -silesia, level 16, old streaming advanced, 4377389 -silesia, level 19, old streaming advanced, 4293262 -silesia, no source size, old streaming advanced, 4849455 -silesia, long distance mode, old streaming advanced, 4849491 -silesia, multithreaded, old streaming advanced, 4849491 -silesia, multithreaded long distance mode, old streaming advanced, 4849491 -silesia, small window log, old streaming advanced, 7105714 -silesia, small hash log, old streaming advanced, 6554898 -silesia, small chain log, old streaming advanced, 4931093 -silesia, explicit params, old streaming advanced, 4797048 -silesia, uncompressed literals, old streaming advanced, 4849491 -silesia, uncompressed literals optimal, old streaming advanced, 4293262 -silesia, huffman literals, old streaming advanced, 6183385 -silesia, multithreaded with advanced params, old streaming advanced, 4849491 -silesia.tar, level -5, old streaming advanced, 6982738 -silesia.tar, level -3, old streaming advanced, 6641264 -silesia.tar, level -1, old streaming advanced, 6190789 -silesia.tar, level 0, old streaming advanced, 4861376 -silesia.tar, level 1, old streaming advanced, 5336879 -silesia.tar, level 3, old streaming advanced, 4861376 -silesia.tar, level 4, old streaming advanced, 4799583 -silesia.tar, level 5, old streaming advanced, 4722276 -silesia.tar, level 6, old streaming advanced, 4672240 -silesia.tar, level 7, old streaming advanced, 4606657 -silesia.tar, level 9, old streaming advanced, 4554106 -silesia.tar, level 13, old streaming advanced, 4491707 -silesia.tar, level 16, old streaming advanced, 4381284 -silesia.tar, level 19, old streaming advanced, 4281511 -silesia.tar, no source size, old streaming advanced, 4861372 -silesia.tar, long distance mode, old streaming advanced, 4861376 -silesia.tar, multithreaded, old streaming advanced, 4861376 -silesia.tar, multithreaded long distance mode, old streaming advanced, 4861376 -silesia.tar, small window log, old streaming advanced, 7112151 -silesia.tar, small hash log, old streaming advanced, 6587834 -silesia.tar, small chain log, old streaming advanced, 4943271 -silesia.tar, explicit params, old streaming advanced, 4808570 -silesia.tar, uncompressed literals, old streaming advanced, 4861376 -silesia.tar, uncompressed literals optimal, old streaming advanced, 4281511 -silesia.tar, huffman literals, old streaming advanced, 6190789 -silesia.tar, multithreaded with advanced params, old streaming advanced, 4861376 +silesia, level -5, old streaming advanced, 6882505 +silesia, level -3, old streaming advanced, 6568376 +silesia, level -1, old streaming advanced, 6183403 +silesia, level 0, old streaming advanced, 4849552 +silesia, level 1, old streaming advanced, 5314162 +silesia, level 3, old streaming advanced, 4849552 +silesia, level 4, old streaming advanced, 4786970 +silesia, level 5, old streaming advanced, 4710236 +silesia, level 6, old streaming advanced, 4660056 +silesia, level 7, old streaming advanced, 4596296 +silesia, level 9, old streaming advanced, 4543925 +silesia, level 13, old streaming advanced, 4482135 +silesia, level 16, old streaming advanced, 4377465 +silesia, level 19, old streaming advanced, 4293330 +silesia, no source size, old streaming advanced, 4849516 +silesia, long distance mode, old streaming advanced, 4849552 +silesia, multithreaded, old streaming advanced, 4849552 +silesia, multithreaded long distance mode, old streaming advanced, 4849552 +silesia, small window log, old streaming advanced, 7112062 +silesia, small hash log, old streaming advanced, 6555021 +silesia, small chain log, old streaming advanced, 4931148 +silesia, explicit params, old streaming advanced, 4797112 +silesia, uncompressed literals, old streaming advanced, 4849552 +silesia, uncompressed literals optimal, old streaming advanced, 4293330 +silesia, huffman literals, old streaming advanced, 6183403 +silesia, multithreaded with advanced params, old streaming advanced, 4849552 +silesia.tar, level -5, old streaming advanced, 6982759 +silesia.tar, level -3, old streaming advanced, 6641283 +silesia.tar, level -1, old streaming advanced, 6190795 +silesia.tar, level 0, old streaming advanced, 4861427 +silesia.tar, level 1, old streaming advanced, 5336939 +silesia.tar, level 3, old streaming advanced, 4861427 +silesia.tar, level 4, old streaming advanced, 4799630 +silesia.tar, level 5, old streaming advanced, 4722329 +silesia.tar, level 6, old streaming advanced, 4672288 +silesia.tar, level 7, old streaming advanced, 4606715 +silesia.tar, level 9, old streaming advanced, 4554154 +silesia.tar, level 13, old streaming advanced, 4491765 +silesia.tar, level 16, old streaming advanced, 4381350 +silesia.tar, level 19, old streaming advanced, 4281562 +silesia.tar, no source size, old streaming advanced, 4861423 +silesia.tar, long distance mode, old streaming advanced, 4861427 +silesia.tar, multithreaded, old streaming advanced, 4861427 +silesia.tar, multithreaded long distance mode, old streaming advanced, 4861427 +silesia.tar, small window log, old streaming advanced, 7118772 +silesia.tar, small hash log, old streaming advanced, 6587952 +silesia.tar, small chain log, old streaming advanced, 4943312 +silesia.tar, explicit params, old streaming advanced, 4808618 +silesia.tar, uncompressed literals, old streaming advanced, 4861427 +silesia.tar, uncompressed literals optimal, old streaming advanced, 4281562 +silesia.tar, huffman literals, old streaming advanced, 6190795 +silesia.tar, multithreaded with advanced params, old streaming advanced, 4861427 github, level -5, old streaming advanced, 216734 github, level -5 with dict, old streaming advanced, 49562 github, level -3, old streaming advanced, 192160 github, level -3 with dict, old streaming advanced, 44956 github, level -1, old streaming advanced, 181108 github, level -1 with dict, old streaming advanced, 42383 -github, level 0, old streaming advanced, 141090 +github, level 0, old streaming advanced, 141104 github, level 0 with dict, old streaming advanced, 41113 -github, level 1, old streaming advanced, 143682 +github, level 1, old streaming advanced, 143692 github, level 1 with dict, old streaming advanced, 42430 -github, level 3, old streaming advanced, 141090 +github, level 3, old streaming advanced, 141104 github, level 3 with dict, old streaming advanced, 41113 -github, level 4, old streaming advanced, 141090 +github, level 4, old streaming advanced, 141104 github, level 4 with dict, old streaming advanced, 41084 -github, level 5, old streaming advanced, 139391 +github, level 5, old streaming advanced, 139399 github, level 5 with dict, old streaming advanced, 39159 -github, level 6, old streaming advanced, 139394 +github, level 6, old streaming advanced, 139402 github, level 6 with dict, old streaming advanced, 38749 -github, level 7, old streaming advanced, 138675 +github, level 7, old streaming advanced, 138676 github, level 7 with dict, old streaming advanced, 38746 -github, level 9, old streaming advanced, 138675 -github, level 9 with dict, old streaming advanced, 38987 -github, level 13, old streaming advanced, 138675 -github, level 13 with dict, old streaming advanced, 39724 -github, level 16, old streaming advanced, 138675 -github, level 16 with dict, old streaming advanced, 40771 -github, level 19, old streaming advanced, 133717 +github, level 9, old streaming advanced, 138676 +github, level 9 with dict, old streaming advanced, 38993 +github, level 13, old streaming advanced, 138676 +github, level 13 with dict, old streaming advanced, 39731 +github, level 16, old streaming advanced, 138676 +github, level 16 with dict, old streaming advanced, 40789 +github, level 19, old streaming advanced, 134064 github, level 19 with dict, old streaming advanced, 37576 -github, no source size, old streaming advanced, 140631 -github, long distance mode, old streaming advanced, 141090 -github, multithreaded, old streaming advanced, 141090 -github, multithreaded long distance mode, old streaming advanced, 141090 -github, small window log, old streaming advanced, 141090 -github, small hash log, old streaming advanced, 141578 -github, small chain log, old streaming advanced, 139258 -github, explicit params, old streaming advanced, 140930 -github, uncompressed literals, old streaming advanced, 141090 -github, uncompressed literals optimal, old streaming advanced, 133717 +github, no source size, old streaming advanced, 140632 +github, long distance mode, old streaming advanced, 141104 +github, multithreaded, old streaming advanced, 141104 +github, multithreaded long distance mode, old streaming advanced, 141104 +github, small window log, old streaming advanced, 141104 +github, small hash log, old streaming advanced, 141597 +github, small chain log, old streaming advanced, 139275 +github, explicit params, old streaming advanced, 140937 +github, uncompressed literals, old streaming advanced, 141104 +github, uncompressed literals optimal, old streaming advanced, 134064 github, huffman literals, old streaming advanced, 181108 -github, multithreaded with advanced params, old streaming advanced, 141090 +github, multithreaded with advanced params, old streaming advanced, 141104 github, level -5 with dict, old streaming cdcit, 46718 github, level -3 with dict, old streaming cdcit, 45395 github, level -1 with dict, old streaming cdcit, 43170 @@ -615,9 +615,9 @@ github, level 3 with dict, old stre github, level 4 with dict, old streaming cdcit, 41251 github, level 5 with dict, old streaming cdcit, 38938 github, level 6 with dict, old streaming cdcit, 38632 -github, level 7 with dict, old streaming cdcit, 38766 -github, level 9 with dict, old streaming cdcit, 39326 -github, level 13 with dict, old streaming cdcit, 39716 +github, level 7 with dict, old streaming cdcit, 38771 +github, level 9 with dict, old streaming cdcit, 39332 +github, level 13 with dict, old streaming cdcit, 39743 github, level 16 with dict, old streaming cdcit, 37577 github, level 19 with dict, old streaming cdcit, 37576 github, level -5 with dict, old streaming advanced cdict, 49562 @@ -627,10 +627,10 @@ github, level 0 with dict, old stre github, level 1 with dict, old streaming advanced cdict, 42430 github, level 3 with dict, old streaming advanced cdict, 41113 github, level 4 with dict, old streaming advanced cdict, 41084 -github, level 5 with dict, old streaming advanced cdict, 39158 -github, level 6 with dict, old streaming advanced cdict, 38748 -github, level 7 with dict, old streaming advanced cdict, 38744 -github, level 9 with dict, old streaming advanced cdict, 38986 -github, level 13 with dict, old streaming advanced cdict, 39724 -github, level 16 with dict, old streaming advanced cdict, 40771 +github, level 5 with dict, old streaming advanced cdict, 39159 +github, level 6 with dict, old streaming advanced cdict, 38749 +github, level 7 with dict, old streaming advanced cdict, 38746 +github, level 9 with dict, old streaming advanced cdict, 38993 +github, level 13 with dict, old streaming advanced cdict, 39731 +github, level 16 with dict, old streaming advanced cdict, 40789 github, level 19 with dict, old streaming advanced cdict, 37576 diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 79d5a828..fa18ea4b 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -31,7 +31,6 @@ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */ #include "zstd.h" /* ZSTD_compressBound */ #include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */ -#include "zstdmt_compress.h" #include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ @@ -274,7 +273,7 @@ static int basicUnitTests(U32 seed, double compressibility) U32 coreSeed = 0; /* this name to conform with CHECK_Z macro display */ ZSTD_CStream* zc = ZSTD_createCStream(); ZSTD_DStream* zd = ZSTD_createDStream(); - ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2); + ZSTD_CCtx* mtctx = ZSTD_createCCtx(); ZSTD_inBuffer inBuff, inBuff2; ZSTD_outBuffer outBuff; @@ -283,12 +282,14 @@ static int basicUnitTests(U32 seed, double compressibility) unsigned dictID = 0; /* Create compressible test buffer */ - if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { + if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd || !mtctx) { DISPLAY("Not enough memory, aborting \n"); goto _output_error; } RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); + CHECK_Z(ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2)); + /* Create dictionary */ DISPLAYLEVEL(3, "creating dictionary for unit tests \n"); dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 3, 16 KB, 48 KB); @@ -753,6 +754,166 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_freeDCtx(dctx); } + /* Compression with ZSTD_c_stable{In,Out}Buffer */ + { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_inBuffer in; + ZSTD_outBuffer out; + size_t cctxSize1; + size_t cctxSize2; + in.src = CNBuffer; + in.size = CNBufferSize; + out.dst = compressedBuffer; + out.size = compressedBufferSize; + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() uses stable input and output : ", testNb++); + CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize)); + CHECK(!(cSize < ZSTD_compressBound(CNBufferSize)), "cSize too large for test"); + CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize + 4, CNBuffer, CNBufferSize)); + CHECK_Z(cctxSize1 = ZSTD_sizeof_CCtx(cctx)); + { ZSTD_CCtx* cctx2 = ZSTD_createCCtx(); + in.pos = out.pos = 0; + CHECK_Z(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_continue)); + CHECK(!(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_end) == 0), "Not finished"); + CHECK_Z(cctxSize2 = ZSTD_sizeof_CCtx(cctx2)); + ZSTD_freeCCtx(cctx2); + } + { ZSTD_CCtx* cctx3 = ZSTD_createCCtx(); + ZSTD_parameters params = ZSTD_getParams(0, CNBufferSize, 0); + size_t cSize3; + params.fParams.checksumFlag = 1; + cSize3 = ZSTD_compress_advanced(cctx3, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, NULL, 0, params); + CHECK_Z(cSize3); + CHECK(!(cSize == cSize3), "Must be same compressed size"); + CHECK(!(cctxSize1 == ZSTD_sizeof_CCtx(cctx3)), "Must be same CCtx size"); + ZSTD_freeCCtx(cctx3); + } + CHECK(!(cctxSize1 < cctxSize2), "Stable buffers means less allocated size"); + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize)); + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() doesn't modify user parameters : ", testNb++); + { + int stableInBuffer; + int stableOutBuffer; + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer)); + CHECK(!(stableInBuffer == 0), "Modified"); + CHECK(!(stableOutBuffer == 0), "Modified"); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1)); + CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer)); + CHECK(!(stableInBuffer == 1), "Modified"); + CHECK(!(stableOutBuffer == 1), "Modified"); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer : ", testNb++); + CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1)); + in.pos = out.pos = 0; + CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished"); + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize)); + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer allocated size : ", testNb++); + { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx); + CHECK(!(cctxSize1 == cctxSize), "Must be the same size as single pass"); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer only : ", testNb++); + CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1)); + in.pos = out.pos = 0; + out.size = cSize / 4; + for (;;) { + size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); + CHECK_Z(ret); + if (ret == 0) + break; + out.size = MIN(out.size + cSize / 4, compressedBufferSize); + } + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize)); + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer modify buffer : ", testNb++); + in.pos = out.pos = 0; + out.size = cSize / 4; + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + in.src = (char const*)in.src + in.pos; + in.size -= in.pos; + in.pos = 0; + { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); + CHECK(!ZSTD_isError(ret), "Must error"); + CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error"); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer with continue and flush : ", testNb++); + in.src = CNBuffer; + in.size = CNBufferSize; + in.pos = 0; + out.pos = 0; + out.size = compressedBufferSize; + CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only)); + { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); + CHECK(!ZSTD_isError(ret), "Must error"); + CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error"); + } + CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only)); + { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush); + CHECK(!ZSTD_isError(ret), "Must error"); + CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error"); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer allocated size : ", testNb++); + { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx); + CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass"); + CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming"); + cctxSize1 = cctxSize; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer only : ", testNb++); + CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1)); + in.pos = out.pos = 0; + in.size = MIN(CNBufferSize, 10); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + in.pos = 0; + in.size = CNBufferSize - in.size; + CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished"); + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize)); + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer modify buffer : ", testNb++); + in.pos = out.pos = 0; + in.size = CNBufferSize; + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); + in.pos = out.pos = 0; + { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); + CHECK(!ZSTD_isError(ret), "Must have errored"); + CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_dstBuffer_wrong), "Must be this error"); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer allocated size : ", testNb++); + { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx); + CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass and stableInBuffer"); + CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming"); + } + DISPLAYLEVEL(3, "OK \n"); + + ZSTD_freeCCtx(cctx); + } + /* CDict scenario */ DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++); { ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ ); @@ -1144,12 +1305,10 @@ static int basicUnitTests(U32 seed, double compressibility) /* Basic multithreading compression test */ DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - { ZSTD_parameters const params = ZSTD_getParams(1, 0, 0); - int jobSize; - CHECK_Z( ZSTDMT_getMTCtxParameter(mtctx, ZSTDMT_p_jobSize, &jobSize)); + { int jobSize; + CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize)); CHECK(jobSize != 0, "job size non-zero"); - CHECK_Z( ZSTDMT_initCStream_advanced(mtctx, CNBuffer, dictSize, params, CNBufferSize) ); - CHECK_Z( ZSTDMT_getMTCtxParameter(mtctx, ZSTDMT_p_jobSize, &jobSize)); + CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize)); CHECK(jobSize != 0, "job size non-zero"); } outBuff.dst = compressedBuffer; @@ -1158,7 +1317,7 @@ static int basicUnitTests(U32 seed, double compressibility) inBuff.src = CNBuffer; inBuff.size = CNBufferSize; inBuff.pos = 0; - { size_t const compressResult = ZSTDMT_compressStream_generic(mtctx, &outBuff, &inBuff, ZSTD_e_end); + { size_t const compressResult = ZSTD_compressStream2(mtctx, &outBuff, &inBuff, ZSTD_e_end); if (compressResult != 0) goto _output_error; /* compression must be completed in a single round */ } if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ @@ -1516,7 +1675,7 @@ _end: FUZ_freeDictionary(dictionary); ZSTD_freeCStream(zc); ZSTD_freeDStream(zd); - ZSTDMT_freeCCtx(mtctx); + ZSTD_freeCCtx(mtctx); free(CNBuffer); free(compressedBuffer); free(decodedBuffer); @@ -1826,283 +1985,6 @@ _output_error: goto _cleanup; } - -/* fuzzing ZSTDMT_* interface */ -static int fuzzerTests_MT(U32 seed, int nbTests, int startTest, - double compressibility, int bigTests) -{ - const U32 maxSrcLog = bigTests ? 24 : 22; - static const U32 maxSampleLog = 19; - size_t const srcBufferSize = (size_t)1<<maxSrcLog; - BYTE* cNoiseBuffer[5]; - size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog); - BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize); - size_t const cBufferSize = ZSTD_compressBound(srcBufferSize); - BYTE* const cBuffer = (BYTE*)malloc (cBufferSize); - size_t const dstBufferSize = srcBufferSize; - BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize); - U32 result = 0; - int testNb = 0; - U32 coreSeed = seed; - int nbThreads = 2; - ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(nbThreads); /* will be reset sometimes */ - ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */ - ZSTD_DStream* const zd_noise = ZSTD_createDStream(); - UTIL_time_t const startClock = UTIL_getTime(); - const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */ - size_t dictSize = 0; - int const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests; - U32 const nbThreadsMax = bigTests ? 4 : 2; - - /* allocations */ - cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); - cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); - cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); - cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); - cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); - CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] || - !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise , - "Not enough memory, fuzzer tests cancelled"); - - /* Create initial samples */ - RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ - RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ - RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); - RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ - RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ - memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */ - ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */ - DISPLAYLEVEL(6, "Creating initial context with %i threads \n", nbThreads); - - /* catch up testNb */ - for (testNb=1; testNb < startTest; testNb++) - FUZ_rand(&coreSeed); - - /* test loop */ - for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) { - U32 lseed; - const BYTE* srcBuffer; - size_t totalTestSize, totalGenSize, cSize; - XXH64_state_t xxhState; - U64 crcOrig; - size_t maxTestSize; - - FUZ_rand(&coreSeed); - if (nbTests >= testNb) { - DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); - } else { - DISPLAYUPDATE(2, "\r%6u ", testNb); - } - lseed = coreSeed ^ prime32; - - /* states full reset (deliberately not synchronized) */ - /* some issues can only happen when reusing states */ - if ((FUZ_rand(&lseed) & 0xFF) == 131) { - nbThreads = (FUZ_rand(&lseed) % nbThreadsMax) + 1; - DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads); - ZSTDMT_freeCCtx(zc); - zc = ZSTDMT_createCCtx(nbThreads); - CHECK(zc==NULL, "ZSTDMT_createCCtx allocation error") - } - if ((FUZ_rand(&lseed) & 0xFF) == 132) { - ZSTD_freeDStream(zd); - zd = ZSTD_createDStream(); - CHECK(zd==NULL, "ZSTDMT_createCCtx allocation error") - ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */ - } - - /* srcBuffer selection [0-4] */ - { U32 buffNb = FUZ_rand(&lseed) & 0x7F; - if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ - else { - buffNb >>= 3; - if (buffNb & 7) { - const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ - buffNb = tnb[buffNb >> 3]; - } else { - const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ - buffNb = tnb[buffNb >> 3]; - } } - srcBuffer = cNoiseBuffer[buffNb]; - } - - /* compression init */ - { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; - int const cLevelCandidate = ( FUZ_rand(&lseed) - % (ZSTD_maxCLevel() - (MAX(testLog, dictLog) / 2)) ) - + 1; - int const cLevelThreadAdjusted = cLevelCandidate - (nbThreads * 2) + 2; /* reduce cLevel when multiple threads to reduce memory consumption */ - int const cLevelMin = MAX(cLevelThreadAdjusted, 1); /* no negative cLevel yet */ - int const cLevel = MIN(cLevelMin, cLevelMax); - maxTestSize = FUZ_rLogLength(&lseed, testLog); - - if (FUZ_rand(&lseed)&1) { /* simple init */ - int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1; - DISPLAYLEVEL(5, "Init with compression level = %i \n", compressionLevel); - CHECK_Z( ZSTDMT_initCStream(zc, compressionLevel) ); - } else { /* advanced init */ - /* random dictionary selection */ - dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; - { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); - dict = srcBuffer + dictStart; - } - { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; - ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize); - DISPLAYLEVEL(5, "Init with windowLog = %u, pledgedSrcSize = %u, dictSize = %u \n", - params.cParams.windowLog, (unsigned)pledgedSrcSize, (unsigned)dictSize); - params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; - params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; - params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1; - DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag); - CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapLog, FUZ_rand(&lseed) % 12) ); - CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custom job size */ - CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); - } } } - - /* multi-segments compression test */ - XXH64_reset(&xxhState, 0); - { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; - U32 n; - for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) { - /* compress random chunks into randomly sized dst buffers */ - { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize); - size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize); - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); - ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 }; - outBuff.size = outBuff.pos + dstBuffSize; - - DISPLAYLEVEL(6, "Sending %u bytes to compress \n", (unsigned)srcSize); - CHECK_Z( ZSTDMT_compressStream(zc, &outBuff, &inBuff) ); - DISPLAYLEVEL(6, "%u bytes read by ZSTDMT_compressStream \n", (unsigned)inBuff.pos); - - XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); - memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); - totalTestSize += inBuff.pos; - } - - /* random flush operation, to mess around */ - if ((FUZ_rand(&lseed) & 15) == 0) { - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); - size_t const previousPos = outBuff.pos; - outBuff.size = outBuff.pos + adjustedDstSize; - DISPLAYLEVEL(5, "Flushing into dst buffer of size %u \n", (unsigned)adjustedDstSize); - CHECK_Z( ZSTDMT_flushStream(zc, &outBuff) ); - assert(outBuff.pos >= previousPos); - DISPLAYLEVEL(6, "%u bytes flushed by ZSTDMT_flushStream \n", (unsigned)(outBuff.pos-previousPos)); - } } - - /* final frame epilogue */ - { size_t remainingToFlush = (size_t)(-1); - while (remainingToFlush) { - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); - size_t const previousPos = outBuff.pos; - outBuff.size = outBuff.pos + adjustedDstSize; - DISPLAYLEVEL(5, "Ending into dst buffer of size %u \n", (unsigned)adjustedDstSize); - remainingToFlush = ZSTDMT_endStream(zc, &outBuff); - CHECK (ZSTD_isError(remainingToFlush), "ZSTDMT_endStream error : %s", ZSTD_getErrorName(remainingToFlush)); - assert(outBuff.pos >= previousPos); - DISPLAYLEVEL(6, "%u bytes flushed by ZSTDMT_endStream \n", (unsigned)(outBuff.pos-previousPos)); - DISPLAYLEVEL(5, "endStream : remainingToFlush : %u \n", (unsigned)remainingToFlush); - } } - crcOrig = XXH64_digest(&xxhState); - cSize = outBuff.pos; - DISPLAYLEVEL(5, "Frame completed : %u bytes compressed into %u bytes \n", - (unsigned)totalTestSize, (unsigned)cSize); - } - - /* multi - fragments decompression test */ - assert(totalTestSize < dstBufferSize); - memset(dstBuffer, 170, totalTestSize); /* init dest area */ - if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { - CHECK_Z( ZSTD_resetDStream(zd) ); - } else { - CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) ); - } - { size_t decompressionResult = 1; - ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; - ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; - for (totalGenSize = 0 ; decompressionResult ; ) { - size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); - inBuff.size = inBuff.pos + readCSrcSize; - outBuff.size = outBuff.pos + dstBuffSize; - DISPLAYLEVEL(6, "ZSTD_decompressStream input %u bytes into outBuff %u bytes \n", - (unsigned)readCSrcSize, (unsigned)dstBuffSize); - decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); - if (ZSTD_isError(decompressionResult)) { - DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult)); - findDiff(copyBuffer, dstBuffer, totalTestSize); - } - CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); - DISPLAYLEVEL(6, "total ingested (inBuff.pos) = %u and produced (outBuff.pos) = %u \n", - (unsigned)inBuff.pos, (unsigned)outBuff.pos); - } - CHECK (outBuff.pos != totalTestSize, - "decompressed data : wrong size (%u != %u)", - (unsigned)outBuff.pos, (unsigned)totalTestSize ); - CHECK (inBuff.pos != cSize, - "compressed data should be fully read (%u != %u)", - (unsigned)inBuff.pos, (unsigned)cSize ); - { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); - if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); - CHECK (crcDest!=crcOrig, "decompressed data corrupted"); - } } - - /*===== noisy/erroneous src decompression test =====*/ - - /* add some noise */ - { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; - U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) { - size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const noiseSize = MIN((cSize/3) , randomNoiseSize); - size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize); - size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize); - memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize); - } } - - /* try decompression on noisy data */ - CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */ - { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; - ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; - while (outBuff.pos < dstBufferSize) { - size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize); - size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize); - outBuff.size = outBuff.pos + adjustedDstSize; - inBuff.size = inBuff.pos + adjustedCSrcSize; - { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff); - if (ZSTD_isError(decompressError)) break; /* error correctly detected */ - /* No forward progress possible */ - if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break; - } } } } - DISPLAY("\r%u fuzzer tests completed \n", testNb); - -_cleanup: - ZSTDMT_freeCCtx(zc); - ZSTD_freeDStream(zd); - ZSTD_freeDStream(zd_noise); - free(cNoiseBuffer[0]); - free(cNoiseBuffer[1]); - free(cNoiseBuffer[2]); - free(cNoiseBuffer[3]); - free(cNoiseBuffer[4]); - free(copyBuffer); - free(cBuffer); - free(dstBuffer); - return result; - -_output_error: - result = 1; - goto _cleanup; -} - /** If useOpaqueAPI, sets param in cctxParams. * Otherwise, sets the param in zc. */ static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams, @@ -2179,6 +2061,8 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, U32 resetAllowed = 1; size_t maxTestSize; ZSTD_parameters savedParams; + int isRefPrefix = 0; + U64 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* init */ if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } @@ -2249,8 +2133,8 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, dict = srcBuffer + dictStart; if (!dictSize) dict=NULL; } - { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; - ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize); + pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; + { ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize); const U32 windowLogMax = bigTests ? 24 : 20; const U32 searchLogMax = bigTests ? 15 : 13; if (dictSize) @@ -2306,6 +2190,8 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, if (FUZ_rand(&lseed) & 1) { DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (unsigned)pledgedSrcSize); CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); + } else { + pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; } /* multi-threading parameters. Only adjust occasionally for small tests. */ @@ -2322,7 +2208,11 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, } } /* Enable rsyncable mode 1 in 4 times. */ - setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, (FUZ_rand(&lseed) % 4 == 0), opaqueAPI); + { + int const rsyncable = (FUZ_rand(&lseed) % 4 == 0); + DISPLAYLEVEL(5, "t%u: rsyncable : %d \n", testNb, rsyncable); + setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, rsyncable, opaqueAPI); + } if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) ); @@ -2339,6 +2229,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) ); } } else { + isRefPrefix = 1; CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) ); } } } @@ -2346,45 +2237,97 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, CHECK_Z(getCCtxParams(zc, &savedParams)); /* multi-segments compression test */ - XXH64_reset(&xxhState, 0); - { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; - for (cSize=0, totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) { - /* compress random chunks into randomly sized dst buffers */ - size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize); - size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize); - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); - size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); - ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush; - ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 }; - outBuff.size = outBuff.pos + dstBuffSize; + { int iter; + int const startSeed = lseed; + XXH64_hash_t compressedCrcs[2]; + for (iter = 0; iter < 2; ++iter, lseed = startSeed) { + ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; + int const singlePass = (FUZ_rand(&lseed) & 3) == 0; + int nbWorkers; + + XXH64_reset(&xxhState, 0); + + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); + if (isRefPrefix) { + DISPLAYLEVEL(6, "t%u: Reloading prefix\n", testNb); + /* Need to reload the prefix because it gets dropped after one compression */ + CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) ); + } - CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) ); - DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n", - testNb, (unsigned)inBuff.pos, (unsigned)(totalTestSize + inBuff.pos), (unsigned)flush, (unsigned)outBuff.pos); + /* Adjust number of workers occassionally - result must be deterministic independent of nbWorkers */ + CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_nbWorkers, &nbWorkers)); + if (nbWorkers > 0 && (FUZ_rand(&lseed) & 7) == 0) { + DISPLAYLEVEL(6, "t%u: Modify nbWorkers: %d -> %d \n", testNb, nbWorkers, nbWorkers + iter); + CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers + iter)); + } - XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); - memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); - totalTestSize += inBuff.pos; - } + if (singlePass) { + ZSTD_inBuffer inBuff = { srcBuffer, maxTestSize, 0 }; + CHECK_Z(ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end)); + DISPLAYLEVEL(6, "t%u: Single pass compression: consumed %u bytes ; produced %u bytes \n", + testNb, (unsigned)inBuff.pos, (unsigned)outBuff.pos); + CHECK(inBuff.pos != inBuff.size, "Input not consumed!"); + crcOrig = XXH64(srcBuffer, maxTestSize, 0); + totalTestSize = maxTestSize; + } else { + outBuff.size = 0; + for (totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) { + /* compress random chunks into randomly sized dst buffers */ + size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize); + size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize); + ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush; + ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 }; + int forwardProgress; + do { + size_t const ipos = inBuff.pos; + size_t const opos = outBuff.pos; + size_t ret; + if (outBuff.pos == outBuff.size) { + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); + size_t const dstBuffSize = MIN(cBufferSize - outBuff.pos, randomDstSize); + outBuff.size = outBuff.pos + dstBuffSize; + } + CHECK_Z( ret = ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) ); + DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n", + testNb, (unsigned)inBuff.pos, (unsigned)(totalTestSize + inBuff.pos), (unsigned)flush, (unsigned)outBuff.pos); + + /* We've completed the flush */ + if (flush == ZSTD_e_flush && ret == 0) + break; + + /* Ensure maximal forward progress for determinism */ + forwardProgress = (inBuff.pos != ipos) || (outBuff.pos != opos); + } while (forwardProgress); + assert(inBuff.pos == inBuff.size); + + XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); + memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); + totalTestSize += inBuff.pos; + } - /* final frame epilogue */ - { size_t remainingToFlush = 1; - while (remainingToFlush) { - ZSTD_inBuffer inBuff = { NULL, 0, 0 }; - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); - size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); - outBuff.size = outBuff.pos + adjustedDstSize; - DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (unsigned)adjustedDstSize); - remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end); - DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (unsigned)outBuff.pos); - CHECK( ZSTD_isError(remainingToFlush), - "ZSTD_compressStream2 w/ ZSTD_e_end error : %s", - ZSTD_getErrorName(remainingToFlush) ); - } } - crcOrig = XXH64_digest(&xxhState); - cSize = outBuff.pos; - DISPLAYLEVEL(5, "Frame completed : %zu bytes \n", cSize); + /* final frame epilogue */ + { size_t remainingToFlush = 1; + while (remainingToFlush) { + ZSTD_inBuffer inBuff = { NULL, 0, 0 }; + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); + size_t const adjustedDstSize = MIN(cBufferSize - outBuff.pos, randomDstSize); + outBuff.size = outBuff.pos + adjustedDstSize; + DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (unsigned)adjustedDstSize); + /* ZSTD_e_end guarantees maximal forward progress */ + remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end); + DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (unsigned)outBuff.pos); + CHECK( ZSTD_isError(remainingToFlush), + "ZSTD_compressStream2 w/ ZSTD_e_end error : %s", + ZSTD_getErrorName(remainingToFlush) ); + } } + crcOrig = XXH64_digest(&xxhState); + } + cSize = outBuff.pos; + compressedCrcs[iter] = XXH64(cBuffer, cSize, 0); + DISPLAYLEVEL(5, "Frame completed : %zu bytes \n", cSize); + } + CHECK(!(compressedCrcs[0] == compressedCrcs[1]), "Compression is not deterministic!"); } CHECK(badParameters(zc, savedParams), "CCtx params are wrong"); @@ -2496,7 +2439,7 @@ static int FUZ_usage(const char* programName) return 0; } -typedef enum { simple_api, mt_api, advanced_api } e_api; +typedef enum { simple_api, advanced_api } e_api; int main(int argc, const char** argv) { @@ -2520,7 +2463,6 @@ int main(int argc, const char** argv) /* Parsing commands. Aggregated commands are allowed */ if (argument[0]=='-') { - if (!strcmp(argument, "--mt")) { selected_api=mt_api; testNb += !testNb; continue; } if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; } if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } @@ -2633,9 +2575,6 @@ int main(int argc, const char** argv) case simple_api : result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, bigTests); break; - case mt_api : - result = fuzzerTests_MT(seed, nbTests, testNb, ((double)proba) / 100, bigTests); - break; case advanced_api : result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests); break; diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index feed5b84..d74c41bc 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -18,6 +18,8 @@ EXAMPLE_PATH = examples PROGRAMS_PATH = ../programs TEST_FILE = ../doc/zstd_compression_format.md +VPATH = $(PROGRAMS_PATH) + CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) STDFLAGS = -std=c89 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat \ @@ -95,7 +97,7 @@ fitblk: $(EXAMPLE_PATH)/fitblk.o zstd_zlibwrapper.o $(ZSTDLIBRARY) fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ -zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY) +zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o zstd_zlibwrapper.o util.o timefn.o datagen.o $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 4d2c7be5..0ae5012a 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -19,13 +19,13 @@ #include <stdlib.h> #include <stdio.h> /* vsprintf */ #include <stdarg.h> /* va_list, for z_gzprintf */ +#include <string.h> #define NO_DUMMY_DECL #define ZLIB_CONST #include <zlib.h> /* without #define Z_PREFIX */ #include "zstd_zlibwrapper.h" -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_isFrame, ZSTD_MAGICNUMBER */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_isFrame, ZSTD_MAGICNUMBER, ZSTD_customMem */ #include "zstd.h" -#include "zstd_internal.h" /* ZSTD_malloc, ZSTD_free */ /* === Constants === */ @@ -42,6 +42,45 @@ #define FINISH_WITH_GZ_ERR(msg) { (void)msg; return Z_STREAM_ERROR; } #define FINISH_WITH_NULL_ERR(msg) { (void)msg; return NULL; } +/* === Utility === */ + +#define MIN(x,y) ((x) < (y) ? (x) : (y)) + +static unsigned ZWRAP_isLittleEndian(void) +{ + const union { unsigned u; char c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +static unsigned ZWRAP_swap32(unsigned in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap32)) + return __builtin_bswap32(in); +#else + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +#endif +} + +static unsigned ZWRAP_readLE32(const void* ptr) +{ + unsigned value; + memcpy(&value, ptr, sizeof(value)); + if (ZWRAP_isLittleEndian()) + return value; + else + return ZWRAP_swap32(value); +} + /* === Wrapper === */ static int g_ZWRAP_useZSTDcompression = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */ @@ -64,8 +103,6 @@ const char * zstdVersion(void) { return ZSTD_VERSION_STRING; } ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } - - static void* ZWRAP_allocFunction(void* opaque, size_t size) { z_streamp strm = (z_streamp) opaque; @@ -81,6 +118,35 @@ static void ZWRAP_freeFunction(void* opaque, void* address) /* if (address) LOG_WRAPPERC("ZWRAP free %p \n", address); */ } +static void* ZWRAP_customMalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return malloc(size); +} + +static void* ZWRAP_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + memset(ptr, 0, size); + return ptr; + } + return calloc(1, size); +} + +static void ZWRAP_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + free(ptr); + } +} + /* === Compression === */ @@ -107,7 +173,7 @@ static size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) { if (zwc==NULL) return 0; /* support free on NULL */ ZSTD_freeCStream(zwc->zbc); - ZSTD_free(zwc, zwc->customMem); + ZWRAP_customFree(zwc, zwc->customMem); return 0; } @@ -115,20 +181,19 @@ static size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) static ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) { ZWRAP_CCtx* zwc; + ZSTD_customMem customMem = { NULL, NULL, NULL }; if (strm->zalloc && strm->zfree) { - zwc = (ZWRAP_CCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_CCtx)); - if (zwc==NULL) return NULL; - memset(zwc, 0, sizeof(ZWRAP_CCtx)); - memcpy(&zwc->allocFunc, strm, sizeof(z_stream)); - { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL }; - ZWRAP_customMem.opaque = &zwc->allocFunc; - zwc->customMem = ZWRAP_customMem; - } - } else { - zwc = (ZWRAP_CCtx*)calloc(1, sizeof(*zwc)); - if (zwc==NULL) return NULL; + customMem.customAlloc = ZWRAP_allocFunction; + customMem.customFree = ZWRAP_freeFunction; } + customMem.opaque = strm; + + zwc = (ZWRAP_CCtx*)ZWRAP_customCalloc(sizeof(ZWRAP_CCtx), customMem); + if (zwc == NULL) return NULL; + zwc->allocFunc = *strm; + customMem.opaque = &zwc->allocFunc; + zwc->customMem = customMem; return zwc; } @@ -181,6 +246,10 @@ int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize) return Z_OK; } +static struct internal_state* convert_into_sis(void* ptr) +{ + return (struct internal_state*) ptr; +} ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, const char *version, int stream_size)) @@ -201,7 +270,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, zwc->streamEnd = 0; zwc->totalInBytes = 0; zwc->compressionLevel = level; - strm->state = (struct internal_state*) zwc; /* use state which in not used by user */ + strm->state = convert_into_sis(zwc); /* use state which in not used by user */ strm->total_in = 0; strm->total_out = 0; strm->adler = 0; @@ -457,21 +526,19 @@ static void ZWRAP_initDCtx(ZWRAP_DCtx* zwd) static ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) { ZWRAP_DCtx* zwd; - MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_HEADERSIZE); /* check static buffer size condition */ + ZSTD_customMem customMem = { NULL, NULL, NULL }; if (strm->zalloc && strm->zfree) { - zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx)); - if (zwd==NULL) return NULL; - memset(zwd, 0, sizeof(ZWRAP_DCtx)); - zwd->allocFunc = *strm; /* just to copy zalloc, zfree & opaque */ - { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL }; - ZWRAP_customMem.opaque = &zwd->allocFunc; - zwd->customMem = ZWRAP_customMem; - } - } else { - zwd = (ZWRAP_DCtx*)calloc(1, sizeof(*zwd)); - if (zwd==NULL) return NULL; + customMem.customAlloc = ZWRAP_allocFunction; + customMem.customFree = ZWRAP_freeFunction; } + customMem.opaque = strm; + + zwd = (ZWRAP_DCtx*)ZWRAP_customCalloc(sizeof(ZWRAP_DCtx), customMem); + if (zwd == NULL) return NULL; + zwd->allocFunc = *strm; + customMem.opaque = &zwd->allocFunc; + zwd->customMem = customMem; ZWRAP_initDCtx(zwd); return zwd; @@ -481,8 +548,8 @@ static size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd) { if (zwd==NULL) return 0; /* support free on null */ ZSTD_freeDStream(zwd->zbd); - ZSTD_free(zwd->version, zwd->customMem); - ZSTD_free(zwd, zwd->customMem); + ZWRAP_customFree(zwd->version, zwd->customMem); + ZWRAP_customFree(zwd, zwd->customMem); return 0; } @@ -524,13 +591,13 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, LOG_WRAPPERD("- inflateInit\n"); if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0); - zwd->version = (char*)ZSTD_malloc(strlen(version)+1, zwd->customMem); + zwd->version = (char*)ZWRAP_customMalloc(strlen(version)+1, zwd->customMem); if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0); strcpy(zwd->version, version); zwd->stream_size = stream_size; zwd->totalInBytes = 0; - strm->state = (struct internal_state*) zwd; + strm->state = convert_into_sis(zwd); strm->total_in = 0; strm->total_out = 0; strm->reserved = ZWRAP_UNKNOWN_STREAM; @@ -670,7 +737,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (zwd->totalInBytes < ZLIB_HEADERSIZE) { if (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) { - if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) { + if (ZWRAP_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) { { int const initErr = (zwd->windowBits) ? inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size) : inflateInit_(strm, zwd->version, zwd->stream_size); @@ -698,7 +765,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) strm->avail_in -= srcSize; if (zwd->totalInBytes < ZLIB_HEADERSIZE) return Z_OK; - if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) { + if (ZWRAP_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) { z_stream strm2; strm2.next_in = strm->next_in; strm2.avail_in = strm->avail_in; |