From 7b65df5ec6a1a59bc1340634a6804d2e9dbc7e88 Mon Sep 17 00:00:00 2001 From: Alex Macdonald Date: Jan 29 2021 19:38:01 +0000 Subject: use system lz4 and xxhash libs instead of bundling --- diff --git a/1-remove-unsafe.patch b/1-remove-unsafe.patch index 9d7b2e7..90adaf3 100644 --- a/1-remove-unsafe.patch +++ b/1-remove-unsafe.patch @@ -728,68 +728,6 @@ index 9a9b4d1..9ccdf76 100644 } /** -diff --git a/src/lz4/contrib/debian/control b/src/lz4/contrib/debian/control -index ac3b460..63b493c 100644 ---- a/src/lz4/contrib/debian/control -+++ b/src/lz4/contrib/debian/control -@@ -13,11 +13,11 @@ Vcs-Browser: https://github.com/lz4/lz4 - - Package: liblz4 - Architecture: any --Depends: ${shlibs:Depends}, ${misc:Depends} -+Depends: ${shlibs:Depends} - Description: Extremely Fast Compression algorithm http://www.lz4.org - - Package: liblz4-dev - Architecture: any --Depends: ${shlibs:Depends}, ${misc:Depends} -+Depends: ${shlibs:Depends} - Description: Extremely Fast Compression algorithm http://www.lz4.org - Development files. -diff --git a/src/lz4/doc/lz4_manual.html b/src/lz4/doc/lz4_manual.html -index a477584..8127877 100644 ---- a/src/lz4/doc/lz4_manual.html -+++ b/src/lz4/doc/lz4_manual.html -@@ -316,10 +316,6 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); - future. They are therefore only safe to depend on when the caller is - statically linked against the library. - -- To protect against unsafe usage, not only are the declarations guarded, -- the definitions are hidden by default -- when building LZ4 as a shared/dynamic library. -- - In order to access these declarations, - define LZ4_STATIC_LINKING_ONLY in your application - before including LZ4's headers. -diff --git a/src/lz4/ossfuzz/Makefile b/src/lz4/ossfuzz/Makefile -index 6875eb6..640d1c7 100644 ---- a/src/lz4/ossfuzz/Makefile -+++ b/src/lz4/ossfuzz/Makefile -@@ -33,8 +33,7 @@ DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL) - - LZ4_CFLAGS = $(CFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) - LZ4_CXXFLAGS = $(CXXFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) --LZ4_CPPFLAGS = $(CPPFLAGS) -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ \ -- -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -+LZ4_CPPFLAGS = $(CPPFLAGS) -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ - - FUZZERS := \ - compress_fuzzer \ -diff --git a/src/lz4/ossfuzz/fuzz.h b/src/lz4/ossfuzz/fuzz.h -index eefac63..b0e9797 100644 ---- a/src/lz4/ossfuzz/fuzz.h -+++ b/src/lz4/ossfuzz/fuzz.h -@@ -19,10 +19,6 @@ - * It can be undefined, or 0 through 2. If it is undefined, it selects - * the method to use based on the compiler. If testing with UBSAN set - * MEM_FORCE_MEMORY_ACCESS=0 to use the standard compliant method. -- * @param FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -- * This is the canonical flag to enable deterministic builds for fuzzing. -- * Changes to zstd for fuzzing are gated behind this define. -- * It is recommended to define this when building zstd for fuzzing. - */ - - #ifndef FUZZ_H diff --git a/src/test/net/jpountz/lz4/Instances.java b/src/test/net/jpountz/lz4/Instances.java index b9caae5..44f7809 100644 --- a/src/test/net/jpountz/lz4/Instances.java diff --git a/2-remove-comments-from-templates.patch b/2-remove-comments-from-templates.patch index 30ec01c..83fb640 100644 --- a/2-remove-comments-from-templates.patch +++ b/2-remove-comments-from-templates.patch @@ -1,94 +1,82 @@ diff --git a/src/build/source_templates/compress.template b/src/build/source_templates/compress.template -index 0dc3c79..ed47f44 100644 +index 32008e9..09a5df4 100644 --- a/src/build/source_templates/compress.template +++ b/src/build/source_templates/compress.template -@@ -16,7 +16,7 @@ +@@ -31,7 +31,6 @@ main: while (true) { - // find a match -+ int forwardOff = sOff; int ref; -@@ -36,15 +36,15 @@ +@@ -51,15 +50,12 @@ ${type}Utils.writeShort(hashTable, h, sOff - srcOff); } while (!LZ4${utils}.readIntEquals(src, ref, sOff)); - // catch up -+ final int excess = LZ4${utils}.commonBytesBackward(src, ref, sOff, srcOff, anchor); sOff -= excess; ref -= excess; - // sequence == refsequence -+ final int runLen = sOff - anchor; - // encode literal length -+ int tokenOff = dOff++; if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) { -@@ -58,16 +58,16 @@ +@@ -73,16 +69,13 @@ ${utils}.writeByte(dest, tokenOff, runLen << ML_BITS); } - // copy literals -+ LZ4${utils}.wildArraycopy(src, anchor, dest, dOff, runLen); dOff += runLen; while (true) { - // encode offset -+ ${utils}.writeShortLE(dest, dOff, (short) (sOff - ref)); dOff += 2; - // count nb matches -+ sOff += MIN_MATCH; ref += MIN_MATCH; final int matchLen = LZ4${utils}.commonBytes(src, ref, sOff, srcLimit); -@@ -76,7 +76,7 @@ +@@ -91,7 +84,6 @@ } sOff += matchLen; - // encode match len -+ if (matchLen >= ML_MASK) { ${utils}.writeByte(dest, tokenOff, ${utils}.readByte(dest, tokenOff) | ML_MASK); dOff = LZ4${utils}.writeLen(matchLen - ML_MASK, dest, dOff); -@@ -84,16 +84,16 @@ +@@ -99,16 +91,13 @@ ${utils}.writeByte(dest, tokenOff, ${utils}.readByte(dest, tokenOff) | matchLen); } - // test end of chunk -+ if (sOff > mflimit) { anchor = sOff; break main; } - // fill table -+ ${type}Utils.writeShort(hashTable, hash64k(${utils}.readInt(src, sOff - 2)), sOff - 2 - srcOff); - // test next position -+ final int h = hash64k(${utils}.readInt(src, sOff)); ref = srcOff + ${type}Utils.readShort(hashTable, h); ${type}Utils.writeShort(hashTable, h, sOff - srcOff); -@@ -106,7 +106,7 @@ +@@ -121,7 +110,6 @@ ${utils}.writeByte(dest, tokenOff, 0); } - // prepare next loop -+ anchor = sOff++; } } -@@ -145,7 +145,6 @@ +@@ -160,7 +148,6 @@ main: while (true) { @@ -96,7 +84,7 @@ index 0dc3c79..ed47f44 100644 int forwardOff = sOff; int ref; -@@ -172,10 +171,8 @@ +@@ -187,10 +174,8 @@ sOff -= excess; ref -= excess; @@ -107,83 +95,73 @@ index 0dc3c79..ed47f44 100644 int tokenOff = dOff++; if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) { -@@ -189,16 +186,16 @@ +@@ -204,16 +189,13 @@ ${utils}.writeByte(dest, tokenOff, runLen << ML_BITS); } - // copy literals -+ LZ4${utils}.wildArraycopy(src, anchor, dest, dOff, runLen); dOff += runLen; while (true) { - // encode offset -+ ${utils}.writeShortLE(dest, dOff, back); dOff += 2; - // count nb matches -+ sOff += MIN_MATCH; final int matchLen = LZ4${utils}.commonBytes(src, ref + MIN_MATCH, sOff, srcLimit); if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) { -@@ -206,7 +203,7 @@ +@@ -221,7 +203,6 @@ } sOff += matchLen; - // encode match len -+ if (matchLen >= ML_MASK) { ${utils}.writeByte(dest, tokenOff, ${utils}.readByte(dest, tokenOff) | ML_MASK); dOff = LZ4${utils}.writeLen(matchLen - ML_MASK, dest, dOff); -@@ -214,16 +211,16 @@ +@@ -229,16 +210,13 @@ ${utils}.writeByte(dest, tokenOff, ${utils}.readByte(dest, tokenOff) | matchLen); } - // test end of chunk -+ if (sOff > mflimit) { anchor = sOff; break main; } - // fill table -+ ${type}Utils.writeInt(hashTable, hash(${utils}.readInt(src, sOff - 2)), sOff - 2); - // test next position -+ final int h = hash(${utils}.readInt(src, sOff)); ref = ${type}Utils.readInt(hashTable, h); ${type}Utils.writeInt(hashTable, h, sOff); -@@ -237,7 +234,7 @@ +@@ -252,7 +230,6 @@ ${utils}.writeByte(dest, tokenOff, 0); } - // prepare next loop -+ anchor = sOff++; } diff --git a/src/build/source_templates/compress_hc.template b/src/build/source_templates/compress_hc.template -index b09b705..b250205 100644 +index 7179db3..7976ad1 100644 --- a/src/build/source_templates/compress_hc.template +++ b/src/build/source_templates/compress_hc.template -@@ -32,7 +32,7 @@ +@@ -47,7 +47,6 @@ continue; } - // saved, in case we would skip too much -+ copyTo(match1, match0); search2: -@@ -40,20 +40,20 @@ +@@ -55,20 +54,19 @@ assert match1.start >= anchor; if (match1.end() >= mfLimit || !ht.insertAndFindWiderMatch(src, match1.end() - 2, match1.start + 1, matchLimit, match1.len, match2)) { - // no better match -+ dOff = LZ4${utils}.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd); anchor = sOff = match1.end(); continue main; @@ -191,32 +169,29 @@ index b09b705..b250205 100644 if (match0.start < match1.start) { - if (match2.start < match1.start + match0.len) { // empirical -+ if (match2.start < match1.start + match0.len) { ++ if (match2.start < match1.start + match0.len) { copyTo(match0, match1); } } assert match2.start > match1.start; - if (match2.start - match1.start < 3) { // First Match too small : removed -+ if (match2.start - match1.start < 3) { ++ if (match2.start - match1.start < 3) { copyTo(match2, match1); continue search2; } -@@ -76,21 +76,21 @@ +@@ -91,21 +89,18 @@ if (match2.start + match2.len >= mfLimit || !ht.insertAndFindWiderMatch(src, match2.end() - 3, match2.start, matchLimit, match2.len, match3)) { - // no better match -> 2 sequences to encode -+ if (match2.start < match1.end()) { match1.len = match2.start - match1.start; } - // encode seq 1 -+ dOff = LZ4${utils}.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd); anchor = sOff = match1.end(); - // encode seq 2 -+ dOff = LZ4${utils}.encodeSequence(src, anchor, match2.start, match2.ref, match2.len, dest, dOff, destEnd); anchor = sOff = match2.end(); continue main; @@ -224,25 +199,24 @@ index b09b705..b250205 100644 - if (match3.start < match1.end() + 3) { // Not enough space for match 2 : remove it - if (match3.start >= match1.end()) { // // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 -+ if (match3.start < match1.end() + 3) { -+ if (match3.start >= match1.end()) { ++ if (match3.start < match1.end() + 3) { ++ if (match3.start >= match1.end()) { if (match2.start < match1.end()) { final int correction = match1.end() - match2.start; match2.fix(correction); -@@ -112,7 +112,7 @@ +@@ -127,7 +122,6 @@ continue search3; } - // OK, now we have 3 ascending matches; let's write at least the first one -+ if (match2.start < match1.end()) { if (match2.start - match1.start < ML_MASK) { if (match1.len > OPTIMAL_ML) { diff --git a/src/build/source_templates/decompress.template b/src/build/source_templates/decompress.template -index 09c0ca8..fce5ace 100644 +index f1c2890..1e2aa23 100644 --- a/src/build/source_templates/decompress.template +++ b/src/build/source_templates/decompress.template -@@ -40,7 +40,6 @@ +@@ -55,7 +55,6 @@ final int token = ${utils}.readByte(src, sOff) & 0xFF; ++sOff; @@ -250,7 +224,7 @@ index 09c0ca8..fce5ace 100644 int literalLen = token >>> ML_BITS; if (literalLen == RUN_MASK) { byte len = (byte) 0xFF; -@@ -66,7 +65,7 @@ +@@ -81,7 +80,7 @@ LZ4${utils}.safeArraycopy(src, sOff, dest, dOff, literalLen); sOff += literalLen; dOff = literalCopyEnd; @@ -259,7 +233,7 @@ index 09c0ca8..fce5ace 100644 } } -@@ -74,7 +73,6 @@ +@@ -89,7 +88,6 @@ sOff += literalLen; dOff = literalCopyEnd; @@ -268,10 +242,10 @@ index 09c0ca8..fce5ace 100644 sOff += 2; int matchOff = dOff - matchDec; diff --git a/src/build/source_templates/hashtable.template b/src/build/source_templates/hashtable.template -index ef4e0ac..f6377d0 100644 +index 174f8e8..91935f5 100644 --- a/src/build/source_templates/hashtable.template +++ b/src/build/source_templates/hashtable.template -@@ -77,8 +77,8 @@ +@@ -92,8 +92,8 @@ int ref = hashPointer(buf, off); @@ -282,7 +256,7 @@ index ef4e0ac..f6377d0 100644 delta = off - ref; repl = match.len = MIN_MATCH + LZ4${utils}.commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit); match.ref = ref; -@@ -104,7 +104,7 @@ +@@ -119,7 +119,7 @@ int ptr = off; final int end = off + repl - (MIN_MATCH - 1); while (ptr < end - delta) { @@ -291,3 +265,47 @@ index ef4e0ac..f6377d0 100644 ++ptr; } do { +diff --git a/src/build/source_templates/xxhash32_streaming.template b/src/build/source_templates/xxhash32_streaming.template +index 6166758..9fa55e8 100644 +--- a/src/build/source_templates/xxhash32_streaming.template ++++ b/src/build/source_templates/xxhash32_streaming.template +@@ -66,7 +66,7 @@ final class StreamingXXHash32Java${type} extends AbstractStreamingXXHash32Java { + + totalLen += len; + +- if (memSize + len < 16) { // fill in tmp buffer ++ if (memSize + len < 16) { + System.arraycopy(buf, off, memory, memSize, len); + memSize += len; + return; +@@ -74,7 +74,7 @@ final class StreamingXXHash32Java${type} extends AbstractStreamingXXHash32Java { + + final int end = off + len; + +- if (memSize > 0) { // data left from previous update ++ if (memSize > 0) { + System.arraycopy(buf, off, memory, memSize, 16 - memSize); + + v1 += readIntLE(memory, 0) * PRIME2; +diff --git a/src/build/source_templates/xxhash64_streaming.template b/src/build/source_templates/xxhash64_streaming.template +index 2789ae0..e781746 100644 +--- a/src/build/source_templates/xxhash64_streaming.template ++++ b/src/build/source_templates/xxhash64_streaming.template +@@ -90,7 +90,7 @@ final class StreamingXXHash64Java${type} extends AbstractStreamingXXHash64Java { + + totalLen += len; + +- if (memSize + len < 32) { // fill in tmp buffer ++ if (memSize + len < 32) { + System.arraycopy(buf, off, memory, memSize, len); + memSize += len; + return; +@@ -98,7 +98,7 @@ final class StreamingXXHash64Java${type} extends AbstractStreamingXXHash64Java { + + final int end = off + len; + +- if (memSize > 0) { // data left from previous update ++ if (memSize > 0) { + System.arraycopy(buf, off, memory, memSize, 32 - memSize); + + v1 += readLongLE(memory, 0) * PRIME64_2; diff --git a/3-use-system-libs.patch b/3-use-system-libs.patch new file mode 100644 index 0000000..1ade925 --- /dev/null +++ b/3-use-system-libs.patch @@ -0,0 +1,20 @@ +diff --git a/build.xml b/build.xml +index 7a63933..7c04e0c 100644 +--- a/build.xml ++++ b/build.xml +@@ -206,14 +206,13 @@ + objdir="${build}/objects" + outfile="${build}/jni/net/jpountz/util/${platform}/${os.arch}/lz4-java" + outtype="shared"> +- + + + + + + +- ++ + + + diff --git a/lz4-java.spec b/lz4-java.spec index df49bfb..51cc04b 100644 --- a/lz4-java.spec +++ b/lz4-java.spec @@ -1,20 +1,16 @@ # empty debuginfo %global debug_package %nil -%global lz4_version 1.9.2 Name: lz4-java Version: 1.7.1 -Release: 6%{?dist} +Release: 7%{?dist} Summary: LZ4 compression for Java -# GPL: -# src/lz4/ -# BSD: -# src/lz4/libs License: ASL 2.0 and (BSD and GPLv2+) URL: https://github.com/lz4/lz4-java Source0: https://github.com/lz4/lz4-java/archive/%{version}.tar.gz -Source1: https://github.com/lz4/lz4/archive/v%{lz4_version}.tar.gz +# lz4-java tests require randomizedtesting, which is not currently +# shipped or maintained in Fedora. Patch0: 0-remove-tests.patch # lz4-java v1.3.0 introduced usage of sun.misc.Unsafe, which would later become # depricated in jdk 9 and kept as an unexposed API in later jdk releases. @@ -27,8 +23,10 @@ Patch1: 1-remove-unsafe.patch # This patch simply removes comments from the templates so the classes can be generated as expected. # Related bug: https://github.com/mvel/mvel/issues/152 Patch2: 2-remove-comments-from-templates.patch +Patch3: 3-use-system-libs.patch # Build tools +BuildRequires: apache-parent BuildRequires: ant BuildRequires: ant-junit BuildRequires: aqute-bnd @@ -37,13 +35,11 @@ BuildRequires: gcc BuildRequires: ivy-local BuildRequires: java-devel BuildRequires: javapackages-local +BuildRequires: lz4-devel BuildRequires: mvel BuildRequires: objectweb-asm BuildRequires: xerces-j2 -BuildRequires: apache-parent -# FPC ticket Bundled Library Exception -# https://fedorahosted.org/fpc/ticket/603 -Provides: bundled(libxxhash) = r37 +BuildRequires: xxhash-devel BuildArch: noarch @@ -77,21 +73,16 @@ This package contains javadoc for %{name}. %prep %setup -q -n %{name}-%{version} -%setup -q -T -D -a 1 - -mv lz4-1.9.2/* src/lz4/ %patch0 -p1 %patch1 -p1 %patch2 -p1 +%patch3 -p1 # Cleanup find -name '*.dylib' -print -delete find -name '*.so' -print -delete -cp -p src/lz4/LICENSE lz4_LICENSE -cp -p src/lz4/lib/LICENSE lz4_lib_LICENSE - %build ant -Divy.mode=local -Divy.revision=1.7.1 -Divy.pom.version=1.7.1 jar docs makepom bnd wrap -p lz4-java.bnd -o dist/lz4-java-%{version}.jar --version %{version} dist/lz4-java.jar @@ -103,12 +94,15 @@ bnd wrap -p lz4-java.bnd -o dist/lz4-java-%{version}.jar --version %{version} di %files -f .mfiles %doc CHANGES.md README.md -%license LICENSE.txt lz4_LICENSE lz4_lib_LICENSE %files javadoc -f .mfiles-javadoc %license LICENSE.txt %changelog +* Mon Nov 16 2020 Alex Macdonald 1.7.1-7 +- cleanup whitespace in the local patch to remove comments from templates +- use system lz4 and xxhash instead of bundling the dependencies + * Tue Oct 06 2020 Alex Macdonald 1.7.1-6 - include patch to strip comments from mvel templates diff --git a/sources b/sources index a2c0121..971053e 100644 --- a/sources +++ b/sources @@ -1,2 +1 @@ SHA512 (1.7.1.tar.gz) = 13fd89a9564f1f504348ce30a51ca22d5581a6661658d55c96f3f0c1e43126de04d2df87683251a8ec5bf53726f529b60bb1ec666863b7e33810df8ed73bb586 -SHA512 (v1.9.2.tar.gz) = ae714c61ec8e33ed91359b63f2896cfa102d66b730dce112b74696ec5850e59d88bd5527173e01e354a70fbe8f036557a47c767ee0766bc5f9c257978116c3c1