Merge pull request #48 from noodlesbad/master

updated native libraries to latest: LZ4 and ZSTD

native/4mc.c

@@ -68,7 +68,7 @@
 #  include <fcntl.h>    // _O_BINARY
 #  include <io.h>       // _setmode, _isatty
 #  ifdef __MINGW32__
-   int _fileno(FILE *stream);   // MINGW somehow forgets to include this windows declaration into <stdio.h>
+   //int _fileno(FILE *stream);   // MINGW somehow forgets to include this windows declaration into <stdio.h>
 #  endif
 #  define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
 #  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
@@ -86,13 +86,13 @@
 #elif GCC_VERSION >= 403
 #  define swap32 __builtin_bswap32
 #else
-  static inline unsigned int swap32(unsigned int x)
-  {
+static inline unsigned int swap32(unsigned int x)
+{
     return ((x << 24) & 0xff000000 ) |
            ((x <<  8) & 0x00ff0000 ) |
            ((x >>  8) & 0x0000ff00 ) |
            ((x >> 24) & 0x000000ff );
-  }
+}
 #endif
 
 
@@ -209,8 +209,8 @@ static int openIOFileHandles(int displayLevel, int overwrite, char* input_filena
 }
 
 
-FORCE_INLINE int LZ4_compress_limitedOutput_local(const char* src, char* dst, int size, int maxOut, int clevel)
-{ (void)clevel; return LZ4_compress_limitedOutput(src, dst, size, maxOut); }
+FORCE_INLINE int LZ4_compress_default_local(const char* src, char* dst, int size, int maxOut, int clevel)
+{ (void)clevel; return LZ4_compress_default(src, dst, size, maxOut); }
 
 
 FORCE_INLINE int LZ4_compressMC_limitedOutput_local(const char* src, char* dst, int size, int maxOut, int clevel)
@@ -220,7 +220,7 @@ FORCE_INLINE int LZ4_compressMC_limitedOutput_local(const char* src, char* dst,
 int fourMCcompressFilename(int displayLevel, int overwrite, char* input_filename, char* output_filename, int compressionLevel)
 {
     int (*compressionFunction)(const char*, char*, int, int, int);
-	int lz4Level=1; unsigned int bi;
+    int lz4Level=1; unsigned int bi;
     unsigned long long filesize = 0;
     unsigned long long compressedfilesize = 0;
     unsigned int checkbits;
@@ -241,14 +241,14 @@ int fourMCcompressFilename(int displayLevel, int overwrite, char* input_filename
     if ((displayLevel==2) && (compressionLevel>1)) displayLevel=3;
 
     if (compressionLevel <= 1) {
-        compressionFunction = LZ4_compress_limitedOutput_local;
+        compressionFunction = LZ4_compress_default_local;
     } else if (compressionLevel == 2) {
         compressionFunction = LZ4_compressMC_limitedOutput_local;
     } else if (compressionLevel == 3) {
-        compressionFunction = LZ4_compressHC2_limitedOutput;
+        compressionFunction = LZ4_compress_HC;
         lz4Level=4;
     } else {
-        compressionFunction = LZ4_compressHC2_limitedOutput;
+        compressionFunction = LZ4_compress_HC;
         lz4Level=8;
     }
 
@@ -317,7 +317,7 @@ int fourMCcompressFilename(int displayLevel, int overwrite, char* input_filename
         }
         else  // Copy Original Uncompressed
         {
-			unsigned int checksum;
+            unsigned int checksum;
             *(unsigned int*)(out_buff) = BIG_ENDIAN_32(readSize);
             *(unsigned int*)(out_buff+4) = BIG_ENDIAN_32(readSize);
             checksum = XXH32(in_buff, (int)readSize, 0);
@@ -373,9 +373,9 @@ int fourMCcompressFilename(int displayLevel, int overwrite, char* input_filename
     end = clock();
     CONSOLE_PRINT_LEVEL(2, "\r%79s\r", "");
     CONSOLE_PRINT_LEVEL(2, "Compressed (%s) %llu bytes into %llu bytes ==> %.2f%% (Ratio=%.3f)\n",
-        compressionLevel<=1?"fast":(compressionLevel==2?"medium":(compressionLevel==3?"high":"ultra")),
-        (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100,
-        (double)100.0/((double)compressedfilesize/filesize*100));
+                        compressionLevel<=1?"fast":(compressionLevel==2?"medium":(compressionLevel==3?"high":"ultra")),
+                        (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100,
+                        (double)100.0/((double)compressedfilesize/filesize*100));
 
     {
         double seconds = (double)(end - start)/CLOCKS_PER_SEC;
@@ -388,7 +388,7 @@ int fourMCcompressFilename(int displayLevel, int overwrite, char* input_filename
 
 int fourMZcompressFilename(int displayLevel, int overwrite, char* input_filename, char* output_filename, int compressionLevel)
 {
-	int zstdLevel=3; unsigned int bi;
+    int zstdLevel=3; unsigned int bi;
     unsigned long long filesize = 0;
     unsigned long long compressedfilesize = 0;
     unsigned int checkbits;
@@ -409,11 +409,11 @@ int fourMZcompressFilename(int displayLevel, int overwrite, char* input_filename
     if ((displayLevel==2) && (compressionLevel>1)) displayLevel=3;
 
     if (compressionLevel <= 1) {
-    	zstdLevel=1;
+        zstdLevel=1;
     } else if (compressionLevel == 2) {
-    	zstdLevel=3;
+        zstdLevel=3;
     } else if (compressionLevel == 3) {
-    	zstdLevel=6;
+        zstdLevel=6;
     } else {
         zstdLevel=12;
     }
@@ -445,7 +445,7 @@ int fourMZcompressFilename(int displayLevel, int overwrite, char* input_filename
     // Main Loop
     while (readSize>0)
     {
-    	size_t outSize;
+        size_t outSize;
 
         // ------------------------------
         if (++blockIndexesCount >= blockIndexesReserved) {
@@ -484,7 +484,7 @@ int fourMZcompressFilename(int displayLevel, int overwrite, char* input_filename
         }
         else  // Copy Original Uncompressed
         {
-			unsigned int checksum;
+            unsigned int checksum;
             *(unsigned int*)(out_buff) = BIG_ENDIAN_32(readSize);
             *(unsigned int*)(out_buff+4) = BIG_ENDIAN_32(readSize);
             checksum = XXH32(in_buff, (int)readSize, 0);
@@ -540,9 +540,9 @@ int fourMZcompressFilename(int displayLevel, int overwrite, char* input_filename
     end = clock();
     CONSOLE_PRINT_LEVEL(2, "\r%79s\r", "");
     CONSOLE_PRINT_LEVEL(2, "Compressed (%s) %llu bytes into %llu bytes ==> %.2f%% (Ratio=%.3f)\n",
-        compressionLevel<=1?"fast":(compressionLevel==2?"medium":(compressionLevel==3?"high":"ultra")),
-        (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100,
-        (double)100.0/((double)compressedfilesize/filesize*100));
+                        compressionLevel<=1?"fast":(compressionLevel==2?"medium":(compressionLevel==3?"high":"ultra")),
+                        (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100,
+                        (double)100.0/((double)compressedfilesize/filesize*100));
 
     {
         double seconds = (double)(end - start)/CLOCKS_PER_SEC;
@@ -567,7 +567,7 @@ static unsigned long long decodeFourMC(int displayLevel, FILE* finput, FILE* fou
     size_t nbReadBytes;
     int decodedBytes=0;
     size_t sizeCheck;
-	unsigned int footerSize, checksum;
+    unsigned int footerSize, checksum;
 
     descriptor = (char*)malloc(FOURMC_HEADERSIZE*2); // will be used for file header(12), block header(12), others(<12)
 
@@ -685,7 +685,7 @@ static unsigned long long decodeFourMC(int displayLevel, FILE* finput, FILE* fou
     if (checksum != BIG_ENDIAN_32(*(unsigned int*)(in_buff+footerSize-4))) EXIT_WITH_FATALERROR_CONTENT("Error : invalid footer checksum detected");
 
     if ( BIG_ENDIAN_32(*(unsigned int*)(in_buff+4)) != 1) // check footer version
-        EXIT_WITH_FATALERROR_CONTENT("Read error : unsupported footer version" );
+    EXIT_WITH_FATALERROR_CONTENT("Read error : unsupported footer version" );
 
     if (displayLevel>=3) {
         unsigned long long absOffset=0;
@@ -716,7 +716,7 @@ static unsigned long long decodeFourMZ(int displayLevel, FILE* finput, FILE* fou
     size_t nbReadBytes;
     int decodedBytes=0;
     size_t sizeCheck;
-	unsigned int footerSize, checksum;
+    unsigned int footerSize, checksum;
 
     descriptor = (char*)malloc(FOURMC_HEADERSIZE*2); // will be used for file header(12), block header(12), others(<12)
 
@@ -835,7 +835,7 @@ static unsigned long long decodeFourMZ(int displayLevel, FILE* finput, FILE* fou
     if (checksum != BIG_ENDIAN_32(*(unsigned int*)(in_buff+footerSize-4))) EXIT_WITH_FATALERROR_CONTENT("Error : invalid footer checksum detected");
 
     if ( BIG_ENDIAN_32(*(unsigned int*)(in_buff+4)) != 1) // check footer version
-        EXIT_WITH_FATALERROR_CONTENT("Read error : unsupported footer version" );
+    EXIT_WITH_FATALERROR_CONTENT("Read error : unsupported footer version" );
 
     if (displayLevel>=3) {
         unsigned long long absOffset=0;
@@ -961,4 +961,4 @@ int fourMZDecompressFileName(int displayLevel, int overwrite, char* input_filena
 
     // Error status = OK
     return 0;
-}
+}

native/4mccli.c

@@ -63,7 +63,7 @@
 #  include <fcntl.h>    // _O_BINARY
 #  include <io.h>       // _setmode, _isatty
 #  ifdef __MINGW32__
-   int _fileno(FILE *stream);   // MINGW somehow forgets to include this windows declaration into <stdio.h>
+   //int _fileno(FILE *stream);   // MINGW somehow forgets to include this windows declaration into <stdio.h>
 #  endif
 #  define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
 #  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))

native/Makefile

@@ -1,7 +1,7 @@
 # ##########################################################################
 # 4MC program - Makefile
 # Copyright (C) Carlo Medas 2014 
-# Used LZ4 code - Copyright (C) Yann Collet 2011-2014
+# Used LZ4 code - Copyright (C) Yann Collet 2011-2016
 # ##########################################################################
 # 4mc : Command Line Utility, supporting gzip-like arguments
 # ##########################################################################
@@ -20,12 +20,13 @@ BINDIR=$(PREFIX)/bin
 MANDIR=$(PREFIX)/share/man/man1
 LZ4DIR=..
 
-LIBVER_MAJOR=1
-LIBVER_MINOR=1
+LIBVER_MAJOR=2
+LIBVER_MINOR=0
 LIBVER_PATCH=0
 LIBVER=$(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
 
-ZSTD_FILES := zstd/common/*.c zstd/compress/*.c zstd/decompress/*.c zstd/dictBuilder/*.c
+ZSTD_COMMON_FILES = $(filter-out zstd/common/xxhash.c, $(wildcard zstd/common/*.c))
+ZSTD_FILES := $(ZSTD_COMMON_FILES) zstd/compress/*.c zstd/decompress/*.c zstd/dictBuilder/*.c
 
 ifeq ($(shell uname), Darwin)
 	SHARED_EXT = dylib

native/cmake/CMakeLists.txt

@@ -11,47 +11,61 @@ cmake_minimum_required (VERSION 2.6)
 INCLUDE (CheckTypeSize)
 check_type_size("void *" SIZEOF_VOID_P)
 IF( ${SIZEOF_VOID_P} STREQUAL  "8" )
-    set (CMAKE_SYSTEM_PROCESSOR "64bit")
-    MESSAGE( STATUS "64 bit architecture detected size of void * is " ${SIZEOF_VOID_P})
+	set (CMAKE_SYSTEM_PROCESSOR "64bit")
+	MESSAGE( STATUS "64 bit architecture detected size of void * is " ${SIZEOF_VOID_P})
 ENDIF()
 
 option(BUILD_TOOLS "Build the command line tools" ON)
 option(BUILD_LIBS  "Build the JNI native library for hadoop-4mc" ON)
 
 if(UNIX AND NOT APPLE AND BUILD_LIBS)
-    if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
-        add_definitions(-fPIC)
-    endif()
+	if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
+		add_definitions(-fPIC)
+	endif()
 endif()
 
 IF (MSVC)
-    # MSVC is logs many inline warnings
-    add_definitions("/wd4710")
-    add_definitions("/wd4711")
+	# MSVC is logs many inline warnings
+	add_definitions("/wd4710")
+	add_definitions("/wd4711")
 ENDIF()
 
 set(LZ4_DIR ../lz4/)
 set(ZSTD_DIR ../zstd/)
 set(PRG_DIR ../)
 set(LZ4_SRCS ${LZ4_DIR}xxhash.h ${LZ4_DIR}xxhash.c ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h ${LZ4_DIR}lz4mc.c ${LZ4_DIR}lz4mc.h)
-set(ZSTD_SRCS ${ZSTD_DIR}common/entropy_common.c 
-		${ZSTD_DIR}common/entropy_common.c 
-		${ZSTD_DIR}common/fse_decompress.c 
-		${ZSTD_DIR}common/zstd_common.c 
-		${ZSTD_DIR}common/xxhash.c 
-		${ZSTD_DIR}compress/fse_compress.c 
+set(ZSTD_SRCS ${ZSTD_DIR}common/entropy_common.c
+		${ZSTD_DIR}common/entropy_common.c
+		${ZSTD_DIR}common/error_private.c
+		${ZSTD_DIR}common/fse_decompress.c
+		${ZSTD_DIR}common/pool.c
+		${ZSTD_DIR}common/zstd_common.c
+		#${ZSTD_DIR}common/xxhash.c
+		${ZSTD_DIR}compress/fse_compress.c
+		${ZSTD_DIR}compress/hist.c
 		${ZSTD_DIR}compress/huf_compress.c
-		${ZSTD_DIR}compress/zbuff_compress.c
-		${ZSTD_DIR}compress/zstd_compress.c					
-		${ZSTD_DIR}decompress/huf_decompress.c 
-		${ZSTD_DIR}decompress/zbuff_decompress.c 
-		${ZSTD_DIR}decompress/zstd_decompress.c 
+		${ZSTD_DIR}compress/zstd_compress.c
+		${ZSTD_DIR}compress/zstd_compress_literals.c
+		${ZSTD_DIR}compress/zstd_compress_sequences.c
+		${ZSTD_DIR}compress/zstd_double_fast.c
+		${ZSTD_DIR}compress/zstd_fast.c
+		${ZSTD_DIR}compress/zstd_lazy.c
+		${ZSTD_DIR}compress/zstd_ldm.c
+		${ZSTD_DIR}compress/zstd_opt.c
+		${ZSTD_DIR}decompress/huf_decompress.c
+		${ZSTD_DIR}decompress/zstd_ddict.c
+		${ZSTD_DIR}decompress/zstd_decompress_block.c
+		${ZSTD_DIR}decompress/zstd_decompress.c
+		${ZSTD_DIR}deprecated/zbuff_compress.c
+		${ZSTD_DIR}deprecated/zbuff_decompress.c
+		${ZSTD_DIR}dictBuilder/cover.c
 		${ZSTD_DIR}dictBuilder/divsufsort.c
+		${ZSTD_DIR}dictBuilder/fastcover.c
 		${ZSTD_DIR}dictBuilder/zdict.c
-)
+		)
 set(4MC_SRCS ${PRG_DIR}4mccli.c ${PRG_DIR}4mc.c)
 set(HADOOP4MC_SRCS ${PRG_DIR}jniCompressor.c ${PRG_DIR}jniDecompressor.c ${PRG_DIR}jniZstdCompressor.c ${PRG_DIR}jniZstdDecompressor.c
-    ${PRG_DIR}jniZstd.c ${PRG_DIR}jniZStreamCompressor.c ${PRG_DIR}jniZStreamDecompressor.c)
+		${PRG_DIR}jniZstd.c ${PRG_DIR}jniZStreamCompressor.c ${PRG_DIR}jniZStreamDecompressor.c)
 
 if(NOT WIN32)
 	add_definitions(-O3)
@@ -61,8 +75,8 @@ endif()
 INCLUDE_DIRECTORIES(../zstd ../zstd/common)
 
 if(BUILD_TOOLS)
-    add_executable(4mc ${LZ4_SRCS} ${ZSTD_SRCS} ${4MC_SRCS})
-    install(TARGETS 4mc RUNTIME DESTINATION "bin/")
+	add_executable(4mc ${LZ4_SRCS} ${ZSTD_SRCS} ${4MC_SRCS})
+	install(TARGETS 4mc RUNTIME DESTINATION "bin/")
 endif()
 
 if(BUILD_LIBS)
@@ -73,16 +87,16 @@ if(BUILD_LIBS)
 		INCLUDE_DIRECTORIES(${JNI_INCLUDE_DIRS})
 	else()
 		MESSAGE(FATAL_ERROR "Found JNI: failed")
-	endif()	
-    
-    add_library(hadoop-4mc SHARED ${LZ4_SRCS} ${ZSTD_SRCS} ${HADOOP4MC_SRCS})
-    install(TARGETS 4mc RUNTIME DESTINATION "lib/")
+	endif()
+
+	add_library(hadoop-4mc SHARED ${LZ4_SRCS} ${ZSTD_SRCS} ${HADOOP4MC_SRCS})
+	install(TARGETS 4mc RUNTIME DESTINATION "lib/")
 endif()
 
 #warnings
 
 ADD_DEFINITIONS("-Wall")
-ADD_DEFINITIONS("-DLZ4_VERSION=\"1.2.1\"")
+ADD_DEFINITIONS("-DLZ4_VERSION=\"1.9.2\"")
 INCLUDE_DIRECTORIES (${LZ4_DIR})
 
 

native/jniZStreamCompressor.c

@@ -37,7 +37,7 @@
 **/
 #include "jnihelper.h"
 #include <zstd.h>
-#include <error_public.h>
+#include <error_private.h>
 
 /* field IDs will not change in the same vm */
 static jfieldID src_pos_id;

native/jniZStreamDecompressor.c

@@ -38,7 +38,7 @@
 
 #include "jnihelper.h"
 #include <zstd.h>
-#include <error_public.h>
+#include <error_private.h>
 
 /* field IDs will not change in the same vm */
 static jfieldID src_pos_id;

native/jniZstd.c

@@ -37,7 +37,8 @@
 **/
 #include <jni.h>
 #include <zstd.h>
-#include <error_public.h>
+#include <error_private.h>
+
 
 
 /*

native/lz4/LICENSE

@@ -1,5 +1,5 @@
 LZ4 Library
-Copyright (c) 2011-2014, Yann Collet
+Copyright (c) 2011-2016, Yann Collet
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,
@@ -21,4 +21,4 @@ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.