From 890019b364ab55a2310e7395e994f9c5ec1e2882 Mon Sep 17 00:00:00 2001 From: rzuo Date: Tue, 28 Mar 2023 15:38:57 +0800 Subject: [PATCH 1/2] BIGTOP-3921: Add ZSTD Codec Support for hadoop --- bigtop-packages/src/common/hadoop/do-component-build | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bigtop-packages/src/common/hadoop/do-component-build b/bigtop-packages/src/common/hadoop/do-component-build index 60ff0de965..3a5fbc9da3 100644 --- a/bigtop-packages/src/common/hadoop/do-component-build +++ b/bigtop-packages/src/common/hadoop/do-component-build @@ -120,6 +120,12 @@ if [ -z "$BUNDLE_SNAPPY" ] ; then [ -f /usr/lib/${HOSTTYPE}-linux-gnu/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib/${HOSTTYPE}-linux-gnu" fi +if [ -z "$BUNDLE_ZSTD" ] ; then + [ -f /usr/lib/libzstd.so ] && BUNDLE_ZSTD="-Dbundle.zstd=true -Dzstd.lib=/usr/lib" + [ -f /usr/lib64/libzstd.so ] && BUNDLE_ZSTD="-Dbundle.zstd=true -Dzstd.lib=/usr/lib64" + [ -f /usr/lib/${HOSTTYPE}-linux-gnu/libzstd.so ] && BUNDLE_ZSTD="-Dbundle.zstd=true -Dzstd.lib=/usr/lib/${HOSTTYPE}-linux-gnu" +fi + mkdir build mkdir build/src @@ -132,7 +138,7 @@ MAVEN_OPTS+="-DskipTests -DskipITs " . $(dirname ${0})/maven_deploy.sh # Build artifacts -mvn $BUNDLE_SNAPPY -Pdist -Pnative -Psrc -Pyarn-ui -Dtar ${MAVEN_OPTS} install ${EXTRA_GOALS} "$@" +mvn $BUNDLE_SNAPPY $BUNDLE_ZSTD -Pdist -Pnative -Psrc -Pyarn-ui -Dtar ${MAVEN_OPTS} install ${EXTRA_GOALS} "$@" mvn site site:stage ${MAVEN_OPTS} $@ (cd build ; tar --strip-components=1 -xzvf ../hadoop-dist/target/hadoop-${HADOOP_VERSION}.tar.gz) From 387377ef333c0238a6b7948eae3a7377858e86d2 Mon Sep 17 00:00:00 2001 From: rzuo Date: Mon, 10 Apr 2023 10:23:06 +0800 Subject: [PATCH 2/2] BIGTOP-3921: Add ZSTD Codec Support for hadoop: add libzstd to hadoop rpm package, so that no system zstd library dependency is needed. --- bigtop-packages/src/common/hadoop/install_hadoop.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigtop-packages/src/common/hadoop/install_hadoop.sh b/bigtop-packages/src/common/hadoop/install_hadoop.sh index cabfa01dc1..6bc5bb61c0 100755 --- a/bigtop-packages/src/common/hadoop/install_hadoop.sh +++ b/bigtop-packages/src/common/hadoop/install_hadoop.sh @@ -275,7 +275,7 @@ cp ${BUILD_DIR}/include/hdfs.h $PREFIX/$SYSTEM_INCLUDE_DIR/ cp -r ${BUILD_DIR}/include/hdfspp $PREFIX/$SYSTEM_INCLUDE_DIR/ cp ${BUILD_DIR}/lib/native/*.a $PREFIX/$HADOOP_NATIVE_LIB_DIR/ -for library in `cd ${BUILD_DIR}/lib/native ; ls libsnappy.so.1.* 2>/dev/null` libhadoop.so.1.0.0 libnativetask.so.1.0.0; do +for library in `cd ${BUILD_DIR}/lib/native ; ls libsnappy.so.1.* 2>/dev/null; ls libzstd.so.1.* 2>/dev/null` libhadoop.so.1.0.0 libnativetask.so.1.0.0; do cp ${BUILD_DIR}/lib/native/${library} $PREFIX/$HADOOP_NATIVE_LIB_DIR/ ldconfig -vlN $PREFIX/$HADOOP_NATIVE_LIB_DIR/${library} ln -s ${library} $PREFIX/$HADOOP_NATIVE_LIB_DIR/${library/.so.*/}.so