#11 Backport zstd
Merged 2 months ago by bruno. Opened 2 months ago by bruno.

@@ -0,0 +1,413 @@ 

+ From 6113361316d5ce5bfdc118d188e5617a1fcd747c Mon Sep 17 00:00:00 2001

+ From: Sean Purcell <me@seanp.xyz>

+ Date: Mon, 14 Aug 2017 22:46:04 -0700

+ Subject: [PATCH 1/4] squashfs-tools: Add zstd support

+ 

+ This patch adds zstd support to squashfs-tools. It works with zstd

+ versions >= 1.0.0. It was originally written by Sean Purcell.

+ 

+ Signed-off-by: Sean Purcell <me@seanp.xyz>

+ Signed-off-by: Nick Terrell <terrelln@fb.com>

+ ---

+  squashfs-tools/Makefile       |  20 +++

+  squashfs-tools/compressor.c   |   8 ++

+  squashfs-tools/squashfs_fs.h  |   1 +

+  squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++

+  squashfs-tools/zstd_wrapper.h |  48 +++++++

+  5 files changed, 331 insertions(+)

+  create mode 100644 squashfs-tools/zstd_wrapper.c

+  create mode 100644 squashfs-tools/zstd_wrapper.h

+ 

+ diff --git a/squashfs-tools/Makefile b/squashfs-tools/Makefile

+ index 52d2582..22fc559 100644

+ --- a/squashfs-tools/Makefile

+ +++ b/squashfs-tools/Makefile

+ @@ -75,6 +75,18 @@ GZIP_SUPPORT = 1

+  #LZMA_SUPPORT = 1

+  #LZMA_DIR = ../../../../LZMA/lzma465

+  

+ +

+ +########### Building ZSTD support ############

+ +#

+ +# The ZSTD library is supported

+ +# ZSTD homepage: http://zstd.net

+ +# ZSTD source repository: https://github.com/facebook/zstd

+ +#

+ +# To build using the ZSTD library - install the library and uncomment the

+ +# ZSTD_SUPPORT line below.

+ +#

+ +#ZSTD_SUPPORT = 1

+ +

+  ######## Specifying default compression ########

+  #

+  # The next line specifies which compression algorithm is used by default

+ @@ -177,6 +189,14 @@ LIBS += -llz4

+  COMPRESSORS += lz4

+  endif

+  

+ +ifeq ($(ZSTD_SUPPORT),1)

+ +CFLAGS += -DZSTD_SUPPORT

+ +MKSQUASHFS_OBJS += zstd_wrapper.o

+ +UNSQUASHFS_OBJS += zstd_wrapper.o

+ +LIBS += -lzstd

+ +COMPRESSORS += zstd

+ +endif

+ +

+  ifeq ($(XATTR_SUPPORT),1)

+  ifeq ($(XATTR_DEFAULT),1)

+  CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT

+ diff --git a/squashfs-tools/compressor.c b/squashfs-tools/compressor.c

+ index 525e316..02b5e90 100644

+ --- a/squashfs-tools/compressor.c

+ +++ b/squashfs-tools/compressor.c

+ @@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = {

+  extern struct compressor xz_comp_ops;

+  #endif

+  

+ +#ifndef ZSTD_SUPPORT

+ +static struct compressor zstd_comp_ops = {

+ +	ZSTD_COMPRESSION, "zstd"

+ +};

+ +#else

+ +extern struct compressor zstd_comp_ops;

+ +#endif

+  

+  static struct compressor unknown_comp_ops = {

+  	0, "unknown"

+ @@ -77,6 +84,7 @@ struct compressor *compressor[] = {

+  	&lzo_comp_ops,

+  	&lz4_comp_ops,

+  	&xz_comp_ops,

+ +	&zstd_comp_ops,

+  	&unknown_comp_ops

+  };

+  

+ diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h

+ index 791fe12..afca918 100644

+ --- a/squashfs-tools/squashfs_fs.h

+ +++ b/squashfs-tools/squashfs_fs.h

+ @@ -277,6 +277,7 @@ typedef long long		squashfs_inode;

+  #define LZO_COMPRESSION		3

+  #define XZ_COMPRESSION		4

+  #define LZ4_COMPRESSION		5

+ +#define ZSTD_COMPRESSION	6

+  

+  struct squashfs_super_block {

+  	unsigned int		s_magic;

+ diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c

+ new file mode 100644

+ index 0000000..dcab75a

+ --- /dev/null

+ +++ b/squashfs-tools/zstd_wrapper.c

+ @@ -0,0 +1,254 @@

+ +/*

+ + * Copyright (c) 2017

+ + * Phillip Lougher <phillip@squashfs.org.uk>

+ + *

+ + * This program is free software; you can redistribute it and/or

+ + * modify it under the terms of the GNU General Public License

+ + * as published by the Free Software Foundation; either version 2,

+ + * or (at your option) any later version.

+ + *

+ + * This program is distributed in the hope that it will be useful,

+ + * but WITHOUT ANY WARRANTY; without even the implied warranty of

+ + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ + * GNU General Public License for more details.

+ + *

+ + * zstd_wrapper.c

+ + *

+ + * Support for ZSTD compression http://zstd.net

+ + */

+ +

+ +#include <stdio.h>

+ +#include <string.h>

+ +#include <stdlib.h>

+ +#include <zstd.h>

+ +#include <zstd_errors.h>

+ +

+ +#include "squashfs_fs.h"

+ +#include "zstd_wrapper.h"

+ +#include "compressor.h"

+ +

+ +static int compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;

+ +

+ +/*

+ + * This function is called by the options parsing code in mksquashfs.c

+ + * to parse any -X compressor option.

+ + *

+ + * This function returns:

+ + *	>=0 (number of additional args parsed) on success

+ + *	-1 if the option was unrecognised, or

+ + *	-2 if the option was recognised, but otherwise bad in

+ + *	   some way (e.g. invalid parameter)

+ + *

+ + * Note: this function sets internal compressor state, but does not

+ + * pass back the results of the parsing other than success/failure.

+ + * The zstd_dump_options() function is called later to get the options in

+ + * a format suitable for writing to the filesystem.

+ + */

+ +static int zstd_options(char *argv[], int argc)

+ +{

+ +	if (strcmp(argv[0], "-Xcompression-level") == 0) {

+ +		if (argc < 2) {

+ +			fprintf(stderr, "zstd: -Xcompression-level missing "

+ +				"compression level\n");

+ +			fprintf(stderr, "zstd: -Xcompression-level it should "

+ +				"be 1 <= n <= %d\n", ZSTD_maxCLevel());

+ +			goto failed;

+ +		}

+ +

+ +		compression_level = atoi(argv[1]);

+ +		if (compression_level < 1 ||

+ +		    compression_level > ZSTD_maxCLevel()) {

+ +			fprintf(stderr, "zstd: -Xcompression-level invalid, it "

+ +				"should be 1 <= n <= %d\n", ZSTD_maxCLevel());

+ +			goto failed;

+ +		}

+ +

+ +		return 1;

+ +	}

+ +

+ +	return -1;

+ +failed:

+ +	return -2;

+ +}

+ +

+ +/*

+ + * This function is called by mksquashfs to dump the parsed

+ + * compressor options in a format suitable for writing to the

+ + * compressor options field in the filesystem (stored immediately

+ + * after the superblock).

+ + *

+ + * This function returns a pointer to the compression options structure

+ + * to be stored (and the size), or NULL if there are no compression

+ + * options.

+ + */

+ +static void *zstd_dump_options(int block_size, int *size)

+ +{

+ +	static struct zstd_comp_opts comp_opts;

+ +

+ +	/* don't return anything if the options are all default */

+ +	if (compression_level == ZSTD_DEFAULT_COMPRESSION_LEVEL)

+ +		return NULL;

+ +

+ +	comp_opts.compression_level = compression_level;

+ +

+ +	SQUASHFS_INSWAP_COMP_OPTS(&comp_opts);

+ +

+ +	*size = sizeof(comp_opts);

+ +	return &comp_opts;

+ +}

+ +

+ +/*

+ + * This function is a helper specifically for the append mode of

+ + * mksquashfs.  Its purpose is to set the internal compressor state

+ + * to the stored compressor options in the passed compressor options

+ + * structure.

+ + *

+ + * In effect this function sets up the compressor options

+ + * to the same state they were when the filesystem was originally

+ + * generated, this is to ensure on appending, the compressor uses

+ + * the same compression options that were used to generate the

+ + * original filesystem.

+ + *

+ + * Note, even if there are no compressor options, this function is still

+ + * called with an empty compressor structure (size == 0), to explicitly

+ + * set the default options, this is to ensure any user supplied

+ + * -X options on the appending mksquashfs command line are over-ridden.

+ + *

+ + * This function returns 0 on sucessful extraction of options, and -1 on error.

+ + */

+ +static int zstd_extract_options(int block_size, void *buffer, int size)

+ +{

+ +	struct zstd_comp_opts *comp_opts = buffer;

+ +

+ +	if (size == 0) {

+ +		/* Set default values */

+ +		compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;

+ +		return 0;

+ +	}

+ +

+ +	/* we expect a comp_opts structure of sufficient size to be present */

+ +	if (size < sizeof(*comp_opts))

+ +		goto failed;

+ +

+ +	SQUASHFS_INSWAP_COMP_OPTS(comp_opts);

+ +

+ +	if (comp_opts->compression_level < 1 ||

+ +	    comp_opts->compression_level > ZSTD_maxCLevel()) {

+ +		fprintf(stderr, "zstd: bad compression level in compression "

+ +			"options structure\n");

+ +		goto failed;

+ +	}

+ +

+ +	compression_level = comp_opts->compression_level;

+ +

+ +	return 0;

+ +

+ +failed:

+ +	fprintf(stderr, "zstd: error reading stored compressor options from "

+ +		"filesystem!\n");

+ +

+ +	return -1;

+ +}

+ +

+ +static void zstd_display_options(void *buffer, int size)

+ +{

+ +	struct zstd_comp_opts *comp_opts = buffer;

+ +

+ +	/* we expect a comp_opts structure of sufficient size to be present */

+ +	if (size < sizeof(*comp_opts))

+ +		goto failed;

+ +

+ +	SQUASHFS_INSWAP_COMP_OPTS(comp_opts);

+ +

+ +	if (comp_opts->compression_level < 1 ||

+ +	    comp_opts->compression_level > ZSTD_maxCLevel()) {

+ +		fprintf(stderr, "zstd: bad compression level in compression "

+ +			"options structure\n");

+ +		goto failed;

+ +	}

+ +

+ +	printf("\tcompression-level %d\n", comp_opts->compression_level);

+ +

+ +	return;

+ +

+ +failed:

+ +	fprintf(stderr, "zstd: error reading stored compressor options from "

+ +		"filesystem!\n");

+ +}

+ +

+ +/*

+ + * This function is called by mksquashfs to initialise the

+ + * compressor, before compress() is called.

+ + *

+ + * This function returns 0 on success, and -1 on error.

+ + */

+ +static int zstd_init(void **strm, int block_size, int datablock)

+ +{

+ +	ZSTD_CCtx *cctx = ZSTD_createCCtx();

+ +

+ +	if (!cctx) {

+ +		fprintf(stderr, "zstd: failed to allocate compression "

+ +			"context!\n");

+ +		return -1;

+ +	}

+ +

+ +	*strm = cctx;

+ +	return 0;

+ +}

+ +

+ +static int zstd_compress(void *strm, void *dest, void *src, int size,

+ +			 int block_size, int *error)

+ +{

+ +	const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size,

+ +					     src, size, compression_level);

+ +

+ +	if (ZSTD_isError(res)) {

+ +		/* FIXME:

+ +		 * zstd does not expose stable error codes. The error enum may

+ +		 * change between versions. Until upstream zstd stablizes the

+ +		 * error codes, we have no way of knowing why the error occurs.

+ +		 * zstd shouldn't fail to compress any input unless there isn't

+ +		 * enough output space. We assume that is the cause and return

+ +		 * the special error code for not enough output space.

+ +		 */

+ +		return 0;

+ +	}

+ +

+ +	return (int)res;

+ +}

+ +

+ +static int zstd_uncompress(void *dest, void *src, int size, int outsize,

+ +			   int *error)

+ +{

+ +	const size_t res = ZSTD_decompress(dest, outsize, src, size);

+ +

+ +	if (ZSTD_isError(res)) {

+ +		fprintf(stderr, "\t%d %d\n", outsize, size);

+ +

+ +		*error = (int)ZSTD_getErrorCode(res);

+ +		return -1;

+ +	}

+ +

+ +	return (int)res;

+ +}

+ +

+ +static void zstd_usage(void)

+ +{

+ +	fprintf(stderr, "\t  -Xcompression-level <compression-level>\n");

+ +	fprintf(stderr, "\t\t<compression-level> should be 1 .. %d (default "

+ +		"%d)\n", ZSTD_maxCLevel(), ZSTD_DEFAULT_COMPRESSION_LEVEL);

+ +}

+ +

+ +struct compressor zstd_comp_ops = {

+ +	.init = zstd_init,

+ +	.compress = zstd_compress,

+ +	.uncompress = zstd_uncompress,

+ +	.options = zstd_options,

+ +	.dump_options = zstd_dump_options,

+ +	.extract_options = zstd_extract_options,

+ +	.display_options = zstd_display_options,

+ +	.usage = zstd_usage,

+ +	.id = ZSTD_COMPRESSION,

+ +	.name = "zstd",

+ +	.supported = 1

+ +};

+ diff --git a/squashfs-tools/zstd_wrapper.h b/squashfs-tools/zstd_wrapper.h

+ new file mode 100644

+ index 0000000..4fbef0a

+ --- /dev/null

+ +++ b/squashfs-tools/zstd_wrapper.h

+ @@ -0,0 +1,48 @@

+ +#ifndef ZSTD_WRAPPER_H

+ +#define ZSTD_WRAPPER_H

+ +/*

+ + * Squashfs

+ + *

+ + * Copyright (c) 2017

+ + * Phillip Lougher <phillip@squashfs.org.uk>

+ + *

+ + * This program is free software; you can redistribute it and/or

+ + * modify it under the terms of the GNU General Public License

+ + * as published by the Free Software Foundation; either version 2,

+ + * or (at your option) any later version.

+ + *

+ + * This program is distributed in the hope that it will be useful,

+ + * but WITHOUT ANY WARRANTY; without even the implied warranty of

+ + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ + * GNU General Public License for more details.

+ + *

+ + * zstd_wrapper.h

+ + *

+ + */

+ +

+ +#ifndef linux

+ +#define __BYTE_ORDER BYTE_ORDER

+ +#define __BIG_ENDIAN BIG_ENDIAN

+ +#define __LITTLE_ENDIAN LITTLE_ENDIAN

+ +#else

+ +#include <endian.h>

+ +#endif

+ +

+ +#if __BYTE_ORDER == __BIG_ENDIAN

+ +extern unsigned int inswap_le16(unsigned short);

+ +extern unsigned int inswap_le32(unsigned int);

+ +

+ +#define SQUASHFS_INSWAP_COMP_OPTS(s) { \

+ +	(s)->compression_level = inswap_le32((s)->compression_level); \

+ +}

+ +#else

+ +#define SQUASHFS_INSWAP_COMP_OPTS(s)

+ +#endif

+ +

+ +/* Default compression */

+ +#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15

+ +

+ +struct zstd_comp_opts {

+ +	int compression_level;

+ +};

+ +#endif

+ -- 

+ 2.19.1

+ 

file modified
+3 -3

@@ -1,3 +1,3 @@ 

- d92ab59aabf5173f2a59089531e30dbf  squashfs4.3.tar.gz

- ed8d2bb2cac678a394815d081f1c465c  mksquashfs.1

- 68bf9bbf3ba00dc6e480a59b24905185  unsquashfs.1

+ SHA512 (squashfs4.3.tar.gz) = 854ed7acc99920f24ecf11e0da807e5a2a162eeda55db971aba63a03f0da2c13b20ec0564a906c4b0e415bd8258b273a10208c7abc0704f2ceea773aa6148a79

+ SHA512 (mksquashfs.1) = 2859790d906263b09502ba062b1c41cfc5f9fd7bdff743021c30b649ed1fe8ff047cdf49b79776fcebac9a750fea58f9348b2f14de8c532175aeaeef3b54b217

+ SHA512 (unsquashfs.1) = 6e1be535d370fb39b2a0e47c98052727bab94ae4f306bb3eb8f7dd07fb84bf985e82ba66bb2030e08261473cffc34d1c1973b27e77cb7127d588e24297f2f0a3

file modified
+10 -2

@@ -1,7 +1,7 @@ 

  Summary: Utility for the creation of squashfs filesystems

  Name: squashfs-tools

  Version: 4.3

- Release: 20%{?dist}

+ Release: 21%{?dist}

  License: GPLv2+

  URL: http://squashfs.sourceforge.net/

  Source0: http://downloads.sourceforge.net/squashfs/squashfs%{version}.tar.gz

@@ -26,12 +26,16 @@ 

  Patch4:  local-cve-fix.patch

  # sys/sysmacros.h is no longer included by sys/types.h

  Patch5:  glibc.patch

+ # zstd compression support from https://github.com/plougher/squashfs-tools

+ Patch6:  0001-squashfs-tools-Add-zstd-support.patch

+ 

  BuildRequires:  gcc

  BuildRequires: zlib-devel

  BuildRequires: xz-devel

  BuildRequires: lzo-devel

  BuildRequires: libattr-devel

  BuildRequires: lz4-devel

+ BuildRequires: libzstd-devel

  

  %description

  Squashfs is a highly compressed read-only filesystem for Linux.  This package

@@ -45,11 +49,12 @@ 

  %patch3 -p1

  %patch4 -p0

  %patch5 -p0

+ %patch6 -p1

  

  %build

  %set_build_flags

  pushd squashfs-tools

- CFLAGS="%{optflags}" XZ_SUPPORT=1 LZO_SUPPORT=1 LZMA_XZ_SUPPORT=1 LZ4_SUPPORT=1 make %{?_smp_mflags}

+ CFLAGS="%{optflags}" XZ_SUPPORT=1 LZO_SUPPORT=1 LZMA_XZ_SUPPORT=1 LZ4_SUPPORT=1 ZSTD_SUPPORT=1 make %{?_smp_mflags}

  

  %install

  mkdir -p %{buildroot}%{_sbindir} %{buildroot}%{_mandir}/man1

@@ -68,6 +73,9 @@ 

  %{_sbindir}/unsquashfs

  

  %changelog

+ * Mon Jun 24 2019 Bruno Wolff III <bruno@wolff.to> - 4.3-21

+ - Add zstd compression support (Sean Purcell via github.com/plougher/squashfs-tools)

+ 

  * Tue May 21 2019 Bruno Wolff III <bruno@wolff.to> - 4.3-20

  - Fix issue with LDFLAGS not being set

  

no initial comment

1 new commit added

  • Update release in spec file
2 months ago

It looks like the CI issue isn't with squashfs-tools. It worked in an outside test.

Pull-Request has been merged by bruno

2 months ago