From 6113361316d5ce5bfdc118d188e5617a1fcd747c Mon Sep 17 00:00:00 2001

From: Sean Purcell <me@seanp.xyz>

Date: Mon, 14 Aug 2017 22:46:04 -0700

Subject: [PATCH 1/4] squashfs-tools: Add zstd support

This patch adds zstd support to squashfs-tools. It works with zstd

versions >= 1.0.0. It was originally written by Sean Purcell.

Signed-off-by: Sean Purcell <me@seanp.xyz>

Signed-off-by: Nick Terrell <terrelln@fb.com>

---

 squashfs-tools/Makefile       |  20 +++

 squashfs-tools/compressor.c   |   8 ++

 squashfs-tools/squashfs_fs.h  |   1 +

 squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++

 squashfs-tools/zstd_wrapper.h |  48 +++++++

 5 files changed, 331 insertions(+)

 create mode 100644 squashfs-tools/zstd_wrapper.c

 create mode 100644 squashfs-tools/zstd_wrapper.h

diff --git a/squashfs-tools/Makefile b/squashfs-tools/Makefile

index 52d2582..22fc559 100644

--- a/squashfs-tools/Makefile

+++ b/squashfs-tools/Makefile

@@ -75,6 +75,18 @@ GZIP_SUPPORT = 1

 #LZMA_SUPPORT = 1

 #LZMA_DIR = ../../../../LZMA/lzma465

+

+########### Building ZSTD support ############

+#

+# The ZSTD library is supported

+# ZSTD homepage: http://zstd.net

+# ZSTD source repository: https://github.com/facebook/zstd

+#

+# To build using the ZSTD library - install the library and uncomment the

+# ZSTD_SUPPORT line below.

+#

+#ZSTD_SUPPORT = 1

+

 ######## Specifying default compression ########

 #

 # The next line specifies which compression algorithm is used by default

@@ -177,6 +189,14 @@ LIBS += -llz4

 COMPRESSORS += lz4

 endif

+ifeq ($(ZSTD_SUPPORT),1)

+CFLAGS += -DZSTD_SUPPORT

+MKSQUASHFS_OBJS += zstd_wrapper.o

+UNSQUASHFS_OBJS += zstd_wrapper.o

+LIBS += -lzstd

+COMPRESSORS += zstd

+endif

+

 ifeq ($(XATTR_SUPPORT),1)

 ifeq ($(XATTR_DEFAULT),1)

 CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT

diff --git a/squashfs-tools/compressor.c b/squashfs-tools/compressor.c

index 525e316..02b5e90 100644

--- a/squashfs-tools/compressor.c

+++ b/squashfs-tools/compressor.c

@@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = {

 extern struct compressor xz_comp_ops;

 #endif

+#ifndef ZSTD_SUPPORT

+static struct compressor zstd_comp_ops = {

+	ZSTD_COMPRESSION, "zstd"

+};

+#else

+extern struct compressor zstd_comp_ops;

+#endif

 static struct compressor unknown_comp_ops = {

 	0, "unknown"

@@ -77,6 +84,7 @@ struct compressor *compressor[] = {

 	&lzo_comp_ops,

 	&lz4_comp_ops,

 	&xz_comp_ops,

+	&zstd_comp_ops,

 	&unknown_comp_ops

 };

diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h

index 791fe12..afca918 100644

--- a/squashfs-tools/squashfs_fs.h

+++ b/squashfs-tools/squashfs_fs.h

@@ -277,6 +277,7 @@ typedef long long		squashfs_inode;

 #define LZO_COMPRESSION		3

 #define XZ_COMPRESSION		4

 #define LZ4_COMPRESSION		5

+#define ZSTD_COMPRESSION	6

 struct squashfs_super_block {

 	unsigned int		s_magic;

diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c

new file mode 100644

index 0000000..dcab75a

--- /dev/null

+++ b/squashfs-tools/zstd_wrapper.c

@@ -0,0 +1,254 @@

+/*

+ * Copyright (c) 2017

+ * Phillip Lougher <phillip@squashfs.org.uk>

+ *

+ * This program is free software; you can redistribute it and/or

+ * modify it under the terms of the GNU General Public License

+ * as published by the Free Software Foundation; either version 2,

+ * or (at your option) any later version.

+ *

+ * This program is distributed in the hope that it will be useful,

+ * but WITHOUT ANY WARRANTY; without even the implied warranty of

+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ * GNU General Public License for more details.

+ *

+ * zstd_wrapper.c

+ *

+ * Support for ZSTD compression http://zstd.net

+ */

+

+#include <stdio.h>

+#include <string.h>

+#include <stdlib.h>

+#include <zstd.h>

+#include <zstd_errors.h>

+

+#include "squashfs_fs.h"

+#include "zstd_wrapper.h"

+#include "compressor.h"

+

+static int compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;

+

+/*

+ * This function is called by the options parsing code in mksquashfs.c

+ * to parse any -X compressor option.

+ *

+ * This function returns:

+ *	>=0 (number of additional args parsed) on success

+ *	-1 if the option was unrecognised, or

+ *	-2 if the option was recognised, but otherwise bad in

+ *	   some way (e.g. invalid parameter)

+ *

+ * Note: this function sets internal compressor state, but does not

+ * pass back the results of the parsing other than success/failure.

+ * The zstd_dump_options() function is called later to get the options in

+ * a format suitable for writing to the filesystem.

+ */

+static int zstd_options(char *argv[], int argc)

+{

+	if (strcmp(argv[0], "-Xcompression-level") == 0) {

+		if (argc < 2) {

+			fprintf(stderr, "zstd: -Xcompression-level missing "

+				"compression level\n");

+			fprintf(stderr, "zstd: -Xcompression-level it should "

+				"be 1 <= n <= %d\n", ZSTD_maxCLevel());

+			goto failed;

+		}

+

+		compression_level = atoi(argv[1]);

+		if (compression_level < 1 ||

+		    compression_level > ZSTD_maxCLevel()) {

+			fprintf(stderr, "zstd: -Xcompression-level invalid, it "

+				"should be 1 <= n <= %d\n", ZSTD_maxCLevel());

+			goto failed;

+		}

+

+		return 1;

+	}

+

+	return -1;

+failed:

+	return -2;

+}

+

+/*

+ * This function is called by mksquashfs to dump the parsed

+ * compressor options in a format suitable for writing to the

+ * compressor options field in the filesystem (stored immediately

+ * after the superblock).

+ *

+ * This function returns a pointer to the compression options structure

+ * to be stored (and the size), or NULL if there are no compression

+ * options.

+ */

+static void *zstd_dump_options(int block_size, int *size)

+{

+	static struct zstd_comp_opts comp_opts;

+

+	/* don't return anything if the options are all default */

+	if (compression_level == ZSTD_DEFAULT_COMPRESSION_LEVEL)

+		return NULL;

+

+	comp_opts.compression_level = compression_level;

+

+	SQUASHFS_INSWAP_COMP_OPTS(&comp_opts);

+

+	*size = sizeof(comp_opts);

+	return &comp_opts;

+}

+

+/*

+ * This function is a helper specifically for the append mode of

+ * mksquashfs.  Its purpose is to set the internal compressor state

+ * to the stored compressor options in the passed compressor options

+ * structure.

+ *

+ * In effect this function sets up the compressor options

+ * to the same state they were when the filesystem was originally

+ * generated, this is to ensure on appending, the compressor uses

+ * the same compression options that were used to generate the

+ * original filesystem.

+ *

+ * Note, even if there are no compressor options, this function is still

+ * called with an empty compressor structure (size == 0), to explicitly

+ * set the default options, this is to ensure any user supplied

+ * -X options on the appending mksquashfs command line are over-ridden.

+ *

+ * This function returns 0 on sucessful extraction of options, and -1 on error.

+ */

+static int zstd_extract_options(int block_size, void *buffer, int size)

+{

+	struct zstd_comp_opts *comp_opts = buffer;

+

+	if (size == 0) {

+		/* Set default values */

+		compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;

+		return 0;

+	}

+

+	/* we expect a comp_opts structure of sufficient size to be present */

+	if (size < sizeof(*comp_opts))

+		goto failed;

+

+	SQUASHFS_INSWAP_COMP_OPTS(comp_opts);

+

+	if (comp_opts->compression_level < 1 ||

+	    comp_opts->compression_level > ZSTD_maxCLevel()) {

+		fprintf(stderr, "zstd: bad compression level in compression "

+			"options structure\n");

+		goto failed;

+	}

+

+	compression_level = comp_opts->compression_level;

+

+	return 0;

+

+failed:

+	fprintf(stderr, "zstd: error reading stored compressor options from "

+		"filesystem!\n");

+

+	return -1;

+}

+

+static void zstd_display_options(void *buffer, int size)

+{

+	struct zstd_comp_opts *comp_opts = buffer;

+

+	/* we expect a comp_opts structure of sufficient size to be present */

+	if (size < sizeof(*comp_opts))

+		goto failed;

+

+	SQUASHFS_INSWAP_COMP_OPTS(comp_opts);

+

+	if (comp_opts->compression_level < 1 ||

+	    comp_opts->compression_level > ZSTD_maxCLevel()) {

+		fprintf(stderr, "zstd: bad compression level in compression "

+			"options structure\n");

+		goto failed;

+	}

+

+	printf("\tcompression-level %d\n", comp_opts->compression_level);

+

+	return;

+

+failed:

+	fprintf(stderr, "zstd: error reading stored compressor options from "

+		"filesystem!\n");

+}

+

+/*

+ * This function is called by mksquashfs to initialise the

+ * compressor, before compress() is called.

+ *

+ * This function returns 0 on success, and -1 on error.

+ */

+static int zstd_init(void **strm, int block_size, int datablock)

+{

+	ZSTD_CCtx *cctx = ZSTD_createCCtx();

+

+	if (!cctx) {

+		fprintf(stderr, "zstd: failed to allocate compression "

+			"context!\n");

+		return -1;

+	}

+

+	*strm = cctx;

+	return 0;

+}

+

+static int zstd_compress(void *strm, void *dest, void *src, int size,

+			 int block_size, int *error)

+{

+	const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size,

+					     src, size, compression_level);

+

+	if (ZSTD_isError(res)) {

+		/* FIXME:

+		 * zstd does not expose stable error codes. The error enum may

+		 * change between versions. Until upstream zstd stablizes the

+		 * error codes, we have no way of knowing why the error occurs.

+		 * zstd shouldn't fail to compress any input unless there isn't

+		 * enough output space. We assume that is the cause and return

+		 * the special error code for not enough output space.

+		 */

+		return 0;

+	}

+

+	return (int)res;

+}

+

+static int zstd_uncompress(void *dest, void *src, int size, int outsize,

+			   int *error)

+{

+	const size_t res = ZSTD_decompress(dest, outsize, src, size);

+

+	if (ZSTD_isError(res)) {

+		fprintf(stderr, "\t%d %d\n", outsize, size);

+

+		*error = (int)ZSTD_getErrorCode(res);

+		return -1;

+	}

+

+	return (int)res;

+}

+

+static void zstd_usage(void)

+{

+	fprintf(stderr, "\t  -Xcompression-level <compression-level>\n");

+	fprintf(stderr, "\t\t<compression-level> should be 1 .. %d (default "

+		"%d)\n", ZSTD_maxCLevel(), ZSTD_DEFAULT_COMPRESSION_LEVEL);

+}

+

+struct compressor zstd_comp_ops = {

+	.init = zstd_init,

+	.compress = zstd_compress,

+	.uncompress = zstd_uncompress,

+	.options = zstd_options,

+	.dump_options = zstd_dump_options,

+	.extract_options = zstd_extract_options,

+	.display_options = zstd_display_options,

+	.usage = zstd_usage,

+	.id = ZSTD_COMPRESSION,

+	.name = "zstd",

+	.supported = 1

+};

diff --git a/squashfs-tools/zstd_wrapper.h b/squashfs-tools/zstd_wrapper.h

new file mode 100644

index 0000000..4fbef0a

--- /dev/null

+++ b/squashfs-tools/zstd_wrapper.h

@@ -0,0 +1,48 @@

+#ifndef ZSTD_WRAPPER_H

+#define ZSTD_WRAPPER_H

+/*

+ * Squashfs

+ *

+ * Copyright (c) 2017

+ * Phillip Lougher <phillip@squashfs.org.uk>

+ *

+ * This program is free software; you can redistribute it and/or

+ * modify it under the terms of the GNU General Public License

+ * as published by the Free Software Foundation; either version 2,

+ * or (at your option) any later version.

+ *

+ * This program is distributed in the hope that it will be useful,

+ * but WITHOUT ANY WARRANTY; without even the implied warranty of

+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ * GNU General Public License for more details.

+ *

+ * zstd_wrapper.h

+ *

+ */

+

+#ifndef linux

+#define __BYTE_ORDER BYTE_ORDER

+#define __BIG_ENDIAN BIG_ENDIAN

+#define __LITTLE_ENDIAN LITTLE_ENDIAN

+#else

+#include <endian.h>

+#endif

+

+#if __BYTE_ORDER == __BIG_ENDIAN

+extern unsigned int inswap_le16(unsigned short);

+extern unsigned int inswap_le32(unsigned int);

+

+#define SQUASHFS_INSWAP_COMP_OPTS(s) { \

+	(s)->compression_level = inswap_le32((s)->compression_level); \

+}

+#else

+#define SQUASHFS_INSWAP_COMP_OPTS(s)

+#endif

+

+/* Default compression */

+#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15

+

+struct zstd_comp_opts {

+	int compression_level;

+};

+#endif

-- 

2.19.1