From patchwork Tue Aug 18 14:31:15 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Joao Marcos Costa X-Patchwork-Id: 1346904 X-Patchwork-Delegate: trini@ti.com Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=lists.denx.de (client-ip=2a01:238:438b:c500:173d:9f52:ddab:ee01; helo=phobos.denx.de; envelope-from=u-boot-bounces@lists.denx.de; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=bootlin.com Received: from phobos.denx.de (phobos.denx.de [IPv6:2a01:238:438b:c500:173d:9f52:ddab:ee01]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 4BWCz102mzz9sTM for ; Wed, 19 Aug 2020 00:31:40 +1000 (AEST) Received: from h2850616.stratoserver.net (localhost [IPv6:::1]) by phobos.denx.de (Postfix) with ESMTP id 3CD3A821D4; Tue, 18 Aug 2020 16:31:26 +0200 (CEST) Authentication-Results: phobos.denx.de; dmarc=none (p=none dis=none) header.from=bootlin.com Authentication-Results: phobos.denx.de; spf=pass smtp.mailfrom=u-boot-bounces@lists.denx.de Received: by phobos.denx.de (Postfix, from userid 109) id 81FA4821E0; Tue, 18 Aug 2020 16:31:23 +0200 (CEST) X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on phobos.denx.de X-Spam-Level: * X-Spam-Status: No, score=1.4 required=5.0 tests=BAYES_00,RCVD_IN_MSPIKE_H3, RCVD_IN_MSPIKE_WL,RCVD_IN_SBL_CSS,SPF_HELO_NONE,URIBL_BLOCKED autolearn=no autolearn_force=no version=3.4.2 Received: from relay2-d.mail.gandi.net (relay2-d.mail.gandi.net [217.70.183.194]) (using TLSv1.2 with cipher ADH-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by phobos.denx.de (Postfix) with ESMTPS id 759AE82176 for ; Tue, 18 Aug 2020 16:31:20 +0200 (CEST) Authentication-Results: phobos.denx.de; dmarc=none (p=none dis=none) header.from=bootlin.com Authentication-Results: phobos.denx.de; spf=pass smtp.mailfrom=joaomarcos.costa@bootlin.com X-Originating-IP: 46.193.64.106 Received: from localhost.localdomain (eth-east-parth2-46-193-64-106.wb.wifirst.net [46.193.64.106]) (Authenticated sender: joaomarcos.costa@bootlin.com) by relay2-d.mail.gandi.net (Postfix) with ESMTPA id B04CF40007; Tue, 18 Aug 2020 14:31:19 +0000 (UTC) From: Joao Marcos Costa To: u-boot@lists.denx.de Cc: joaomarcos.costa@bootlin.com, miquel.raynal@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v3 1/4] fs/squashfs: Add init. and clean-up functions to decompression Date: Tue, 18 Aug 2020 16:31:15 +0200 Message-Id: <20200818143118.16422-2-joaomarcos.costa@bootlin.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20200818143118.16422-1-joaomarcos.costa@bootlin.com> References: <20200818143118.16422-1-joaomarcos.costa@bootlin.com> X-BeenThere: u-boot@lists.denx.de X-Mailman-Version: 2.1.34 Precedence: list List-Id: U-Boot discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: u-boot-bounces@lists.denx.de Sender: "U-Boot" X-Virus-Scanned: clamav-milter 0.102.3 at phobos.denx.de X-Virus-Status: Clean Add sqfs_decompressor_init() and sqfs_decompressor_cleanup(). These functions are called respectively in sqfs_probe() and sqfs_close(). For now, only ZSTD requires an initialization logic. ZSTD support will be added in a follow-up commit. Move squashfs_ctxt definition to sqfs_filesystem.h. This structure is passed to sqfs_decompressor_init() and sqfs_decompressor_cleanup(), so it can no longer be local to sqfs.c. Signed-off-by: Joao Marcos Costa --- Changes in v3: - Remove memory leak from sqfs_zstd_decompress() - Rename structure in sqfs.c and remove the 'static' so it can be exported to sqfs_decompressor.c Changes in v2: - This patch was not present in the previous version. fs/squashfs/sqfs.c | 87 +++++++++++++++++---------------- fs/squashfs/sqfs_decompressor.c | 36 ++++++++++++-- fs/squashfs/sqfs_decompressor.h | 3 ++ fs/squashfs/sqfs_filesystem.h | 9 +++- 4 files changed, 89 insertions(+), 46 deletions(-) diff --git a/fs/squashfs/sqfs.c b/fs/squashfs/sqfs.c index 340e5ebdb9..2836b3de8b 100644 --- a/fs/squashfs/sqfs.c +++ b/fs/squashfs/sqfs.c @@ -23,22 +23,16 @@ #include "sqfs_filesystem.h" #include "sqfs_utils.h" -struct squashfs_ctxt { - struct disk_partition cur_part_info; - struct blk_desc *cur_dev; - struct squashfs_super_block *sblk; -}; - -static struct squashfs_ctxt ctxt; +struct squashfs_ctxt sqfs_ctxt; static int sqfs_disk_read(__u32 block, __u32 nr_blocks, void *buf) { ulong ret; - if (!ctxt.cur_dev) + if (!sqfs_ctxt.cur_dev) return -1; - ret = blk_dread(ctxt.cur_dev, ctxt.cur_part_info.start + block, + ret = blk_dread(sqfs_ctxt.cur_dev, sqfs_ctxt.cur_part_info.start + block, nr_blocks, buf); if (ret != nr_blocks) @@ -49,7 +43,7 @@ static int sqfs_disk_read(__u32 block, __u32 nr_blocks, void *buf) static int sqfs_read_sblk(struct squashfs_super_block **sblk) { - *sblk = malloc_cache_aligned(ctxt.cur_dev->blksz); + *sblk = malloc_cache_aligned(sqfs_ctxt.cur_dev->blksz); if (!*sblk) return -ENOMEM; @@ -91,10 +85,10 @@ static int sqfs_calc_n_blks(__le64 start, __le64 end, u64 *offset) u64 start_, table_size; table_size = le64_to_cpu(end) - le64_to_cpu(start); - start_ = le64_to_cpu(start) / ctxt.cur_dev->blksz; - *offset = le64_to_cpu(start) - (start_ * ctxt.cur_dev->blksz); + start_ = le64_to_cpu(start) / sqfs_ctxt.cur_dev->blksz; + *offset = le64_to_cpu(start) - (start_ * sqfs_ctxt.cur_dev->blksz); - return DIV_ROUND_UP(table_size + *offset, ctxt.cur_dev->blksz); + return DIV_ROUND_UP(table_size + *offset, sqfs_ctxt.cur_dev->blksz); } /* @@ -107,7 +101,7 @@ static int sqfs_frag_lookup(u32 inode_fragment_index, u64 start, n_blks, src_len, table_offset, start_block; unsigned char *metadata_buffer, *metadata, *table; struct squashfs_fragment_block_entry *entries; - struct squashfs_super_block *sblk = ctxt.sblk; + struct squashfs_super_block *sblk = sqfs_ctxt.sblk; unsigned long dest_len; int block, offset, ret; u16 header, comp_type; @@ -118,13 +112,13 @@ static int sqfs_frag_lookup(u32 inode_fragment_index, return -EINVAL; start = get_unaligned_le64(&sblk->fragment_table_start) / - ctxt.cur_dev->blksz; + sqfs_ctxt.cur_dev->blksz; n_blks = sqfs_calc_n_blks(sblk->fragment_table_start, sblk->export_table_start, &table_offset); /* Allocate a proper sized buffer to store the fragment index table */ - table = malloc_cache_aligned(n_blks * ctxt.cur_dev->blksz); + table = malloc_cache_aligned(n_blks * sqfs_ctxt.cur_dev->blksz); if (!table) return -ENOMEM; @@ -143,11 +137,11 @@ static int sqfs_frag_lookup(u32 inode_fragment_index, start_block = get_unaligned_le64(table + table_offset + block * sizeof(u64)); - start = start_block / ctxt.cur_dev->blksz; + start = start_block / sqfs_ctxt.cur_dev->blksz; n_blks = sqfs_calc_n_blks(cpu_to_le64(start_block), sblk->fragment_table_start, &table_offset); - metadata_buffer = malloc_cache_aligned(n_blks * ctxt.cur_dev->blksz); + metadata_buffer = malloc_cache_aligned(n_blks * sqfs_ctxt.cur_dev->blksz); if (!metadata_buffer) { ret = -ENOMEM; goto free_table; @@ -434,7 +428,7 @@ static char *sqfs_resolve_symlink(struct squashfs_symlink_inode *sym, static int sqfs_search_dir(struct squashfs_dir_stream *dirs, char **token_list, int token_count, u32 *m_list, int m_count) { - struct squashfs_super_block *sblk = ctxt.sblk; + struct squashfs_super_block *sblk = sqfs_ctxt.sblk; char *path, *target, **sym_tokens, *res, *rem; int j, ret, new_inode_number, offset; struct squashfs_symlink_inode *sym; @@ -633,7 +627,7 @@ static int sqfs_get_metablk_pos(u32 *pos_list, void *table, u32 offset, static int sqfs_read_inode_table(unsigned char **inode_table) { - struct squashfs_super_block *sblk = ctxt.sblk; + struct squashfs_super_block *sblk = sqfs_ctxt.sblk; u64 start, n_blks, table_offset, table_size; int j, ret = 0, metablks_count, comp_type; unsigned char *src_table, *itb; @@ -645,12 +639,12 @@ static int sqfs_read_inode_table(unsigned char **inode_table) table_size = get_unaligned_le64(&sblk->directory_table_start) - get_unaligned_le64(&sblk->inode_table_start); start = get_unaligned_le64(&sblk->inode_table_start) / - ctxt.cur_dev->blksz; + sqfs_ctxt.cur_dev->blksz; n_blks = sqfs_calc_n_blks(sblk->inode_table_start, sblk->directory_table_start, &table_offset); /* Allocate a proper sized buffer (itb) to store the inode table */ - itb = malloc_cache_aligned(n_blks * ctxt.cur_dev->blksz); + itb = malloc_cache_aligned(n_blks * sqfs_ctxt.cur_dev->blksz); if (!itb) return -ENOMEM; @@ -718,7 +712,7 @@ static int sqfs_read_directory_table(unsigned char **dir_table, u32 **pos_list) { u64 start, n_blks, table_offset, table_size; int j, ret = 0, metablks_count = -1, comp_type; - struct squashfs_super_block *sblk = ctxt.sblk; + struct squashfs_super_block *sblk = sqfs_ctxt.sblk; unsigned char *src_table, *dtb; u32 src_len, dest_offset = 0; unsigned long dest_len; @@ -730,12 +724,12 @@ static int sqfs_read_directory_table(unsigned char **dir_table, u32 **pos_list) table_size = get_unaligned_le64(&sblk->fragment_table_start) - get_unaligned_le64(&sblk->directory_table_start); start = get_unaligned_le64(&sblk->directory_table_start) / - ctxt.cur_dev->blksz; + sqfs_ctxt.cur_dev->blksz; n_blks = sqfs_calc_n_blks(sblk->directory_table_start, sblk->fragment_table_start, &table_offset); /* Allocate a proper sized buffer (dtb) to store the directory table */ - dtb = malloc_cache_aligned(n_blks * ctxt.cur_dev->blksz); + dtb = malloc_cache_aligned(n_blks * sqfs_ctxt.cur_dev->blksz); if (!dtb) return -ENOMEM; @@ -891,7 +885,7 @@ free_path: int sqfs_readdir(struct fs_dir_stream *fs_dirs, struct fs_dirent **dentp) { - struct squashfs_super_block *sblk = ctxt.sblk; + struct squashfs_super_block *sblk = sqfs_ctxt.sblk; struct squashfs_dir_stream *dirs; struct squashfs_lreg_inode *lreg; struct squashfs_base_inode *base; @@ -1007,8 +1001,8 @@ int sqfs_probe(struct blk_desc *fs_dev_desc, struct disk_partition *fs_partition struct squashfs_super_block *sblk; int ret; - ctxt.cur_dev = fs_dev_desc; - ctxt.cur_part_info = *fs_partition; + sqfs_ctxt.cur_dev = fs_dev_desc; + sqfs_ctxt.cur_part_info = *fs_partition; ret = sqfs_read_sblk(&sblk); if (ret) @@ -1017,11 +1011,19 @@ int sqfs_probe(struct blk_desc *fs_dev_desc, struct disk_partition *fs_partition /* Make sure it has a valid SquashFS magic number*/ if (get_unaligned_le32(&sblk->s_magic) != SQFS_MAGIC_NUMBER) { printf("Bad magic number for SquashFS image.\n"); - ctxt.cur_dev = NULL; + sqfs_ctxt.cur_dev = NULL; return -EINVAL; } - ctxt.sblk = sblk; + sqfs_ctxt.sblk = sblk; + + ret = sqfs_decompressor_init(&sqfs_ctxt); + + if (ret) { + sqfs_ctxt.cur_dev = NULL; + free(sqfs_ctxt.sblk); + return -EINVAL; + } return 0; } @@ -1196,7 +1198,7 @@ int sqfs_read(const char *filename, void *buf, loff_t offset, loff_t len, char *fragment, *file, *resolved, *data; u64 start, n_blks, table_size, data_offset, table_offset; int ret, j, i_number, comp_type, datablk_count = 0; - struct squashfs_super_block *sblk = ctxt.sblk; + struct squashfs_super_block *sblk = sqfs_ctxt.sblk; struct squashfs_fragment_block_entry frag_entry; struct squashfs_file_info finfo = {0}; struct squashfs_symlink_inode *symlink; @@ -1315,13 +1317,13 @@ int sqfs_read(const char *filename, void *buf, loff_t offset, loff_t len, } for (j = 0; j < datablk_count; j++) { - start = data_offset / ctxt.cur_dev->blksz; + start = data_offset / sqfs_ctxt.cur_dev->blksz; table_size = SQFS_BLOCK_SIZE(finfo.blk_sizes[j]); - table_offset = data_offset - (start * ctxt.cur_dev->blksz); + table_offset = data_offset - (start * sqfs_ctxt.cur_dev->blksz); n_blks = DIV_ROUND_UP(table_size + table_offset, - ctxt.cur_dev->blksz); + sqfs_ctxt.cur_dev->blksz); - data_buffer = malloc_cache_aligned(n_blks * ctxt.cur_dev->blksz); + data_buffer = malloc_cache_aligned(n_blks * sqfs_ctxt.cur_dev->blksz); if (!data_buffer) { ret = -ENOMEM; @@ -1369,12 +1371,12 @@ int sqfs_read(const char *filename, void *buf, loff_t offset, loff_t len, goto free_buffer; } - start = frag_entry.start / ctxt.cur_dev->blksz; + start = frag_entry.start / sqfs_ctxt.cur_dev->blksz; table_size = SQFS_BLOCK_SIZE(frag_entry.size); - table_offset = frag_entry.start - (start * ctxt.cur_dev->blksz); - n_blks = DIV_ROUND_UP(table_size + table_offset, ctxt.cur_dev->blksz); + table_offset = frag_entry.start - (start * sqfs_ctxt.cur_dev->blksz); + n_blks = DIV_ROUND_UP(table_size + table_offset, sqfs_ctxt.cur_dev->blksz); - fragment = malloc_cache_aligned(n_blks * ctxt.cur_dev->blksz); + fragment = malloc_cache_aligned(n_blks * sqfs_ctxt.cur_dev->blksz); if (!fragment) { ret = -ENOMEM; @@ -1435,7 +1437,7 @@ free_paths: int sqfs_size(const char *filename, loff_t *size) { - struct squashfs_super_block *sblk = ctxt.sblk; + struct squashfs_super_block *sblk = sqfs_ctxt.sblk; struct squashfs_symlink_inode *symlink; struct fs_dir_stream *dirsp = NULL; struct squashfs_base_inode *base; @@ -1523,8 +1525,9 @@ free_strings: void sqfs_close(void) { - free(ctxt.sblk); - ctxt.cur_dev = NULL; + free(sqfs_ctxt.sblk); + sqfs_ctxt.cur_dev = NULL; + sqfs_decompressor_cleanup(&sqfs_ctxt); } void sqfs_closedir(struct fs_dir_stream *dirs) diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c index 09ca6cf6d0..577cffd8bb 100644 --- a/fs/squashfs/sqfs_decompressor.c +++ b/fs/squashfs/sqfs_decompressor.c @@ -14,9 +14,39 @@ #endif #include "sqfs_decompressor.h" -#include "sqfs_filesystem.h" #include "sqfs_utils.h" +extern struct squashfs_ctxt sqfs_ctxt; + +int sqfs_decompressor_init(struct squashfs_ctxt *ctxt) +{ + u16 comp_type = get_unaligned_le16(&ctxt->sblk->compression); + + switch (comp_type) { +#if IS_ENABLED(CONFIG_ZLIB) + case SQFS_COMP_ZLIB: + break; +#endif + default: + printf("Error: unknown compression type.\n"); + return -EINVAL; + } + + return 0; +} + +void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt) +{ + u16 comp_type = get_unaligned_le16(&ctxt->sblk->compression); + + switch (comp_type) { +#if IS_ENABLED(CONFIG_ZLIB) + case SQFS_COMP_ZLIB: + break; +#endif + } +} + #if IS_ENABLED(CONFIG_ZLIB) static void zlib_decompression_status(int ret) { @@ -35,14 +65,14 @@ static void zlib_decompression_status(int ret) #endif int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len, - void *source, u32 lenp) + void *source, u32 src_len) { int ret = 0; switch (comp_type) { #if IS_ENABLED(CONFIG_ZLIB) case SQFS_COMP_ZLIB: - ret = uncompress(dest, dest_len, source, lenp); + ret = uncompress(dest, dest_len, source, src_len); if (ret) { zlib_decompression_status(ret); return -EINVAL; diff --git a/fs/squashfs/sqfs_decompressor.h b/fs/squashfs/sqfs_decompressor.h index 378965dda8..450257e5ce 100644 --- a/fs/squashfs/sqfs_decompressor.h +++ b/fs/squashfs/sqfs_decompressor.h @@ -9,6 +9,7 @@ #define SQFS_DECOMPRESSOR_H #include +#include "sqfs_filesystem.h" #define SQFS_COMP_ZLIB 1 #define SQFS_COMP_LZMA 2 @@ -54,5 +55,7 @@ union squashfs_compression_opts { int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len, void *source, u32 lenp); +int sqfs_decompressor_init(struct squashfs_ctxt *ctxt); +void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt); #endif /* SQFS_DECOMPRESSOR_H */ diff --git a/fs/squashfs/sqfs_filesystem.h b/fs/squashfs/sqfs_filesystem.h index d63e3a41ad..ff2b0b1d34 100644 --- a/fs/squashfs/sqfs_filesystem.h +++ b/fs/squashfs/sqfs_filesystem.h @@ -9,8 +9,9 @@ #define SQFS_FILESYSTEM_H #include -#include #include +#include +#include #define SQFS_UNCOMPRESSED_DATA 0x0002 #define SQFS_MAGIC_NUMBER 0x73717368 @@ -72,6 +73,12 @@ struct squashfs_super_block { __le64 export_table_start; }; +struct squashfs_ctxt { + struct disk_partition cur_part_info; + struct blk_desc *cur_dev; + struct squashfs_super_block *sblk; +}; + struct squashfs_directory_index { u32 index; u32 start; From patchwork Tue Aug 18 14:31:16 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Joao Marcos Costa X-Patchwork-Id: 1346906 X-Patchwork-Delegate: trini@ti.com Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=lists.denx.de (client-ip=2a01:238:438b:c500:173d:9f52:ddab:ee01; helo=phobos.denx.de; envelope-from=u-boot-bounces@lists.denx.de; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=bootlin.com Received: from phobos.denx.de (phobos.denx.de [IPv6:2a01:238:438b:c500:173d:9f52:ddab:ee01]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 4BWCzG32JRz9sPC for ; Wed, 19 Aug 2020 00:31:53 +1000 (AEST) Received: from h2850616.stratoserver.net (localhost [IPv6:::1]) by phobos.denx.de (Postfix) with ESMTP id 9BE158222C; Tue, 18 Aug 2020 16:31:27 +0200 (CEST) Authentication-Results: phobos.denx.de; dmarc=none (p=none dis=none) header.from=bootlin.com Authentication-Results: phobos.denx.de; spf=pass smtp.mailfrom=u-boot-bounces@lists.denx.de Received: by phobos.denx.de (Postfix, from userid 109) id 16A2182229; Tue, 18 Aug 2020 16:31:25 +0200 (CEST) X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on phobos.denx.de X-Spam-Level: * X-Spam-Status: No, score=1.4 required=5.0 tests=BAYES_00,RCVD_IN_MSPIKE_H3, RCVD_IN_MSPIKE_WL,RCVD_IN_SBL_CSS,SPF_HELO_NONE,URIBL_BLOCKED autolearn=no autolearn_force=no version=3.4.2 Received: from relay2-d.mail.gandi.net (relay2-d.mail.gandi.net [217.70.183.194]) (using TLSv1.2 with cipher ADH-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by phobos.denx.de (Postfix) with ESMTPS id 3E98F821BE for ; Tue, 18 Aug 2020 16:31:21 +0200 (CEST) Authentication-Results: phobos.denx.de; dmarc=none (p=none dis=none) header.from=bootlin.com Authentication-Results: phobos.denx.de; spf=pass smtp.mailfrom=joaomarcos.costa@bootlin.com X-Originating-IP: 46.193.64.106 Received: from localhost.localdomain (eth-east-parth2-46-193-64-106.wb.wifirst.net [46.193.64.106]) (Authenticated sender: joaomarcos.costa@bootlin.com) by relay2-d.mail.gandi.net (Postfix) with ESMTPA id 3058B40011; Tue, 18 Aug 2020 14:31:20 +0000 (UTC) From: Joao Marcos Costa To: u-boot@lists.denx.de Cc: joaomarcos.costa@bootlin.com, miquel.raynal@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v3 2/4] fs/squashfs: add support for ZSTD decompression Date: Tue, 18 Aug 2020 16:31:16 +0200 Message-Id: <20200818143118.16422-3-joaomarcos.costa@bootlin.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20200818143118.16422-1-joaomarcos.costa@bootlin.com> References: <20200818143118.16422-1-joaomarcos.costa@bootlin.com> X-BeenThere: u-boot@lists.denx.de X-Mailman-Version: 2.1.34 Precedence: list List-Id: U-Boot discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: u-boot-bounces@lists.denx.de Sender: "U-Boot" X-Virus-Scanned: clamav-milter 0.102.3 at phobos.denx.de X-Virus-Status: Clean Add call to ZSTD's ZSTD_decompressDCtx(). In this use case, the caller can upper bound the decompressed size, which will be the SquashFS data block (or metadata block) size, so there is no need to use streaming API. Add ZSTD's worskpace to squashfs_ctxt structure. Signed-off-by: Joao Marcos Costa --- Changes in v3: - Remove memory leak from sqfs_zstd_decompress() Changes in v2: - No changes since last version. fs/squashfs/sqfs_decompressor.c | 42 +++++++++++++++++++++++++++++++++ fs/squashfs/sqfs_filesystem.h | 3 +++ 2 files changed, 45 insertions(+) diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c index 577cffd8bb..8fdaaa853c 100644 --- a/fs/squashfs/sqfs_decompressor.c +++ b/fs/squashfs/sqfs_decompressor.c @@ -13,6 +13,10 @@ #include #endif +#if IS_ENABLED(CONFIG_ZSTD) +#include +#endif + #include "sqfs_decompressor.h" #include "sqfs_utils.h" @@ -26,6 +30,13 @@ int sqfs_decompressor_init(struct squashfs_ctxt *ctxt) #if IS_ENABLED(CONFIG_ZLIB) case SQFS_COMP_ZLIB: break; +#endif +#if IS_ENABLED(CONFIG_ZSTD) + case SQFS_COMP_ZSTD: + ctxt->zstd_workspace = malloc(ZSTD_DCtxWorkspaceBound()); + if (!ctxt->zstd_workspace) + return -ENOMEM; + break; #endif default: printf("Error: unknown compression type.\n"); @@ -43,6 +54,11 @@ void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt) #if IS_ENABLED(CONFIG_ZLIB) case SQFS_COMP_ZLIB: break; +#endif +#if IS_ENABLED(CONFIG_ZSTD) + case SQFS_COMP_ZSTD: + free(ctxt->zstd_workspace); + break; #endif } } @@ -64,6 +80,22 @@ static void zlib_decompression_status(int ret) } #endif +#if IS_ENABLED(CONFIG_ZSTD) +static int sqfs_zstd_decompress(void *dest, unsigned long dest_len, + void *source, u32 src_len) +{ + ZSTD_DCtx *ctx; + size_t wsize; + int ret; + + wsize = ZSTD_DCtxWorkspaceBound(); + ctx = ZSTD_initDCtx(sqfs_ctxt.zstd_workspace, wsize); + ret = ZSTD_decompressDCtx(ctx, dest, dest_len, source, src_len); + + return ZSTD_isError(ret); +} +#endif /* CONFIG_ZSTD */ + int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len, void *source, u32 src_len) { @@ -80,6 +112,16 @@ int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len, break; #endif +#if IS_ENABLED(CONFIG_ZSTD) + case SQFS_COMP_ZSTD: + ret = sqfs_zstd_decompress(dest, *dest_len, source, src_len); + if (ret) { + printf("ZSTD Error code: %d\n", ZSTD_getErrorCode(ret)); + return -EINVAL; + } + + break; +#endif default: printf("Error: unknown compression type.\n"); return -EINVAL; diff --git a/fs/squashfs/sqfs_filesystem.h b/fs/squashfs/sqfs_filesystem.h index ff2b0b1d34..856cd15e34 100644 --- a/fs/squashfs/sqfs_filesystem.h +++ b/fs/squashfs/sqfs_filesystem.h @@ -77,6 +77,9 @@ struct squashfs_ctxt { struct disk_partition cur_part_info; struct blk_desc *cur_dev; struct squashfs_super_block *sblk; +#if IS_ENABLED(CONFIG_ZSTD) + void *zstd_workspace; +#endif }; struct squashfs_directory_index { From patchwork Tue Aug 18 14:31:17 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Joao Marcos Costa X-Patchwork-Id: 1346907 X-Patchwork-Delegate: trini@ti.com Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=lists.denx.de (client-ip=85.214.62.61; helo=phobos.denx.de; envelope-from=u-boot-bounces@lists.denx.de; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=bootlin.com Received: from phobos.denx.de (phobos.denx.de [85.214.62.61]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 4BWCzT297Vz9sR4 for ; Wed, 19 Aug 2020 00:32:05 +1000 (AEST) Received: from h2850616.stratoserver.net (localhost [IPv6:::1]) by phobos.denx.de (Postfix) with ESMTP id 7894A8224F; Tue, 18 Aug 2020 16:31:28 +0200 (CEST) Authentication-Results: phobos.denx.de; dmarc=none (p=none dis=none) header.from=bootlin.com Authentication-Results: phobos.denx.de; spf=pass smtp.mailfrom=u-boot-bounces@lists.denx.de Received: by phobos.denx.de (Postfix, from userid 109) id 32F82821F1; Tue, 18 Aug 2020 16:31:25 +0200 (CEST) X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on phobos.denx.de X-Spam-Level: * X-Spam-Status: No, score=1.4 required=5.0 tests=BAYES_00,RCVD_IN_MSPIKE_H3, RCVD_IN_MSPIKE_WL,RCVD_IN_SBL_CSS,SPF_HELO_NONE,URIBL_BLOCKED autolearn=no autolearn_force=no version=3.4.2 Received: from relay2-d.mail.gandi.net (relay2-d.mail.gandi.net [217.70.183.194]) (using TLSv1.2 with cipher ADH-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by phobos.denx.de (Postfix) with ESMTPS id BE32F821D4 for ; Tue, 18 Aug 2020 16:31:21 +0200 (CEST) Authentication-Results: phobos.denx.de; dmarc=none (p=none dis=none) header.from=bootlin.com Authentication-Results: phobos.denx.de; spf=pass smtp.mailfrom=joaomarcos.costa@bootlin.com X-Originating-IP: 46.193.64.106 Received: from localhost.localdomain (eth-east-parth2-46-193-64-106.wb.wifirst.net [46.193.64.106]) (Authenticated sender: joaomarcos.costa@bootlin.com) by relay2-d.mail.gandi.net (Postfix) with ESMTPA id 0F18740009; Tue, 18 Aug 2020 14:31:20 +0000 (UTC) From: Joao Marcos Costa To: u-boot@lists.denx.de Cc: joaomarcos.costa@bootlin.com, miquel.raynal@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v3 3/4] fs/squashfs: add support for LZO decompression Date: Tue, 18 Aug 2020 16:31:17 +0200 Message-Id: <20200818143118.16422-4-joaomarcos.costa@bootlin.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20200818143118.16422-1-joaomarcos.costa@bootlin.com> References: <20200818143118.16422-1-joaomarcos.costa@bootlin.com> X-BeenThere: u-boot@lists.denx.de X-Mailman-Version: 2.1.34 Precedence: list List-Id: U-Boot discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: u-boot-bounces@lists.denx.de Sender: "U-Boot" X-Virus-Scanned: clamav-milter 0.102.3 at phobos.denx.de X-Virus-Status: Clean Add call to lzo's lzo1x_decompress_safe() into sqfs_decompress(). U-Boot's LZO sources may still have some unsolved issues that could make the decompression crash when dealing with fragmented files, so those should be avoided. The "-no-fragments" option can be passed to mksquashfs. Signed-off-by: Joao Marcos Costa --- Changes in v3: - Change commit message. Changes in v2: - Change commit message. fs/squashfs/sqfs_decompressor.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c index 8fdaaa853c..f57d7e2626 100644 --- a/fs/squashfs/sqfs_decompressor.c +++ b/fs/squashfs/sqfs_decompressor.c @@ -9,6 +9,11 @@ #include #include #include + +#if IS_ENABLED(CONFIG_LZO) +#include +#endif + #if IS_ENABLED(CONFIG_ZLIB) #include #endif @@ -27,6 +32,10 @@ int sqfs_decompressor_init(struct squashfs_ctxt *ctxt) u16 comp_type = get_unaligned_le16(&ctxt->sblk->compression); switch (comp_type) { +#if IS_ENABLED(CONFIG_LZO) + case SQFS_COMP_LZO: + break; +#endif #if IS_ENABLED(CONFIG_ZLIB) case SQFS_COMP_ZLIB: break; @@ -51,6 +60,10 @@ void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt) u16 comp_type = get_unaligned_le16(&ctxt->sblk->compression); switch (comp_type) { +#if IS_ENABLED(CONFIG_LZO) + case SQFS_COMP_LZO: + break; +#endif #if IS_ENABLED(CONFIG_ZLIB) case SQFS_COMP_ZLIB: break; @@ -102,6 +115,18 @@ int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len, int ret = 0; switch (comp_type) { +#if IS_ENABLED(CONFIG_LZO) + case SQFS_COMP_LZO: { + size_t lzo_dest_len = *dest_len; + ret = lzo1x_decompress_safe(source, src_len, dest, &lzo_dest_len); + if (ret) { + printf("LZO decompression failed. Error code: %d\n", ret); + return -EINVAL; + } + + break; + } +#endif #if IS_ENABLED(CONFIG_ZLIB) case SQFS_COMP_ZLIB: ret = uncompress(dest, dest_len, source, src_len); From patchwork Tue Aug 18 14:31:18 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Joao Marcos Costa X-Patchwork-Id: 1346908 X-Patchwork-Delegate: trini@ti.com Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=lists.denx.de (client-ip=85.214.62.61; helo=phobos.denx.de; envelope-from=u-boot-bounces@lists.denx.de; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=bootlin.com Received: from phobos.denx.de (phobos.denx.de [85.214.62.61]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 4BWCzj23Mnz9sPC for ; Wed, 19 Aug 2020 00:32:17 +1000 (AEST) Received: from h2850616.stratoserver.net (localhost [IPv6:::1]) by phobos.denx.de (Postfix) with ESMTP id B13F482258; Tue, 18 Aug 2020 16:31:32 +0200 (CEST) Authentication-Results: phobos.denx.de; dmarc=none (p=none dis=none) header.from=bootlin.com Authentication-Results: phobos.denx.de; spf=pass smtp.mailfrom=u-boot-bounces@lists.denx.de Received: by phobos.denx.de (Postfix, from userid 109) id 76032821D4; Tue, 18 Aug 2020 16:31:25 +0200 (CEST) X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on phobos.denx.de X-Spam-Level: * X-Spam-Status: No, score=1.4 required=5.0 tests=BAYES_00,RCVD_IN_MSPIKE_H3, RCVD_IN_MSPIKE_WL,RCVD_IN_SBL_CSS,SPF_HELO_NONE,URIBL_BLOCKED autolearn=no autolearn_force=no version=3.4.2 Received: from relay2-d.mail.gandi.net (relay2-d.mail.gandi.net [217.70.183.194]) (using TLSv1.2 with cipher ADH-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by phobos.denx.de (Postfix) with ESMTPS id 5DCFE821D5 for ; Tue, 18 Aug 2020 16:31:22 +0200 (CEST) Authentication-Results: phobos.denx.de; dmarc=none (p=none dis=none) header.from=bootlin.com Authentication-Results: phobos.denx.de; spf=pass smtp.mailfrom=joaomarcos.costa@bootlin.com X-Originating-IP: 46.193.64.106 Received: from localhost.localdomain (eth-east-parth2-46-193-64-106.wb.wifirst.net [46.193.64.106]) (Authenticated sender: joaomarcos.costa@bootlin.com) by relay2-d.mail.gandi.net (Postfix) with ESMTPA id 9148440011; Tue, 18 Aug 2020 14:31:21 +0000 (UTC) From: Joao Marcos Costa To: u-boot@lists.denx.de Cc: joaomarcos.costa@bootlin.com, miquel.raynal@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v3 4/4] test/py: Add tests for LZO and ZSTD Date: Tue, 18 Aug 2020 16:31:18 +0200 Message-Id: <20200818143118.16422-5-joaomarcos.costa@bootlin.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20200818143118.16422-1-joaomarcos.costa@bootlin.com> References: <20200818143118.16422-1-joaomarcos.costa@bootlin.com> X-BeenThere: u-boot@lists.denx.de X-Mailman-Version: 2.1.34 Precedence: list List-Id: U-Boot discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: u-boot-bounces@lists.denx.de Sender: "U-Boot" X-Virus-Scanned: clamav-milter 0.102.3 at phobos.denx.de X-Virus-Status: Clean Improve SquashFS tests architecture. Add 'Compression' class. LZO algorithm may crash if the file is fragmented, so the fragments are disabled when testing LZO. Signed-off-by: Joao Marcos Costa --- Changes in v3: - No changes since the last version. Changes in v2: - This patch was not present in the previous version. .../test_fs/test_squashfs/sqfs_common.py | 76 ++++++++++++++----- .../test_fs/test_squashfs/test_sqfs_load.py | 40 ++++++---- .../test_fs/test_squashfs/test_sqfs_ls.py | 33 +++++--- 3 files changed, 102 insertions(+), 47 deletions(-) diff --git a/test/py/tests/test_fs/test_squashfs/sqfs_common.py b/test/py/tests/test_fs/test_squashfs/sqfs_common.py index 2dc344d1b2..c96f92c1d8 100644 --- a/test/py/tests/test_fs/test_squashfs/sqfs_common.py +++ b/test/py/tests/test_fs/test_squashfs/sqfs_common.py @@ -5,6 +5,7 @@ import os import random import string +import subprocess def sqfs_get_random_letters(size): letters = [] @@ -19,24 +20,57 @@ def sqfs_generate_file(path, size): file.write(content) file.close() -# generate image with three files and a symbolic link -def sqfs_generate_image(cons): - src = os.path.join(cons.config.build_dir, "sqfs_src/") - dest = os.path.join(cons.config.build_dir, "sqfs") - os.mkdir(src) - sqfs_generate_file(src + "frag_only", 100) - sqfs_generate_file(src + "blks_frag", 5100) - sqfs_generate_file(src + "blks_only", 4096) - os.symlink("frag_only", src + "sym") - os.system("mksquashfs " + src + " " + dest + " -b 4096 -always-use-fragments") - -# removes all files created by sqfs_generate_image() -def sqfs_clean(cons): - src = os.path.join(cons.config.build_dir, "sqfs_src/") - dest = os.path.join(cons.config.build_dir, "sqfs") - os.remove(src + "frag_only") - os.remove(src + "blks_frag") - os.remove(src + "blks_only") - os.remove(src + "sym") - os.rmdir(src) - os.remove(dest) +class Compression: + def __init__(self, name, files, sizes, block_size = 4096): + self.name = name + self.files = files + self.sizes = sizes + self.mksquashfs_opts = " -b " + str(block_size) + " -comp " + self.name + + def add_opt(self, opt): + self.mksquashfs_opts += " " + opt + + def gen_image(self, build_dir): + src = os.path.join(build_dir, "sqfs_src/") + os.mkdir(src) + for (f, s) in zip(self.files, self.sizes): + sqfs_generate_file(src + f, s) + + # the symbolic link always targets the first file + os.symlink(self.files[0], src + "sym") + + sqfs_img = os.path.join(build_dir, "sqfs-" + self.name) + i_o = src + " " + sqfs_img + opts = self.mksquashfs_opts + try: + subprocess.run(["mksquashfs " + i_o + opts], shell = True, check = True) + except: + print("mksquashfs error. Compression type: " + self.name) + raise RuntimeError + + def clean_source(self, build_dir): + src = os.path.join(build_dir, "sqfs_src/") + for f in self.files: + os.remove(src + f) + os.remove(src + "sym") + os.rmdir(src) + + def cleanup(self, build_dir): + self.clean_source(build_dir) + sqfs_img = os.path.join(build_dir, "sqfs-" + self.name) + os.remove(sqfs_img) + +files = ["blks_only", "blks_frag", "frag_only"] +sizes = [4096, 5100, 100] +gzip = Compression("gzip", files, sizes) +zstd = Compression("zstd", files, sizes) +lzo = Compression("lzo", files, sizes) + +# use fragment blocks for files larger than block_size +gzip.add_opt("-always-use-fragments") +zstd.add_opt("-always-use-fragments") + +# avoid fragments if lzo is used +lzo.add_opt("-no-fragments") + +comp_opts = [gzip, zstd, lzo] diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py index eb1baae5c5..9e90062384 100644 --- a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py +++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py @@ -12,23 +12,35 @@ from sqfs_common import * @pytest.mark.buildconfigspec('fs_squashfs') @pytest.mark.requiredtool('mksquashfs') def test_sqfs_load(u_boot_console): - cons = u_boot_console - sqfs_generate_image(cons) + build_dir = u_boot_console.config.build_dir command = "sqfsload host 0 $kernel_addr_r " - path = os.path.join(cons.config.build_dir, "sqfs") - try: + for opt in comp_opts: + # generate and load the squashfs image + try: + opt.gen_image(build_dir) + except RuntimeError: + opt.clean_source(build_dir) + # skip unsupported compression types + continue + + path = os.path.join(build_dir, "sqfs-" + opt.name) output = u_boot_console.run_command("host bind 0 " + path) + output = u_boot_console.run_command(command + "xxx") assert "File not found." in output - output = u_boot_console.run_command(command + "frag_only") - assert "100 bytes read in" in output - output = u_boot_console.run_command(command + "blks_frag") - assert "5100 bytes read in" in output - output = u_boot_console.run_command(command + "blks_only") - assert "4096 bytes read in" in output + + for (f, s) in zip(opt.files, opt.sizes): + try: + output = u_boot_console.run_command(command + f) + assert str(s) in output + except: + assert False + opt.cleanup(build_dir) + + # test symbolic link output = u_boot_console.run_command(command + "sym") - assert "100 bytes read in" in output - except: - sqfs_clean(cons) - sqfs_clean(cons) + assert str(opt.sizes[0]) in output + + # remove generated files + opt.cleanup(build_dir) diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py index 3a7b75c778..a0dca2e2fc 100644 --- a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py +++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py @@ -12,16 +12,25 @@ from sqfs_common import * @pytest.mark.buildconfigspec('fs_squashfs') @pytest.mark.requiredtool('mksquashfs') def test_sqfs_ls(u_boot_console): - cons = u_boot_console - sqfs_generate_image(cons) - path = os.path.join(cons.config.build_dir, "sqfs") - try: + build_dir = u_boot_console.config.build_dir + for opt in comp_opts: + try: + opt.gen_image(build_dir) + except RuntimeError: + opt.clean_source(build_dir) + # skip unsupported compression types + continue + path = os.path.join(build_dir, "sqfs-" + opt.name) output = u_boot_console.run_command("host bind 0 " + path) - output = u_boot_console.run_command("sqfsls host 0") - assert "4 file(s), 0 dir(s)" in output - assert " sym" in output - output = u_boot_console.run_command("sqfsls host 0 xxx") - assert "** Cannot find directory. **" in output - except: - sqfs_clean(cons) - sqfs_clean(cons) + + try: + # list files in root directory + output = u_boot_console.run_command("sqfsls host 0") + assert str(len(opt.files) + 1) + " file(s), 0 dir(s)" in output + assert " sym" in output + output = u_boot_console.run_command("sqfsls host 0 xxx") + assert "** Cannot find directory. **" in output + except: + opt.cleanup(build_dir) + assert False + opt.cleanup(build_dir)