util/cbfstool: Add cbfs-compression-tool
cbfs-compression-tool provides a way to benchmark the compression algorithms as used by cbfstool (and coreboot) and allows to pre-compress data for later consumption by cbfstool (once it supports the format). For an impression, the benchmark's results on my machine: measuring 'none' compressing 10485760 bytes to 10485760 took 0 seconds measuring 'LZMA' compressing 10485760 bytes to 1736 took 2 seconds measuring 'LZ4' compressing 10485760 bytes to 41880 took 0 seconds And a possible use for external compression, parallel and non-parallel (60MB in 53 files compressed to 650KB on a machine with 40 threads): $ time (ls -1 *.* |xargs -n 1 -P $(nproc) -I '{}' cbfs-compression-tool compress '{}' out/'{}' LZMA) real 0m0.786s user 0m11.440s sys 0m0.044s $ time (ls -1 *.* |xargs -n 1 -P 1 -I '{}' cbfs-compression-tool compress '{}' out/'{}' LZMA) real 0m10.444s user 0m10.280s sys 0m0.064s Change-Id: I40be087e85d09a895b1ed277270350ab65a4d6d4 Signed-off-by: Patrick Georgi <pgeorgi@chromium.org> Reviewed-on: https://review.coreboot.org/18099 Tested-by: build bot (Jenkins) Reviewed-by: Martin Roth <martinroth@google.com>
This commit is contained in:
parent
8474e7d7e8
commit
c88d16baaf
|
@ -9,7 +9,7 @@ OBJCOPY ?= objcopy
|
||||||
VBOOT_SOURCE ?= $(top)/3rdparty/vboot
|
VBOOT_SOURCE ?= $(top)/3rdparty/vboot
|
||||||
|
|
||||||
.PHONY: all
|
.PHONY: all
|
||||||
all: cbfstool fmaptool rmodtool ifwitool
|
all: cbfstool fmaptool rmodtool ifwitool cbfs-compression-tool
|
||||||
|
|
||||||
cbfstool: $(objutil)/cbfstool/cbfstool
|
cbfstool: $(objutil)/cbfstool/cbfstool
|
||||||
|
|
||||||
|
@ -19,13 +19,16 @@ rmodtool: $(objutil)/cbfstool/rmodtool
|
||||||
|
|
||||||
ifwitool: $(objutil)/cbfstool/ifwitool
|
ifwitool: $(objutil)/cbfstool/ifwitool
|
||||||
|
|
||||||
.PHONY: clean cbfstool fmaptool rmodtool ifwitool
|
cbfs-compression-tool: $(objutil)/cbfstool/cbfs-compression-tool
|
||||||
|
|
||||||
|
.PHONY: clean cbfstool fmaptool rmodtool ifwitool cbfs-compression-tool
|
||||||
clean:
|
clean:
|
||||||
$(RM) fmd_parser.c fmd_parser.h fmd_scanner.c fmd_scanner.h
|
$(RM) fmd_parser.c fmd_parser.h fmd_scanner.c fmd_scanner.h
|
||||||
$(RM) $(objutil)/cbfstool/cbfstool $(cbfsobj)
|
$(RM) $(objutil)/cbfstool/cbfstool $(cbfsobj)
|
||||||
$(RM) $(objutil)/cbfstool/fmaptool $(fmapobj)
|
$(RM) $(objutil)/cbfstool/fmaptool $(fmapobj)
|
||||||
$(RM) $(objutil)/cbfstool/rmodtool $(rmodobj)
|
$(RM) $(objutil)/cbfstool/rmodtool $(rmodobj)
|
||||||
$(RM) $(objutil)/cbfstool/ifwitool $(ifwiobj)
|
$(RM) $(objutil)/cbfstool/ifwitool $(ifwiobj)
|
||||||
|
$(RM) $(objutil)/cbfstool/cbfs-compression-tool $(cbfscompobj)
|
||||||
|
|
||||||
linux_trampoline.c: linux_trampoline.S
|
linux_trampoline.c: linux_trampoline.S
|
||||||
rm -f linux_trampoline.c
|
rm -f linux_trampoline.c
|
||||||
|
|
|
@ -1,7 +1,20 @@
|
||||||
|
compressionobj :=
|
||||||
|
compressionobj += compress.o
|
||||||
|
# LZ4
|
||||||
|
compressionobj += lz4.o
|
||||||
|
compressionobj += lz4hc.o
|
||||||
|
compressionobj += lz4frame.o
|
||||||
|
compressionobj += xxhash.o
|
||||||
|
compressionobj += lz4_wrapper.o
|
||||||
|
# LZMA
|
||||||
|
compressionobj += lzma.o
|
||||||
|
compressionobj += LzFind.o
|
||||||
|
compressionobj += LzmaDec.o
|
||||||
|
compressionobj += LzmaEnc.o
|
||||||
|
|
||||||
cbfsobj :=
|
cbfsobj :=
|
||||||
cbfsobj += cbfstool.o
|
cbfsobj += cbfstool.o
|
||||||
cbfsobj += common.o
|
cbfsobj += common.o
|
||||||
cbfsobj += compress.o
|
|
||||||
cbfsobj += cbfs_image.o
|
cbfsobj += cbfs_image.o
|
||||||
cbfsobj += cbfs-mkstage.o
|
cbfsobj += cbfs-mkstage.o
|
||||||
cbfsobj += cbfs-mkpayload.o
|
cbfsobj += cbfs-mkpayload.o
|
||||||
|
@ -13,24 +26,13 @@ cbfsobj += partitioned_file.o
|
||||||
# COMMONLIB
|
# COMMONLIB
|
||||||
cbfsobj += cbfs.o
|
cbfsobj += cbfs.o
|
||||||
cbfsobj += fsp_relocate.o
|
cbfsobj += fsp_relocate.o
|
||||||
cbfsobj += lz4_wrapper.o
|
|
||||||
cbfsobj += mem_pool.o
|
cbfsobj += mem_pool.o
|
||||||
cbfsobj += region.o
|
cbfsobj += region.o
|
||||||
# LZMA
|
|
||||||
cbfsobj += lzma.o
|
|
||||||
cbfsobj += LzFind.o
|
|
||||||
cbfsobj += LzmaDec.o
|
|
||||||
cbfsobj += LzmaEnc.o
|
|
||||||
# CRYPTOLIB
|
# CRYPTOLIB
|
||||||
cbfsobj += 2sha_utility.o
|
cbfsobj += 2sha_utility.o
|
||||||
cbfsobj += 2sha1.o
|
cbfsobj += 2sha1.o
|
||||||
cbfsobj += 2sha256.o
|
cbfsobj += 2sha256.o
|
||||||
cbfsobj += 2sha512.o
|
cbfsobj += 2sha512.o
|
||||||
# LZ4
|
|
||||||
cbfsobj += lz4.o
|
|
||||||
cbfsobj += lz4hc.o
|
|
||||||
cbfsobj += lz4frame.o
|
|
||||||
cbfsobj += xxhash.o
|
|
||||||
# FMAP
|
# FMAP
|
||||||
cbfsobj += fmap.o
|
cbfsobj += fmap.o
|
||||||
cbfsobj += kv_pair.o
|
cbfsobj += kv_pair.o
|
||||||
|
@ -38,6 +40,8 @@ cbfsobj += valstr.o
|
||||||
# linux as payload
|
# linux as payload
|
||||||
cbfsobj += linux_trampoline.o
|
cbfsobj += linux_trampoline.o
|
||||||
cbfsobj += cbfs-payload-linux.o
|
cbfsobj += cbfs-payload-linux.o
|
||||||
|
# compression algorithms
|
||||||
|
cbfsobj += $(compressionobj)
|
||||||
|
|
||||||
fmapobj :=
|
fmapobj :=
|
||||||
fmapobj += fmaptool.o
|
fmapobj += fmaptool.o
|
||||||
|
@ -62,6 +66,10 @@ ifwiobj :=
|
||||||
ifwiobj += ifwitool.o
|
ifwiobj += ifwitool.o
|
||||||
ifwiobj += common.o
|
ifwiobj += common.o
|
||||||
|
|
||||||
|
cbfscompobj :=
|
||||||
|
cbfscompobj += $(compressionobj)
|
||||||
|
cbfscompobj += cbfscomptool.o
|
||||||
|
|
||||||
TOOLCFLAGS ?= -Werror -Wall -Wextra
|
TOOLCFLAGS ?= -Werror -Wall -Wextra
|
||||||
TOOLCFLAGS += -Wcast-qual -Wmissing-prototypes -Wredundant-decls -Wshadow
|
TOOLCFLAGS += -Wcast-qual -Wmissing-prototypes -Wredundant-decls -Wshadow
|
||||||
TOOLCFLAGS += -Wstrict-prototypes -Wwrite-strings
|
TOOLCFLAGS += -Wstrict-prototypes -Wwrite-strings
|
||||||
|
@ -141,6 +149,10 @@ $(objutil)/cbfstool/ifwitool: $(addprefix $(objutil)/cbfstool/,$(ifwiobj))
|
||||||
printf " HOSTCC $(subst $(objutil)/,,$(@)) (link)\n"
|
printf " HOSTCC $(subst $(objutil)/,,$(@)) (link)\n"
|
||||||
$(HOSTCC) $(TOOLLDFLAGS) -o $@ $(addprefix $(objutil)/cbfstool/,$(ifwiobj))
|
$(HOSTCC) $(TOOLLDFLAGS) -o $@ $(addprefix $(objutil)/cbfstool/,$(ifwiobj))
|
||||||
|
|
||||||
|
$(objutil)/cbfstool/cbfs-compression-tool: $(addprefix $(objutil)/cbfstool/,$(cbfscompobj))
|
||||||
|
printf " HOSTCC $(subst $(objutil)/,,$(@)) (link)\n"
|
||||||
|
$(HOSTCC) $(TOOLLDFLAGS) -o $@ $(addprefix $(objutil)/cbfstool/,$(cbfscompobj))
|
||||||
|
|
||||||
# Yacc source is superset of header
|
# Yacc source is superset of header
|
||||||
$(objutil)/cbfstool/fmd.o: TOOLCFLAGS += -Wno-redundant-decls
|
$(objutil)/cbfstool/fmd.o: TOOLCFLAGS += -Wno-redundant-decls
|
||||||
$(objutil)/cbfstool/fmd_parser.o: TOOLCFLAGS += -Wno-redundant-decls
|
$(objutil)/cbfstool/fmd_parser.o: TOOLCFLAGS += -Wno-redundant-decls
|
||||||
|
|
|
@ -204,11 +204,6 @@ struct cbfs_payload {
|
||||||
*/
|
*/
|
||||||
#define CBFS_COMPONENT_NULL 0xFFFFFFFF
|
#define CBFS_COMPONENT_NULL 0xFFFFFFFF
|
||||||
|
|
||||||
struct typedesc_t {
|
|
||||||
uint32_t type;
|
|
||||||
const char *name;
|
|
||||||
};
|
|
||||||
|
|
||||||
static struct typedesc_t filetypes[] unused = {
|
static struct typedesc_t filetypes[] unused = {
|
||||||
{CBFS_COMPONENT_BOOTBLOCK, "bootblock"},
|
{CBFS_COMPONENT_BOOTBLOCK, "bootblock"},
|
||||||
{CBFS_COMPONENT_CBFSHEADER, "cbfs header"},
|
{CBFS_COMPONENT_CBFSHEADER, "cbfs header"},
|
||||||
|
|
|
@ -50,15 +50,6 @@
|
||||||
* (old) cbfstool. */
|
* (old) cbfstool. */
|
||||||
#define CBFS_FILENAME_ALIGN (16)
|
#define CBFS_FILENAME_ALIGN (16)
|
||||||
|
|
||||||
/* Type and format */
|
|
||||||
|
|
||||||
static const struct typedesc_t types_cbfs_compression[] = {
|
|
||||||
{CBFS_COMPRESS_NONE, "none"},
|
|
||||||
{CBFS_COMPRESS_LZMA, "LZMA"},
|
|
||||||
{CBFS_COMPRESS_LZ4, "LZ4"},
|
|
||||||
{0, NULL},
|
|
||||||
};
|
|
||||||
|
|
||||||
static const char *lookup_name_by_type(const struct typedesc_t *desc, uint32_t type,
|
static const char *lookup_name_by_type(const struct typedesc_t *desc, uint32_t type,
|
||||||
const char *default_value)
|
const char *default_value)
|
||||||
{
|
{
|
||||||
|
|
|
@ -0,0 +1,189 @@
|
||||||
|
/*
|
||||||
|
* cbfs-compression-tool, CLI utility for dealing with CBFS compressed data
|
||||||
|
*
|
||||||
|
* Copyright (C) 2017 Google, Inc.
|
||||||
|
*
|
||||||
|
* This program is free software; you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU General Public License as published by
|
||||||
|
* the Free Software Foundation; version 2 of the License.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <time.h>
|
||||||
|
|
||||||
|
#include "common.h"
|
||||||
|
|
||||||
|
void usage(void);
|
||||||
|
int benchmark(void);
|
||||||
|
int compress(char *infile, char *outfile, char *algoname);
|
||||||
|
|
||||||
|
const char *usage_text = "cbfs-compression-tool benchmark\n"
|
||||||
|
" runs benchmarks for all implemented algorithms\n"
|
||||||
|
"cbfs-compression-tool compress inFile outFile algo\n"
|
||||||
|
" compresses inFile with algo and stores in outFile\n"
|
||||||
|
"\n"
|
||||||
|
"'compress' file format:\n"
|
||||||
|
" 4 bytes little endian: algorithm ID (as used in CBFS)\n"
|
||||||
|
" 4 bytes little endian: uncompressed size\n"
|
||||||
|
" ...: compressed data stream\n";
|
||||||
|
|
||||||
|
void usage()
|
||||||
|
{
|
||||||
|
puts(usage_text);
|
||||||
|
}
|
||||||
|
|
||||||
|
int benchmark()
|
||||||
|
{
|
||||||
|
const int bufsize = 10*1024*1024;
|
||||||
|
char *data = malloc(bufsize);
|
||||||
|
if (!data) {
|
||||||
|
fprintf(stderr, "out of memory\n");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
char *compressed_data = malloc(bufsize);
|
||||||
|
if (!compressed_data) {
|
||||||
|
fprintf(stderr, "out of memory\n");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
int i, l = strlen(usage_text) + 1;
|
||||||
|
for (i = 0; i + l < bufsize; i += l) {
|
||||||
|
memcpy(data + i, usage_text, l);
|
||||||
|
}
|
||||||
|
memset(data + i, 0, bufsize - i);
|
||||||
|
const struct typedesc_t *algo;
|
||||||
|
for (algo = &types_cbfs_compression[0]; algo->name != NULL; algo++) {
|
||||||
|
int outsize = bufsize;
|
||||||
|
printf("measuring '%s'\n", algo->name);
|
||||||
|
comp_func_ptr comp = compression_function(algo->type);
|
||||||
|
if (comp == NULL) {
|
||||||
|
printf("no handler associated with algorithm\n");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct timespec t_s, t_e;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &t_s);
|
||||||
|
|
||||||
|
if (comp(data, bufsize, compressed_data, &outsize)) {
|
||||||
|
printf("compression failed");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &t_e);
|
||||||
|
printf("compressing %d bytes to %d took %ld seconds\n",
|
||||||
|
bufsize, outsize,
|
||||||
|
t_e.tv_sec - t_s.tv_sec);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int compress(char *infile, char *outfile, char *algoname)
|
||||||
|
{
|
||||||
|
int err = 1;
|
||||||
|
FILE *fin = NULL;
|
||||||
|
FILE *fout = NULL;
|
||||||
|
void *indata = NULL;
|
||||||
|
|
||||||
|
const struct typedesc_t *algo = &types_cbfs_compression[0];
|
||||||
|
while (algo->name != NULL) {
|
||||||
|
if (strcmp(algo->name, algoname) == 0) break;
|
||||||
|
algo++;
|
||||||
|
}
|
||||||
|
if (algo->name == NULL) {
|
||||||
|
fprintf(stderr, "algo '%s' is not supported.\n", algoname);
|
||||||
|
}
|
||||||
|
|
||||||
|
comp_func_ptr comp = compression_function(algo->type);
|
||||||
|
if (comp == NULL) {
|
||||||
|
printf("no handler associated with algorithm\n");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
fin = fopen(infile, "rb");
|
||||||
|
if (!fin) {
|
||||||
|
fprintf(stderr, "could not open '%s'\n", infile);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
fout = fopen(outfile, "wb");
|
||||||
|
if (!fout) {
|
||||||
|
fprintf(stderr, "could not open '%s' for writing\n", outfile);
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fseek(fin, 0, SEEK_END) != 0) {
|
||||||
|
fprintf(stderr, "could not seek in input\n");
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
long insize = ftell(fin);
|
||||||
|
if (insize < 0) {
|
||||||
|
fprintf(stderr, "could not determine input size\n");
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
rewind(fin);
|
||||||
|
|
||||||
|
indata = malloc(insize);
|
||||||
|
if (!indata) {
|
||||||
|
fprintf(stderr, "out of memory\n");
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
void *outdata = malloc(insize);
|
||||||
|
if (!outdata) {
|
||||||
|
fprintf(stderr, "out of memory\n");
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
int outsize;
|
||||||
|
|
||||||
|
int remsize = insize;
|
||||||
|
while (remsize > 0) {
|
||||||
|
int readsz = fread(indata, 1, remsize, fin);
|
||||||
|
if (readsz < 0) {
|
||||||
|
fprintf(stderr, "failed to read input with %d bytes left\n", remsize);
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
remsize -= readsz;
|
||||||
|
}
|
||||||
|
|
||||||
|
comp(indata, insize, outdata, &outsize);
|
||||||
|
|
||||||
|
char header[8];
|
||||||
|
header[0] = algo->type & 0xff;
|
||||||
|
header[1] = (algo->type >> 8) & 0xff;
|
||||||
|
header[2] = (algo->type >> 16) & 0xff;
|
||||||
|
header[3] = (algo->type >> 24) & 0xff;
|
||||||
|
header[4] = insize & 0xff;
|
||||||
|
header[5] = (insize >> 8) & 0xff;
|
||||||
|
header[6] = (insize >> 16) & 0xff;
|
||||||
|
header[7] = (insize >> 24) & 0xff;
|
||||||
|
if (fwrite(header, 8, 1, fout) != 1) {
|
||||||
|
fprintf(stderr, "failed writing header\n");
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
if (fwrite(outdata, outsize, 1, fout) != 1) {
|
||||||
|
fprintf(stderr, "failed writing compressed data\n");
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
err = 0;
|
||||||
|
out:
|
||||||
|
if (fin) fclose(fin);
|
||||||
|
if (fout) fclose(fout);
|
||||||
|
if (indata) free(indata);
|
||||||
|
return err;
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char **argv)
|
||||||
|
{
|
||||||
|
if ((argc == 2) && (strcmp(argv[1], "benchmark") == 0))
|
||||||
|
return benchmark();
|
||||||
|
if ((argc == 5) && (strcmp(argv[1], "compress") == 0))
|
||||||
|
return compress(argv[2], argv[3], argv[4]);
|
||||||
|
usage();
|
||||||
|
return 1;
|
||||||
|
}
|
|
@ -166,6 +166,18 @@ enum comp_algo {
|
||||||
CBFS_COMPRESS_LZ4 = 2,
|
CBFS_COMPRESS_LZ4 = 2,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct typedesc_t {
|
||||||
|
uint32_t type;
|
||||||
|
const char *name;
|
||||||
|
};
|
||||||
|
|
||||||
|
static const struct typedesc_t types_cbfs_compression[] = {
|
||||||
|
{CBFS_COMPRESS_NONE, "none"},
|
||||||
|
{CBFS_COMPRESS_LZMA, "LZMA"},
|
||||||
|
{CBFS_COMPRESS_LZ4, "LZ4"},
|
||||||
|
{0, NULL},
|
||||||
|
};
|
||||||
|
|
||||||
comp_func_ptr compression_function(enum comp_algo algo);
|
comp_func_ptr compression_function(enum comp_algo algo);
|
||||||
decomp_func_ptr decompression_function(enum comp_algo algo);
|
decomp_func_ptr decompression_function(enum comp_algo algo);
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue