From e61685253608c340920ca66619b2068c22280e10 Mon Sep 17 00:00:00 2001
From: Janne Grunau <janne-vlc@jannau.net>
Date: Tue, 5 Jan 2021 14:31:43 +0100
Subject: [PATCH] tools: add optional xxh3 based muxer
The required 'xxhash.h' header can either be in system include directory
or can be copied to 'tools/output'.
The xxh3_128bits based muxer shows no significant slowdown compared to
the null muxer. Decoding times Chimera-AV1-8bit-1920x1080-6736kbps.ivf
with 4 frame and 4 tile threads on a core i7-8550U (disabled turbo boost):
null: 72.5 s
md5: 99.8 s
xxh3: 73.8 s
Decoding Chimera-AV1-10bit-1920x1080-6191kbps.ivf with 6 frame and 4 tile
threads on a m1 mc mini:
null: 27.8 s
md5: 105.9 s
xxh3: 28.3 s
---
.gitignore | 1 +
meson_options.txt | 4 ++
tools/dav1d_cli_parse.c | 21 ++++--
tools/meson.build | 14 ++++
tools/output/output.c | 5 ++
tools/output/xxhash.c | 142 ++++++++++++++++++++++++++++++++++++++++
6 files changed, 182 insertions(+), 5 deletions(-)
create mode 100644 tools/output/xxhash.c
diff --git a/.gitignore b/.gitignore
index 2bbd7c48a..b00c21ac2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,4 @@ tags
.DS_Store
/tests/dav1d-test-data
*.snap
+/tools/output/xxhash.h
diff --git a/meson_options.txt b/meson_options.txt
index 37bd08433..b9a791299 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -53,3 +53,7 @@ option('fuzzer_ldflags',
option('stack_alignment',
type: 'integer',
value: 0)
+
+option('xxhash_muxer',
+ type : 'feature',
+ value : 'auto')
diff --git a/tools/dav1d_cli_parse.c b/tools/dav1d_cli_parse.c
index b229d546b..1b4466df7 100644
--- a/tools/dav1d_cli_parse.c
+++ b/tools/dav1d_cli_parse.c
@@ -26,6 +26,7 @@
*/
#include "config.h"
+#include "cli_config.h"
#include <getopt.h>
#include <limits.h>
@@ -84,6 +85,12 @@ static const struct option long_opts[] = {
{ NULL, 0, NULL, 0 },
};
+#if HAVE_XXHASH_H
+#define AVAILABLE_MUXERS "'md5', 'xxh3', 'yuv', 'yuv4mpeg2' or 'null'"
+#else
+#define AVAILABLE_MUXERS "'md5', 'yuv', 'yuv4mpeg2' or 'null'"
+#endif
+
#if ARCH_AARCH64 || ARCH_ARM
#define ALLOWED_CPU_MASKS " or 'neon'"
#elif ARCH_PPC64LE
@@ -109,7 +116,7 @@ static void usage(const char *const app, const char *const reason, ...) {
" --input/-i $file: input file\n"
" --output/-o $file: output file\n"
" --demuxer $name: force demuxer type ('ivf', 'section5' or 'annexb'; default: detect from content)\n"
- " --muxer $name: force muxer type ('md5', 'yuv', 'yuv4mpeg2' or 'null'; default: detect from extension)\n"
+ " --muxer $name: force muxer type (" AVAILABLE_MUXERS "; default: detect from extension)\n"
" --quiet/-q: disable status messages\n"
" --frametimes $file: dump frame times to file\n"
" --limit/-l $num: stop decoding after $num frames\n"
@@ -120,7 +127,7 @@ static void usage(const char *const app, const char *const reason, ...) {
" --framethreads $num: number of frame threads (default: 1)\n"
" --tilethreads $num: number of tile threads (default: 1)\n"
" --pfthreads $num: number of postfilter threads (default: 1)\n"
- " --filmgrain $num: enable film grain application (default: 1, except if muxer is md5)\n"
+ " --filmgrain $num: enable film grain application (default: 1, except if muxer is md5 or xxh3)\n"
" --oppoint $num: select an operating point of a scalable AV1 bitstream (0 - 31)\n"
" --alllayers $num: output all spatial layers of a scalable AV1 bitstream (default: 1)\n"
" --sizelimit $num: stop decoding if the frame size exceeds the specified limit\n"
@@ -345,8 +352,11 @@ void parse(const int argc, char *const *const argv,
if (cli_settings->verify) {
if (cli_settings->outputfile)
usage(argv[0], "Verification (--verify) requires output file (-o/--output) to not set");
- if (cli_settings->muxer && strcmp(cli_settings->muxer, "md5"))
- usage(argv[0], "Verification (--verify) requires the md5 muxer (--muxer md5)");
+ if (cli_settings->muxer && strcmp(cli_settings->muxer, "md5") &&
+ strcmp(cli_settings->muxer, "xxh3"))
+ {
+ usage(argv[0], "Verification (--verify) requires a checksum muxer (md5 or xxh3)");
+ }
cli_settings->outputfile = "-";
if (!cli_settings->muxer)
@@ -354,7 +364,8 @@ void parse(const int argc, char *const *const argv,
}
if (!grain_specified && cli_settings->muxer &&
- !strcmp(cli_settings->muxer, "md5"))
+ (!strcmp(cli_settings->muxer, "md5") ||
+ !strcmp(cli_settings->muxer, "xxh3")))
{
lib_settings->apply_grain = 0;
}
diff --git a/tools/meson.build b/tools/meson.build
index 032e27282..59b145828 100644
--- a/tools/meson.build
+++ b/tools/meson.build
@@ -39,6 +39,18 @@ dav1d_output_sources = files(
'output/yuv.c',
)
+# hacky check for xxhash.h to allow copying it to tools/output
+if not get_option('xxhash_muxer').disabled()
+ xxhash_include = '-I' + meson.current_source_dir() / 'output'
+ if cc.has_header_symbol('xxhash.h', 'XXH3_createState', args : xxhash_include)
+ dav1d_output_sources += 'output/xxhash.c'
+ xxh3_found = true
+ elif get_option('xxhash_muxer').enabled()
+ # manual error since 'required' kw arg in has_header_symbol() was only added in meson 0.50
+ error( 'C symbol XXH3_createState not found in header xxhash.h')
+ endif
+endif
+
dav1d_input_objs = static_library('dav1d_input',
dav1d_input_sources,
@@ -69,6 +81,8 @@ endif
# Configuratin data for cli_config.h
cli_cdata = configuration_data()
+cli_cdata.set10('HAVE_XXHASH_H', get_variable('xxh3_found', false))
+
cli_config_h_target = configure_file(output: 'cli_config.h', configuration: cli_cdata)
# dav1d cli tool sources
diff --git a/tools/output/output.c b/tools/output/output.c
index 368d07915..fe4e07e8a 100644
--- a/tools/output/output.c
+++ b/tools/output/output.c
@@ -26,6 +26,7 @@
*/
#include "config.h"
+#include "cli_config.h"
#include <errno.h>
#include <stdio.h>
@@ -44,11 +45,15 @@ struct MuxerContext {
extern const Muxer null_muxer;
extern const Muxer md5_muxer;
+extern const Muxer xxh3_muxer;
extern const Muxer yuv_muxer;
extern const Muxer y4m2_muxer;
static const Muxer *muxers[] = {
&null_muxer,
&md5_muxer,
+#if HAVE_XXHASH_H
+ &xxh3_muxer,
+#endif
&yuv_muxer,
&y4m2_muxer,
NULL
diff --git a/tools/output/xxhash.c b/tools/output/xxhash.c
new file mode 100644
index 000000000..7d38c09bb
--- /dev/null
+++ b/tools/output/xxhash.c
@@ -0,0 +1,142 @@
+/*
+ * Copyright © 2018-2021, VideoLAN and dav1d authors
+ * Copyright © 2018-2021, Two Orioles, LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define XXH_INLINE_ALL
+#include "xxhash.h"
+
+#include "output/muxer.h"
+
+typedef struct MuxerPriv {
+ XXH3_state_t* state;
+ FILE *f;
+} xxh3Context;
+
+static int xxh3_open(xxh3Context *const xxh3, const char *const file,
+ const Dav1dPictureParameters *const p,
+ const unsigned fps[2])
+{
+ xxh3->state = XXH3_createState();
+ if (!xxh3->state) return DAV1D_ERR(ENOMEM);
+ XXH_errorcode err = XXH3_128bits_reset(xxh3->state);
+ if (err != XXH_OK) {
+ XXH3_freeState(xxh3->state);
+ xxh3->state = NULL;
+ return DAV1D_ERR(ENOMEM);
+ }
+
+ if (!strcmp(file, "-")) {
+ xxh3->f = stdout;
+ } else if (!(xxh3->f = fopen(file, "wb"))) {
+ XXH3_freeState(xxh3->state);
+ xxh3->state = NULL;
+ fprintf(stderr, "Failed to open %s: %s\n", file, strerror(errno));
+ return -1;
+ }
+
+ return 0;
+}
+
+static int xxh3_write(xxh3Context *const xxh3, Dav1dPicture *const p) {
+ const int hbd = p->p.bpc > 8;
+ const int w = p->p.w, h = p->p.h;
+ uint8_t *yptr = p->data[0];
+
+ for (int y = 0; y < h; y++) {
+ XXH3_128bits_update(xxh3->state, yptr, w << hbd);
+ yptr += p->stride[0];
+ }
+
+ if (p->p.layout != DAV1D_PIXEL_LAYOUT_I400) {
+ const int ss_ver = p->p.layout == DAV1D_PIXEL_LAYOUT_I420;
+ const int ss_hor = p->p.layout != DAV1D_PIXEL_LAYOUT_I444;
+ const int cw = (w + ss_hor) >> ss_hor;
+ const int ch = (h + ss_ver) >> ss_ver;
+ for (int pl = 1; pl <= 2; pl++) {
+ uint8_t *uvptr = p->data[pl];
+
+ for (int y = 0; y < ch; y++) {
+ XXH3_128bits_update(xxh3->state, uvptr, cw << hbd);
+ uvptr += p->stride[1];
+ }
+ }
+ }
+
+ dav1d_picture_unref(p);
+
+ return 0;
+}
+
+static void xxh3_close(xxh3Context *const xxh3) {
+ XXH128_hash_t hash = XXH3_128bits_digest(xxh3->state);
+ XXH3_freeState(xxh3->state);
+ XXH128_canonical_t c;
+ XXH128_canonicalFromHash(&c, hash);
+
+ for (int i = 0; i < 16; i++)
+ fprintf(xxh3->f, "%2.2x", c.digest[i]);
+ fprintf(xxh3->f, "\n");
+
+ if (xxh3->f != stdout)
+ fclose(xxh3->f);
+}
+
+static int xxh3_verify(xxh3Context *const xxh3, const char * xxh3_str) {
+ XXH128_hash_t hash = XXH3_128bits_digest(xxh3->state);
+ XXH3_freeState(xxh3->state);
+
+ if (strlen(xxh3_str) < 32)
+ return -1;
+
+ XXH128_canonical_t c;
+ char t[3] = { 0 };
+ for (int i = 0; i < 16; i++) {
+ char *ignore;
+ memcpy(t, xxh3_str, 2);
+ xxh3_str += 2;
+ c.digest[i] = strtoul(t, &ignore, 16);
+ }
+ XXH128_hash_t verify = XXH128_hashFromCanonical(&c);
+
+ return !XXH128_isEqual(hash, verify);
+}
+
+const Muxer xxh3_muxer = {
+ .priv_data_size = sizeof(xxh3Context),
+ .name = "xxh3",
+ .extension = "xxh3",
+ .write_header = xxh3_open,
+ .write_picture = xxh3_write,
+ .write_trailer = xxh3_close,
+ .verify = xxh3_verify,
+};
--
GitLab