Skip to content
Snippets Groups Projects
Commit e6168525 authored by Janne Grunau's avatar Janne Grunau Committed by Jean-Baptiste Kempf
Browse files

tools: add optional xxh3 based muxer

The required 'xxhash.h' header can either be in system include directory
or can be copied to 'tools/output'.

The xxh3_128bits based muxer shows no significant slowdown compared to
the null muxer. Decoding times Chimera-AV1-8bit-1920x1080-6736kbps.ivf
with 4 frame and 4 tile threads on a core i7-8550U (disabled turbo boost):

null:  72.5 s
md5:   99.8 s
xxh3:  73.8 s

Decoding Chimera-AV1-10bit-1920x1080-6191kbps.ivf with 6 frame and 4 tile
threads on a m1 mc mini:

null:  27.8 s
md5:  105.9 s
xxh3:  28.3 s
parent 061ac9ae
No related branches found
No related tags found
1 merge request!1117tools/dav1d: add xxhash 3 muxer
Pipeline #65634 passed with stages
in 10 minutes and 47 seconds
......@@ -6,3 +6,4 @@ tags
.DS_Store
/tests/dav1d-test-data
*.snap
/tools/output/xxhash.h
......@@ -53,3 +53,7 @@ option('fuzzer_ldflags',
option('stack_alignment',
type: 'integer',
value: 0)
option('xxhash_muxer',
type : 'feature',
value : 'auto')
......@@ -26,6 +26,7 @@
*/
#include "config.h"
#include "cli_config.h"
#include <getopt.h>
#include <limits.h>
......@@ -84,6 +85,12 @@ static const struct option long_opts[] = {
{ NULL, 0, NULL, 0 },
};
#if HAVE_XXHASH_H
#define AVAILABLE_MUXERS "'md5', 'xxh3', 'yuv', 'yuv4mpeg2' or 'null'"
#else
#define AVAILABLE_MUXERS "'md5', 'yuv', 'yuv4mpeg2' or 'null'"
#endif
#if ARCH_AARCH64 || ARCH_ARM
#define ALLOWED_CPU_MASKS " or 'neon'"
#elif ARCH_PPC64LE
......@@ -109,7 +116,7 @@ static void usage(const char *const app, const char *const reason, ...) {
" --input/-i $file: input file\n"
" --output/-o $file: output file\n"
" --demuxer $name: force demuxer type ('ivf', 'section5' or 'annexb'; default: detect from content)\n"
" --muxer $name: force muxer type ('md5', 'yuv', 'yuv4mpeg2' or 'null'; default: detect from extension)\n"
" --muxer $name: force muxer type (" AVAILABLE_MUXERS "; default: detect from extension)\n"
" --quiet/-q: disable status messages\n"
" --frametimes $file: dump frame times to file\n"
" --limit/-l $num: stop decoding after $num frames\n"
......@@ -120,7 +127,7 @@ static void usage(const char *const app, const char *const reason, ...) {
" --framethreads $num: number of frame threads (default: 1)\n"
" --tilethreads $num: number of tile threads (default: 1)\n"
" --pfthreads $num: number of postfilter threads (default: 1)\n"
" --filmgrain $num: enable film grain application (default: 1, except if muxer is md5)\n"
" --filmgrain $num: enable film grain application (default: 1, except if muxer is md5 or xxh3)\n"
" --oppoint $num: select an operating point of a scalable AV1 bitstream (0 - 31)\n"
" --alllayers $num: output all spatial layers of a scalable AV1 bitstream (default: 1)\n"
" --sizelimit $num: stop decoding if the frame size exceeds the specified limit\n"
......@@ -345,8 +352,11 @@ void parse(const int argc, char *const *const argv,
if (cli_settings->verify) {
if (cli_settings->outputfile)
usage(argv[0], "Verification (--verify) requires output file (-o/--output) to not set");
if (cli_settings->muxer && strcmp(cli_settings->muxer, "md5"))
usage(argv[0], "Verification (--verify) requires the md5 muxer (--muxer md5)");
if (cli_settings->muxer && strcmp(cli_settings->muxer, "md5") &&
strcmp(cli_settings->muxer, "xxh3"))
{
usage(argv[0], "Verification (--verify) requires a checksum muxer (md5 or xxh3)");
}
cli_settings->outputfile = "-";
if (!cli_settings->muxer)
......@@ -354,7 +364,8 @@ void parse(const int argc, char *const *const argv,
}
if (!grain_specified && cli_settings->muxer &&
!strcmp(cli_settings->muxer, "md5"))
(!strcmp(cli_settings->muxer, "md5") ||
!strcmp(cli_settings->muxer, "xxh3")))
{
lib_settings->apply_grain = 0;
}
......
......@@ -39,6 +39,18 @@ dav1d_output_sources = files(
'output/yuv.c',
)
# hacky check for xxhash.h to allow copying it to tools/output
if not get_option('xxhash_muxer').disabled()
xxhash_include = '-I' + meson.current_source_dir() / 'output'
if cc.has_header_symbol('xxhash.h', 'XXH3_createState', args : xxhash_include)
dav1d_output_sources += 'output/xxhash.c'
xxh3_found = true
elif get_option('xxhash_muxer').enabled()
# manual error since 'required' kw arg in has_header_symbol() was only added in meson 0.50
error( 'C symbol XXH3_createState not found in header xxhash.h')
endif
endif
dav1d_input_objs = static_library('dav1d_input',
dav1d_input_sources,
......@@ -69,6 +81,8 @@ endif
# Configuratin data for cli_config.h
cli_cdata = configuration_data()
cli_cdata.set10('HAVE_XXHASH_H', get_variable('xxh3_found', false))
cli_config_h_target = configure_file(output: 'cli_config.h', configuration: cli_cdata)
# dav1d cli tool sources
......
......@@ -26,6 +26,7 @@
*/
#include "config.h"
#include "cli_config.h"
#include <errno.h>
#include <stdio.h>
......@@ -44,11 +45,15 @@ struct MuxerContext {
extern const Muxer null_muxer;
extern const Muxer md5_muxer;
extern const Muxer xxh3_muxer;
extern const Muxer yuv_muxer;
extern const Muxer y4m2_muxer;
static const Muxer *muxers[] = {
&null_muxer,
&md5_muxer,
#if HAVE_XXHASH_H
&xxh3_muxer,
#endif
&yuv_muxer,
&y4m2_muxer,
NULL
......
/*
* Copyright © 2018-2021, VideoLAN and dav1d authors
* Copyright © 2018-2021, Two Orioles, LLC
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define XXH_INLINE_ALL
#include "xxhash.h"
#include "output/muxer.h"
typedef struct MuxerPriv {
XXH3_state_t* state;
FILE *f;
} xxh3Context;
static int xxh3_open(xxh3Context *const xxh3, const char *const file,
const Dav1dPictureParameters *const p,
const unsigned fps[2])
{
xxh3->state = XXH3_createState();
if (!xxh3->state) return DAV1D_ERR(ENOMEM);
XXH_errorcode err = XXH3_128bits_reset(xxh3->state);
if (err != XXH_OK) {
XXH3_freeState(xxh3->state);
xxh3->state = NULL;
return DAV1D_ERR(ENOMEM);
}
if (!strcmp(file, "-")) {
xxh3->f = stdout;
} else if (!(xxh3->f = fopen(file, "wb"))) {
XXH3_freeState(xxh3->state);
xxh3->state = NULL;
fprintf(stderr, "Failed to open %s: %s\n", file, strerror(errno));
return -1;
}
return 0;
}
static int xxh3_write(xxh3Context *const xxh3, Dav1dPicture *const p) {
const int hbd = p->p.bpc > 8;
const int w = p->p.w, h = p->p.h;
uint8_t *yptr = p->data[0];
for (int y = 0; y < h; y++) {
XXH3_128bits_update(xxh3->state, yptr, w << hbd);
yptr += p->stride[0];
}
if (p->p.layout != DAV1D_PIXEL_LAYOUT_I400) {
const int ss_ver = p->p.layout == DAV1D_PIXEL_LAYOUT_I420;
const int ss_hor = p->p.layout != DAV1D_PIXEL_LAYOUT_I444;
const int cw = (w + ss_hor) >> ss_hor;
const int ch = (h + ss_ver) >> ss_ver;
for (int pl = 1; pl <= 2; pl++) {
uint8_t *uvptr = p->data[pl];
for (int y = 0; y < ch; y++) {
XXH3_128bits_update(xxh3->state, uvptr, cw << hbd);
uvptr += p->stride[1];
}
}
}
dav1d_picture_unref(p);
return 0;
}
static void xxh3_close(xxh3Context *const xxh3) {
XXH128_hash_t hash = XXH3_128bits_digest(xxh3->state);
XXH3_freeState(xxh3->state);
XXH128_canonical_t c;
XXH128_canonicalFromHash(&c, hash);
for (int i = 0; i < 16; i++)
fprintf(xxh3->f, "%2.2x", c.digest[i]);
fprintf(xxh3->f, "\n");
if (xxh3->f != stdout)
fclose(xxh3->f);
}
static int xxh3_verify(xxh3Context *const xxh3, const char * xxh3_str) {
XXH128_hash_t hash = XXH3_128bits_digest(xxh3->state);
XXH3_freeState(xxh3->state);
if (strlen(xxh3_str) < 32)
return -1;
XXH128_canonical_t c;
char t[3] = { 0 };
for (int i = 0; i < 16; i++) {
char *ignore;
memcpy(t, xxh3_str, 2);
xxh3_str += 2;
c.digest[i] = strtoul(t, &ignore, 16);
}
XXH128_hash_t verify = XXH128_hashFromCanonical(&c);
return !XXH128_isEqual(hash, verify);
}
const Muxer xxh3_muxer = {
.priv_data_size = sizeof(xxh3Context),
.name = "xxh3",
.extension = "xxh3",
.write_header = xxh3_open,
.write_picture = xxh3_write,
.write_trailer = xxh3_close,
.verify = xxh3_verify,
};
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment