Commit ea88b8d6 authored by Filip Roséen's avatar Filip Roséen Committed by Hugo Beauzée-Luyssen
Browse files

playlist/fetcher: refactor



The following changes refactors the fetcher to take advantange of the
newly introduced background_worker helper. The new implementation
should not only be easier to maintain, but it also adds some
advantages over the old implementation:

 - The implementation has shrunk in size.

 - A fetch-request can include a timeout.

 - Given that there now is a background worker associated with each of
   the different fetcher types (local, network, download):

    - A slow download does not prevent the network-fetcher from
      working the queue.

    - A slow network-fetcher does not prevent further work in regards
      of pending requests in the local fetcher.

 - A fetch request can now be properly cancelled (most importantly
   during VLC close).

 - We no longer invoke modules with "meta fetcher" capability if the
   item already has all metadata in terms of title, album, and artist.

 - We no longer invoke modules with "art finder" capability of the
   item already has vlc_meta_ArtworkUrl.

fixes: #18150
Signed-off-by: Hugo Beauzée-Luyssen's avatarHugo Beauzée-Luyssen <hugo@beauzee.fr>
parent 5c42881e
/*****************************************************************************
* fetcher.c: Art fetcher thread.
* fetcher.c
*****************************************************************************
* Copyright © 1999-2009 VLC authors and VideoLAN
* Copyright © 2017-2017 VLC authors and VideoLAN
* $Id$
*
* Authors: Samuel Hocevar <sam@zoy.org>
* Clément Stenac <zorglub@videolan.org>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation; either version 2.1 of the License, or
......@@ -21,555 +18,442 @@
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
*****************************************************************************/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif
#include <limits.h>
#include <assert.h>
#include <vlc_common.h>
#include <vlc_stream.h>
#include <vlc_meta_fetcher.h>
#include <vlc_memory.h>
#include <vlc_demux.h>
#include <vlc_modules.h>
#include <vlc_interrupt.h>
#include <vlc_arrays.h>
#include <vlc_atomic.h>
#include <vlc_threads.h>
#include <vlc_memstream.h>
#include <vlc_meta_fetcher.h>
#include "libvlc.h"
#include "art.h"
#include "libvlc.h"
#include "fetcher.h"
#include "input/input_interface.h"
#include "misc/background_worker.h"
#include "misc/interrupt.h"
/*****************************************************************************
* Structures/definitions
*****************************************************************************/
typedef enum
{
PASS1_LOCAL = 0,
PASS2_NETWORK
} fetcher_pass_t;
#define PASS_COUNT 2
struct playlist_fetcher_t {
struct background_worker* local;
struct background_worker* network;
struct background_worker* downloader;
typedef struct
{
char *psz_artist;
char *psz_album;
char *psz_arturl;
bool b_found;
meta_fetcher_scope_t e_scope; /* max scope */
} playlist_album_t;
typedef struct fetcher_entry_t fetcher_entry_t;
vlc_dictionary_t album_cache;
vlc_object_t* owner;
vlc_mutex_t lock;
};
struct fetcher_entry_t
{
input_item_t *p_item;
input_item_meta_request_option_t i_options;
fetcher_entry_t *p_next;
struct fetcher_request {
input_item_t* item;
atomic_uint refs;
int options;
};
struct playlist_fetcher_t
{
vlc_object_t *object;
vlc_mutex_t lock;
vlc_cond_t wait;
bool b_live;
vlc_interrupt_t *interrupt;
struct fetcher_thread {
void (*pf_worker)( playlist_fetcher_t*, struct fetcher_request* );
fetcher_entry_t *p_waiting_head[PASS_COUNT];
fetcher_entry_t *p_waiting_tail[PASS_COUNT];
struct background_worker* worker;
struct fetcher_request* req;
playlist_fetcher_t* fetcher;
DECL_ARRAY(playlist_album_t) albums;
meta_fetcher_scope_t e_scope;
vlc_interrupt_t interrupt;
vlc_thread_t thread;
atomic_bool active;
};
static void *Thread( void * );
/*****************************************************************************
* Public functions
*****************************************************************************/
playlist_fetcher_t *playlist_fetcher_New( vlc_object_t *parent )
static char* CreateCacheKey( input_item_t* item )
{
playlist_fetcher_t *p_fetcher = malloc( sizeof(*p_fetcher) );
if( !p_fetcher )
return NULL;
vlc_mutex_lock( &item->lock );
p_fetcher->interrupt = vlc_interrupt_create();
if( unlikely(p_fetcher->interrupt == NULL) )
if( !item->p_meta )
{
free( p_fetcher );
vlc_mutex_unlock( &item->lock );
return NULL;
}
p_fetcher->object = parent;
vlc_mutex_init( &p_fetcher->lock );
vlc_cond_init( &p_fetcher->wait );
p_fetcher->b_live = false;
if( var_InheritBool( parent, "metadata-network-access" ) )
p_fetcher->e_scope = FETCHER_SCOPE_ANY;
else
p_fetcher->e_scope = FETCHER_SCOPE_LOCAL;
char const* artist = vlc_meta_Get( item->p_meta, vlc_meta_Artist );
char const* album = vlc_meta_Get( item->p_meta, vlc_meta_Album );
char* key;
memset( p_fetcher->p_waiting_head, 0, PASS_COUNT * sizeof(fetcher_entry_t *) );
memset( p_fetcher->p_waiting_tail, 0, PASS_COUNT * sizeof(fetcher_entry_t *) );
ARRAY_INIT( p_fetcher->albums );
/**
* Simple concatenation of artist and album can lead to the same key
* for entities that should not have such. Imagine { dogs, tick } and
* { dog, stick } */
if( !artist || !album || asprintf( &key, "%s:%zu:%s:%zu",
artist, strlen( artist ), album, strlen( album ) ) < 0 )
{
key = NULL;
}
vlc_mutex_unlock( &item->lock );
return p_fetcher;
return key;
}
void playlist_fetcher_Push( playlist_fetcher_t *p_fetcher, input_item_t *p_item,
input_item_meta_request_option_t i_options )
static void FreeCacheEntry( void* data, void* obj )
{
fetcher_entry_t *p_entry = malloc( sizeof(fetcher_entry_t) );
if ( !p_entry ) return;
vlc_gc_incref( p_item );
p_entry->p_item = p_item;
p_entry->p_next = NULL;
p_entry->i_options = i_options;
vlc_mutex_lock( &p_fetcher->lock );
/* Append last */
if ( p_fetcher->p_waiting_head[PASS1_LOCAL] )
p_fetcher->p_waiting_tail[PASS1_LOCAL]->p_next = p_entry;
else
p_fetcher->p_waiting_head[PASS1_LOCAL] = p_entry;
p_fetcher->p_waiting_tail[PASS1_LOCAL] = p_entry;
if( !p_fetcher->b_live )
{
assert( p_fetcher->p_waiting_head[PASS1_LOCAL] );
if( vlc_clone_detach( NULL, Thread, p_fetcher,
VLC_THREAD_PRIORITY_LOW ) )
msg_Err( p_fetcher->object,
"cannot spawn secondary preparse thread" );
else
p_fetcher->b_live = true;
}
vlc_mutex_unlock( &p_fetcher->lock );
free( data );
VLC_UNUSED( obj );
}
void playlist_fetcher_Delete( playlist_fetcher_t *p_fetcher )
static int ReadAlbumCache( playlist_fetcher_t* fetcher, input_item_t* item )
{
fetcher_entry_t *p_next;
char* key = CreateCacheKey( item );
vlc_interrupt_kill(p_fetcher->interrupt);
if( key == NULL )
return VLC_EGENERIC;
vlc_mutex_lock( &fetcher->lock );
char const* art = vlc_dictionary_value_for_key( &fetcher->album_cache,
key );
if( art )
input_item_SetArtURL( item, art );
vlc_mutex_unlock( &fetcher->lock );
free( key );
return art ? VLC_SUCCESS : VLC_EGENERIC;
}
static void AddAlbumCache( playlist_fetcher_t* fetcher, input_item_t* item,
bool overwrite )
{
char* art = input_item_GetArtURL( item );
char* key = CreateCacheKey( item );
vlc_mutex_lock( &p_fetcher->lock );
/* Remove any left-over item, the fetcher will exit */
for ( int i_queue=0; i_queue<PASS_COUNT; i_queue++ )
if( key && art && strncasecmp( art, "attachment://", 13 ) )
{
while( p_fetcher->p_waiting_head[i_queue] )
vlc_mutex_lock( &fetcher->lock );
if( overwrite || !vlc_dictionary_has_key( &fetcher->album_cache, key ) )
{
p_next = p_fetcher->p_waiting_head[i_queue]->p_next;
vlc_gc_decref( p_fetcher->p_waiting_head[i_queue]->p_item );
free( p_fetcher->p_waiting_head[i_queue] );
p_fetcher->p_waiting_head[i_queue] = p_next;
vlc_dictionary_insert( &fetcher->album_cache, key, art );
art = NULL;
}
p_fetcher->p_waiting_head[i_queue] = NULL;
vlc_mutex_unlock( &fetcher->lock );
}
while( p_fetcher->b_live )
vlc_cond_wait( &p_fetcher->wait, &p_fetcher->lock );
vlc_mutex_unlock( &p_fetcher->lock );
free( art );
free( key );
}
vlc_cond_destroy( &p_fetcher->wait );
vlc_mutex_destroy( &p_fetcher->lock );
static int InvokeModule( playlist_fetcher_t* fetcher, input_item_t* item,
int scope, char const* type )
{
meta_fetcher_t* mf = vlc_custom_create( fetcher->owner,
sizeof( *mf ), type );
if( unlikely( !mf ) )
return VLC_ENOMEM;
vlc_interrupt_destroy( p_fetcher->interrupt );
mf->e_scope = scope;
mf->p_item = item;
playlist_album_t album;
FOREACH_ARRAY( album, p_fetcher->albums )
free( album.psz_album );
free( album.psz_artist );
free( album.psz_arturl );
FOREACH_END()
ARRAY_RESET( p_fetcher->albums );
module_t* mf_module = module_need( mf, type, NULL, false );
free( p_fetcher );
}
if( mf_module )
module_unneed( mf, mf_module );
vlc_object_release( mf );
/*****************************************************************************
* Privates functions
*****************************************************************************/
/**
* This function locates the art associated to an input item.
* Return codes:
* 0 : Art is in cache or is a local file
* 1 : Art found, need to download
* -X : Error/not found
*/
static int FindArt( playlist_fetcher_t *p_fetcher, input_item_t *p_item )
return VLC_SUCCESS;
}
static int CheckMeta( input_item_t* item )
{
int i_ret;
vlc_mutex_lock( &item->lock );
bool error = !item->p_meta ||
!vlc_meta_Get( item->p_meta, vlc_meta_Title ) ||
!vlc_meta_Get( item->p_meta, vlc_meta_Artist ) ||
!vlc_meta_Get( item->p_meta, vlc_meta_Album );
vlc_mutex_unlock( &item->lock );
return error;
}
playlist_album_t *p_album = NULL;
char *psz_artist = input_item_GetArtist( p_item );
char *psz_album = input_item_GetAlbum( p_item );
char *psz_title = input_item_GetTitle( p_item );
if( !psz_title )
psz_title = input_item_GetName( p_item );
static int CheckArt( input_item_t* item )
{
vlc_mutex_lock( &item->lock );
bool error = !item->p_meta ||
!vlc_meta_Get( item->p_meta, vlc_meta_ArtworkURL );
vlc_mutex_unlock( &item->lock );
return error;
}
if( !psz_title && !psz_artist && !psz_album )
return VLC_EGENERIC;
static int SearchArt( playlist_fetcher_t* fetcher, input_item_t* item, int scope)
{
InvokeModule( fetcher, item, scope, "art finder" );
return CheckArt( item );
}
free( psz_title );
static int SearchByScope( playlist_fetcher_t* fetcher,
struct fetcher_request* req, int scope )
{
input_item_t* item = req->item;
/* If we already checked this album in this session, skip */
if( psz_artist && psz_album )
if( CheckMeta( item ) &&
InvokeModule( fetcher, req->item, scope, "meta fetcher" ) )
{
FOREACH_ARRAY( playlist_album_t album, p_fetcher->albums )
if( !strcmp( album.psz_artist, psz_artist ) &&
!strcmp( album.psz_album, psz_album ) )
{
msg_Dbg( p_fetcher->object,
" %s - %s has already been searched",
psz_artist, psz_album );
/* TODO-fenrir if we cache art filename too, we can go faster */
free( psz_artist );
free( psz_album );
if( album.b_found )
{
if( !strncmp( album.psz_arturl, "file://", 7 ) )
input_item_SetArtURL( p_item, album.psz_arturl );
else /* Actually get URL from cache */
playlist_FindArtInCache( p_item );
return 0;
}
else if ( album.e_scope >= p_fetcher->e_scope )
{
return VLC_EGENERIC;
}
msg_Dbg( p_fetcher->object,
" will search at higher scope, if possible" );
p_album = &p_fetcher->albums.p_elems[fe_idx];
psz_artist = psz_album = NULL;
break;
}
FOREACH_END();
return VLC_EGENERIC;
}
free( psz_artist );
free( psz_album );
if( ! CheckArt( item ) ||
! ReadAlbumCache( fetcher, item ) ||
! playlist_FindArtInCacheUsingItemUID( item ) ||
! playlist_FindArtInCache( item ) ||
! SearchArt( fetcher, item, scope ) )
{
AddAlbumCache( fetcher, req->item, false );
background_worker_Push( fetcher->downloader, req, NULL, 0 );
return VLC_SUCCESS;
}
if ( playlist_FindArtInCacheUsingItemUID( p_item ) != VLC_SUCCESS )
playlist_FindArtInCache( p_item );
else
msg_Dbg( p_fetcher->object, "successfully retrieved arturl by uid" );
return VLC_EGENERIC;
}
char *psz_arturl = input_item_GetArtURL( p_item );
if( psz_arturl )
{
/* We already have a URL */
if( !strncmp( psz_arturl, "file://", strlen( "file://" ) ) )
{
free( psz_arturl );
return 0; /* Art is in cache, no need to go further */
}
static void Downloader( playlist_fetcher_t* fetcher,
struct fetcher_request* req )
{
ReadAlbumCache( fetcher, req->item );
free( psz_arturl );
char *psz_arturl = input_item_GetArtURL( req->item );
if( !psz_arturl )
goto error;
/* Art need to be put in cache */
return 1;
}
if( !strncasecmp( psz_arturl, "file://", 7 ) ||
!strncasecmp( psz_arturl, "attachment://", 13 ) )
goto out; /* no fetch required */
/* */
psz_album = input_item_GetAlbum( p_item );
psz_artist = input_item_GetArtist( p_item );
if( psz_album && psz_artist )
{
msg_Dbg( p_fetcher->object, "searching art for %s - %s",
psz_artist, psz_album );
}
else
{
psz_title = input_item_GetTitle( p_item );
if( !psz_title )
psz_title = input_item_GetName( p_item );
stream_t* source = vlc_stream_NewURL( fetcher->owner, psz_arturl );
msg_Dbg( p_fetcher->object, "searching art for %s", psz_title );
free( psz_title );
}
if( !source )
goto error;
/* Fetch the art url */
i_ret = VLC_EGENERIC;
struct vlc_memstream output_stream;
vlc_memstream_open( &output_stream );
vlc_object_t *p_parent = p_fetcher->object;
meta_fetcher_t *p_finder =
vlc_custom_create( p_parent, sizeof( *p_finder ), "art finder" );
if( p_finder != NULL)
for( ;; )
{
module_t *p_module;
char buffer[2048];
p_finder->p_item = p_item;
p_finder->e_scope = p_fetcher->e_scope;
int read = vlc_stream_Read( source, buffer, sizeof( buffer ) );
if( read <= 0 )
break;
p_module = module_need( p_finder, "art finder", NULL, false );
if( p_module )
{
module_unneed( p_finder, p_module );
/* Try immediately if found in cache by download URL */
if( !playlist_FindArtInCache( p_item ) )
i_ret = 0;
else
i_ret = 1;
}
vlc_object_release( p_finder );
if( (int)vlc_memstream_write( &output_stream, buffer, read ) < read )
break;
}
/* Record this album */
if( psz_artist && psz_album )
vlc_stream_Delete( source );
if( vlc_memstream_close( &output_stream ) )
goto error;
if( vlc_killed() )
{
if ( p_album )
{
p_album->e_scope = p_fetcher->e_scope;
free( p_album->psz_arturl );
p_album->psz_arturl = input_item_GetArtURL( p_item );
p_album->b_found = (i_ret == VLC_EGENERIC ? false : true );
free( psz_artist );
free( psz_album );
}
else
{
playlist_album_t a;
a.psz_artist = psz_artist;
a.psz_album = psz_album;
a.psz_arturl = input_item_GetArtURL( p_item );
a.b_found = (i_ret == VLC_EGENERIC ? false : true );
a.e_scope = p_fetcher->e_scope;
ARRAY_APPEND( p_fetcher->albums, a );
}
free( output_stream.ptr );
goto error;
}
else
playlist_SaveArt( fetcher->owner, req->item, output_stream.ptr,
output_stream.length, NULL );
free( output_stream.ptr );
AddAlbumCache( fetcher, req->item, true );
out:
if( psz_arturl )
{
free( psz_artist );
free( psz_album );
var_SetAddress( fetcher->owner, "item-change", req->item );
input_item_SetArtFetched( req->item, true );
}
return i_ret;
free( psz_arturl );
return;
error:
FREENULL( psz_arturl );
goto out;
}
/**
* Download the art using the URL or an art downloaded
* This function should be called only if data is not already in cache
*/
static int DownloadArt( playlist_fetcher_t *p_fetcher, input_item_t *p_item )
static void SearchLocal( playlist_fetcher_t* fetcher, struct fetcher_request* req )
{
char *psz_arturl = input_item_GetArtURL( p_item );
assert( *psz_arturl );
if( SearchByScope( fetcher, req, FETCHER_SCOPE_LOCAL ) == VLC_SUCCESS )
return; /* done */
if( !strncasecmp( psz_arturl , "file://", 7 ) )
if( var_InheritBool( fetcher->owner, "metadata-network-access" ) ||
req->options & META_REQUEST_OPTION_SCOPE_NETWORK )
{
msg_Dbg( p_fetcher->object,
"Album art is local file, no need to cache" );
free( psz_arturl );
return VLC_SUCCESS;
background_worker_Push( fetcher->network, req, NULL, 0 );
}
else input_item_SetArtNotFound( req->item, true );
}
if( !strncmp( psz_arturl , "APIC", 4 ) )
{
msg_Warn( p_fetcher->object, "APIC fetch not supported yet" );
goto error;
}
static void SearchNetwork( playlist_fetcher_t* fetcher, struct fetcher_request* req )
{
if( SearchByScope( fetcher, req, FETCHER_SCOPE_NETWORK ) )
input_item_SetArtNotFound( req->item, true );
}
stream_t *p_stream = vlc_stream_NewURL( p_fetcher->object, psz_arturl );
if( !p_stream )
goto error;
static void RequestRelease( void* req_ )
{
struct fetcher_request* req = req_;
uint8_t *p_data = NULL;
int i_data = 0;
for( ;; )
{
int i_read = 65536;
if( atomic_fetch_sub( &req->refs, 1 ) != 1 )
return;
if( i_data >= INT_MAX - i_read )
break;
input_item_Release( req->item );
free( req );
}
p_data = realloc_or_free( p_data, i_data + i_read );
if( !p_data )
break;
static void RequestHold( void* req_ )
{
struct fetcher_request* req = req_;
atomic_fetch_add_explicit( &req->refs, 1, memory_order_relaxed );
}
i_read = vlc_stream_Read( p_stream, &p_data[i_data], i_read );
if( i_read <= 0 )
break;
static void* FetcherThread( void* handle )
{
struct fetcher_thread* th = handle;
vlc_interrupt_set( &th->interrupt );