MediaLibrary.cpp 47.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2015 Hugo Beauzée-Luyssen, Videolabs
 *
 * Authors: Hugo Beauzée-Luyssen<hugo@beauzee.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

23 24 25 26
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

27 28
#include <algorithm>
#include <functional>
29
#include <utility>
30
#include <sys/stat.h>
31
#include <unistd.h>
32

33 34
#include "Album.h"
#include "AlbumTrack.h"
35
#include "Artist.h"
36
#include "AudioTrack.h"
37
#include "discoverer/DiscovererWorker.h"
38
#include "discoverer/probe/CrawlerProbe.h"
39
#include "utils/ModificationsNotifier.h"
40
#include "Device.h"
41
#include "File.h"
42
#include "Folder.h"
43
#include "Genre.h"
44
#include "History.h"
45
#include "Media.h"
46
#include "MediaLibrary.h"
47
#include "Label.h"
48
#include "logging/Logger.h"
49
#include "Movie.h"
50
#include "parser/Parser.h"
51
#include "Playlist.h"
52 53
#include "Show.h"
#include "ShowEpisode.h"
54
#include "Thumbnail.h"
55
#include "database/SqliteTools.h"
56
#include "database/SqliteConnection.h"
57
#include "database/SqliteQuery.h"
58
#include "utils/Filename.h"
59
#include "utils/Url.h"
60
#include "VideoTrack.h"
61

62 63 64
// Discoverers:
#include "discoverer/FsDiscoverer.h"

65
// Metadata services:
66
#ifdef HAVE_LIBVLC
67 68
#include "metadata_services/vlc/VLCMetadataService.h"
#include "metadata_services/vlc/VLCThumbnailer.h"
69
#endif
70
#include "metadata_services/MetadataParser.h"
71

72 73
// FileSystem
#include "factory/DeviceListerFactory.h"
74
#include "factory/FileSystemFactory.h"
75 76

#ifdef HAVE_LIBVLC
77
#include "factory/NetworkFileSystemFactory.h"
78 79
#endif

80
#include "medialibrary/filesystem/IDevice.h"
81

82 83 84
namespace medialibrary
{

85
const char* const MediaLibrary::supportedExtensions[] = {
86
    "3g2", "3gp", "a52", "aac", "ac3", "aif", "aifc", "aiff", "alac", "amr",
87
    "amv", "aob", "ape", "asf", "asx", "avi", "b4s", "conf", /*"cue",*/
88 89 90 91 92 93 94 95 96 97
    "divx", "dts", "dv", "flac", "flv", "gxf", "ifo", "iso",
    "it", "itml",  "m1v", "m2t", "m2ts", "m2v", "m3u", "m3u8",
    "m4a", "m4b", "m4p", "m4v", "mid", "mka", "mkv", "mlp",
    "mod", "mov", "mp1", "mp2", "mp3", "mp4", "mpc", "mpeg",
    "mpeg1", "mpeg2", "mpeg4", "mpg", "mts", "mxf", "nsv",
    "nuv", "oga", "ogg", "ogm", "ogv", "ogx", "oma", "opus",
    "pls", "ps", "qtl", "ram", "rec", "rm", "rmi", "rmvb",
    "s3m", "sdp", "spx", "tod", "trp", "ts", "tta", "vlc",
    "vob", "voc", "vqf", "vro", "w64", "wav", "wax", "webm",
    "wma", "wmv", "wmx", "wpl", "wv", "wvx", "xa", "xm", "xspf"
98 99
};

100 101
const size_t MediaLibrary::NbSupportedExtensions = sizeof(supportedExtensions) / sizeof(supportedExtensions[0]);

102
MediaLibrary::MediaLibrary()
103 104
    : m_callback( nullptr )
    , m_verbosity( LogLevel::Error )
105
    , m_settings( this )
106
    , m_initialized( false )
107 108
    , m_discovererIdle( true )
    , m_parserIdle( true )
109
{
110
    Log::setLogLevel( m_verbosity );
111 112
}

113 114
MediaLibrary::~MediaLibrary()
{
115
    // Explicitely stop the discoverer, to avoid it writting while tearing down.
116 117
    if ( m_discovererWorker != nullptr )
        m_discovererWorker->stop();
118 119
    if ( m_parser != nullptr )
        m_parser->stop();
120 121 122 123 124
    clearCache();
}

void MediaLibrary::clearCache()
{
125
    Media::clear();
126
    Folder::clear();
127 128 129 130 131
    Label::clear();
    Album::clear();
    AlbumTrack::clear();
    Show::clear();
    ShowEpisode::clear();
132
    Movie::clear();
133
    VideoTrack::clear();
134
    AudioTrack::clear();
135
    Artist::clear();
136
    Device::clear();
137
    File::clear();
138
    Playlist::clear();
139
    History::clear();
140
    Genre::clear();
141
    Thumbnail::clear();
142 143
}

144
void MediaLibrary::createAllTables()
145
{
146 147 148 149 150
    // We need to create the tables in order of triggers creation
    // Device is the "root of all evil". When a device is modified,
    // we will trigger an update on folder, which will trigger
    // an update on files, and so on.

151 152
    Device::createTable( m_dbConnection.get() );
    Folder::createTable( m_dbConnection.get() );
153
    Thumbnail::createTable( m_dbConnection.get() );
154 155 156 157 158 159 160 161 162 163 164 165 166 167
    Media::createTable( m_dbConnection.get() );
    File::createTable( m_dbConnection.get() );
    Label::createTable( m_dbConnection.get() );
    Playlist::createTable( m_dbConnection.get() );
    Genre::createTable( m_dbConnection.get() );
    Album::createTable( m_dbConnection.get() );
    AlbumTrack::createTable( m_dbConnection.get() );
    Show::createTable( m_dbConnection.get() );
    ShowEpisode::createTable( m_dbConnection.get() );
    Movie::createTable( m_dbConnection.get() );
    VideoTrack::createTable( m_dbConnection.get() );
    AudioTrack::createTable( m_dbConnection.get() );
    Artist::createTable( m_dbConnection.get() );
    Artist::createDefaultArtists( m_dbConnection.get() );
168 169
    History::createTable( m_dbConnection.get() );
    Settings::createTable( m_dbConnection.get() );
170
    parser::Task::createTable( m_dbConnection.get() );
171 172 173 174
}

void MediaLibrary::createAllTriggers()
{
175
    auto dbModelVersion = m_settings.dbModelVersion();
176
    Folder::createTriggers( m_dbConnection.get() );
177
    Album::createTriggers( m_dbConnection.get() );
178
    AlbumTrack::createTriggers( m_dbConnection.get() );
179
    Artist::createTriggers( m_dbConnection.get(), dbModelVersion );
180
    Media::createTriggers( m_dbConnection.get() );
181
    File::createTriggers( m_dbConnection.get() );
182 183
    Genre::createTriggers( m_dbConnection.get() );
    Playlist::createTriggers( m_dbConnection.get() );
184
    History::createTriggers( m_dbConnection.get() );
185
    Label::createTriggers( m_dbConnection.get() );
186 187
}

188
template <typename T>
189
static void propagateDeletionToCache( sqlite::Connection::HookReason reason, int64_t rowId )
190
{
191
    if ( reason != sqlite::Connection::HookReason::Delete )
192 193 194 195
        return;
    T::removeFromCache( rowId );
}

196 197
void MediaLibrary::registerEntityHooks()
{
198
    if ( m_modificationNotifier == nullptr )
199 200
        return;

201
    m_dbConnection->registerUpdateHook( policy::MediaTable::Name,
202 203
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
204 205
            return;
        Media::removeFromCache( rowId );
206
        m_modificationNotifier->notifyMediaRemoval( rowId );
207
    });
208
    m_dbConnection->registerUpdateHook( policy::ArtistTable::Name,
209 210
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
211 212 213 214
            return;
        Artist::removeFromCache( rowId );
        m_modificationNotifier->notifyArtistRemoval( rowId );
    });
215
    m_dbConnection->registerUpdateHook( policy::AlbumTable::Name,
216 217
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
218 219 220 221
            return;
        Album::removeFromCache( rowId );
        m_modificationNotifier->notifyAlbumRemoval( rowId );
    });
222
    m_dbConnection->registerUpdateHook( policy::AlbumTrackTable::Name,
223 224
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
225 226 227 228
            return;
        AlbumTrack::removeFromCache( rowId );
        m_modificationNotifier->notifyAlbumTrackRemoval( rowId );
    });
229
    m_dbConnection->registerUpdateHook( policy::PlaylistTable::Name,
230 231
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
232 233 234 235
            return;
        Playlist::removeFromCache( rowId );
        m_modificationNotifier->notifyPlaylistRemoval( rowId );
    });
236 237 238 239 240 241 242 243 244 245
    m_dbConnection->registerUpdateHook( policy::DeviceTable::Name, &propagateDeletionToCache<Device> );
    m_dbConnection->registerUpdateHook( policy::FileTable::Name, &propagateDeletionToCache<File> );
    m_dbConnection->registerUpdateHook( policy::FolderTable::Name, &propagateDeletionToCache<Folder> );
    m_dbConnection->registerUpdateHook( policy::GenreTable::Name, &propagateDeletionToCache<Genre> );
    m_dbConnection->registerUpdateHook( policy::LabelTable::Name, &propagateDeletionToCache<Label> );
    m_dbConnection->registerUpdateHook( policy::MovieTable::Name, &propagateDeletionToCache<Movie> );
    m_dbConnection->registerUpdateHook( policy::ShowTable::Name, &propagateDeletionToCache<Show> );
    m_dbConnection->registerUpdateHook( policy::ShowEpisodeTable::Name, &propagateDeletionToCache<ShowEpisode> );
    m_dbConnection->registerUpdateHook( policy::AudioTrackTable::Name, &propagateDeletionToCache<AudioTrack> );
    m_dbConnection->registerUpdateHook( policy::VideoTrackTable::Name, &propagateDeletionToCache<VideoTrack> );
246 247
}

248 249 250 251 252
bool MediaLibrary::validateSearchPattern( const std::string& pattern )
{
    return pattern.size() >= 3;
}

253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283
bool MediaLibrary::createThumbnailFolder( const std::string& thumbnailPath ) const
{
    auto paths = utils::file::splitPath( thumbnailPath, true );
#ifndef _WIN32
    std::string fullPath{ "/" };
#else
    std::string fullPath;
#endif
    while ( paths.empty() == false )
    {
        fullPath += paths.top();

#ifdef _WIN32
        if ( mkdir( fullPath.c_str() ) != 0 )
#else
        if ( mkdir( fullPath.c_str(), S_IRWXU ) != 0 )
#endif
        {
            if ( errno != EEXIST )
                return false;
        }
        paths.pop();
#ifndef _WIN32
        fullPath += "/";
#else
        fullPath += "\\";
#endif
    }
    return true;
}

284 285 286
InitializeResult MediaLibrary::initialize( const std::string& dbPath,
                                           const std::string& thumbnailPath,
                                           IMediaLibraryCb* mlCallback )
287
{
288
    LOG_INFO( "Initializing medialibrary..." );
289
    if ( m_initialized == true )
290 291
    {
        LOG_INFO( "...Already initialized" );
292
        return InitializeResult::AlreadyInitialized;
293
    }
294 295 296 297
    if ( m_deviceLister == nullptr )
    {
        m_deviceLister = factory::createDeviceLister();
        if ( m_deviceLister == nullptr )
298 299
        {
            LOG_ERROR( "No available IDeviceLister was found." );
300
            return InitializeResult::Failed;
301
        }
302
    }
303
    addLocalFsFactory();
304
    if ( createThumbnailFolder( thumbnailPath ) == false )
305
    {
306 307 308
        LOG_ERROR( "Failed to create thumbnail directory (", thumbnailPath,
                    ": ", strerror( errno ) );
        return InitializeResult::Failed;
309
    }
310
    m_thumbnailPath = thumbnailPath;
311
    m_callback = mlCallback;
312
    m_dbConnection = sqlite::Connection::connect( dbPath );
313

314 315 316
    // Give a chance to test overloads to reject the creation of a notifier
    startDeletionNotifier();
    // Which allows us to register hooks, or not, depending on the presence of a notifier
317 318
    registerEntityHooks();

319
    auto res = InitializeResult::Success;
320
    try
321
    {
322
        auto t = m_dbConnection->newTransaction();
323
        createAllTables();
324
        if ( m_settings.load() == false )
325 326
        {
            LOG_ERROR( "Failed to load settings" );
327
            return InitializeResult::Failed;
328
        }
329 330 331
        createAllTriggers();
        t->commit();

332 333
        if ( m_settings.dbModelVersion() != Settings::DbModelVersion )
        {
334 335
            res = updateDatabaseModel( m_settings.dbModelVersion(), dbPath );
            if ( res == InitializeResult::Failed )
336 337
            {
                LOG_ERROR( "Failed to update database model" );
338
                return res;
339 340 341 342 343 344
            }
        }
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Can't initialize medialibrary: ", ex.what() );
345
        return InitializeResult::Failed;
346
    }
347 348
    m_initialized = true;
    LOG_INFO( "Successfuly initialized" );
349
    return res;
350 351 352 353
}

bool MediaLibrary::start()
{
354
    assert( m_initialized == true );
355 356 357
    if ( m_parser != nullptr )
        return false;

358
    populateFsFactories();
359 360
    for ( auto& fsFactory : m_fsFactories )
        refreshDevices( *fsFactory );
361
    startDiscoverer();
362 363
    if ( startParser() == false )
        return false;
364
    startThumbnailer();
365
    return true;
366 367
}

368
void MediaLibrary::setVerbosity( LogLevel v )
369 370 371 372 373
{
    m_verbosity = v;
    Log::setLogLevel( v );
}

374 375 376 377 378
MediaPtr MediaLibrary::media( int64_t mediaId ) const
{
    return Media::fetch( this, mediaId );
}

379 380
MediaPtr MediaLibrary::media( const std::string& mrl ) const
{
381
    LOG_INFO( "Fetching media from mrl: ", mrl );
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
382
    auto file = File::fromExternalMrl( this, mrl );
383 384 385 386 387
    if ( file != nullptr )
    {
        LOG_INFO( "Found external media: ", mrl );
        return file->media();
    }
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
388
    auto fsFactory = fsFactoryForMrl( mrl );
389
    if ( fsFactory == nullptr )
390
    {
391 392
        LOG_WARN( "Failed to create FS factory for path ", mrl );
        return nullptr;
393
    }
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
394
    auto device = fsFactory->createDeviceFromMrl( mrl );
395 396 397 398 399 400
    if ( device == nullptr )
    {
        LOG_WARN( "Failed to create a device associated with mrl ", mrl );
        return nullptr;
    }
    if ( device->isRemovable() == false )
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
401
        file = File::fromMrl( this, mrl );
402 403
    else
    {
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
404
        auto folder = Folder::fromMrl( this, utils::file::directory( mrl ) );
405 406 407 408 409 410 411 412 413 414 415 416 417 418
        if ( folder == nullptr )
        {
            LOG_WARN( "Failed to find folder containing ", mrl );
            return nullptr;
        }
        if ( folder->isPresent() == false )
        {
            LOG_INFO( "Found a folder containing ", mrl, " but it is not present" );
            return nullptr;
        }
        file = File::fromFileName( this, utils::file::fileName( mrl ), folder->id() );
    }
    if ( file == nullptr )
    {
419
        LOG_WARN( "Failed to fetch file for ", mrl, " (device ", device->uuid(), " was ",
420
                  device->isRemovable() ? "" : "NOT ", "removable)");
421 422 423 424 425
        return nullptr;
    }
    return file->media();
}

426 427
MediaPtr MediaLibrary::addMedia( const std::string& mrl )
{
428 429 430 431
    try
    {
        return sqlite::Tools::withRetries( 3, [this, &mrl]() -> MediaPtr {
            auto t = m_dbConnection->newTransaction();
432
            auto media = Media::create( this, IMedia::Type::External, utils::file::fileName( mrl ) );
433 434 435 436 437 438 439 440 441 442 443 444 445
            if ( media == nullptr )
                return nullptr;
            if ( media->addExternalMrl( mrl, IFile::Type::Main ) == nullptr )
                return nullptr;
            t->commit();
            return media;
        });
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create external media: ", ex.what() );
        return nullptr;
    }
446 447
}

448
Query<IMedia> MediaLibrary::audioFiles( const QueryParameters* params ) const
449
{
450
    return Media::listAll( this, IMedia::Type::Audio, params );
451 452
}

453
Query<IMedia> MediaLibrary::videoFiles( const QueryParameters* params ) const
454
{
455
    return Media::listAll( this, IMedia::Type::Video, params );
456 457
}

458 459 460 461 462 463 464 465
bool MediaLibrary::isExtensionSupported( const char* ext )
{
    return std::binary_search( std::begin( supportedExtensions ),
        std::end( supportedExtensions ), ext, [](const char* l, const char* r) {
            return strcasecmp( l, r ) < 0;
        });
}

466 467
void MediaLibrary::addDiscoveredFile( std::shared_ptr<fs::IFile> fileFs,
                                      std::shared_ptr<Folder> parentFolder,
468 469
                                      std::shared_ptr<fs::IDirectory> parentFolderFs,
                                      std::pair<std::shared_ptr<Playlist>, unsigned int> parentPlaylist )
470
{
471 472
    try
    {
473 474 475 476 477 478 479 480 481 482 483 484 485 486
        std::shared_ptr<parser::Task> task;
        if ( parentPlaylist.first == nullptr )
        {
            // Sqlite won't ensure uniqueness for Task with the same (mrl, parent_playlist_id)
            // when parent_playlist_id is null, so we have to ensure of it ourselves
            const std::string req = "SELECT * FROM " + policy::TaskTable::Name + " "
                    "WHERE mrl = ? AND parent_playlist_id IS NULL";
            task = parser::Task::fetch( this, req, fileFs->mrl() );
            if ( task != nullptr )
            {
                LOG_INFO( "Not creating duplicated task for mrl: ", fileFs->mrl() );
                return;
            }
        }
487
        // Don't move the file as we might need it for error handling
488
        task = parser::Task::create( this, fileFs, std::move( parentFolder ),
489 490 491 492 493 494 495 496 497 498 499
                                          std::move( parentFolderFs ), std::move( parentPlaylist ) );
        if ( task != nullptr && m_parser != nullptr )
            m_parser->parse( task );
    }
    catch(sqlite::errors::ConstraintViolation& ex)
    {
        // Most likely the file is already scheduled and we restarted the
        // discovery after a crash.
        LOG_WARN( "Failed to insert ", fileFs->mrl(), ": ", ex.what(), ". "
                  "Assuming the file is already scheduled for discovery" );
    }
500 501
}

502
bool MediaLibrary::deleteFolder( const Folder& folder )
503
{
504
    LOG_INFO( "deleting folder ", folder.mrl() );
505
    if ( Folder::destroy( this, folder.id() ) == false )
506
        return false;
507
    Media::clear();
508
    return true;
509 510
}

511 512
LabelPtr MediaLibrary::createLabel( const std::string& label )
{
513 514 515 516 517 518 519 520 521
    try
    {
        return Label::create( this, label );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create a label: ", ex.what() );
        return nullptr;
    }
522
}
523 524 525

bool MediaLibrary::deleteLabel( LabelPtr label )
{
526 527 528 529 530 531 532 533 534
    try
    {
        return Label::destroy( this, label->id() );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to delete label: ", ex.what() );
        return false;
    }
535
}
536

537
AlbumPtr MediaLibrary::album( int64_t id ) const
538
{
539
    return Album::fetch( this, id );
540 541
}

542
std::shared_ptr<Album> MediaLibrary::createAlbum( const std::string& title, int64_t thumbnailId )
543
{
544
    return Album::create( this, title, thumbnailId );
545 546
}

547
Query<IAlbum> MediaLibrary::albums( const QueryParameters* params ) const
548
{
549
    return Album::listAll( this, params );
550 551
}

552
Query<IGenre> MediaLibrary::genres( const QueryParameters* params ) const
553
{
554
    return Genre::listAll( this, params );
555 556
}

557
GenrePtr MediaLibrary::genre( int64_t id ) const
558 559 560 561
{
    return Genre::fetch( this, id );
}

562 563 564 565 566
ShowPtr MediaLibrary::show( int64_t id ) const
{
    return Show::fetch( this, id );
}

567
ShowPtr MediaLibrary::show( const std::string& name ) const
568 569 570
{
    static const std::string req = "SELECT * FROM " + policy::ShowTable::Name
            + " WHERE name = ?";
571
    return Show::fetch( this, req, name );
572 573
}

574
std::shared_ptr<Show> MediaLibrary::createShow( const std::string& name )
575
{
576
    return Show::create( this, name );
577 578
}

579 580 581 582 583
MoviePtr MediaLibrary::movie( int64_t id ) const
{
    return Movie::fetch( this, id );
}

584
MoviePtr MediaLibrary::movie( const std::string& title ) const
585 586 587
{
    static const std::string req = "SELECT * FROM " + policy::MovieTable::Name
            + " WHERE title = ?";
588
    return Movie::fetch( this, req, title );
589 590
}

591
std::shared_ptr<Movie> MediaLibrary::createMovie( Media& media, const std::string& title )
592
{
593
    auto movie = Movie::create( this, media.id(), title );
594
    media.setMovie( movie );
595
    media.save();
596
    return movie;
597 598
}

599
ArtistPtr MediaLibrary::artist( int64_t id ) const
600
{
601
    return Artist::fetch( this, id );
602 603 604
}

ArtistPtr MediaLibrary::artist( const std::string& name )
605 606
{
    static const std::string req = "SELECT * FROM " + policy::ArtistTable::Name
607
            + " WHERE name = ? AND is_present != 0";
608
    return Artist::fetch( this, req, name );
609 610
}

611
std::shared_ptr<Artist> MediaLibrary::createArtist( const std::string& name )
612
{
613 614
    try
    {
615
        return Artist::create( this, name );
616 617 618 619 620 621
    }
    catch( sqlite::errors::ConstraintViolation &ex )
    {
        LOG_WARN( "ContraintViolation while creating an artist (", ex.what(), ") attempting to fetch it instead" );
        return std::static_pointer_cast<Artist>( artist( name ) );
    }
622 623
}

624
Query<IArtist> MediaLibrary::artists( bool includeAll, const QueryParameters* params ) const
625
{
626
    return Artist::listAll( this, includeAll, params );
627 628
}

629 630
PlaylistPtr MediaLibrary::createPlaylist( const std::string& name )
{
631 632 633 634 635 636 637 638 639
    try
    {
        return Playlist::create( this, name );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create a playlist: ", ex.what() );
        return nullptr;
    }
640 641
}

642
Query<IPlaylist> MediaLibrary::playlists( const QueryParameters* params )
643
{
644
    return Playlist::listAll( this, params );
645 646
}

647 648 649 650 651
PlaylistPtr MediaLibrary::playlist( int64_t id ) const
{
    return Playlist::fetch( this, id );
}

652
bool MediaLibrary::deletePlaylist( int64_t playlistId )
653
{
654 655 656 657 658 659 660 661 662
    try
    {
        return Playlist::destroy( this, playlistId );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to delete playlist: ", ex.what() );
        return false;
    }
663
}
664

665
bool MediaLibrary::addToStreamHistory( MediaPtr media )
666
{
667 668 669 670 671 672 673 674 675
    try
    {
        return History::insert( getConn(), media->id() );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to add stream to history: ", ex.what() );
        return false;
    }
676 677
}

678
Query<IHistoryEntry> MediaLibrary::lastStreamsPlayed() const
679 680 681 682
{
    return History::fetch( this );
}

683
Query<IMedia> MediaLibrary::lastMediaPlayed() const
684
{
685
    return Media::fetchHistory( this );
686 687
}

688 689
bool MediaLibrary::clearHistory()
{
690 691 692 693 694
    try
    {
        return sqlite::Tools::withRetries( 3, [this]() {
            auto t = getConn()->newTransaction();
            Media::clearHistory( this );
695
            History::clearStreams( this );
696 697 698 699 700 701 702 703 704
            t->commit();
            return true;
        });
    }
    catch ( sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to clear history: ", ex.what() );
        return false;
    }
705 706
}

707
MediaSearchAggregate MediaLibrary::searchMedia( const std::string& title,
708
                                                const QueryParameters* params ) const
709
{
710 711
    if ( validateSearchPattern( title ) == false )
        return {};
712
    MediaSearchAggregate res;
713
    res.episodes = Media::search( this, title, IMedia::SubType::ShowEpisode,
714
                                params );
715
    res.movies = Media::search( this, title, IMedia::SubType::Movie,
716
                                params );
717
    res.others = Media::search( this, title, IMedia::SubType::Unknown,
718
                                params );
719
    res.tracks = Media::search( this, title, IMedia::SubType::AlbumTrack,
720
                                params );
721
    return res;
722 723
}

724
Query<IPlaylist> MediaLibrary::searchPlaylists( const std::string& name,
725
                                                const QueryParameters* params ) const
726
{
727 728
    if ( validateSearchPattern( name ) == false )
        return {};
729
    return Playlist::search( this, name, params );
730 731
}

732
Query<IAlbum> MediaLibrary::searchAlbums( const std::string& pattern,
733
                                          const QueryParameters* params ) const
734
{
735 736
    if ( validateSearchPattern( pattern ) == false )
        return {};
737
    return Album::search( this, pattern, params );
738 739
}

740 741
Query<IGenre> MediaLibrary::searchGenre( const std::string& genre,
                                         const QueryParameters* params ) const
742
{
743 744
    if ( validateSearchPattern( genre ) == false )
        return {};
745
    return Genre::search( this, genre, params );
746 747
}

748
Query<IArtist> MediaLibrary::searchArtists( const std::string& name,
749
                                            const QueryParameters* params ) const
750 751 752
{
    if ( validateSearchPattern( name ) == false )
        return {};
753
    return Artist::search( this, name, params );
754 755
}

756
SearchAggregate MediaLibrary::search( const std::string& pattern,
757
                                      const QueryParameters* params ) const
758
{
759
    SearchAggregate res;
760 761 762 763 764
    res.albums = searchAlbums( pattern, params );
    res.artists = searchArtists( pattern, params );
    res.genres = searchGenre( pattern, params );
    res.media = searchMedia( pattern, params );
    res.playlists = searchPlaylists( pattern, params );
765 766 767
    return res;
}

768
bool MediaLibrary::startParser()
769
{
770
    m_parser.reset( new parser::Parser( this ) );
771

772 773
    if ( m_services.size() == 0 )
    {
774
#ifdef HAVE_LIBVLC
775
        m_parser->addService( std::make_shared<parser::VLCMetadataService>() );
776 777 778
#else
        return false;
#endif
779 780 781 782 783 784 785
    }
    else
    {
        assert( m_services[0]->targetedStep() == parser::Step::MetadataExtraction );
        m_parser->addService( m_services[0] );
    }
    m_parser->addService( std::make_shared<parser::MetadataAnalyzer>() );
786
    m_parser->start();
787
    return true;
788 789
}

790
void MediaLibrary::startDiscoverer()
791
{
792
    m_discovererWorker.reset( new DiscovererWorker( this ) );
793
    for ( const auto& fsFactory : m_fsFactories )
794
    {
795
        std::unique_ptr<prober::CrawlerProbe> probePtr( new prober::CrawlerProbe{} );
796 797 798
        m_discovererWorker->addDiscoverer( std::unique_ptr<IDiscoverer>( new FsDiscoverer( fsFactory, this, m_callback,
                                                                                           std::move ( probePtr ) ) ) );
    }
799 800
}

801 802
void MediaLibrary::startDeletionNotifier()
{
803 804
    m_modificationNotifier.reset( new ModificationNotifier( this ) );
    m_modificationNotifier->start();
805 806
}

807
void MediaLibrary::startThumbnailer()
808
{
809
#ifdef HAVE_LIBVLC
810
    m_thumbnailer = std::unique_ptr<VLCThumbnailer>( new VLCThumbnailer( this ) );
811
#endif
812 813
}

814 815 816 817 818 819 820
void MediaLibrary::populateFsFactories()
{
#ifdef HAVE_LIBVLC
    m_externalFsFactories.emplace_back( std::make_shared<factory::NetworkFileSystemFactory>( "smb", "dsm-sd" ) );
#endif
}

821 822
void MediaLibrary::addLocalFsFactory()
{
823
    m_fsFactories.emplace( begin( m_fsFactories ), std::make_shared<factory::FileSystemFactory>( m_deviceLister ) );
824 825
}

826
InitializeResult MediaLibrary::updateDatabaseModel( unsigned int previousVersion,
827
                                        const std::string& dbPath )
828
{
829 830 831
    LOG_INFO( "Updating database model from ", previousVersion, " to ", Settings::DbModelVersion );
    // Up until model 3, it's safer (and potentially more efficient with index changes) to drop the DB
    // It's also way simpler to implement
832
    // In case of downgrade, just recreate the database
833
    for ( auto i = 0u; i < 3; ++i )
834
    {
835 836
        try
        {
837
            bool needRescan = false;
838 839 840
            // Up until model 3, it's safer (and potentially more efficient with index changes) to drop the DB
            // It's also way simpler to implement
            // In case of downgrade, just recreate the database
841 842
            // We might also have some special cases for failed upgrade (see
            // comments below for per-version details)
843
            if ( previousVersion < 3 ||
844 845
                 previousVersion > Settings::DbModelVersion ||
                 previousVersion == 4 )
846
            {
847
                if( recreateDatabase( dbPath ) == false )
848
                    throw std::runtime_error( "Failed to recreate the database" );
849
                return InitializeResult::DbReset;
850
            }
851 852 853 854 855 856 857 858
            /**
             * Migration from 3 to 4 didn't happen so well and broke a few
             * users DB. So:
             * - Any v4 database will be dropped and recreated in v5
             * - Any v3 database will be upgraded to v5
             * V4 database is only used by VLC-android 2.5.6 // 2.5.8, which are
             * beta versions.
             */
859 860
            if ( previousVersion == 3 )
            {
861
                migrateModel3to5();
862
                previousVersion = 5;
863
            }
864 865
            if ( previousVersion == 5 )
            {
866
                migrateModel5to6();
867 868
                previousVersion = 6;
            }
Alexandre Fernandez's avatar
Alexandre Fernandez committed
869 870
            if ( previousVersion == 6 )
            {
871 872 873
                // Force a rescan to solve metadata analysis problems.
                // The insertion is fixed, but won't edit already inserted data.
                forceRescan();
Alexandre Fernandez's avatar
Alexandre Fernandez committed
874 875
                previousVersion = 7;
            }
876 877 878 879 880 881 882 883 884
            /**
             * V7 introduces artist.nb_tracks and an associated trigger to delete
             * artists when it has no track/album left.
             */
            if ( previousVersion == 7 )
            {
                migrateModel7to8();
                previousVersion = 8;
            }
885 886
            if ( previousVersion == 8 )
            {
887 888 889 890 891 892 893 894 895

                // Multiple changes justify the rescan:
                // - Changes in the way we chose to encode or not MRL, meaning
                //   potentially all MRL are wrong (more precisely, will
                //   mismatch what VLC expects, which makes playlist analysis
                //   break.
                // - Fix in the way we chose album candidates, meaning some
                //   albums were likely to be wrongfully created.
                needRescan = true;
896
                migrateModel8to9();
897 898
                previousVersion = 9;
            }
899 900
            if ( previousVersion == 9 )
            {
901
                needRescan = true;
902 903 904
                migrateModel9to10();
                previousVersion = 10;
            }
905 906
            if ( previousVersion == 10 )
            {
907
                needRescan = true;
908 909 910
                migrateModel10to11();
                previousVersion = 11;
            }
911 912 913 914 915
            if ( previousVersion == 11 )
            {
                parser::Task::recoverUnscannedFiles( this );
                previousVersion = 12;
            }
916 917 918 919 920
            if ( previousVersion == 12 )
            {
                migrateModel12to13();
                previousVersion = 13;
            }
921 922
            if ( previousVersion == 13 )
            {
923 924 925
                // We need to recreate many thumbnail records, and hopefully
                // generate better ones
                needRescan = true;
926 927 928
                migrateModel13to14();
                previousVersion = 14;
            }
929 930
            // To be continued in the future!

931 932 933
            if ( needRescan == true )
                forceRescan();

934 935 936
            // Safety check: ensure we didn't forget a migration along the way
            assert( previousVersion == Settings::DbModelVersion );
            m_settings.setDbModelVersion( Settings::DbModelVersion );
937
            if ( m_settings.save() == false )
938 939
                return InitializeResult::Failed;
            return InitializeResult::Success;
940 941 942 943 944 945 946 947 948 949 950
        }
        catch( const std::exception& ex )
        {
            LOG_ERROR( "An error occured during the database upgrade: ",
                       ex.what() );
        }
        catch( ... )
        {
            LOG_ERROR( "An unknown error occured during the database upgrade." );
        }
        LOG_WARN( "Retrying database migration, attempt ", i + 1, " / 3" );
951
    }
952 953
    LOG_ERROR( "Failed to upgrade database, recreating it" );
    for ( auto i = 0u; i < 3; ++i )
954
    {
955 956
        try
        {
957
            if( recreateDatabase( dbPath ) == true )
958
                return InitializeResult::DbReset;
959 960 961 962 963 964 965 966 967 968
        }
        catch( const std::exception& ex )
        {
            LOG_ERROR( "Failed to recreate database: ", ex.what() );
        }
        catch(...)
        {
            LOG_ERROR( "Unknown error while trying to recreate the database." );
        }
        LOG_WARN( "Retrying to recreate the database, attempt ", i + 1, " / 3" );
969
    }
970
    return InitializeResult::Failed;
971 972
}

973
bool MediaLibrary::recreateDatabase( const std::string& dbPath )
974
{
975 976 977 978
    // Close all active connections, flushes all previously run statements.
    m_dbConnection.reset();
    unlink( dbPath.c_str() );
    m_dbConnection = sqlite::Connection::connect( dbPath );
979
    createAllTables();
980 981 982
    // We dropped the database, there is no setting to be read anymore
    if( m_settings.load() == false )
        return false;
983 984 985
    return true;
}

986
void MediaLibrary::migrateModel3to5()
987 988
{
    /*
989 990
     * Disable Foreign Keys & recursive triggers to avoid cascading deletion
     * while remodeling the database into the transaction.
991
     */
992
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
993
    auto t = getConn()->newTransaction();
994 995 996 997
    using namespace policy;
    // As SQLite do not allow us to remove or add some constraints,
    // we use the method described here https://www.sqlite.org/faq.html#q11
    std::string reqs[] = {
998
#               include "database/migrations/migration3-5.sql"
999 1000 1001
    };

    for ( const auto& req : reqs )
1002
        sqlite::Tools::executeRequest( getConn(), req );
1003
    // Re-create triggers removed in the process
1004 1005
    Media::createTriggers( getConn() );
    Playlist::createTriggers( getConn() );
1006 1007 1008
    t->commit();
}

1009
void MediaLibrary::migrateModel5to6()
1010
{
1011 1012 1013
    std::string req = "DELETE FROM " + policy::MediaTable::Name + " WHERE type = ?";
    sqlite::Tools::executeRequest( getConn(), req, Media::Type::Unknown );

1014 1015
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
    using namespace policy;
1016
    req = "UPDATE " + MediaTable::Name + " SET is_present = 1 WHERE is_present != 0";
1017
    sqlite::Tools::executeRequest( getConn(), req );
Alexandre Fernandez's avatar
Alexandre Fernandez committed
1018 1019
}

1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
void MediaLibrary::migrateModel7to8()
{
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
    auto t = getConn()->newTransaction();
    using namespace policy;
    std::string reqs[] = {
#               include "database/migrations/migration7-8.sql"
    };

    for ( const auto& req : reqs )
        sqlite::Tools::executeRequest( getConn(), req );
    // Re-create triggers removed in the process
1032
    Artist::createTriggers( getConn(), 8u );
1033 1034
    Media::createTriggers( getConn() );
    File::createTriggers( getConn() );
1035 1036 1037
    t->commit();
}

1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054
void MediaLibrary::migrateModel8to9()
{
    // A bug in a previous migration caused our triggers to be missing for the
    // first application run (after the migration).
    // This could have caused media associated to deleted files not to be
    // deleted as well, so let's do that now.
    const std::string req = "DELETE FROM " + policy::MediaTable::Name + " "
            "WHERE id_media IN "
            "(SELECT id_media FROM " + policy::MediaTable::Name + " m LEFT JOIN " +
                policy::FileTable::Name + " f ON f.media_id = m.id_media "
                "WHERE f.media_id IS NULL)";

    // Don't check for the return value, we don't mind if nothing deleted.
    // Quite the opposite actually :)
    sqlite::Tools::executeDelete( getConn(), req );
}

1055 1056 1057 1058 1059 1060 1061 1062
void MediaLibrary::migrateModel9to10()
{
    const std::string req = "SELECT * FROM " + policy::FileTable::Name +
            " WHERE mrl LIKE '%#%%' ESCAPE '#'";
    auto files = File::fetchAll<File>( this, req );
    auto t = getConn()->newTransaction();
    for ( const auto& f : files )
    {
1063 1064 1065 1066
        // We must not call mrl() from here. We might not have all devices yet,
        // and calling mrl would crash for files stored on removable devices.
        auto newMrl = utils::url::encode( utils::url::decode( f->rawMrl() ) );
        LOG_INFO( "Converting ", f->rawMrl(), " to ", newMrl );
1067 1068 1069 1070 1071
        f->setMrl( newMrl );
    }
    t->commit();
}

1072 1073 1074 1075
void MediaLibrary::migrateModel10to11()
{
    const std::string req = "SELECT * FROM " + policy::TaskTable::Name +
            " WHERE mrl LIKE '%#%%' ESCAPE '#'";
1076 1077
    const std::string folderReq = "SELECT * FROM " + policy::FolderTable::Name +
            " WHERE path LIKE '%#%%' ESCAPE '#'";
1078
    auto tasks = parser::Task::fetchAll<parser::Task>( this, req );
1079
    auto folders = Folder::fetchAll<Folder>( this, folderReq );
1080 1081 1082
    auto t = getConn()->newTransaction();
    for ( const auto& t : tasks )
    {
1083 1084
        auto newMrl = utils::url::encode( utils::url::decode( t->item().mrl() ) );
        LOG_INFO( "Converting task mrl: ", t->item().mrl(), " to ", newMrl );
1085 1086
        t->setMrl( std::move( newMrl ) );
    }
1087 1088 1089 1090 1091 1092 1093
    for ( const auto &f : folders )
    {
        // We must not call mrl() from here. We might not have all devices yet,
        // and calling mrl would crash for files stored on removable devices.
        auto newMrl = utils::url::encode( utils::url::decode( f->rawMrl() ) );
        f->setMrl( std::move( newMrl ) );
    }
1094 1095 1096
    t->commit();
}

1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108
/*
 * - Some is_present related triggers were fixed in model 6 to 7 migration, but
 *   they were not recreated if already existing. The has_file_present trigger
 *   was recreated as part of model 7 to 8 migration, but we need to ensure
 *   has_album_present (Artist) & is_album_present (Album) triggers are
 *   recreated to behave as expected
 * - Due to a typo, is_track_present was named is_track_presentAFTER, and was
 *   executed BEFORE the update took place, thus using the wrong is_present value.
 *   The trigger will be recreated as part of this migration, and the values
 *   will be enforced, causing the entire update chain to be triggered, and
 *   restoring correct is_present values for all AlbumTrack/Album/Artist entries
 */
1109 1110
void MediaLibrary::migrateModel12to13()
{
1111
    auto t = getConn()->newTransaction();
1112 1113 1114 1115 1116 1117 1118 1119 1120
    const std::string reqs[] = {
        "DROP TRIGGER IF EXISTS is_track_presentAFTER",
        "DROP TRIGGER has_album_present",
        "DROP TRIGGER is_album_present",
    };

    for ( const auto& req : reqs )
        sqlite::Tools::executeDelete( getConn(), req );

1121
    AlbumTrack::createTriggers( getConn() );
1122 1123
    Album::createTriggers( getConn() );
    Artist::createTriggers( getConn(), 13 );
1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139
    // Leave the weak context as we now need to update is_present fields, which
    // are propagated through recursive triggers
    const std::string migrateData = "UPDATE " + policy::AlbumTrackTable::Name +
            " SET is_present = (SELECT is_present FROM " + policy::MediaTable::Name +
            " WHERE id_media = media_id)";
    sqlite::Tools::executeUpdate( getConn(), migrateData );
    t->commit();
}

/*
 * - Remove the Media.thumbnail
 * - Add Media.thumbnail_id
 * - Add Media.thumbnail_generated
 */
void MediaLibrary::migrateModel13to14()
{
1140 1141 1142
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
    auto t = getConn()->newTransaction();
    using namespace policy;
1143
    using ThumbnailType = typename std::underlying_type<Thumbnail::Origin>::type;
1144
    std::string reqs[] = {
1145
#               include "database/migrations/migration13-14.sql"
1146 1147 1148 1149
    };

    for ( const auto& req : reqs )
        sqlite::Tools::executeRequest( getConn(), req );
1150 1151 1152
    // Re-create tables that we just removed
    // We will run a re-scan, so we don't care about keeping their content
    Album::createTable( getConn() );
1153
    Artist::createTable( getConn() );
1154 1155 1156
    // Re-create triggers removed in the process
    Media::createTriggers( getConn() );
    AlbumTrack::createTriggers( getConn() );
1157 1158
    Album::createTriggers( getConn() );
    Artist::createTriggers( getConn(), 14 );
1159 1160 1161
    t->commit();
}

1162 1163
void MediaLibrary::reload()
{
1164 1165
    if ( m_discovererWorker != nullptr )
        m_discovererWorker->reload();
1166 1167
}

1168 1169
void MediaLibrary::reload( const std::string& entryPoint )
{