MediaLibrary.cpp 45.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2015 Hugo Beauzée-Luyssen, Videolabs
 *
 * Authors: Hugo Beauzée-Luyssen<hugo@beauzee.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

23 24 25 26
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

27 28
#include <algorithm>
#include <functional>
29
#include <utility>
30
#include <sys/stat.h>
31
#include <unistd.h>
32

33 34
#include "Album.h"
#include "AlbumTrack.h"
35
#include "Artist.h"
36
#include "AudioTrack.h"
37
#include "discoverer/DiscovererWorker.h"
38
#include "discoverer/probe/CrawlerProbe.h"
39
#include "utils/ModificationsNotifier.h"
40
#include "Device.h"
41
#include "File.h"
42
#include "Folder.h"
43
#include "Genre.h"
44
#include "History.h"
45
#include "Media.h"
46
#include "MediaLibrary.h"
47
#include "Label.h"
48
#include "logging/Logger.h"
49
#include "Movie.h"
50
#include "parser/Parser.h"
51
#include "Playlist.h"
52 53
#include "Show.h"
#include "ShowEpisode.h"
54
#include "Thumbnail.h"
55
#include "database/SqliteTools.h"
56
#include "database/SqliteConnection.h"
57
#include "database/SqliteQuery.h"
58
#include "utils/Filename.h"
59
#include "utils/Url.h"
60
#include "VideoTrack.h"
61

62 63 64
// Discoverers:
#include "discoverer/FsDiscoverer.h"

65 66 67
// Metadata services:
#include "metadata_services/vlc/VLCMetadataService.h"
#include "metadata_services/vlc/VLCThumbnailer.h"
68
#include "metadata_services/MetadataParser.h"
69

70 71
// FileSystem
#include "factory/DeviceListerFactory.h"
72
#include "factory/FileSystemFactory.h"
73
#include "factory/NetworkFileSystemFactory.h"
74
#include "medialibrary/filesystem/IDevice.h"
75

76 77 78
namespace medialibrary
{

79
const char* const MediaLibrary::supportedExtensions[] = {
80 81
    "3gp", "a52", "aac", "ac3", "aif", "aifc", "aiff", "alac", "amr",
    "amv", "aob", "ape", "asf", "asx", "avi", "b4s", "conf", /*"cue",*/
82 83 84 85 86 87 88 89 90 91
    "divx", "dts", "dv", "flac", "flv", "gxf", "ifo", "iso",
    "it", "itml",  "m1v", "m2t", "m2ts", "m2v", "m3u", "m3u8",
    "m4a", "m4b", "m4p", "m4v", "mid", "mka", "mkv", "mlp",
    "mod", "mov", "mp1", "mp2", "mp3", "mp4", "mpc", "mpeg",
    "mpeg1", "mpeg2", "mpeg4", "mpg", "mts", "mxf", "nsv",
    "nuv", "oga", "ogg", "ogm", "ogv", "ogx", "oma", "opus",
    "pls", "ps", "qtl", "ram", "rec", "rm", "rmi", "rmvb",
    "s3m", "sdp", "spx", "tod", "trp", "ts", "tta", "vlc",
    "vob", "voc", "vqf", "vro", "w64", "wav", "wax", "webm",
    "wma", "wmv", "wmx", "wpl", "wv", "wvx", "xa", "xm", "xspf"
92 93
};

94 95
const size_t MediaLibrary::NbSupportedExtensions = sizeof(supportedExtensions) / sizeof(supportedExtensions[0]);

96
MediaLibrary::MediaLibrary()
97 98
    : m_callback( nullptr )
    , m_verbosity( LogLevel::Error )
99
    , m_settings( this )
100
    , m_initialized( false )
101 102
    , m_discovererIdle( true )
    , m_parserIdle( true )
103
{
104
    Log::setLogLevel( m_verbosity );
105 106
}

107 108
MediaLibrary::~MediaLibrary()
{
109
    // Explicitely stop the discoverer, to avoid it writting while tearing down.
110 111
    if ( m_discovererWorker != nullptr )
        m_discovererWorker->stop();
112 113
    if ( m_parser != nullptr )
        m_parser->stop();
114 115 116 117 118
    clearCache();
}

void MediaLibrary::clearCache()
{
119
    Media::clear();
120
    Folder::clear();
121 122 123 124 125
    Label::clear();
    Album::clear();
    AlbumTrack::clear();
    Show::clear();
    ShowEpisode::clear();
126
    Movie::clear();
127
    VideoTrack::clear();
128
    AudioTrack::clear();
129
    Artist::clear();
130
    Device::clear();
131
    File::clear();
132
    Playlist::clear();
133
    History::clear();
134
    Genre::clear();
135
    Thumbnail::clear();
136 137
}

138
void MediaLibrary::createAllTables()
139
{
140 141 142 143 144
    // We need to create the tables in order of triggers creation
    // Device is the "root of all evil". When a device is modified,
    // we will trigger an update on folder, which will trigger
    // an update on files, and so on.

145 146
    Device::createTable( m_dbConnection.get() );
    Folder::createTable( m_dbConnection.get() );
147
    Thumbnail::createTable( m_dbConnection.get() );
148 149 150 151 152 153 154 155 156 157 158 159 160 161
    Media::createTable( m_dbConnection.get() );
    File::createTable( m_dbConnection.get() );
    Label::createTable( m_dbConnection.get() );
    Playlist::createTable( m_dbConnection.get() );
    Genre::createTable( m_dbConnection.get() );
    Album::createTable( m_dbConnection.get() );
    AlbumTrack::createTable( m_dbConnection.get() );
    Show::createTable( m_dbConnection.get() );
    ShowEpisode::createTable( m_dbConnection.get() );
    Movie::createTable( m_dbConnection.get() );
    VideoTrack::createTable( m_dbConnection.get() );
    AudioTrack::createTable( m_dbConnection.get() );
    Artist::createTable( m_dbConnection.get() );
    Artist::createDefaultArtists( m_dbConnection.get() );
162 163
    History::createTable( m_dbConnection.get() );
    Settings::createTable( m_dbConnection.get() );
164
    parser::Task::createTable( m_dbConnection.get() );
165 166 167 168
}

void MediaLibrary::createAllTriggers()
{
169
    auto dbModelVersion = m_settings.dbModelVersion();
170
    Folder::createTriggers( m_dbConnection.get() );
171
    Album::createTriggers( m_dbConnection.get() );
172
    AlbumTrack::createTriggers( m_dbConnection.get() );
173
    Artist::createTriggers( m_dbConnection.get(), dbModelVersion );
174
    Media::createTriggers( m_dbConnection.get() );
175
    File::createTriggers( m_dbConnection.get() );
176 177
    Genre::createTriggers( m_dbConnection.get() );
    Playlist::createTriggers( m_dbConnection.get() );
178
    History::createTriggers( m_dbConnection.get() );
179
    Label::createTriggers( m_dbConnection.get() );
180 181
}

182
template <typename T>
183
static void propagateDeletionToCache( sqlite::Connection::HookReason reason, int64_t rowId )
184
{
185
    if ( reason != sqlite::Connection::HookReason::Delete )
186 187 188 189
        return;
    T::removeFromCache( rowId );
}

190 191
void MediaLibrary::registerEntityHooks()
{
192
    if ( m_modificationNotifier == nullptr )
193 194
        return;

195
    m_dbConnection->registerUpdateHook( policy::MediaTable::Name,
196 197
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
198 199
            return;
        Media::removeFromCache( rowId );
200
        m_modificationNotifier->notifyMediaRemoval( rowId );
201
    });
202
    m_dbConnection->registerUpdateHook( policy::ArtistTable::Name,
203 204
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
205 206 207 208
            return;
        Artist::removeFromCache( rowId );
        m_modificationNotifier->notifyArtistRemoval( rowId );
    });
209
    m_dbConnection->registerUpdateHook( policy::AlbumTable::Name,
210 211
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
212 213 214 215
            return;
        Album::removeFromCache( rowId );
        m_modificationNotifier->notifyAlbumRemoval( rowId );
    });
216
    m_dbConnection->registerUpdateHook( policy::AlbumTrackTable::Name,
217 218
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
219 220 221 222
            return;
        AlbumTrack::removeFromCache( rowId );
        m_modificationNotifier->notifyAlbumTrackRemoval( rowId );
    });
223
    m_dbConnection->registerUpdateHook( policy::PlaylistTable::Name,
224 225
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
226 227 228 229
            return;
        Playlist::removeFromCache( rowId );
        m_modificationNotifier->notifyPlaylistRemoval( rowId );
    });
230 231 232 233 234 235 236 237 238 239
    m_dbConnection->registerUpdateHook( policy::DeviceTable::Name, &propagateDeletionToCache<Device> );
    m_dbConnection->registerUpdateHook( policy::FileTable::Name, &propagateDeletionToCache<File> );
    m_dbConnection->registerUpdateHook( policy::FolderTable::Name, &propagateDeletionToCache<Folder> );
    m_dbConnection->registerUpdateHook( policy::GenreTable::Name, &propagateDeletionToCache<Genre> );
    m_dbConnection->registerUpdateHook( policy::LabelTable::Name, &propagateDeletionToCache<Label> );
    m_dbConnection->registerUpdateHook( policy::MovieTable::Name, &propagateDeletionToCache<Movie> );
    m_dbConnection->registerUpdateHook( policy::ShowTable::Name, &propagateDeletionToCache<Show> );
    m_dbConnection->registerUpdateHook( policy::ShowEpisodeTable::Name, &propagateDeletionToCache<ShowEpisode> );
    m_dbConnection->registerUpdateHook( policy::AudioTrackTable::Name, &propagateDeletionToCache<AudioTrack> );
    m_dbConnection->registerUpdateHook( policy::VideoTrackTable::Name, &propagateDeletionToCache<VideoTrack> );
240 241
}

242 243 244 245 246
bool MediaLibrary::validateSearchPattern( const std::string& pattern )
{
    return pattern.size() >= 3;
}

247 248 249
InitializeResult MediaLibrary::initialize( const std::string& dbPath,
                                           const std::string& thumbnailPath,
                                           IMediaLibraryCb* mlCallback )
250
{
251
    LOG_INFO( "Initializing medialibrary..." );
252
    if ( m_initialized == true )
253 254
    {
        LOG_INFO( "...Already initialized" );
255
        return InitializeResult::AlreadyInitialized;
256
    }
257 258 259 260
    if ( m_deviceLister == nullptr )
    {
        m_deviceLister = factory::createDeviceLister();
        if ( m_deviceLister == nullptr )
261 262
        {
            LOG_ERROR( "No available IDeviceLister was found." );
263
            return InitializeResult::Failed;
264
        }
265
    }
266
    addLocalFsFactory();
267 268 269
#ifdef _WIN32
    if ( mkdir( thumbnailPath.c_str() ) != 0 )
#else
270
    if ( mkdir( thumbnailPath.c_str(), S_IRWXU ) != 0 )
271
#endif
272 273
    {
        if ( errno != EEXIST )
274 275
        {
            LOG_ERROR( "Failed to create thumbnail directory: ", strerror( errno ) );
276
            return InitializeResult::Failed;
277
        }
278
    }
279
    m_thumbnailPath = thumbnailPath;
280
    m_callback = mlCallback;
281
    m_dbConnection = sqlite::Connection::connect( dbPath );
282

283 284 285
    // Give a chance to test overloads to reject the creation of a notifier
    startDeletionNotifier();
    // Which allows us to register hooks, or not, depending on the presence of a notifier
286 287
    registerEntityHooks();

288
    auto res = InitializeResult::Success;
289
    try
290
    {
291
        auto t = m_dbConnection->newTransaction();
292
        createAllTables();
293
        if ( m_settings.load() == false )
294 295
        {
            LOG_ERROR( "Failed to load settings" );
296
            return InitializeResult::Failed;
297
        }
298 299 300
        createAllTriggers();
        t->commit();

301 302
        if ( m_settings.dbModelVersion() != Settings::DbModelVersion )
        {
303 304
            res = updateDatabaseModel( m_settings.dbModelVersion(), dbPath );
            if ( res == InitializeResult::Failed )
305 306
            {
                LOG_ERROR( "Failed to update database model" );
307
                return res;
308 309 310 311 312 313
            }
        }
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Can't initialize medialibrary: ", ex.what() );
314
        return InitializeResult::Failed;
315
    }
316 317
    m_initialized = true;
    LOG_INFO( "Successfuly initialized" );
318
    return res;
319 320 321 322
}

bool MediaLibrary::start()
{
323
    assert( m_initialized == true );
324 325 326
    if ( m_parser != nullptr )
        return false;

327 328
    for ( auto& fsFactory : m_fsFactories )
        refreshDevices( *fsFactory );
329
    startDiscoverer();
330 331
    if ( startParser() == false )
        return false;
332 333
    if ( startThumbnailer() == false )
        return false;
334
    return true;
335 336
}

337
void MediaLibrary::setVerbosity( LogLevel v )
338 339 340 341 342
{
    m_verbosity = v;
    Log::setLogLevel( v );
}

343 344 345 346 347
MediaPtr MediaLibrary::media( int64_t mediaId ) const
{
    return Media::fetch( this, mediaId );
}

348 349
MediaPtr MediaLibrary::media( const std::string& mrl ) const
{
350
    LOG_INFO( "Fetching media from mrl: ", mrl );
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
351
    auto file = File::fromExternalMrl( this, mrl );
352 353 354 355 356
    if ( file != nullptr )
    {
        LOG_INFO( "Found external media: ", mrl );
        return file->media();
    }
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
357
    auto fsFactory = fsFactoryForMrl( mrl );
358
    if ( fsFactory == nullptr )
359
    {
360 361
        LOG_WARN( "Failed to create FS factory for path ", mrl );
        return nullptr;
362
    }
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
363
    auto device = fsFactory->createDeviceFromMrl( mrl );
364 365 366 367 368 369
    if ( device == nullptr )
    {
        LOG_WARN( "Failed to create a device associated with mrl ", mrl );
        return nullptr;
    }
    if ( device->isRemovable() == false )
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
370
        file = File::fromMrl( this, mrl );
371 372
    else
    {
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
373
        auto folder = Folder::fromMrl( this, utils::file::directory( mrl ) );
374 375 376 377 378 379 380 381 382 383 384 385 386 387
        if ( folder == nullptr )
        {
            LOG_WARN( "Failed to find folder containing ", mrl );
            return nullptr;
        }
        if ( folder->isPresent() == false )
        {
            LOG_INFO( "Found a folder containing ", mrl, " but it is not present" );
            return nullptr;
        }
        file = File::fromFileName( this, utils::file::fileName( mrl ), folder->id() );
    }
    if ( file == nullptr )
    {
388
        LOG_WARN( "Failed to fetch file for ", mrl, " (device ", device->uuid(), " was ",
389
                  device->isRemovable() ? "" : "NOT ", "removable)");
390 391 392 393 394
        return nullptr;
    }
    return file->media();
}

395 396
MediaPtr MediaLibrary::addMedia( const std::string& mrl )
{
397 398 399 400
    try
    {
        return sqlite::Tools::withRetries( 3, [this, &mrl]() -> MediaPtr {
            auto t = m_dbConnection->newTransaction();
401
            auto media = Media::create( this, IMedia::Type::External, utils::file::fileName( mrl ) );
402 403 404 405 406 407 408 409 410 411 412 413 414
            if ( media == nullptr )
                return nullptr;
            if ( media->addExternalMrl( mrl, IFile::Type::Main ) == nullptr )
                return nullptr;
            t->commit();
            return media;
        });
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create external media: ", ex.what() );
        return nullptr;
    }
415 416
}

417
Query<IMedia> MediaLibrary::audioFiles( SortingCriteria sort, bool desc ) const
418
{
419
    return Media::listAll( this, IMedia::Type::Audio, sort, desc );
420 421
}

422
Query<IMedia> MediaLibrary::videoFiles( SortingCriteria sort, bool desc ) const
423
{
424
    return Media::listAll( this, IMedia::Type::Video, sort, desc );
425 426
}

427 428 429 430 431 432 433 434
bool MediaLibrary::isExtensionSupported( const char* ext )
{
    return std::binary_search( std::begin( supportedExtensions ),
        std::end( supportedExtensions ), ext, [](const char* l, const char* r) {
            return strcasecmp( l, r ) < 0;
        });
}

435 436
void MediaLibrary::addDiscoveredFile( std::shared_ptr<fs::IFile> fileFs,
                                      std::shared_ptr<Folder> parentFolder,
437 438
                                      std::shared_ptr<fs::IDirectory> parentFolderFs,
                                      std::pair<std::shared_ptr<Playlist>, unsigned int> parentPlaylist )
439
{
440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
    try
    {
        // Don't move the file as we might need it for error handling
        auto task = parser::Task::create( this, fileFs, std::move( parentFolder ),
                                          std::move( parentFolderFs ), std::move( parentPlaylist ) );
        if ( task != nullptr && m_parser != nullptr )
            m_parser->parse( task );
    }
    catch(sqlite::errors::ConstraintViolation& ex)
    {
        // Most likely the file is already scheduled and we restarted the
        // discovery after a crash.
        LOG_WARN( "Failed to insert ", fileFs->mrl(), ": ", ex.what(), ". "
                  "Assuming the file is already scheduled for discovery" );
    }
455 456
}

457
bool MediaLibrary::deleteFolder( const Folder& folder )
458
{
459
    LOG_INFO( "deleting folder ", folder.mrl() );
460
    if ( Folder::destroy( this, folder.id() ) == false )
461
        return false;
462
    Media::clear();
463
    return true;
464 465
}

466 467
LabelPtr MediaLibrary::createLabel( const std::string& label )
{
468 469 470 471 472 473 474 475 476
    try
    {
        return Label::create( this, label );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create a label: ", ex.what() );
        return nullptr;
    }
477
}
478 479 480

bool MediaLibrary::deleteLabel( LabelPtr label )
{
481 482 483 484 485 486 487 488 489
    try
    {
        return Label::destroy( this, label->id() );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to delete label: ", ex.what() );
        return false;
    }
490
}
491

492
AlbumPtr MediaLibrary::album( int64_t id ) const
493
{
494
    return Album::fetch( this, id );
495 496
}

497
std::shared_ptr<Album> MediaLibrary::createAlbum( const std::string& title, int64_t thumbnailId )
498
{
499
    return Album::create( this, title, thumbnailId );
500 501
}

502
Query<IAlbum> MediaLibrary::albums( SortingCriteria sort, bool desc ) const
503
{
504
    return Album::listAll( this, sort, desc );
505 506
}

507
Query<IGenre> MediaLibrary::genres( SortingCriteria sort, bool desc ) const
508
{
509
    return Genre::listAll( this, sort, desc );
510 511
}

512
GenrePtr MediaLibrary::genre( int64_t id ) const
513 514 515 516
{
    return Genre::fetch( this, id );
}

517
ShowPtr MediaLibrary::show( const std::string& name ) const
518 519 520
{
    static const std::string req = "SELECT * FROM " + policy::ShowTable::Name
            + " WHERE name = ?";
521
    return Show::fetch( this, req, name );
522 523
}

524
std::shared_ptr<Show> MediaLibrary::createShow( const std::string& name )
525
{
526
    return Show::create( this, name );
527 528
}

529
MoviePtr MediaLibrary::movie( const std::string& title ) const
530 531 532
{
    static const std::string req = "SELECT * FROM " + policy::MovieTable::Name
            + " WHERE title = ?";
533
    return Movie::fetch( this, req, title );
534 535
}

536
std::shared_ptr<Movie> MediaLibrary::createMovie( Media& media, const std::string& title )
537
{
538
    auto movie = Movie::create( this, media.id(), title );
539
    media.setMovie( movie );
540
    media.save();
541
    return movie;
542 543
}

544
ArtistPtr MediaLibrary::artist( int64_t id ) const
545
{
546
    return Artist::fetch( this, id );
547 548 549
}

ArtistPtr MediaLibrary::artist( const std::string& name )
550 551
{
    static const std::string req = "SELECT * FROM " + policy::ArtistTable::Name
552
            + " WHERE name = ? AND is_present != 0";
553
    return Artist::fetch( this, req, name );
554 555
}

556
std::shared_ptr<Artist> MediaLibrary::createArtist( const std::string& name )
557
{
558 559
    try
    {
560
        return Artist::create( this, name );
561 562 563 564 565 566
    }
    catch( sqlite::errors::ConstraintViolation &ex )
    {
        LOG_WARN( "ContraintViolation while creating an artist (", ex.what(), ") attempting to fetch it instead" );
        return std::static_pointer_cast<Artist>( artist( name ) );
    }
567 568
}

569 570
Query<IArtist> MediaLibrary::artists( bool includeAll, SortingCriteria sort,
                                      bool desc ) const
571
{
572
    return Artist::listAll( this, includeAll, sort, desc );
573 574
}

575 576
PlaylistPtr MediaLibrary::createPlaylist( const std::string& name )
{
577 578 579 580 581 582 583 584 585
    try
    {
        return Playlist::create( this, name );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create a playlist: ", ex.what() );
        return nullptr;
    }
586 587
}

588
Query<IPlaylist> MediaLibrary::playlists( SortingCriteria sort, bool desc )
589
{
590
    return Playlist::listAll( this, sort, desc );
591 592
}

593 594 595 596 597
PlaylistPtr MediaLibrary::playlist( int64_t id ) const
{
    return Playlist::fetch( this, id );
}

598
bool MediaLibrary::deletePlaylist( int64_t playlistId )
599
{
600 601 602 603 604 605 606 607 608
    try
    {
        return Playlist::destroy( this, playlistId );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to delete playlist: ", ex.what() );
        return false;
    }
609
}
610

611
bool MediaLibrary::addToStreamHistory( MediaPtr media )
612
{
613 614 615 616 617 618 619 620 621
    try
    {
        return History::insert( getConn(), media->id() );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to add stream to history: ", ex.what() );
        return false;
    }
622 623
}

624
Query<IHistoryEntry> MediaLibrary::lastStreamsPlayed() const
625 626 627 628
{
    return History::fetch( this );
}

629
Query<IMedia> MediaLibrary::lastMediaPlayed() const
630
{
631
    return Media::fetchHistory( this );
632 633
}

634 635
bool MediaLibrary::clearHistory()
{
636 637 638 639 640
    try
    {
        return sqlite::Tools::withRetries( 3, [this]() {
            auto t = getConn()->newTransaction();
            Media::clearHistory( this );
641
            History::clearStreams( this );
642 643 644 645 646 647 648 649 650
            t->commit();
            return true;
        });
    }
    catch ( sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to clear history: ", ex.what() );
        return false;
    }
651 652
}

653 654
MediaSearchAggregate MediaLibrary::searchMedia( const std::string& title,
                                                SortingCriteria sort, bool desc ) const
655
{
656 657
    if ( validateSearchPattern( title ) == false )
        return {};
658
    MediaSearchAggregate res;
659 660 661 662 663 664 665 666
    res.episodes = Media::search( this, title, IMedia::SubType::ShowEpisode,
                                sort, desc );
    res.movies = Media::search( this, title, IMedia::SubType::Movie,
                                sort, desc );
    res.others = Media::search( this, title, IMedia::SubType::Unknown,
                                sort, desc );
    res.tracks = Media::search( this, title, IMedia::SubType::AlbumTrack,
                                sort, desc );
667
    return res;
668 669
}

670
Query<IPlaylist> MediaLibrary::searchPlaylists( const std::string& name,
671 672
                                                        SortingCriteria sort,
                                                        bool desc ) const
673
{
674 675
    if ( validateSearchPattern( name ) == false )
        return {};
676
    return Playlist::search( this, name, sort, desc );
677 678
}

679
Query<IAlbum> MediaLibrary::searchAlbums( const std::string& pattern,
680
                                                  SortingCriteria sort, bool desc ) const
681
{
682 683
    if ( validateSearchPattern( pattern ) == false )
        return {};
684
    return Album::search( this, pattern, sort, desc );
685 686
}

687
Query<IGenre> MediaLibrary::searchGenre( const std::string& genre ) const
688
{
689 690
    if ( validateSearchPattern( genre ) == false )
        return {};
691
    return Genre::search( this, genre );
692 693
}

694
Query<IArtist> MediaLibrary::searchArtists( const std::string& name,
695 696
                                                    SortingCriteria sort,
                                                    bool desc ) const
697 698 699
{
    if ( validateSearchPattern( name ) == false )
        return {};
700
    return Artist::search( this, name, sort, desc );
701 702
}

703 704
SearchAggregate MediaLibrary::search( const std::string& pattern,
                                      SortingCriteria sort, bool desc ) const
705
{
706
    SearchAggregate res;
707 708
    res.albums = searchAlbums( pattern, sort, desc );
    res.artists = searchArtists( pattern, sort, desc );
709
    res.genres = searchGenre( pattern );
710 711
    res.media = searchMedia( pattern, sort, desc );
    res.playlists = searchPlaylists( pattern, sort, desc );
712 713 714
    return res;
}

715
bool MediaLibrary::startParser()
716
{
717
    m_parser.reset( new parser::Parser( this ) );
718

719 720 721 722 723 724 725 726 727 728
    if ( m_services.size() == 0 )
    {
        m_parser->addService( std::make_shared<parser::VLCMetadataService>() );
    }
    else
    {
        assert( m_services[0]->targetedStep() == parser::Step::MetadataExtraction );
        m_parser->addService( m_services[0] );
    }
    m_parser->addService( std::make_shared<parser::MetadataAnalyzer>() );
729
    m_parser->start();
730
    return true;
731 732
}

733
void MediaLibrary::startDiscoverer()
734
{
735
    m_discovererWorker.reset( new DiscovererWorker( this ) );
736
    for ( const auto& fsFactory : m_fsFactories )
737
    {
738
        std::unique_ptr<prober::CrawlerProbe> probePtr( new prober::CrawlerProbe{} );
739 740 741
        m_discovererWorker->addDiscoverer( std::unique_ptr<IDiscoverer>( new FsDiscoverer( fsFactory, this, m_callback,
                                                                                           std::move ( probePtr ) ) ) );
    }
742 743
}

744 745
void MediaLibrary::startDeletionNotifier()
{
746 747
    m_modificationNotifier.reset( new ModificationNotifier( this ) );
    m_modificationNotifier->start();
748 749
}

750
bool MediaLibrary::startThumbnailer()
751 752
{
    m_thumbnailer = std::unique_ptr<VLCThumbnailer>( new VLCThumbnailer( this ) );
753
    return true;
754 755
}

756 757 758 759 760
void MediaLibrary::addLocalFsFactory()
{
    m_fsFactories.insert( begin( m_fsFactories ), std::make_shared<factory::FileSystemFactory>( m_deviceLister ) );
}

761
InitializeResult MediaLibrary::updateDatabaseModel( unsigned int previousVersion,
762
                                        const std::string& dbPath )
763
{
764 765 766
    LOG_INFO( "Updating database model from ", previousVersion, " to ", Settings::DbModelVersion );
    // Up until model 3, it's safer (and potentially more efficient with index changes) to drop the DB
    // It's also way simpler to implement
767
    // In case of downgrade, just recreate the database
768
    for ( auto i = 0u; i < 3; ++i )
769
    {
770 771
        try
        {
772
            bool needRescan = false;
773 774 775
            // Up until model 3, it's safer (and potentially more efficient with index changes) to drop the DB
            // It's also way simpler to implement
            // In case of downgrade, just recreate the database
776 777
            // We might also have some special cases for failed upgrade (see
            // comments below for per-version details)
778
            if ( previousVersion < 3 ||
779 780
                 previousVersion > Settings::DbModelVersion ||
                 previousVersion == 4 )
781
            {
782
                if( recreateDatabase( dbPath ) == false )
783
                    throw std::runtime_error( "Failed to recreate the database" );
784
                return InitializeResult::DbReset;
785
            }
786 787 788 789 790 791 792 793
            /**
             * Migration from 3 to 4 didn't happen so well and broke a few
             * users DB. So:
             * - Any v4 database will be dropped and recreated in v5
             * - Any v3 database will be upgraded to v5
             * V4 database is only used by VLC-android 2.5.6 // 2.5.8, which are
             * beta versions.
             */
794 795
            if ( previousVersion == 3 )
            {
796
                migrateModel3to5();
797
                previousVersion = 5;
798
            }
799 800
            if ( previousVersion == 5 )
            {
801
                migrateModel5to6();
802 803
                previousVersion = 6;
            }
Alexandre Fernandez's avatar
Alexandre Fernandez committed
804 805
            if ( previousVersion == 6 )
            {
806 807 808
                // Force a rescan to solve metadata analysis problems.
                // The insertion is fixed, but won't edit already inserted data.
                forceRescan();
Alexandre Fernandez's avatar
Alexandre Fernandez committed
809 810
                previousVersion = 7;
            }
811 812 813 814 815 816 817 818 819
            /**
             * V7 introduces artist.nb_tracks and an associated trigger to delete
             * artists when it has no track/album left.
             */
            if ( previousVersion == 7 )
            {
                migrateModel7to8();
                previousVersion = 8;
            }
820 821
            if ( previousVersion == 8 )
            {
822 823 824 825 826 827 828 829 830

                // Multiple changes justify the rescan:
                // - Changes in the way we chose to encode or not MRL, meaning
                //   potentially all MRL are wrong (more precisely, will
                //   mismatch what VLC expects, which makes playlist analysis
                //   break.
                // - Fix in the way we chose album candidates, meaning some
                //   albums were likely to be wrongfully created.
                needRescan = true;
831
                migrateModel8to9();
832 833
                previousVersion = 9;
            }
834 835
            if ( previousVersion == 9 )
            {
836
                needRescan = true;
837 838 839
                migrateModel9to10();
                previousVersion = 10;
            }
840 841
            if ( previousVersion == 10 )
            {
842
                needRescan = true;
843 844 845
                migrateModel10to11();
                previousVersion = 11;
            }
846 847 848 849 850
            if ( previousVersion == 11 )
            {
                parser::Task::recoverUnscannedFiles( this );
                previousVersion = 12;
            }
851 852 853 854 855
            if ( previousVersion == 12 )
            {
                migrateModel12to13();
                previousVersion = 13;
            }
856 857
            if ( previousVersion == 13 )
            {
858 859 860
                // We need to recreate many thumbnail records, and hopefully
                // generate better ones
                needRescan = true;
861 862 863
                migrateModel13to14();
                previousVersion = 14;
            }
864 865
            // To be continued in the future!

866 867 868
            if ( needRescan == true )
                forceRescan();

869 870 871
            // Safety check: ensure we didn't forget a migration along the way
            assert( previousVersion == Settings::DbModelVersion );
            m_settings.setDbModelVersion( Settings::DbModelVersion );
872
            if ( m_settings.save() == false )
873 874
                return InitializeResult::Failed;
            return InitializeResult::Success;
875 876 877 878 879 880 881 882 883 884 885
        }
        catch( const std::exception& ex )
        {
            LOG_ERROR( "An error occured during the database upgrade: ",
                       ex.what() );
        }
        catch( ... )
        {
            LOG_ERROR( "An unknown error occured during the database upgrade." );
        }
        LOG_WARN( "Retrying database migration, attempt ", i + 1, " / 3" );
886
    }
887 888
    LOG_ERROR( "Failed to upgrade database, recreating it" );
    for ( auto i = 0u; i < 3; ++i )
889
    {
890 891
        try
        {
892
            if( recreateDatabase( dbPath ) == true )
893
                return InitializeResult::DbReset;
894 895 896 897 898 899 900 901 902 903
        }
        catch( const std::exception& ex )
        {
            LOG_ERROR( "Failed to recreate database: ", ex.what() );
        }
        catch(...)
        {
            LOG_ERROR( "Unknown error while trying to recreate the database." );
        }
        LOG_WARN( "Retrying to recreate the database, attempt ", i + 1, " / 3" );
904
    }
905
    return InitializeResult::Failed;
906 907
}

908
bool MediaLibrary::recreateDatabase( const std::string& dbPath )
909
{
910 911 912 913
    // Close all active connections, flushes all previously run statements.
    m_dbConnection.reset();
    unlink( dbPath.c_str() );
    m_dbConnection = sqlite::Connection::connect( dbPath );
914
    createAllTables();
915 916 917
    // We dropped the database, there is no setting to be read anymore
    if( m_settings.load() == false )
        return false;
918 919 920
    return true;
}

921
void MediaLibrary::migrateModel3to5()
922 923
{
    /*
924 925
     * Disable Foreign Keys & recursive triggers to avoid cascading deletion
     * while remodeling the database into the transaction.
926
     */
927
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
928
    auto t = getConn()->newTransaction();
929 930 931 932
    using namespace policy;
    // As SQLite do not allow us to remove or add some constraints,
    // we use the method described here https://www.sqlite.org/faq.html#q11
    std::string reqs[] = {
933
#               include "database/migrations/migration3-5.sql"
934 935 936
    };

    for ( const auto& req : reqs )
937
        sqlite::Tools::executeRequest( getConn(), req );
938
    // Re-create triggers removed in the process
939 940
    Media::createTriggers( getConn() );
    Playlist::createTriggers( getConn() );
941 942 943
    t->commit();
}

944
void MediaLibrary::migrateModel5to6()
945
{
946 947 948
    std::string req = "DELETE FROM " + policy::MediaTable::Name + " WHERE type = ?";
    sqlite::Tools::executeRequest( getConn(), req, Media::Type::Unknown );

949 950
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
    using namespace policy;
951
    req = "UPDATE " + MediaTable::Name + " SET is_present = 1 WHERE is_present != 0";
952
    sqlite::Tools::executeRequest( getConn(), req );
Alexandre Fernandez's avatar
Alexandre Fernandez committed
953 954
}

955 956 957 958 959 960 961 962 963 964 965 966
void MediaLibrary::migrateModel7to8()
{
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
    auto t = getConn()->newTransaction();
    using namespace policy;
    std::string reqs[] = {
#               include "database/migrations/migration7-8.sql"
    };

    for ( const auto& req : reqs )
        sqlite::Tools::executeRequest( getConn(), req );
    // Re-create triggers removed in the process
967
    Artist::createTriggers( getConn(), 8u );
968 969
    Media::createTriggers( getConn() );
    File::createTriggers( getConn() );
970 971 972
    t->commit();
}

973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989
void MediaLibrary::migrateModel8to9()
{
    // A bug in a previous migration caused our triggers to be missing for the
    // first application run (after the migration).
    // This could have caused media associated to deleted files not to be
    // deleted as well, so let's do that now.
    const std::string req = "DELETE FROM " + policy::MediaTable::Name + " "
            "WHERE id_media IN "
            "(SELECT id_media FROM " + policy::MediaTable::Name + " m LEFT JOIN " +
                policy::FileTable::Name + " f ON f.media_id = m.id_media "
                "WHERE f.media_id IS NULL)";

    // Don't check for the return value, we don't mind if nothing deleted.
    // Quite the opposite actually :)
    sqlite::Tools::executeDelete( getConn(), req );
}

990 991 992 993 994 995 996 997
void MediaLibrary::migrateModel9to10()
{
    const std::string req = "SELECT * FROM " + policy::FileTable::Name +
            " WHERE mrl LIKE '%#%%' ESCAPE '#'";
    auto files = File::fetchAll<File>( this, req );
    auto t = getConn()->newTransaction();
    for ( const auto& f : files )
    {
998 999 1000 1001
        // We must not call mrl() from here. We might not have all devices yet,
        // and calling mrl would crash for files stored on removable devices.
        auto newMrl = utils::url::encode( utils::url::decode( f->rawMrl() ) );
        LOG_INFO( "Converting ", f->rawMrl(), " to ", newMrl );
1002 1003 1004 1005 1006
        f->setMrl( newMrl );
    }
    t->commit();
}

1007 1008 1009 1010
void MediaLibrary::migrateModel10to11()
{
    const std::string req = "SELECT * FROM " + policy::TaskTable::Name +
            " WHERE mrl LIKE '%#%%' ESCAPE '#'";
1011 1012
    const std::string folderReq = "SELECT * FROM " + policy::FolderTable::Name +
            " WHERE path LIKE '%#%%' ESCAPE '#'";
1013
    auto tasks = parser::Task::fetchAll<parser::Task>( this, req );
1014
    auto folders = Folder::fetchAll<Folder>( this, folderReq );
1015 1016 1017
    auto t = getConn()->newTransaction();
    for ( const auto& t : tasks )
    {
1018 1019
        auto newMrl = utils::url::encode( utils::url::decode( t->item().mrl() ) );
        LOG_INFO( "Converting task mrl: ", t->item().mrl(), " to ", newMrl );
1020 1021
        t->setMrl( std::move( newMrl ) );
    }
1022 1023 1024 1025 1026 1027 1028
    for ( const auto &f : folders )
    {
        // We must not call mrl() from here. We might not have all devices yet,
        // and calling mrl would crash for files stored on removable devices.
        auto newMrl = utils::url::encode( utils::url::decode( f->rawMrl() ) );
        f->setMrl( std::move( newMrl ) );
    }
1029 1030 1031
    t->commit();
}

1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
/*
 * - Some is_present related triggers were fixed in model 6 to 7 migration, but
 *   they were not recreated if already existing. The has_file_present trigger
 *   was recreated as part of model 7 to 8 migration, but we need to ensure
 *   has_album_present (Artist) & is_album_present (Album) triggers are
 *   recreated to behave as expected
 * - Due to a typo, is_track_present was named is_track_presentAFTER, and was
 *   executed BEFORE the update took place, thus using the wrong is_present value.
 *   The trigger will be recreated as part of this migration, and the values
 *   will be enforced, causing the entire update chain to be triggered, and
 *   restoring correct is_present values for all AlbumTrack/Album/Artist entries
 */
1044 1045
void MediaLibrary::migrateModel12to13()
{
1046
    auto t = getConn()->newTransaction();
1047 1048 1049 1050 1051 1052 1053 1054 1055
    const std::string reqs[] = {
        "DROP TRIGGER IF EXISTS is_track_presentAFTER",
        "DROP TRIGGER has_album_present",
        "DROP TRIGGER is_album_present",
    };

    for ( const auto& req : reqs )
        sqlite::Tools::executeDelete( getConn(), req );

1056
    AlbumTrack::createTriggers( getConn() );
1057 1058
    Album::createTriggers( getConn() );
    Artist::createTriggers( getConn(), 13 );
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074
    // Leave the weak context as we now need to update is_present fields, which
    // are propagated through recursive triggers
    const std::string migrateData = "UPDATE " + policy::AlbumTrackTable::Name +
            " SET is_present = (SELECT is_present FROM " + policy::MediaTable::Name +
            " WHERE id_media = media_id)";
    sqlite::Tools::executeUpdate( getConn(), migrateData );
    t->commit();
}

/*
 * - Remove the Media.thumbnail
 * - Add Media.thumbnail_id
 * - Add Media.thumbnail_generated
 */
void MediaLibrary::migrateModel13to14()
{
1075 1076 1077
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
    auto t = getConn()->newTransaction();
    using namespace policy;
1078
    using ThumbnailType = typename std::underlying_type<Thumbnail::Origin>::type;
1079
    std::string reqs[] = {
1080
#               include "database/migrations/migration13-14.sql"
1081 1082 1083 1084
    };

    for ( const auto& req : reqs )
        sqlite::Tools::executeRequest( getConn(), req );
1085 1086 1087
    // Re-create tables that we just removed
    // We will run a re-scan, so we don't care about keeping their content
    Album::createTable( getConn() );
1088
    Artist::createTable( getConn() );
1089 1090 1091
    // Re-create triggers removed in the process
    Media::createTriggers( getConn() );
    AlbumTrack::createTriggers( getConn() );
1092 1093
    Album::createTriggers( getConn() );
    Artist::createTriggers( getConn(), 14 );
1094 1095 1096
    t->commit();
}

1097 1098
void MediaLibrary::reload()
{
1099 1100
    if ( m_discovererWorker != nullptr )
        m_discovererWorker->reload();
1101 1102
}

1103 1104
void MediaLibrary::reload( const std::string& entryPoint )
{
1105 1106
    if ( m_discovererWorker != nullptr )
        m_discovererWorker->reload( entryPoint );
1107 1108
}

1109
bool MediaLibrary::forceParserRetry()
1110
{
1111 1112
    try
    {
1113
        parser::Task::resetRetryCount( this );
1114 1115 1116 1117 1118 1119 1120
        return true;
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to force parser retry: ", ex.what() );
        return false;
    }
1121 1122
}

1123 1124
void MediaLibrary::pauseBackgroundOperations()
{
1125 1126
    if ( m_parser != nullptr )
        m_parser->pause();
1127 1128 1129 1130
}

void MediaLibrary::resumeBackgroundOperations()
{
1131 1132
    if ( m_parser != nullptr )
        m_parser->resume();
1133 1134
}

1135 1136 1137 1138 1139
void MediaLibrary::onDiscovererIdleChanged( bool idle )
{
    bool expected = !idle;
    if ( m_discovererIdle.compare_exchange_strong( expected, idle ) == true )
    {
1140 1141
        // If any idle state changed to false, then we need to trigger the callback.
        // If switching to idle == true, then both background workers need to be idle before signaling.
1142
        LOG_INFO( idle ? "Discoverer thread went idle" : "Discover thread was resumed" );
1143
        if ( idle == false || m_parserIdle == true )
1144 1145 1146
        {
            LOG_INFO( "Setting background idle state to ",
                      idle ? "true" : "false" );
1147
            m_callback->onBackgroundTasksIdleChanged( idle );
1148
        }
1149 1150 1151 1152 1153 1154 1155 1156
    }
}

void MediaLibrary::onParserIdleChanged( bool idle )
{
    bool expected = !idle;
    if ( m_parserIdle.compare_exchange_strong( expected, idle ) == true )
    {
1157
        LOG_INFO( idle ? "All parser services went idle" : "Parse services were resumed" );
1158
        if ( idle == false || m_discovererIdle == true )
1159 1160 1161
        {
            LOG_INFO( "Setting background idle state to ",
                      idle ? "true" : "false" );
1162
            m_callback->onBackgroundTasksIdleChanged( idle );
1163
        }
1164 1165 1166
    }