MediaLibrary.cpp 50.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2015 Hugo Beauzée-Luyssen, Videolabs
 *
 * Authors: Hugo Beauzée-Luyssen<hugo@beauzee.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

23 24 25 26
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

27 28
#include <algorithm>
#include <functional>
29
#include <utility>
30
#include <sys/stat.h>
31
#include <unistd.h>
32

33 34
#include "Album.h"
#include "AlbumTrack.h"
35
#include "Artist.h"
36
#include "AudioTrack.h"
37
#include "discoverer/DiscovererWorker.h"
38
#include "discoverer/probe/CrawlerProbe.h"
39
#include "utils/ModificationsNotifier.h"
40
#include "Device.h"
41
#include "File.h"
42
#include "Folder.h"
43
#include "Genre.h"
44
#include "Media.h"
45
#include "MediaLibrary.h"
46
#include "Label.h"
47
#include "logging/Logger.h"
48
#include "Movie.h"
49
#include "parser/Parser.h"
50
#include "Playlist.h"
51 52
#include "Show.h"
#include "ShowEpisode.h"
53
#include "Thumbnail.h"
54
#include "database/SqliteTools.h"
55
#include "database/SqliteConnection.h"
56
#include "database/SqliteQuery.h"
57
#include "utils/Filename.h"
58
#include "utils/Url.h"
59
#include "VideoTrack.h"
60
#include "Metadata.h"
61
#include "parser/Task.h"
62
#include "utils/Charsets.h"
63

64 65 66
// Discoverers:
#include "discoverer/FsDiscoverer.h"

67
// Metadata services:
68
#ifdef HAVE_LIBVLC
69 70
#include "metadata_services/vlc/VLCMetadataService.h"
#include "metadata_services/vlc/VLCThumbnailer.h"
71
#endif
72
#include "metadata_services/MetadataParser.h"
73

74 75
// FileSystem
#include "factory/DeviceListerFactory.h"
76
#include "factory/FileSystemFactory.h"
77 78

#ifdef HAVE_LIBVLC
79
#include "factory/NetworkFileSystemFactory.h"
80 81
#endif

82
#include "medialibrary/filesystem/IDevice.h"
83

84 85 86
namespace medialibrary
{

87
const char* const MediaLibrary::supportedExtensions[] = {
88
    "3g2", "3gp", "a52", "aac", "ac3", "aif", "aifc", "aiff", "alac", "amr",
89
    "amv", "aob", "ape", "asf", "asx", "avi", "b4s", "conf", /*"cue",*/
90 91 92 93 94 95 96 97 98 99
    "divx", "dts", "dv", "flac", "flv", "gxf", "ifo", "iso",
    "it", "itml",  "m1v", "m2t", "m2ts", "m2v", "m3u", "m3u8",
    "m4a", "m4b", "m4p", "m4v", "mid", "mka", "mkv", "mlp",
    "mod", "mov", "mp1", "mp2", "mp3", "mp4", "mpc", "mpeg",
    "mpeg1", "mpeg2", "mpeg4", "mpg", "mts", "mxf", "nsv",
    "nuv", "oga", "ogg", "ogm", "ogv", "ogx", "oma", "opus",
    "pls", "ps", "qtl", "ram", "rec", "rm", "rmi", "rmvb",
    "s3m", "sdp", "spx", "tod", "trp", "ts", "tta", "vlc",
    "vob", "voc", "vqf", "vro", "w64", "wav", "wax", "webm",
    "wma", "wmv", "wmx", "wpl", "wv", "wvx", "xa", "xm", "xspf"
100 101
};

102 103
const size_t MediaLibrary::NbSupportedExtensions = sizeof(supportedExtensions) / sizeof(supportedExtensions[0]);

104
MediaLibrary::MediaLibrary()
105 106
    : m_callback( nullptr )
    , m_verbosity( LogLevel::Error )
107
    , m_settings( this )
108
    , m_initialized( false )
109 110
    , m_discovererIdle( true )
    , m_parserIdle( true )
111
{
112
    Log::setLogLevel( m_verbosity );
113 114
}

115 116
MediaLibrary::~MediaLibrary()
{
117
    // Explicitely stop the discoverer, to avoid it writting while tearing down.
118 119
    if ( m_discovererWorker != nullptr )
        m_discovererWorker->stop();
120 121
    if ( m_parser != nullptr )
        m_parser->stop();
122 123 124 125 126
    clearCache();
}

void MediaLibrary::clearCache()
{
127
    Media::clear();
128
    Folder::clear();
129 130 131 132 133
    Label::clear();
    Album::clear();
    AlbumTrack::clear();
    Show::clear();
    ShowEpisode::clear();
134
    Movie::clear();
135
    VideoTrack::clear();
136
    AudioTrack::clear();
137
    Artist::clear();
138
    Device::clear();
139
    File::clear();
140
    Playlist::clear();
141
    Genre::clear();
142
    Thumbnail::clear();
143 144
}

145
void MediaLibrary::createAllTables()
146
{
147
    auto dbModelVersion = m_settings.dbModelVersion();
148 149 150 151 152
    // We need to create the tables in order of triggers creation
    // Device is the "root of all evil". When a device is modified,
    // we will trigger an update on folder, which will trigger
    // an update on files, and so on.

153 154
    Device::createTable( m_dbConnection.get() );
    Folder::createTable( m_dbConnection.get() );
155
    Thumbnail::createTable( m_dbConnection.get() );
156
    Media::createTable( m_dbConnection.get(), dbModelVersion );
157 158
    File::createTable( m_dbConnection.get() );
    Label::createTable( m_dbConnection.get() );
159
    Playlist::createTable( m_dbConnection.get(), dbModelVersion );
160 161 162 163 164 165 166 167 168 169
    Genre::createTable( m_dbConnection.get() );
    Album::createTable( m_dbConnection.get() );
    AlbumTrack::createTable( m_dbConnection.get() );
    Show::createTable( m_dbConnection.get() );
    ShowEpisode::createTable( m_dbConnection.get() );
    Movie::createTable( m_dbConnection.get() );
    VideoTrack::createTable( m_dbConnection.get() );
    AudioTrack::createTable( m_dbConnection.get() );
    Artist::createTable( m_dbConnection.get() );
    Artist::createDefaultArtists( m_dbConnection.get() );
170
    Settings::createTable( m_dbConnection.get() );
171
    parser::Task::createTable( m_dbConnection.get() );
172
    Metadata::createTable( m_dbConnection.get() );
173 174 175 176
}

void MediaLibrary::createAllTriggers()
{
177
    auto dbModelVersion = m_settings.dbModelVersion();
178
    Folder::createTriggers( m_dbConnection.get() );
179
    Album::createTriggers( m_dbConnection.get() );
180
    AlbumTrack::createTriggers( m_dbConnection.get() );
181
    Artist::createTriggers( m_dbConnection.get(), dbModelVersion );
182
    Media::createTriggers( m_dbConnection.get(), dbModelVersion );
183
    File::createTriggers( m_dbConnection.get() );
184 185
    Genre::createTriggers( m_dbConnection.get() );
    Playlist::createTriggers( m_dbConnection.get() );
186
    Label::createTriggers( m_dbConnection.get() );
187
    Show::createTriggers( m_dbConnection.get() );
188 189
}

190
template <typename T>
191
static void propagateDeletionToCache( sqlite::Connection::HookReason reason, int64_t rowId )
192
{
193
    if ( reason != sqlite::Connection::HookReason::Delete )
194 195 196 197
        return;
    T::removeFromCache( rowId );
}

198 199
void MediaLibrary::registerEntityHooks()
{
200
    if ( m_modificationNotifier == nullptr )
201 202
        return;

203
    m_dbConnection->registerUpdateHook( Media::Table::Name,
204 205
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
206 207
            return;
        Media::removeFromCache( rowId );
208
        m_modificationNotifier->notifyMediaRemoval( rowId );
209
    });
210
    m_dbConnection->registerUpdateHook( Artist::Table::Name,
211 212
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
213 214 215 216
            return;
        Artist::removeFromCache( rowId );
        m_modificationNotifier->notifyArtistRemoval( rowId );
    });
217
    m_dbConnection->registerUpdateHook( Album::Table::Name,
218 219
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
220 221 222 223
            return;
        Album::removeFromCache( rowId );
        m_modificationNotifier->notifyAlbumRemoval( rowId );
    });
224
    m_dbConnection->registerUpdateHook( AlbumTrack::Table::Name,
225 226
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
227 228 229 230
            return;
        AlbumTrack::removeFromCache( rowId );
        m_modificationNotifier->notifyAlbumTrackRemoval( rowId );
    });
231
    m_dbConnection->registerUpdateHook( Playlist::Table::Name,
232 233
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
234 235 236 237
            return;
        Playlist::removeFromCache( rowId );
        m_modificationNotifier->notifyPlaylistRemoval( rowId );
    });
238 239 240 241 242 243 244 245 246 247
    m_dbConnection->registerUpdateHook( Device::Table::Name, &propagateDeletionToCache<Device> );
    m_dbConnection->registerUpdateHook( File::Table::Name, &propagateDeletionToCache<File> );
    m_dbConnection->registerUpdateHook( Folder::Table::Name, &propagateDeletionToCache<Folder> );
    m_dbConnection->registerUpdateHook( Genre::Table::Name, &propagateDeletionToCache<Genre> );
    m_dbConnection->registerUpdateHook( Label::Table::Name, &propagateDeletionToCache<Label> );
    m_dbConnection->registerUpdateHook( Movie::Table::Name, &propagateDeletionToCache<Movie> );
    m_dbConnection->registerUpdateHook( Show::Table::Name, &propagateDeletionToCache<Show> );
    m_dbConnection->registerUpdateHook( ShowEpisode::Table::Name, &propagateDeletionToCache<ShowEpisode> );
    m_dbConnection->registerUpdateHook( AudioTrack::Table::Name, &propagateDeletionToCache<AudioTrack> );
    m_dbConnection->registerUpdateHook( VideoTrack::Table::Name, &propagateDeletionToCache<VideoTrack> );
248 249
}

250 251 252 253 254
bool MediaLibrary::validateSearchPattern( const std::string& pattern )
{
    return pattern.size() >= 3;
}

255 256 257 258 259 260 261 262 263 264 265 266 267
bool MediaLibrary::createThumbnailFolder( const std::string& thumbnailPath ) const
{
    auto paths = utils::file::splitPath( thumbnailPath, true );
#ifndef _WIN32
    std::string fullPath{ "/" };
#else
    std::string fullPath;
#endif
    while ( paths.empty() == false )
    {
        fullPath += paths.top();

#ifdef _WIN32
268 269 270 271 272 273 274 275 276
        // Don't try to create C: or various other drives
        if ( isalpha( fullPath[0] ) && fullPath[1] == ':' && fullPath.length() == 2 )
        {
            fullPath += "\\";
            paths.pop();
            continue;
        }
        auto wFullPath = charset::ToWide( fullPath.c_str() );
        if ( _wmkdir( wFullPath.get() ) != 0 )
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
#else
        if ( mkdir( fullPath.c_str(), S_IRWXU ) != 0 )
#endif
        {
            if ( errno != EEXIST )
                return false;
        }
        paths.pop();
#ifndef _WIN32
        fullPath += "/";
#else
        fullPath += "\\";
#endif
    }
    return true;
}

294 295 296
InitializeResult MediaLibrary::initialize( const std::string& dbPath,
                                           const std::string& thumbnailPath,
                                           IMediaLibraryCb* mlCallback )
297
{
298
    LOG_INFO( "Initializing medialibrary..." );
299
    if ( m_initialized == true )
300 301
    {
        LOG_INFO( "...Already initialized" );
302
        return InitializeResult::AlreadyInitialized;
303
    }
304 305 306 307
    if ( m_deviceLister == nullptr )
    {
        m_deviceLister = factory::createDeviceLister();
        if ( m_deviceLister == nullptr )
308 309
        {
            LOG_ERROR( "No available IDeviceLister was found." );
310
            return InitializeResult::Failed;
311
        }
312
    }
313
    addLocalFsFactory();
314
    if ( createThumbnailFolder( thumbnailPath ) == false )
315
    {
316 317 318
        LOG_ERROR( "Failed to create thumbnail directory (", thumbnailPath,
                    ": ", strerror( errno ) );
        return InitializeResult::Failed;
319
    }
320
    m_thumbnailPath = thumbnailPath;
321
    m_callback = mlCallback;
322
    m_dbConnection = sqlite::Connection::connect( dbPath );
323

324 325 326
    // Give a chance to test overloads to reject the creation of a notifier
    startDeletionNotifier();
    // Which allows us to register hooks, or not, depending on the presence of a notifier
327 328
    registerEntityHooks();

329
    auto res = InitializeResult::Success;
330
    try
331
    {
332
        auto t = m_dbConnection->newTransaction();
333
        createAllTables();
334
        if ( m_settings.load() == false )
335 336
        {
            LOG_ERROR( "Failed to load settings" );
337
            return InitializeResult::Failed;
338
        }
339 340 341
        createAllTriggers();
        t->commit();

342 343
        if ( m_settings.dbModelVersion() != Settings::DbModelVersion )
        {
344 345
            res = updateDatabaseModel( m_settings.dbModelVersion(), dbPath );
            if ( res == InitializeResult::Failed )
346 347
            {
                LOG_ERROR( "Failed to update database model" );
348
                return res;
349 350 351 352 353 354
            }
        }
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Can't initialize medialibrary: ", ex.what() );
355
        return InitializeResult::Failed;
356
    }
357 358
    m_initialized = true;
    LOG_INFO( "Successfuly initialized" );
359
    return res;
360 361 362 363
}

bool MediaLibrary::start()
{
364
    assert( m_initialized == true );
365 366 367
    if ( m_parser != nullptr )
        return false;

368
    populateFsFactories();
369 370
    for ( auto& fsFactory : m_fsFactories )
        refreshDevices( *fsFactory );
371 372 373
    // Now that we know which devices are plugged, check for outdated devices
    // Approximate 6 months for old device precision.
    Device::removeOldDevices( this, std::chrono::seconds{ 3600 * 24 * 30 * 6 } );
374
    Media::removeOldMedia( this, std::chrono::seconds{ 3600 * 24 * 30 * 6 } );
375

376
    startDiscoverer();
377 378
    if ( startParser() == false )
        return false;
379
    startThumbnailer();
380
    return true;
381 382
}

383
void MediaLibrary::setVerbosity( LogLevel v )
384 385 386 387 388
{
    m_verbosity = v;
    Log::setLogLevel( v );
}

389 390 391 392 393
MediaPtr MediaLibrary::media( int64_t mediaId ) const
{
    return Media::fetch( this, mediaId );
}

394 395
MediaPtr MediaLibrary::media( const std::string& mrl ) const
{
396
    LOG_INFO( "Fetching media from mrl: ", mrl );
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
397
    auto file = File::fromExternalMrl( this, mrl );
398 399 400 401 402
    if ( file != nullptr )
    {
        LOG_INFO( "Found external media: ", mrl );
        return file->media();
    }
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
403
    auto fsFactory = fsFactoryForMrl( mrl );
404
    if ( fsFactory == nullptr )
405
    {
406 407
        LOG_WARN( "Failed to create FS factory for path ", mrl );
        return nullptr;
408
    }
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
409
    auto device = fsFactory->createDeviceFromMrl( mrl );
410 411 412 413 414 415
    if ( device == nullptr )
    {
        LOG_WARN( "Failed to create a device associated with mrl ", mrl );
        return nullptr;
    }
    if ( device->isRemovable() == false )
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
416
        file = File::fromMrl( this, mrl );
417 418
    else
    {
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
419
        auto folder = Folder::fromMrl( this, utils::file::directory( mrl ) );
420 421 422 423 424 425 426 427 428 429 430 431 432 433
        if ( folder == nullptr )
        {
            LOG_WARN( "Failed to find folder containing ", mrl );
            return nullptr;
        }
        if ( folder->isPresent() == false )
        {
            LOG_INFO( "Found a folder containing ", mrl, " but it is not present" );
            return nullptr;
        }
        file = File::fromFileName( this, utils::file::fileName( mrl ), folder->id() );
    }
    if ( file == nullptr )
    {
434
        LOG_WARN( "Failed to fetch file for ", mrl, " (device ", device->uuid(), " was ",
435
                  device->isRemovable() ? "" : "NOT ", "removable)");
436 437 438 439 440
        return nullptr;
    }
    return file->media();
}

441
MediaPtr MediaLibrary::addExternalMedia( const std::string& mrl, IMedia::Type type )
442
{
443 444
    try
    {
445
        return sqlite::Tools::withRetries( 3, [this, &mrl, type]() -> MediaPtr {
446
            auto t = m_dbConnection->newTransaction();
447 448
            auto fileName = utils::file::fileName( mrl );
            auto media = Media::create( this, type, utils::url::decode( fileName ) );
449 450 451 452 453 454 455 456 457 458 459 460 461
            if ( media == nullptr )
                return nullptr;
            if ( media->addExternalMrl( mrl, IFile::Type::Main ) == nullptr )
                return nullptr;
            t->commit();
            return media;
        });
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create external media: ", ex.what() );
        return nullptr;
    }
462 463
}

464 465 466 467 468 469 470 471 472 473
MediaPtr MediaLibrary::addExternalMedia( const std::string& mrl )
{
    return addExternalMedia( mrl, IMedia::Type::External );
}

MediaPtr MediaLibrary::addStream( const std::string& mrl )
{
    return addExternalMedia( mrl, IMedia::Type::Stream );
}

474
Query<IMedia> MediaLibrary::audioFiles( const QueryParameters* params ) const
475
{
476
    return Media::listAll( this, IMedia::Type::Audio, params );
477 478
}

479
Query<IMedia> MediaLibrary::videoFiles( const QueryParameters* params ) const
480
{
481
    return Media::listAll( this, IMedia::Type::Video, params );
482 483
}

484 485 486 487 488 489 490 491
bool MediaLibrary::isExtensionSupported( const char* ext )
{
    return std::binary_search( std::begin( supportedExtensions ),
        std::end( supportedExtensions ), ext, [](const char* l, const char* r) {
            return strcasecmp( l, r ) < 0;
        });
}

492
void MediaLibrary::onDiscoveredFile( std::shared_ptr<fs::IFile> fileFs,
493
                                      std::shared_ptr<Folder> parentFolder,
494 495
                                      std::shared_ptr<fs::IDirectory> parentFolderFs,
                                      std::pair<std::shared_ptr<Playlist>, unsigned int> parentPlaylist )
496
{
497
    auto mrl = fileFs->mrl();
498 499
    try
    {
500 501 502 503 504
        std::shared_ptr<parser::Task> task;
        if ( parentPlaylist.first == nullptr )
        {
            // Sqlite won't ensure uniqueness for Task with the same (mrl, parent_playlist_id)
            // when parent_playlist_id is null, so we have to ensure of it ourselves
505
            const std::string req = "SELECT * FROM " + parser::Task::Table::Name + " "
506
                    "WHERE mrl = ? AND parent_playlist_id IS NULL";
507
            task = parser::Task::fetch( this, req, mrl );
508 509
            if ( task != nullptr )
            {
510
                LOG_INFO( "Not creating duplicated task for mrl: ", mrl );
511 512 513
                return;
            }
        }
514
        task = parser::Task::create( this, std::move( fileFs ), std::move( parentFolder ),
515 516
                                     std::move( parentFolderFs ),
                                     std::move( parentPlaylist ) );
517 518 519 520 521 522 523
        if ( task != nullptr && m_parser != nullptr )
            m_parser->parse( task );
    }
    catch(sqlite::errors::ConstraintViolation& ex)
    {
        // Most likely the file is already scheduled and we restarted the
        // discovery after a crash.
524
        LOG_WARN( "Failed to insert ", mrl, ": ", ex.what(), ". "
525 526
                  "Assuming the file is already scheduled for discovery" );
    }
527 528
}

529
bool MediaLibrary::deleteFolder( const Folder& folder )
530
{
531
    LOG_INFO( "deleting folder ", folder.mrl() );
532
    if ( Folder::destroy( this, folder.id() ) == false )
533
        return false;
534
    Media::clear();
535
    return true;
536 537
}

538 539
LabelPtr MediaLibrary::createLabel( const std::string& label )
{
540 541 542 543 544 545 546 547 548
    try
    {
        return Label::create( this, label );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create a label: ", ex.what() );
        return nullptr;
    }
549
}
550 551 552

bool MediaLibrary::deleteLabel( LabelPtr label )
{
553 554 555 556 557 558 559 560 561
    try
    {
        return Label::destroy( this, label->id() );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to delete label: ", ex.what() );
        return false;
    }
562
}
563

564
AlbumPtr MediaLibrary::album( int64_t id ) const
565
{
566
    return Album::fetch( this, id );
567 568
}

569
std::shared_ptr<Album> MediaLibrary::createAlbum( const std::string& title, int64_t thumbnailId )
570
{
571
    return Album::create( this, title, thumbnailId );
572 573
}

574
Query<IAlbum> MediaLibrary::albums( const QueryParameters* params ) const
575
{
576
    return Album::listAll( this, params );
577 578
}

579
Query<IGenre> MediaLibrary::genres( const QueryParameters* params ) const
580
{
581
    return Genre::listAll( this, params );
582 583
}

584
GenrePtr MediaLibrary::genre( int64_t id ) const
585 586 587 588
{
    return Genre::fetch( this, id );
}

589 590 591 592 593
ShowPtr MediaLibrary::show( int64_t id ) const
{
    return Show::fetch( this, id );
}

594
std::shared_ptr<Show> MediaLibrary::createShow( const std::string& name )
595
{
596
    return Show::create( this, name );
597 598
}

599 600 601 602 603
Query<IShow> MediaLibrary::shows(const QueryParameters* params) const
{
    return Show::listAll( this, params );
}

604 605 606 607 608
MoviePtr MediaLibrary::movie( int64_t id ) const
{
    return Movie::fetch( this, id );
}

609
std::shared_ptr<Movie> MediaLibrary::createMovie( Media& media )
610
{
611
    auto movie = Movie::create( this, media.id() );
612
    media.setMovie( movie );
613
    media.save();
614
    return movie;
615 616
}

617
ArtistPtr MediaLibrary::artist( int64_t id ) const
618
{
619
    return Artist::fetch( this, id );
620 621 622
}

ArtistPtr MediaLibrary::artist( const std::string& name )
623
{
624
    static const std::string req = "SELECT * FROM " + Artist::Table::Name
625
            + " WHERE name = ? AND is_present != 0";
626
    return Artist::fetch( this, req, name );
627 628
}

629
std::shared_ptr<Artist> MediaLibrary::createArtist( const std::string& name )
630
{
631 632
    try
    {
633
        return Artist::create( this, name );
634 635 636 637 638 639
    }
    catch( sqlite::errors::ConstraintViolation &ex )
    {
        LOG_WARN( "ContraintViolation while creating an artist (", ex.what(), ") attempting to fetch it instead" );
        return std::static_pointer_cast<Artist>( artist( name ) );
    }
640 641
}

642
Query<IArtist> MediaLibrary::artists( bool includeAll, const QueryParameters* params ) const
643
{
644
    return Artist::listAll( this, includeAll, params );
645 646
}

647 648
PlaylistPtr MediaLibrary::createPlaylist( const std::string& name )
{
649 650
    try
    {
651 652 653 654
        auto pl = Playlist::create( this, name );
        if ( pl != nullptr && m_modificationNotifier != nullptr )
            m_modificationNotifier->notifyPlaylistCreation( pl );
        return pl;
655 656 657 658 659 660
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create a playlist: ", ex.what() );
        return nullptr;
    }
661 662
}

663
Query<IPlaylist> MediaLibrary::playlists( const QueryParameters* params )
664
{
665
    return Playlist::listAll( this, params );
666 667
}

668 669 670 671 672
PlaylistPtr MediaLibrary::playlist( int64_t id ) const
{
    return Playlist::fetch( this, id );
}

673
bool MediaLibrary::deletePlaylist( int64_t playlistId )
674
{
675 676 677 678 679 680 681 682 683
    try
    {
        return Playlist::destroy( this, playlistId );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to delete playlist: ", ex.what() );
        return false;
    }
684
}
685

686
Query<IMedia> MediaLibrary::history() const
687
{
688
    return Media::fetchHistory( this );
689 690
}

691
Query<IMedia> MediaLibrary::streamHistory() const
692
{
693
    return Media::fetchStreamHistory( this );
694 695
}

696 697
bool MediaLibrary::clearHistory()
{
698 699 700 701 702 703 704 705 706 707 708 709
    try
    {
        return sqlite::Tools::withRetries( 3, [this]() {
            Media::clearHistory( this );
            return true;
        });
    }
    catch ( sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to clear history: ", ex.what() );
        return false;
    }
710 711
}

712
Query<IMedia> MediaLibrary::searchMedia( const std::string& title,
713
                                                const QueryParameters* params ) const
714
{
715 716
    if ( validateSearchPattern( title ) == false )
        return {};
717
    return Media::search( this, title, params );
718 719
}

720 721 722 723 724 725 726 727 728 729 730
Query<IMedia> MediaLibrary::searchAudio( const std::string& pattern, const QueryParameters* params ) const
{
    if ( validateSearchPattern( pattern ) == false )
        return {};
    return Media::search( this, pattern, IMedia::Type::Audio, params );
}

Query<IMedia> MediaLibrary::searchVideo( const std::string& pattern, const QueryParameters* params ) const
{
    if ( validateSearchPattern( pattern ) == false )
        return {};
731
    return Media::search( this, pattern, IMedia::Type::Video, params );
732 733
}

734
Query<IPlaylist> MediaLibrary::searchPlaylists( const std::string& name,
735
                                                const QueryParameters* params ) const
736
{
737 738
    if ( validateSearchPattern( name ) == false )
        return {};
739
    return Playlist::search( this, name, params );
740 741
}

742
Query<IAlbum> MediaLibrary::searchAlbums( const std::string& pattern,
743
                                          const QueryParameters* params ) const
744
{
745 746
    if ( validateSearchPattern( pattern ) == false )
        return {};
747
    return Album::search( this, pattern, params );
748 749
}

750 751
Query<IGenre> MediaLibrary::searchGenre( const std::string& genre,
                                         const QueryParameters* params ) const
752
{
753 754
    if ( validateSearchPattern( genre ) == false )
        return {};
755
    return Genre::search( this, genre, params );
756 757
}

758
Query<IArtist> MediaLibrary::searchArtists( const std::string& name,
759
                                            const QueryParameters* params ) const
760 761 762
{
    if ( validateSearchPattern( name ) == false )
        return {};
763
    return Artist::search( this, name, params );
764 765
}

766 767 768 769 770 771
Query<IShow> MediaLibrary::searchShows( const std::string& pattern,
                                        const QueryParameters* params ) const
{
    return Show::search( this, pattern, params );
}

772
SearchAggregate MediaLibrary::search( const std::string& pattern,
773
                                      const QueryParameters* params ) const
774
{
775
    SearchAggregate res;
776 777 778 779 780
    res.albums = searchAlbums( pattern, params );
    res.artists = searchArtists( pattern, params );
    res.genres = searchGenre( pattern, params );
    res.media = searchMedia( pattern, params );
    res.playlists = searchPlaylists( pattern, params );
781
    res.shows = searchShows( pattern, params );
782 783 784
    return res;
}

785
bool MediaLibrary::startParser()
786
{
787
    m_parser.reset( new parser::Parser( this ) );
788

789 790
    if ( m_services.size() == 0 )
    {
791
#ifdef HAVE_LIBVLC
792
        m_parser->addService( std::make_shared<parser::VLCMetadataService>() );
793 794 795
#else
        return false;
#endif
796 797 798 799 800 801 802
    }
    else
    {
        assert( m_services[0]->targetedStep() == parser::Step::MetadataExtraction );
        m_parser->addService( m_services[0] );
    }
    m_parser->addService( std::make_shared<parser::MetadataAnalyzer>() );
803
    m_parser->start();
804
    return true;
805 806
}

807
void MediaLibrary::startDiscoverer()
808
{
809
    m_discovererWorker.reset( new DiscovererWorker( this ) );
810
    for ( const auto& fsFactory : m_fsFactories )
811
    {
812
        std::unique_ptr<prober::CrawlerProbe> probePtr( new prober::CrawlerProbe{} );
813 814 815
        m_discovererWorker->addDiscoverer( std::unique_ptr<IDiscoverer>( new FsDiscoverer( fsFactory, this, m_callback,
                                                                                           std::move ( probePtr ) ) ) );
    }
816 817
}

818 819
void MediaLibrary::startDeletionNotifier()
{
820 821
    m_modificationNotifier.reset( new ModificationNotifier( this ) );
    m_modificationNotifier->start();
822 823
}

824
void MediaLibrary::startThumbnailer()
825
{
826
#ifdef HAVE_LIBVLC
827
    m_thumbnailer = std::unique_ptr<VLCThumbnailer>( new VLCThumbnailer( this ) );
828
#endif
829 830
}

831 832 833 834 835 836 837
void MediaLibrary::populateFsFactories()
{
#ifdef HAVE_LIBVLC
    m_externalFsFactories.emplace_back( std::make_shared<factory::NetworkFileSystemFactory>( "smb", "dsm-sd" ) );
#endif
}

838 839
void MediaLibrary::addLocalFsFactory()
{
840
    m_fsFactories.emplace( begin( m_fsFactories ), std::make_shared<factory::FileSystemFactory>( m_deviceLister ) );
841 842
}

843
InitializeResult MediaLibrary::updateDatabaseModel( unsigned int previousVersion,
844
                                        const std::string& dbPath )
845
{
846
    LOG_INFO( "Updating database model from ", previousVersion, " to ", Settings::DbModelVersion );
847
    auto originalPreviousVersion = previousVersion;
848 849
    // Up until model 3, it's safer (and potentially more efficient with index changes) to drop the DB
    // It's also way simpler to implement
850
    // In case of downgrade, just recreate the database
851
    for ( auto i = 0u; i < 3; ++i )
852
    {
853 854
        try
        {
855
            bool needRescan = false;
856 857 858
            // Up until model 3, it's safer (and potentially more efficient with index changes) to drop the DB
            // It's also way simpler to implement
            // In case of downgrade, just recreate the database
859 860
            // We might also have some special cases for failed upgrade (see
            // comments below for per-version details)
861
            if ( previousVersion < 3 ||
862 863
                 previousVersion > Settings::DbModelVersion ||
                 previousVersion == 4 )
864
            {
865
                if( recreateDatabase( dbPath ) == false )
866
                    throw std::runtime_error( "Failed to recreate the database" );
867
                return InitializeResult::DbReset;
868
            }
869 870 871 872 873 874 875 876
            /**
             * Migration from 3 to 4 didn't happen so well and broke a few
             * users DB. So:
             * - Any v4 database will be dropped and recreated in v5
             * - Any v3 database will be upgraded to v5
             * V4 database is only used by VLC-android 2.5.6 // 2.5.8, which are
             * beta versions.
             */
877 878
            if ( previousVersion == 3 )
            {
879
                migrateModel3to5();
880
                previousVersion = 5;
881
            }
882 883
            if ( previousVersion == 5 )
            {
884
                migrateModel5to6();
885 886
                previousVersion = 6;
            }
Alexandre Fernandez's avatar
Alexandre Fernandez committed
887 888
            if ( previousVersion == 6 )
            {
889 890 891
                // Force a rescan to solve metadata analysis problems.
                // The insertion is fixed, but won't edit already inserted data.
                forceRescan();
Alexandre Fernandez's avatar
Alexandre Fernandez committed
892 893
                previousVersion = 7;
            }
894 895 896 897 898 899 900 901 902
            /**
             * V7 introduces artist.nb_tracks and an associated trigger to delete
             * artists when it has no track/album left.
             */
            if ( previousVersion == 7 )
            {
                migrateModel7to8();
                previousVersion = 8;
            }
903 904
            if ( previousVersion == 8 )
            {
905 906 907 908 909 910 911 912 913

                // Multiple changes justify the rescan:
                // - Changes in the way we chose to encode or not MRL, meaning
                //   potentially all MRL are wrong (more precisely, will
                //   mismatch what VLC expects, which makes playlist analysis
                //   break.
                // - Fix in the way we chose album candidates, meaning some
                //   albums were likely to be wrongfully created.
                needRescan = true;
914
                migrateModel8to9();
915 916
                previousVersion = 9;
            }
917 918
            if ( previousVersion == 9 )
            {
919
                needRescan = true;
920 921 922
                migrateModel9to10();
                previousVersion = 10;
            }
923 924
            if ( previousVersion == 10 )
            {
925
                needRescan = true;
926 927 928
                migrateModel10to11();
                previousVersion = 11;
            }
929 930 931 932 933
            if ( previousVersion == 11 )
            {
                parser::Task::recoverUnscannedFiles( this );
                previousVersion = 12;
            }
934 935 936 937 938
            if ( previousVersion == 12 )
            {
                migrateModel12to13();
                previousVersion = 13;
            }
939 940
            if ( previousVersion == 13 )
            {
941 942 943
                // We need to recreate many thumbnail records, and hopefully
                // generate better ones
                needRescan = true;
944
                migrateModel13to14( originalPreviousVersion );
945 946
                previousVersion = 14;
            }
947 948
            // To be continued in the future!

949 950 951
            if ( needRescan == true )
                forceRescan();

952 953 954
            // Safety check: ensure we didn't forget a migration along the way
            assert( previousVersion == Settings::DbModelVersion );
            m_settings.setDbModelVersion( Settings::DbModelVersion );
955
            if ( m_settings.save() == false )
956 957
                return InitializeResult::Failed;
            return InitializeResult::Success;
958 959 960 961 962 963 964 965 966 967 968
        }
        catch( const std::exception& ex )
        {
            LOG_ERROR( "An error occured during the database upgrade: ",
                       ex.what() );
        }
        catch( ... )
        {
            LOG_ERROR( "An unknown error occured during the database upgrade." );
        }
        LOG_WARN( "Retrying database migration, attempt ", i + 1, " / 3" );
969
    }
970 971
    LOG_ERROR( "Failed to upgrade database, recreating it" );
    for ( auto i = 0u; i < 3; ++i )
972
    {
973 974
        try
        {
975
            if( recreateDatabase( dbPath ) == true )
976
                return InitializeResult::DbReset;
977 978 979 980 981 982 983 984 985 986
        }
        catch( const std::exception& ex )
        {
            LOG_ERROR( "Failed to recreate database: ", ex.what() );
        }
        catch(...)
        {
            LOG_ERROR( "Unknown error while trying to recreate the database." );
        }
        LOG_WARN( "Retrying to recreate the database, attempt ", i + 1, " / 3" );
987
    }
988
    return InitializeResult::Failed;
989 990
}

991
bool MediaLibrary::recreateDatabase( const std::string& dbPath )
992
{
993 994 995 996
    // Close all active connections, flushes all previously run statements.
    m_dbConnection.reset();
    unlink( dbPath.c_str() );
    m_dbConnection = sqlite::Connection::connect( dbPath );
997
    createAllTables();
998 999 1000
    // We dropped the database, there is no setting to be read anymore
    if( m_settings.load() == false )
        return false;
1001 1002 1003
    return true;
}

1004
void MediaLibrary::migrateModel3to5()
1005 1006
{
    /*
1007 1008
     * Disable Foreign Keys & recursive triggers to avoid cascading deletion
     * while remodeling the database into the transaction.
1009
     */
1010
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
1011
    auto t = getConn()->newTransaction();
1012 1013 1014
    // As SQLite do not allow us to remove or add some constraints,
    // we use the method described here https://www.sqlite.org/faq.html#q11
    std::string reqs[] = {
1015
#               include "database/migrations/migration3-5.sql"
1016 1017 1018
    };

    for ( const auto& req : reqs )
1019
        sqlite::Tools::executeRequest( getConn(), req );
1020
    // Re-create triggers removed in the process
1021
    Media::createTriggers( getConn(), 5 );
1022
    Playlist::createTriggers( getConn() );
1023 1024 1025
    t->commit();
}

1026
void MediaLibrary::migrateModel5to6()
1027
{
1028
    std::string req = "DELETE FROM " + Media::Table::Name + " WHERE type = ?";
1029 1030
    sqlite::Tools::executeRequest( getConn(), req, Media::Type::Unknown );

1031
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
1032
    req = "UPDATE " + Media::Table::Name + " SET is_present = 1 WHERE is_present != 0";
1033
    sqlite::Tools::executeRequest( getConn(), req );
Alexandre Fernandez's avatar
Alexandre Fernandez committed
1034 1035
}

1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046
void MediaLibrary::migrateModel7to8()
{
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
    auto t = getConn()->newTransaction();
    std::string reqs[] = {
#               include "database/migrations/migration7-8.sql"
    };

    for ( const auto& req : reqs )
        sqlite::Tools::executeRequest( getConn(), req );
    // Re-create triggers removed in the process
1047
    Artist::createTriggers( getConn(), 8u );
1048
    Media::createTriggers( getConn(), 5 );
1049
    File::createTriggers( getConn() );
1050 1051 1052
    t->commit();
}

1053 1054 1055 1056 1057 1058
void MediaLibrary::migrateModel8to9()
{
    // A bug in a previous migration caused our triggers to be missing for the
    // first application run (after the migration).
    // This could have caused media associated to deleted files not to be
    // deleted as well, so let's do that now.
1059
    const std::string req = "DELETE FROM " + Media::Table::Name + " "
1060
            "WHERE id_media IN "
1061 1062
            "(SELECT id_media FROM " + Media::Table::Name + " m LEFT JOIN " +
                File::Table::Name + " f ON f.media_id = m.id_media "
1063 1064 1065 1066 1067 1068 1069
                "WHERE f.media_id IS NULL)";

    // Don't check for the return value, we don't mind if nothing deleted.
    // Quite the opposite actually :)
    sqlite::Tools::executeDelete( getConn(), req );
}

1070 1071
void MediaLibrary::migrateModel9to10()
{
1072
    const std::string req = "SELECT * FROM " + File::Table::Name +
1073 1074 1075 1076 1077
            " WHERE mrl LIKE '%#%%' ESCAPE '#'";
    auto files = File::fetchAll<File>( this, req );
    auto t = getConn()->newTransaction();
    for ( const auto& f : files )
    {
1078 1079 1080 1081
        // We must not call mrl() from here. We might not have all devices yet,
        // and calling mrl would crash for files stored on removable devices.
        auto newMrl = utils::url::encode( utils::url::decode( f->rawMrl() ) );
        LOG_INFO( "Converting ", f->rawMrl(), " to ", newMrl );
1082 1083 1084 1085 1086
        f->setMrl( newMrl );
    }
    t->commit();
}

1087 1088
void MediaLibrary::migrateModel10to11()
{
1089
    const std::string req = "SELECT * FROM " + parser::Task::Table::Name +
1090
            " WHERE mrl LIKE '%#%%' ESCAPE '#'";
1091
    const std::string folderReq = "SELECT * FROM " + Folder::Table::Name +
1092
            " WHERE path LIKE '%#%%' ESCAPE '#'";
1093
    auto tasks = parser::Task::fetchAll<parser::Task>( this, req );
1094
    auto folders = Folder::fetchAll<Folder>( this, folderReq );
1095 1096 1097
    auto t = getConn()->newTransaction();
    for ( const auto& t : tasks )
    {
1098 1099
        auto newMrl = utils::url::encode( utils::url::decode( t->item().mrl() ) );
        LOG_INFO( "Converting task mrl: ", t->item().mrl(), " to ", newMrl );
1100 1101
        t->setMrl( std::move( newMrl ) );
    }
1102 1103 1104 1105 1106 1107 1108
    for ( const auto &f : folders )
    {
        // We must not call mrl() from here. We might not have all devices yet,
        // and calling mrl would crash for files stored on removable devices.
        auto newMrl = utils::url::encode( utils::url::decode( f->rawMrl() ) );
        f->setMrl( std::move( newMrl ) );
    }
1109 1110 1111
    t->commit();
}

1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123
/*
 * - Some is_present related triggers were fixed in model 6 to 7 migration, but
 *   they were not recreated if already existing. The has_file_present trigger
 *   was recreated as part of model 7 to 8 migration, but we need to ensure
 *   has_album_present (Artist) & is_album_present (Album) triggers are
 *   recreated to behave as expected
 * - Due to a typo, is_track_present was named is_track_presentAFTER, and was
 *   executed BEFORE the update took place, thus using the wrong is_present value.
 *   The trigger will be recreated as part of this migration, and the values
 *   will be enforced, causing the entire update chain to be triggered, and
 *   restoring correct is_present values for all AlbumTrack/Album/Artist entries
 */
1124 1125
void MediaLibrary::migrateModel12to13()
{
1126
    auto t = getConn()->newTransaction();
1127 1128 1129 1130 1131 1132