MediaLibrary.cpp 47.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2015 Hugo Beauzée-Luyssen, Videolabs
 *
 * Authors: Hugo Beauzée-Luyssen<hugo@beauzee.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

23 24 25 26
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

27 28
#include <algorithm>
#include <functional>
29
#include <utility>
30
#include <sys/stat.h>
31
#include <unistd.h>
32

33 34
#include "Album.h"
#include "AlbumTrack.h"
35
#include "Artist.h"
36
#include "AudioTrack.h"
37
#include "discoverer/DiscovererWorker.h"
38
#include "discoverer/probe/CrawlerProbe.h"
39
#include "utils/ModificationsNotifier.h"
40
#include "Device.h"
41
#include "File.h"
42
#include "Folder.h"
43
#include "Genre.h"
44
#include "Media.h"
45
#include "MediaLibrary.h"
46
#include "Label.h"
47
#include "logging/Logger.h"
48
#include "Movie.h"
49
#include "parser/Parser.h"
50
#include "Playlist.h"
51 52
#include "Show.h"
#include "ShowEpisode.h"
53
#include "Thumbnail.h"
54
#include "database/SqliteTools.h"
55
#include "database/SqliteConnection.h"
56
#include "database/SqliteQuery.h"
57
#include "utils/Filename.h"
58
#include "utils/Url.h"
59
#include "VideoTrack.h"
60
#include "Metadata.h"
61
#include "utils/Charsets.h"
62

63 64 65
// Discoverers:
#include "discoverer/FsDiscoverer.h"

66
// Metadata services:
67
#ifdef HAVE_LIBVLC
68 69
#include "metadata_services/vlc/VLCMetadataService.h"
#include "metadata_services/vlc/VLCThumbnailer.h"
70
#endif
71
#include "metadata_services/MetadataParser.h"
72

73 74
// FileSystem
#include "factory/DeviceListerFactory.h"
75
#include "factory/FileSystemFactory.h"
76 77

#ifdef HAVE_LIBVLC
78
#include "factory/NetworkFileSystemFactory.h"
79 80
#endif

81
#include "medialibrary/filesystem/IDevice.h"
82

83 84 85
namespace medialibrary
{

86
const char* const MediaLibrary::supportedExtensions[] = {
87
    "3g2", "3gp", "a52", "aac", "ac3", "aif", "aifc", "aiff", "alac", "amr",
88
    "amv", "aob", "ape", "asf", "asx", "avi", "b4s", "conf", /*"cue",*/
89 90 91 92 93 94 95 96 97 98
    "divx", "dts", "dv", "flac", "flv", "gxf", "ifo", "iso",
    "it", "itml",  "m1v", "m2t", "m2ts", "m2v", "m3u", "m3u8",
    "m4a", "m4b", "m4p", "m4v", "mid", "mka", "mkv", "mlp",
    "mod", "mov", "mp1", "mp2", "mp3", "mp4", "mpc", "mpeg",
    "mpeg1", "mpeg2", "mpeg4", "mpg", "mts", "mxf", "nsv",
    "nuv", "oga", "ogg", "ogm", "ogv", "ogx", "oma", "opus",
    "pls", "ps", "qtl", "ram", "rec", "rm", "rmi", "rmvb",
    "s3m", "sdp", "spx", "tod", "trp", "ts", "tta", "vlc",
    "vob", "voc", "vqf", "vro", "w64", "wav", "wax", "webm",
    "wma", "wmv", "wmx", "wpl", "wv", "wvx", "xa", "xm", "xspf"
99 100
};

101 102
const size_t MediaLibrary::NbSupportedExtensions = sizeof(supportedExtensions) / sizeof(supportedExtensions[0]);

103
MediaLibrary::MediaLibrary()
104 105
    : m_callback( nullptr )
    , m_verbosity( LogLevel::Error )
106
    , m_settings( this )
107
    , m_initialized( false )
108 109
    , m_discovererIdle( true )
    , m_parserIdle( true )
110
{
111
    Log::setLogLevel( m_verbosity );
112 113
}

114 115
MediaLibrary::~MediaLibrary()
{
116
    // Explicitely stop the discoverer, to avoid it writting while tearing down.
117 118
    if ( m_discovererWorker != nullptr )
        m_discovererWorker->stop();
119 120
    if ( m_parser != nullptr )
        m_parser->stop();
121 122 123 124 125
    clearCache();
}

void MediaLibrary::clearCache()
{
126
    Media::clear();
127
    Folder::clear();
128 129 130 131 132
    Label::clear();
    Album::clear();
    AlbumTrack::clear();
    Show::clear();
    ShowEpisode::clear();
133
    Movie::clear();
134
    VideoTrack::clear();
135
    AudioTrack::clear();
136
    Artist::clear();
137
    Device::clear();
138
    File::clear();
139
    Playlist::clear();
140
    Genre::clear();
141
    Thumbnail::clear();
142 143
}

144
void MediaLibrary::createAllTables()
145
{
146 147 148 149 150
    // We need to create the tables in order of triggers creation
    // Device is the "root of all evil". When a device is modified,
    // we will trigger an update on folder, which will trigger
    // an update on files, and so on.

151 152
    Device::createTable( m_dbConnection.get() );
    Folder::createTable( m_dbConnection.get() );
153
    Thumbnail::createTable( m_dbConnection.get() );
154 155 156 157 158 159 160 161 162 163 164 165 166 167
    Media::createTable( m_dbConnection.get() );
    File::createTable( m_dbConnection.get() );
    Label::createTable( m_dbConnection.get() );
    Playlist::createTable( m_dbConnection.get() );
    Genre::createTable( m_dbConnection.get() );
    Album::createTable( m_dbConnection.get() );
    AlbumTrack::createTable( m_dbConnection.get() );
    Show::createTable( m_dbConnection.get() );
    ShowEpisode::createTable( m_dbConnection.get() );
    Movie::createTable( m_dbConnection.get() );
    VideoTrack::createTable( m_dbConnection.get() );
    AudioTrack::createTable( m_dbConnection.get() );
    Artist::createTable( m_dbConnection.get() );
    Artist::createDefaultArtists( m_dbConnection.get() );
168
    Settings::createTable( m_dbConnection.get() );
169
    parser::Task::createTable( m_dbConnection.get() );
170
    Metadata::createTable( m_dbConnection.get() );
171 172 173 174
}

void MediaLibrary::createAllTriggers()
{
175
    auto dbModelVersion = m_settings.dbModelVersion();
176
    Folder::createTriggers( m_dbConnection.get() );
177
    Album::createTriggers( m_dbConnection.get() );
178
    AlbumTrack::createTriggers( m_dbConnection.get() );
179
    Artist::createTriggers( m_dbConnection.get(), dbModelVersion );
180
    Media::createTriggers( m_dbConnection.get() );
181
    File::createTriggers( m_dbConnection.get() );
182 183
    Genre::createTriggers( m_dbConnection.get() );
    Playlist::createTriggers( m_dbConnection.get() );
184
    Label::createTriggers( m_dbConnection.get() );
185
    Show::createTriggers( m_dbConnection.get() );
186 187
}

188
template <typename T>
189
static void propagateDeletionToCache( sqlite::Connection::HookReason reason, int64_t rowId )
190
{
191
    if ( reason != sqlite::Connection::HookReason::Delete )
192 193 194 195
        return;
    T::removeFromCache( rowId );
}

196 197
void MediaLibrary::registerEntityHooks()
{
198
    if ( m_modificationNotifier == nullptr )
199 200
        return;

201
    m_dbConnection->registerUpdateHook( policy::MediaTable::Name,
202 203
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
204 205
            return;
        Media::removeFromCache( rowId );
206
        m_modificationNotifier->notifyMediaRemoval( rowId );
207
    });
208
    m_dbConnection->registerUpdateHook( policy::ArtistTable::Name,
209 210
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
211 212 213 214
            return;
        Artist::removeFromCache( rowId );
        m_modificationNotifier->notifyArtistRemoval( rowId );
    });
215
    m_dbConnection->registerUpdateHook( policy::AlbumTable::Name,
216 217
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
218 219 220 221
            return;
        Album::removeFromCache( rowId );
        m_modificationNotifier->notifyAlbumRemoval( rowId );
    });
222
    m_dbConnection->registerUpdateHook( policy::AlbumTrackTable::Name,
223 224
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
225 226 227 228
            return;
        AlbumTrack::removeFromCache( rowId );
        m_modificationNotifier->notifyAlbumTrackRemoval( rowId );
    });
229
    m_dbConnection->registerUpdateHook( policy::PlaylistTable::Name,
230 231
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
232 233 234 235
            return;
        Playlist::removeFromCache( rowId );
        m_modificationNotifier->notifyPlaylistRemoval( rowId );
    });
236 237 238 239 240 241 242 243 244 245
    m_dbConnection->registerUpdateHook( policy::DeviceTable::Name, &propagateDeletionToCache<Device> );
    m_dbConnection->registerUpdateHook( policy::FileTable::Name, &propagateDeletionToCache<File> );
    m_dbConnection->registerUpdateHook( policy::FolderTable::Name, &propagateDeletionToCache<Folder> );
    m_dbConnection->registerUpdateHook( policy::GenreTable::Name, &propagateDeletionToCache<Genre> );
    m_dbConnection->registerUpdateHook( policy::LabelTable::Name, &propagateDeletionToCache<Label> );
    m_dbConnection->registerUpdateHook( policy::MovieTable::Name, &propagateDeletionToCache<Movie> );
    m_dbConnection->registerUpdateHook( policy::ShowTable::Name, &propagateDeletionToCache<Show> );
    m_dbConnection->registerUpdateHook( policy::ShowEpisodeTable::Name, &propagateDeletionToCache<ShowEpisode> );
    m_dbConnection->registerUpdateHook( policy::AudioTrackTable::Name, &propagateDeletionToCache<AudioTrack> );
    m_dbConnection->registerUpdateHook( policy::VideoTrackTable::Name, &propagateDeletionToCache<VideoTrack> );
246 247
}

248 249 250 251 252
bool MediaLibrary::validateSearchPattern( const std::string& pattern )
{
    return pattern.size() >= 3;
}

253 254 255 256 257 258 259 260 261 262 263 264 265
bool MediaLibrary::createThumbnailFolder( const std::string& thumbnailPath ) const
{
    auto paths = utils::file::splitPath( thumbnailPath, true );
#ifndef _WIN32
    std::string fullPath{ "/" };
#else
    std::string fullPath;
#endif
    while ( paths.empty() == false )
    {
        fullPath += paths.top();

#ifdef _WIN32
266 267 268 269 270 271 272 273 274
        // Don't try to create C: or various other drives
        if ( isalpha( fullPath[0] ) && fullPath[1] == ':' && fullPath.length() == 2 )
        {
            fullPath += "\\";
            paths.pop();
            continue;
        }
        auto wFullPath = charset::ToWide( fullPath.c_str() );
        if ( _wmkdir( wFullPath.get() ) != 0 )
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291
#else
        if ( mkdir( fullPath.c_str(), S_IRWXU ) != 0 )
#endif
        {
            if ( errno != EEXIST )
                return false;
        }
        paths.pop();
#ifndef _WIN32
        fullPath += "/";
#else
        fullPath += "\\";
#endif
    }
    return true;
}

292 293 294
InitializeResult MediaLibrary::initialize( const std::string& dbPath,
                                           const std::string& thumbnailPath,
                                           IMediaLibraryCb* mlCallback )
295
{
296
    LOG_INFO( "Initializing medialibrary..." );
297
    if ( m_initialized == true )
298 299
    {
        LOG_INFO( "...Already initialized" );
300
        return InitializeResult::AlreadyInitialized;
301
    }
302 303 304 305
    if ( m_deviceLister == nullptr )
    {
        m_deviceLister = factory::createDeviceLister();
        if ( m_deviceLister == nullptr )
306 307
        {
            LOG_ERROR( "No available IDeviceLister was found." );
308
            return InitializeResult::Failed;
309
        }
310
    }
311
    addLocalFsFactory();
312
    if ( createThumbnailFolder( thumbnailPath ) == false )
313
    {
314 315 316
        LOG_ERROR( "Failed to create thumbnail directory (", thumbnailPath,
                    ": ", strerror( errno ) );
        return InitializeResult::Failed;
317
    }
318
    m_thumbnailPath = thumbnailPath;
319
    m_callback = mlCallback;
320
    m_dbConnection = sqlite::Connection::connect( dbPath );
321

322 323 324
    // Give a chance to test overloads to reject the creation of a notifier
    startDeletionNotifier();
    // Which allows us to register hooks, or not, depending on the presence of a notifier
325 326
    registerEntityHooks();

327
    auto res = InitializeResult::Success;
328
    try
329
    {
330
        auto t = m_dbConnection->newTransaction();
331
        createAllTables();
332
        if ( m_settings.load() == false )
333 334
        {
            LOG_ERROR( "Failed to load settings" );
335
            return InitializeResult::Failed;
336
        }
337 338 339
        createAllTriggers();
        t->commit();

340 341
        if ( m_settings.dbModelVersion() != Settings::DbModelVersion )
        {
342 343
            res = updateDatabaseModel( m_settings.dbModelVersion(), dbPath );
            if ( res == InitializeResult::Failed )
344 345
            {
                LOG_ERROR( "Failed to update database model" );
346
                return res;
347 348 349 350 351 352
            }
        }
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Can't initialize medialibrary: ", ex.what() );
353
        return InitializeResult::Failed;
354
    }
355 356
    m_initialized = true;
    LOG_INFO( "Successfuly initialized" );
357
    return res;
358 359 360 361
}

bool MediaLibrary::start()
{
362
    assert( m_initialized == true );
363 364 365
    if ( m_parser != nullptr )
        return false;

366
    populateFsFactories();
367 368
    for ( auto& fsFactory : m_fsFactories )
        refreshDevices( *fsFactory );
369
    startDiscoverer();
370 371
    if ( startParser() == false )
        return false;
372
    startThumbnailer();
373
    return true;
374 375
}

376
void MediaLibrary::setVerbosity( LogLevel v )
377 378 379 380 381
{
    m_verbosity = v;
    Log::setLogLevel( v );
}

382 383 384 385 386
MediaPtr MediaLibrary::media( int64_t mediaId ) const
{
    return Media::fetch( this, mediaId );
}

387 388
MediaPtr MediaLibrary::media( const std::string& mrl ) const
{
389
    LOG_INFO( "Fetching media from mrl: ", mrl );
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
390
    auto file = File::fromExternalMrl( this, mrl );
391 392 393 394 395
    if ( file != nullptr )
    {
        LOG_INFO( "Found external media: ", mrl );
        return file->media();
    }
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
396
    auto fsFactory = fsFactoryForMrl( mrl );
397
    if ( fsFactory == nullptr )
398
    {
399 400
        LOG_WARN( "Failed to create FS factory for path ", mrl );
        return nullptr;
401
    }
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
402
    auto device = fsFactory->createDeviceFromMrl( mrl );
403 404 405 406 407 408
    if ( device == nullptr )
    {
        LOG_WARN( "Failed to create a device associated with mrl ", mrl );
        return nullptr;
    }
    if ( device->isRemovable() == false )
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
409
        file = File::fromMrl( this, mrl );
410 411
    else
    {
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
412
        auto folder = Folder::fromMrl( this, utils::file::directory( mrl ) );
413 414 415 416 417 418 419 420 421 422 423 424 425 426
        if ( folder == nullptr )
        {
            LOG_WARN( "Failed to find folder containing ", mrl );
            return nullptr;
        }
        if ( folder->isPresent() == false )
        {
            LOG_INFO( "Found a folder containing ", mrl, " but it is not present" );
            return nullptr;
        }
        file = File::fromFileName( this, utils::file::fileName( mrl ), folder->id() );
    }
    if ( file == nullptr )
    {
427
        LOG_WARN( "Failed to fetch file for ", mrl, " (device ", device->uuid(), " was ",
428
                  device->isRemovable() ? "" : "NOT ", "removable)");
429 430 431 432 433
        return nullptr;
    }
    return file->media();
}

434
MediaPtr MediaLibrary::addExternalMedia( const std::string& mrl, IMedia::Type type )
435
{
436 437
    try
    {
438
        return sqlite::Tools::withRetries( 3, [this, &mrl, type]() -> MediaPtr {
439
            auto t = m_dbConnection->newTransaction();
440
            auto media = Media::create( this, type, utils::file::fileName( mrl ) );
441 442 443 444 445 446 447 448 449 450 451 452 453
            if ( media == nullptr )
                return nullptr;
            if ( media->addExternalMrl( mrl, IFile::Type::Main ) == nullptr )
                return nullptr;
            t->commit();
            return media;
        });
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create external media: ", ex.what() );
        return nullptr;
    }
454 455
}

456 457 458 459 460 461 462 463 464 465
MediaPtr MediaLibrary::addExternalMedia( const std::string& mrl )
{
    return addExternalMedia( mrl, IMedia::Type::External );
}

MediaPtr MediaLibrary::addStream( const std::string& mrl )
{
    return addExternalMedia( mrl, IMedia::Type::Stream );
}

466
Query<IMedia> MediaLibrary::audioFiles( const QueryParameters* params ) const
467
{
468
    return Media::listAll( this, IMedia::Type::Audio, params );
469 470
}

471
Query<IMedia> MediaLibrary::videoFiles( const QueryParameters* params ) const
472
{
473
    return Media::listAll( this, IMedia::Type::Video, params );
474 475
}

476 477 478 479 480 481 482 483
bool MediaLibrary::isExtensionSupported( const char* ext )
{
    return std::binary_search( std::begin( supportedExtensions ),
        std::end( supportedExtensions ), ext, [](const char* l, const char* r) {
            return strcasecmp( l, r ) < 0;
        });
}

484 485
void MediaLibrary::addDiscoveredFile( std::shared_ptr<fs::IFile> fileFs,
                                      std::shared_ptr<Folder> parentFolder,
486 487
                                      std::shared_ptr<fs::IDirectory> parentFolderFs,
                                      std::pair<std::shared_ptr<Playlist>, unsigned int> parentPlaylist )
488
{
489 490
    try
    {
491 492 493 494 495 496 497 498 499 500 501 502 503 504
        std::shared_ptr<parser::Task> task;
        if ( parentPlaylist.first == nullptr )
        {
            // Sqlite won't ensure uniqueness for Task with the same (mrl, parent_playlist_id)
            // when parent_playlist_id is null, so we have to ensure of it ourselves
            const std::string req = "SELECT * FROM " + policy::TaskTable::Name + " "
                    "WHERE mrl = ? AND parent_playlist_id IS NULL";
            task = parser::Task::fetch( this, req, fileFs->mrl() );
            if ( task != nullptr )
            {
                LOG_INFO( "Not creating duplicated task for mrl: ", fileFs->mrl() );
                return;
            }
        }
505
        // Don't move the file as we might need it for error handling
506
        task = parser::Task::create( this, fileFs, std::move( parentFolder ),
507 508 509 510 511 512 513 514 515 516 517
                                          std::move( parentFolderFs ), std::move( parentPlaylist ) );
        if ( task != nullptr && m_parser != nullptr )
            m_parser->parse( task );
    }
    catch(sqlite::errors::ConstraintViolation& ex)
    {
        // Most likely the file is already scheduled and we restarted the
        // discovery after a crash.
        LOG_WARN( "Failed to insert ", fileFs->mrl(), ": ", ex.what(), ". "
                  "Assuming the file is already scheduled for discovery" );
    }
518 519
}

520
bool MediaLibrary::deleteFolder( const Folder& folder )
521
{
522
    LOG_INFO( "deleting folder ", folder.mrl() );
523
    if ( Folder::destroy( this, folder.id() ) == false )
524
        return false;
525
    Media::clear();
526
    return true;
527 528
}

529 530
LabelPtr MediaLibrary::createLabel( const std::string& label )
{
531 532 533 534 535 536 537 538 539
    try
    {
        return Label::create( this, label );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create a label: ", ex.what() );
        return nullptr;
    }
540
}
541 542 543

bool MediaLibrary::deleteLabel( LabelPtr label )
{
544 545 546 547 548 549 550 551 552
    try
    {
        return Label::destroy( this, label->id() );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to delete label: ", ex.what() );
        return false;
    }
553
}
554

555
AlbumPtr MediaLibrary::album( int64_t id ) const
556
{
557
    return Album::fetch( this, id );
558 559
}

560
std::shared_ptr<Album> MediaLibrary::createAlbum( const std::string& title, int64_t thumbnailId )
561
{
562
    return Album::create( this, title, thumbnailId );
563 564
}

565
Query<IAlbum> MediaLibrary::albums( const QueryParameters* params ) const
566
{
567
    return Album::listAll( this, params );
568 569
}

570
Query<IGenre> MediaLibrary::genres( const QueryParameters* params ) const
571
{
572
    return Genre::listAll( this, params );
573 574
}

575
GenrePtr MediaLibrary::genre( int64_t id ) const
576 577 578 579
{
    return Genre::fetch( this, id );
}

580 581 582 583 584
ShowPtr MediaLibrary::show( int64_t id ) const
{
    return Show::fetch( this, id );
}

585
std::shared_ptr<Show> MediaLibrary::createShow( const std::string& name )
586
{
587
    return Show::create( this, name );
588 589
}

590 591 592 593 594
Query<IShow> MediaLibrary::shows(const QueryParameters* params) const
{
    return Show::listAll( this, params );
}

595 596 597 598 599
MoviePtr MediaLibrary::movie( int64_t id ) const
{
    return Movie::fetch( this, id );
}

600
std::shared_ptr<Movie> MediaLibrary::createMovie( Media& media )
601
{
602
    auto movie = Movie::create( this, media.id() );
603
    media.setMovie( movie );
604
    media.save();
605
    return movie;
606 607
}

608
ArtistPtr MediaLibrary::artist( int64_t id ) const
609
{
610
    return Artist::fetch( this, id );
611 612 613
}

ArtistPtr MediaLibrary::artist( const std::string& name )
614 615
{
    static const std::string req = "SELECT * FROM " + policy::ArtistTable::Name
616
            + " WHERE name = ? AND is_present != 0";
617
    return Artist::fetch( this, req, name );
618 619
}

620
std::shared_ptr<Artist> MediaLibrary::createArtist( const std::string& name )
621
{
622 623
    try
    {
624
        return Artist::create( this, name );
625 626 627 628 629 630
    }
    catch( sqlite::errors::ConstraintViolation &ex )
    {
        LOG_WARN( "ContraintViolation while creating an artist (", ex.what(), ") attempting to fetch it instead" );
        return std::static_pointer_cast<Artist>( artist( name ) );
    }
631 632
}

633
Query<IArtist> MediaLibrary::artists( bool includeAll, const QueryParameters* params ) const
634
{
635
    return Artist::listAll( this, includeAll, params );
636 637
}

638 639
PlaylistPtr MediaLibrary::createPlaylist( const std::string& name )
{
640 641 642 643 644 645 646 647 648
    try
    {
        return Playlist::create( this, name );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create a playlist: ", ex.what() );
        return nullptr;
    }
649 650
}

651
Query<IPlaylist> MediaLibrary::playlists( const QueryParameters* params )
652
{
653
    return Playlist::listAll( this, params );
654 655
}

656 657 658 659 660
PlaylistPtr MediaLibrary::playlist( int64_t id ) const
{
    return Playlist::fetch( this, id );
}

661
bool MediaLibrary::deletePlaylist( int64_t playlistId )
662
{
663 664 665 666 667 668 669 670 671
    try
    {
        return Playlist::destroy( this, playlistId );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to delete playlist: ", ex.what() );
        return false;
    }
672
}
673

674
Query<IMedia> MediaLibrary::history() const
675
{
676
    return Media::fetchHistory( this );
677 678
}

679
Query<IMedia> MediaLibrary::streamHistory() const
680
{
681
    return Media::fetchStreamHistory( this );
682 683
}

684 685
bool MediaLibrary::clearHistory()
{
686 687 688 689 690 691 692 693 694 695 696 697
    try
    {
        return sqlite::Tools::withRetries( 3, [this]() {
            Media::clearHistory( this );
            return true;
        });
    }
    catch ( sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to clear history: ", ex.what() );
        return false;
    }
698 699
}

700
Query<IMedia> MediaLibrary::searchMedia( const std::string& title,
701
                                                const QueryParameters* params ) const
702
{
703 704
    if ( validateSearchPattern( title ) == false )
        return {};
705
    return Media::search( this, title, params );
706 707
}

708 709 710 711 712 713 714 715 716 717 718
Query<IMedia> MediaLibrary::searchAudio( const std::string& pattern, const QueryParameters* params ) const
{
    if ( validateSearchPattern( pattern ) == false )
        return {};
    return Media::search( this, pattern, IMedia::Type::Audio, params );
}

Query<IMedia> MediaLibrary::searchVideo( const std::string& pattern, const QueryParameters* params ) const
{
    if ( validateSearchPattern( pattern ) == false )
        return {};
719
    return Media::search( this, pattern, IMedia::Type::Video, params );
720 721
}

722
Query<IPlaylist> MediaLibrary::searchPlaylists( const std::string& name,
723
                                                const QueryParameters* params ) const
724
{
725 726
    if ( validateSearchPattern( name ) == false )
        return {};
727
    return Playlist::search( this, name, params );
728 729
}

730
Query<IAlbum> MediaLibrary::searchAlbums( const std::string& pattern,
731
                                          const QueryParameters* params ) const
732
{
733 734
    if ( validateSearchPattern( pattern ) == false )
        return {};
735
    return Album::search( this, pattern, params );
736 737
}

738 739
Query<IGenre> MediaLibrary::searchGenre( const std::string& genre,
                                         const QueryParameters* params ) const
740
{
741 742
    if ( validateSearchPattern( genre ) == false )
        return {};
743
    return Genre::search( this, genre, params );
744 745
}

746
Query<IArtist> MediaLibrary::searchArtists( const std::string& name,
747
                                            const QueryParameters* params ) const
748 749 750
{
    if ( validateSearchPattern( name ) == false )
        return {};
751
    return Artist::search( this, name, params );
752 753
}

754 755 756 757 758 759
Query<IShow> MediaLibrary::searchShows( const std::string& pattern,
                                        const QueryParameters* params ) const
{
    return Show::search( this, pattern, params );
}

760
SearchAggregate MediaLibrary::search( const std::string& pattern,
761
                                      const QueryParameters* params ) const
762
{
763
    SearchAggregate res;
764 765 766 767 768
    res.albums = searchAlbums( pattern, params );
    res.artists = searchArtists( pattern, params );
    res.genres = searchGenre( pattern, params );
    res.media = searchMedia( pattern, params );
    res.playlists = searchPlaylists( pattern, params );
769
    res.shows = searchShows( pattern, params );
770 771 772
    return res;
}

773
bool MediaLibrary::startParser()
774
{
775
    m_parser.reset( new parser::Parser( this ) );
776

777 778
    if ( m_services.size() == 0 )
    {
779
#ifdef HAVE_LIBVLC
780
        m_parser->addService( std::make_shared<parser::VLCMetadataService>() );
781 782 783
#else
        return false;
#endif
784 785 786 787 788 789 790
    }
    else
    {
        assert( m_services[0]->targetedStep() == parser::Step::MetadataExtraction );
        m_parser->addService( m_services[0] );
    }
    m_parser->addService( std::make_shared<parser::MetadataAnalyzer>() );
791
    m_parser->start();
792
    return true;
793 794
}

795
void MediaLibrary::startDiscoverer()
796
{
797
    m_discovererWorker.reset( new DiscovererWorker( this ) );
798
    for ( const auto& fsFactory : m_fsFactories )
799
    {
800
        std::unique_ptr<prober::CrawlerProbe> probePtr( new prober::CrawlerProbe{} );
801 802 803
        m_discovererWorker->addDiscoverer( std::unique_ptr<IDiscoverer>( new FsDiscoverer( fsFactory, this, m_callback,
                                                                                           std::move ( probePtr ) ) ) );
    }
804 805
}

806 807
void MediaLibrary::startDeletionNotifier()
{
808 809
    m_modificationNotifier.reset( new ModificationNotifier( this ) );
    m_modificationNotifier->start();
810 811
}

812
void MediaLibrary::startThumbnailer()
813
{
814
#ifdef HAVE_LIBVLC
815
    m_thumbnailer = std::unique_ptr<VLCThumbnailer>( new VLCThumbnailer( this ) );
816
#endif
817 818
}

819 820 821 822 823 824 825
void MediaLibrary::populateFsFactories()
{
#ifdef HAVE_LIBVLC
    m_externalFsFactories.emplace_back( std::make_shared<factory::NetworkFileSystemFactory>( "smb", "dsm-sd" ) );
#endif
}

826 827
void MediaLibrary::addLocalFsFactory()
{
828
    m_fsFactories.emplace( begin( m_fsFactories ), std::make_shared<factory::FileSystemFactory>( m_deviceLister ) );
829 830
}

831
InitializeResult MediaLibrary::updateDatabaseModel( unsigned int previousVersion,
832
                                        const std::string& dbPath )
833
{
834 835 836
    LOG_INFO( "Updating database model from ", previousVersion, " to ", Settings::DbModelVersion );
    // Up until model 3, it's safer (and potentially more efficient with index changes) to drop the DB
    // It's also way simpler to implement
837
    // In case of downgrade, just recreate the database
838
    for ( auto i = 0u; i < 3; ++i )
839
    {
840 841
        try
        {
842
            bool needRescan = false;
843 844 845
            // Up until model 3, it's safer (and potentially more efficient with index changes) to drop the DB
            // It's also way simpler to implement
            // In case of downgrade, just recreate the database
846 847
            // We might also have some special cases for failed upgrade (see
            // comments below for per-version details)
848
            if ( previousVersion < 3 ||
849 850
                 previousVersion > Settings::DbModelVersion ||
                 previousVersion == 4 )
851
            {
852
                if( recreateDatabase( dbPath ) == false )
853
                    throw std::runtime_error( "Failed to recreate the database" );
854
                return InitializeResult::DbReset;
855
            }
856 857 858 859 860 861 862 863
            /**
             * Migration from 3 to 4 didn't happen so well and broke a few
             * users DB. So:
             * - Any v4 database will be dropped and recreated in v5
             * - Any v3 database will be upgraded to v5
             * V4 database is only used by VLC-android 2.5.6 // 2.5.8, which are
             * beta versions.
             */
864 865
            if ( previousVersion == 3 )
            {
866
                migrateModel3to5();
867
                previousVersion = 5;
868
            }
869 870
            if ( previousVersion == 5 )
            {
871
                migrateModel5to6();
872 873
                previousVersion = 6;
            }
Alexandre Fernandez's avatar
Alexandre Fernandez committed
874 875
            if ( previousVersion == 6 )
            {
876 877 878
                // Force a rescan to solve metadata analysis problems.
                // The insertion is fixed, but won't edit already inserted data.
                forceRescan();
Alexandre Fernandez's avatar
Alexandre Fernandez committed
879 880
                previousVersion = 7;
            }
881 882 883 884 885 886 887 888 889
            /**
             * V7 introduces artist.nb_tracks and an associated trigger to delete
             * artists when it has no track/album left.
             */
            if ( previousVersion == 7 )
            {
                migrateModel7to8();
                previousVersion = 8;
            }
890 891
            if ( previousVersion == 8 )
            {
892 893 894 895 896 897 898 899 900

                // Multiple changes justify the rescan:
                // - Changes in the way we chose to encode or not MRL, meaning
                //   potentially all MRL are wrong (more precisely, will
                //   mismatch what VLC expects, which makes playlist analysis
                //   break.
                // - Fix in the way we chose album candidates, meaning some
                //   albums were likely to be wrongfully created.
                needRescan = true;
901
                migrateModel8to9();
902 903
                previousVersion = 9;
            }
904 905
            if ( previousVersion == 9 )
            {
906
                needRescan = true;
907 908 909
                migrateModel9to10();
                previousVersion = 10;
            }
910 911
            if ( previousVersion == 10 )
            {
912
                needRescan = true;
913 914 915
                migrateModel10to11();
                previousVersion = 11;
            }
916 917 918 919 920
            if ( previousVersion == 11 )
            {
                parser::Task::recoverUnscannedFiles( this );
                previousVersion = 12;
            }
921 922 923 924 925
            if ( previousVersion == 12 )
            {
                migrateModel12to13();
                previousVersion = 13;
            }
926 927
            if ( previousVersion == 13 )
            {
928 929 930
                // We need to recreate many thumbnail records, and hopefully
                // generate better ones
                needRescan = true;
931 932 933
                migrateModel13to14();
                previousVersion = 14;
            }
934 935
            // To be continued in the future!

936 937 938
            if ( needRescan == true )
                forceRescan();

939 940 941
            // Safety check: ensure we didn't forget a migration along the way
            assert( previousVersion == Settings::DbModelVersion );
            m_settings.setDbModelVersion( Settings::DbModelVersion );
942
            if ( m_settings.save() == false )
943 944
                return InitializeResult::Failed;
            return InitializeResult::Success;
945 946 947 948 949 950 951 952 953 954 955
        }
        catch( const std::exception& ex )
        {
            LOG_ERROR( "An error occured during the database upgrade: ",
                       ex.what() );
        }
        catch( ... )
        {
            LOG_ERROR( "An unknown error occured during the database upgrade." );
        }
        LOG_WARN( "Retrying database migration, attempt ", i + 1, " / 3" );
956
    }
957 958
    LOG_ERROR( "Failed to upgrade database, recreating it" );
    for ( auto i = 0u; i < 3; ++i )
959
    {
960 961
        try
        {
962
            if( recreateDatabase( dbPath ) == true )
963
                return InitializeResult::DbReset;
964 965 966 967 968 969 970 971 972 973
        }
        catch( const std::exception& ex )
        {
            LOG_ERROR( "Failed to recreate database: ", ex.what() );
        }
        catch(...)
        {
            LOG_ERROR( "Unknown error while trying to recreate the database." );
        }
        LOG_WARN( "Retrying to recreate the database, attempt ", i + 1, " / 3" );
974
    }
975
    return InitializeResult::Failed;
976 977
}

978
bool MediaLibrary::recreateDatabase( const std::string& dbPath )
979
{
980 981 982 983
    // Close all active connections, flushes all previously run statements.
    m_dbConnection.reset();
    unlink( dbPath.c_str() );
    m_dbConnection = sqlite::Connection::connect( dbPath );
984
    createAllTables();
985 986 987
    // We dropped the database, there is no setting to be read anymore
    if( m_settings.load() == false )
        return false;
988 989 990
    return true;
}

991
void MediaLibrary::migrateModel3to5()
992 993
{
    /*
994 995
     * Disable Foreign Keys & recursive triggers to avoid cascading deletion
     * while remodeling the database into the transaction.
996
     */
997
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
998
    auto t = getConn()->newTransaction();
999 1000 1001 1002
    using namespace policy;
    // As SQLite do not allow us to remove or add some constraints,
    // we use the method described here https://www.sqlite.org/faq.html#q11
    std::string reqs[] = {
1003
#               include "database/migrations/migration3-5.sql"
1004 1005 1006
    };

    for ( const auto& req : reqs )
1007
        sqlite::Tools::executeRequest( getConn(), req );
1008
    // Re-create triggers removed in the process
1009 1010
    Media::createTriggers( getConn() );
    Playlist::createTriggers( getConn() );
1011 1012 1013
    t->commit();
}

1014
void MediaLibrary::migrateModel5to6()
1015
{
1016 1017 1018
    std::string req = "DELETE FROM " + policy::MediaTable::Name + " WHERE type = ?";
    sqlite::Tools::executeRequest( getConn(), req, Media::Type::Unknown );

1019 1020
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
    using namespace policy;
1021
    req = "UPDATE " + MediaTable::Name + " SET is_present = 1 WHERE is_present != 0";
1022
    sqlite::Tools::executeRequest( getConn(), req );
Alexandre Fernandez's avatar
Alexandre Fernandez committed
1023 1024
}

1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036
void MediaLibrary::migrateModel7to8()
{
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
    auto t = getConn()->newTransaction();
    using namespace policy;
    std::string reqs[] = {
#               include "database/migrations/migration7-8.sql"
    };

    for ( const auto& req : reqs )
        sqlite::Tools::executeRequest( getConn(), req );
    // Re-create triggers removed in the process
1037
    Artist::createTriggers( getConn(), 8u );
1038 1039
    Media::createTriggers( getConn() );
    File::createTriggers( getConn() );
1040 1041 1042
    t->commit();
}

1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
void MediaLibrary::migrateModel8to9()
{
    // A bug in a previous migration caused our triggers to be missing for the
    // first application run (after the migration).
    // This could have caused media associated to deleted files not to be
    // deleted as well, so let's do that now.
    const std::string req = "DELETE FROM " + policy::MediaTable::Name + " "
            "WHERE id_media IN "
            "(SELECT id_media FROM " + policy::MediaTable::Name + " m LEFT JOIN " +
                policy::FileTable::Name + " f ON f.media_id = m.id_media "
                "WHERE f.media_id IS NULL)";

    // Don't check for the return value, we don't mind if nothing deleted.
    // Quite the opposite actually :)
    sqlite::Tools::executeDelete( getConn(), req );
}

1060 1061 1062 1063 1064 1065 1066 1067
void MediaLibrary::migrateModel9to10()
{
    const std::string req = "SELECT * FROM " + policy::FileTable::Name +
            " WHERE mrl LIKE '%#%%' ESCAPE '#'";
    auto files = File::fetchAll<File>( this, req );
    auto t = getConn()->newTransaction();
    for ( const auto& f : files )
    {
1068 1069 1070 1071
        // We must not call mrl() from here. We might not have all devices yet,
        // and calling mrl would crash for files stored on removable devices.
        auto newMrl = utils::url::encode( utils::url::decode( f->rawMrl() ) );
        LOG_INFO( "Converting ", f->rawMrl(), " to ", newMrl );
1072 1073 1074 1075 1076
        f->setMrl( newMrl );
    }
    t->commit();
}

1077 1078 1079 1080
void MediaLibrary::migrateModel10to11()
{
    const std::string req = "SELECT * FROM " + policy::TaskTable::Name +
            " WHERE mrl LIKE '%#%%' ESCAPE '#'";
1081 1082
    const std::string folderReq = "SELECT * FROM " + policy::FolderTable::Name +
            " WHERE path LIKE '%#%%' ESCAPE '#'";
1083
    auto tasks = parser::Task::fetchAll<parser::Task>( this, req );
1084
    auto folders = Folder::fetchAll<Folder>( this, folderReq );
1085 1086 1087
    auto t = getConn()->newTransaction();
    for ( const auto& t : tasks )
    {
1088 1089
        auto newMrl = utils::url::encode( utils::url::decode( t->item().mrl() ) );
        LOG_INFO( "Converting task mrl: ", t->item().mrl(), " to ", newMrl );
1090 1091
        t->setMrl( std::move( newMrl ) );
    }
1092 1093 1094 1095 1096 1097 1098
    for ( const auto &f : folders )
    {
        // We must not call mrl() from here. We might not have all devices yet,
        // and calling mrl would crash for files stored on removable devices.
        auto newMrl = utils::url::encode( utils::url::decode( f->rawMrl() ) );
        f->setMrl( std::move( newMrl ) );
    }
1099 1100 1101
    t->commit();
}

1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
/*
 * - Some is_present related triggers were fixed in model 6 to 7 migration, but
 *   they were not recreated if already existing. The has_file_present trigger
 *   was recreated as part of model 7 to 8 migration, but we need to ensure
 *   has_album_present (Artist) & is_album_present (Album) triggers are
 *   recreated to behave as expected
 * - Due to a typo, is_track_present was named is_track_presentAFTER, and was
 *   executed BEFORE the update took place, thus using the wrong is_present value.
 *   The trigger will be recreated as part of this migration, and the values
 *   will be enforced, causing the entire update chain to be triggered, and
 *   restoring correct is_present values for all AlbumTrack/Album/Artist entries
 */
1114 1115
void MediaLibrary::migrateModel12to13()
{
1116
    auto t = getConn()->newTransaction();
1117 1118 1119 1120 1121 1122 1123 1124 1125
    const std::string reqs[] = {
        "DROP TRIGGER IF EXISTS is_track_presentAFTER",
        "DROP TRIGGER has_album_present",
        "DROP TRIGGER is_album_present",
    };

    for ( const auto& req : reqs )
        sqlite::Tools::executeDelete( getConn(), req );

1126
    AlbumTrack::createTriggers( getConn() );
1127 1128
    Album::createTriggers( getConn() );
    Artist::createTriggers( getConn(), 13 );
1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
    // Leave the weak context as we now need to update is_present fields, which
    // are propagated through recursive triggers
    const std::string migrateData = "UPDATE " + policy::AlbumTrackTable::Name +
            " SET is_present = (SELECT is_present FROM " + policy::MediaTable::Name +
            " WHERE id_media = media_id)";
    sqlite::Tools::executeUpdate( getConn(), migrateData );
    t->commit();
}

/*
 * - Remove the Media.thumbnail
 * - Add Media.thumbnail_id
 * - Add Media.thumbnail_generated
 */
void MediaLibrary::migrateModel13to14()
{
1145 1146 1147
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
    auto t = getConn()->newTransaction();
    using namespace policy;
1148
    using ThumbnailType = typename std::underlying_type<Thumbnail::Origin>::type;
1149
    std::string reqs[] = {
1150
#               include "database/migrations/migration13-14.sql"
1151 1152 1153 1154
    };

    for ( const auto& req : reqs )
        sqlite::Tools::executeRequest( getConn(), req );
1155 1156 1157
    // Re-create tables that we just removed
    // We will run a re-scan, so we don't care about keeping their content
    Album::createTable( getConn() );
1158
    Artist::createTable( getConn() );
1159 1160
    Movie::createTable( getConn() );
    Show::createTable( getConn() );
1161
    VideoTrack::createTable( getConn() );
1162 1163 1164
    // Re-create triggers removed in the process
    Media::createTriggers( getConn() );
    AlbumTrack::createTriggers( getConn() );
1165 1166
    Album::createTriggers( getConn() );
    Artist::createTriggers( getConn(), 14 );
1167
    Show::createTriggers( getConn() );
1168 1169
    Playlist::createTriggers( getConn() );

1170 1171 1172
    t->commit();
}

1173 1174
void MediaLibrary::reload()
{
1175 1176
    if ( m_discovererWorker != nullptr )
        m_discovererWorker->reload();
1177 1178
}

1179 1180
void MediaLibrary::reload( const std::string& entryPoint )
{