MediaLibrary.cpp 57.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2015 Hugo Beauzée-Luyssen, Videolabs
 *
 * Authors: Hugo Beauzée-Luyssen<hugo@beauzee.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

23 24 25 26
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

27 28
#include <algorithm>
#include <functional>
29
#include <utility>
30
#include <sys/stat.h>
31
#include <unistd.h>
32

33 34
#include "Album.h"
#include "AlbumTrack.h"
35
#include "Artist.h"
36
#include "AudioTrack.h"
37
#include "discoverer/DiscovererWorker.h"
38
#include "discoverer/probe/CrawlerProbe.h"
39
#include "utils/ModificationsNotifier.h"
40
#include "Device.h"
41
#include "File.h"
42
#include "Folder.h"
43
#include "Genre.h"
44
#include "Media.h"
45
#include "MediaLibrary.h"
46
#include "Label.h"
47
#include "logging/Logger.h"
48
#include "Movie.h"
49
#include "parser/Parser.h"
50
#include "Playlist.h"
51 52
#include "Show.h"
#include "ShowEpisode.h"
53
#include "SubtitleTrack.h"
54
#include "Thumbnail.h"
55
#include "database/SqliteTools.h"
56
#include "database/SqliteConnection.h"
57
#include "database/SqliteQuery.h"
58
#include "utils/Filename.h"
59
#include "utils/Url.h"
60
#include "VideoTrack.h"
61
#include "Metadata.h"
62
#include "parser/Task.h"
63
#include "utils/Charsets.h"
64

65 66 67
// Discoverers:
#include "discoverer/FsDiscoverer.h"

68
// Metadata services:
69
#ifdef HAVE_LIBVLC
70
#include "metadata_services/vlc/VLCMetadataService.h"
71
#endif
72
#include "metadata_services/MetadataParser.h"
73
#include "metadata_services/ThumbnailerWorker.h"
74

75 76
// FileSystem
#include "factory/DeviceListerFactory.h"
77
#include "factory/FileSystemFactory.h"
78 79

#ifdef HAVE_LIBVLC
80
#include "factory/NetworkFileSystemFactory.h"
81 82 83 84 85 86 87 88 89

#include <vlcpp/vlc.hpp>
#if LIBVLC_VERSION_INT >= LIBVLC_VERSION(4, 0, 0, 0)
#include "metadata_services/vlc/CoreThumbnailer.h"
using ThumbnailerType = medialibrary::CoreThumbnailer;
#else
#include "metadata_services/vlc/VmemThumbnailer.h"
using ThumbnailerType = medialibrary::VmemThumbnailer;
#endif
90 91
#endif

92
#include "medialibrary/filesystem/IDevice.h"
93

94 95 96
namespace medialibrary
{

97
const char* const MediaLibrary::supportedExtensions[] = {
98 99 100 101 102 103 104 105 106 107 108 109 110
    "3g2", "3gp", "a52", "aac", "ac3", "adx", "aif", "aifc",
    "aiff", "alac", "amr", "amv", "aob", "ape", "asf", "asx",
    "avi", "b4s", "conf", /*"cue",*/ "divx", "dts", "dv",
    "flac", "flv", "gxf", "ifo", "iso", "it", "itml",
    "m1v", "m2t", "m2ts", "m2v", "m3u", "m3u8", "m4a", "m4b",
    "m4p", "m4v", "mid", "mka", "mkv", "mlp", "mod", "mov",
    "mp1", "mp2", "mp3", "mp4", "mpc", "mpeg", "mpeg1", "mpeg2",
    "mpeg4", "mpg", "mts", "mxf", "nsv", "nuv", "oga", "ogg",
    "ogm", "ogv", "ogx", "oma", "opus", "pls", "ps", "qtl",
    "ram", "rec", "rm", "rmi", "rmvb", "s3m", "sdp", "spx",
    "tod", "trp", "ts", "tta", "vlc", "vob", "voc", "vqf",
    "vro", "w64", "wav", "wax", "webm", "wma", "wmv", "wmx",
    "wpl", "wv", "wvx", "xa", "xm", "xspf"
111 112
};

113 114
const size_t MediaLibrary::NbSupportedExtensions = sizeof(supportedExtensions) / sizeof(supportedExtensions[0]);

115
MediaLibrary::MediaLibrary()
116
    : m_callback( nullptr )
117
    , m_fsFactoryCb( this )
118
    , m_deviceListerCbImpl( this )
119
    , m_verbosity( LogLevel::Error )
120
    , m_settings( this )
121
    , m_initialized( false )
122 123
    , m_discovererIdle( true )
    , m_parserIdle( true )
124
{
125
    Log::setLogLevel( m_verbosity );
126 127
}

128 129
MediaLibrary::~MediaLibrary()
{
130
    // Explicitely stop the discoverer, to avoid it writting while tearing down.
131 132
    if ( m_discovererWorker != nullptr )
        m_discovererWorker->stop();
133 134
    if ( m_parser != nullptr )
        m_parser->stop();
135 136
}

137
void MediaLibrary::createAllTables()
138
{
139
    auto dbModelVersion = m_settings.dbModelVersion();
140 141 142 143 144
    // We need to create the tables in order of triggers creation
    // Device is the "root of all evil". When a device is modified,
    // we will trigger an update on folder, which will trigger
    // an update on files, and so on.

145 146
    Device::createTable( m_dbConnection.get() );
    Folder::createTable( m_dbConnection.get() );
147
    Thumbnail::createTable( m_dbConnection.get() );
148
    Media::createTable( m_dbConnection.get(), dbModelVersion );
149 150
    File::createTable( m_dbConnection.get() );
    Label::createTable( m_dbConnection.get() );
151
    Playlist::createTable( m_dbConnection.get(), dbModelVersion );
152 153 154 155 156 157 158 159 160 161
    Genre::createTable( m_dbConnection.get() );
    Album::createTable( m_dbConnection.get() );
    AlbumTrack::createTable( m_dbConnection.get() );
    Show::createTable( m_dbConnection.get() );
    ShowEpisode::createTable( m_dbConnection.get() );
    Movie::createTable( m_dbConnection.get() );
    VideoTrack::createTable( m_dbConnection.get() );
    AudioTrack::createTable( m_dbConnection.get() );
    Artist::createTable( m_dbConnection.get() );
    Artist::createDefaultArtists( m_dbConnection.get() );
162
    Settings::createTable( m_dbConnection.get() );
163
    parser::Task::createTable( m_dbConnection.get() );
164
    Metadata::createTable( m_dbConnection.get() );
165
    SubtitleTrack::createTable( m_dbConnection.get() );
166 167 168 169
}

void MediaLibrary::createAllTriggers()
{
170
    auto dbModelVersion = m_settings.dbModelVersion();
171
    Folder::createTriggers( m_dbConnection.get(), dbModelVersion );
172
    Album::createTriggers( m_dbConnection.get() );
173
    AlbumTrack::createTriggers( m_dbConnection.get() );
174
    Artist::createTriggers( m_dbConnection.get(), dbModelVersion );
175
    Media::createTriggers( m_dbConnection.get(), dbModelVersion );
176
    File::createTriggers( m_dbConnection.get() );
177 178
    Genre::createTriggers( m_dbConnection.get() );
    Playlist::createTriggers( m_dbConnection.get() );
179
    Label::createTriggers( m_dbConnection.get() );
180
    Show::createTriggers( m_dbConnection.get() );
181 182
}

183 184
void MediaLibrary::registerEntityHooks()
{
185
    if ( m_modificationNotifier == nullptr )
186 187
        return;

188
    m_dbConnection->registerUpdateHook( Media::Table::Name,
189 190
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
191
            return;
192
        m_modificationNotifier->notifyMediaRemoval( rowId );
193
    });
194
    m_dbConnection->registerUpdateHook( Artist::Table::Name,
195 196
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
197 198 199
            return;
        m_modificationNotifier->notifyArtistRemoval( rowId );
    });
200
    m_dbConnection->registerUpdateHook( Album::Table::Name,
201 202
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
203 204 205
            return;
        m_modificationNotifier->notifyAlbumRemoval( rowId );
    });
206
    m_dbConnection->registerUpdateHook( Playlist::Table::Name,
207 208
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
209 210 211
            return;
        m_modificationNotifier->notifyPlaylistRemoval( rowId );
    });
212 213 214 215 216 217
    m_dbConnection->registerUpdateHook( Genre::Table::Name,
                                        [this]( sqlite::Connection::HookReason reason, int64_t rowId ) {
        if ( reason != sqlite::Connection::HookReason::Delete )
            return;
        m_modificationNotifier->notifyGenreRemoval( rowId );
    });
218 219
}

220 221 222 223 224
bool MediaLibrary::validateSearchPattern( const std::string& pattern )
{
    return pattern.size() >= 3;
}

225 226 227 228 229 230 231 232 233 234 235 236 237
bool MediaLibrary::createThumbnailFolder( const std::string& thumbnailPath ) const
{
    auto paths = utils::file::splitPath( thumbnailPath, true );
#ifndef _WIN32
    std::string fullPath{ "/" };
#else
    std::string fullPath;
#endif
    while ( paths.empty() == false )
    {
        fullPath += paths.top();

#ifdef _WIN32
238 239 240 241 242 243 244 245 246
        // Don't try to create C: or various other drives
        if ( isalpha( fullPath[0] ) && fullPath[1] == ':' && fullPath.length() == 2 )
        {
            fullPath += "\\";
            paths.pop();
            continue;
        }
        auto wFullPath = charset::ToWide( fullPath.c_str() );
        if ( _wmkdir( wFullPath.get() ) != 0 )
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
#else
        if ( mkdir( fullPath.c_str(), S_IRWXU ) != 0 )
#endif
        {
            if ( errno != EEXIST )
                return false;
        }
        paths.pop();
#ifndef _WIN32
        fullPath += "/";
#else
        fullPath += "\\";
#endif
    }
    return true;
}

264 265 266
InitializeResult MediaLibrary::initialize( const std::string& dbPath,
                                           const std::string& thumbnailPath,
                                           IMediaLibraryCb* mlCallback )
267
{
268
    LOG_INFO( "Initializing medialibrary..." );
269
    if ( m_initialized == true )
270 271
    {
        LOG_INFO( "...Already initialized" );
272
        return InitializeResult::AlreadyInitialized;
273
    }
274 275 276 277
    if ( m_deviceLister == nullptr )
    {
        m_deviceLister = factory::createDeviceLister();
        if ( m_deviceLister == nullptr )
278 279
        {
            LOG_ERROR( "No available IDeviceLister was found." );
280
            return InitializeResult::Failed;
281
        }
282
    }
283
    addLocalFsFactory();
284
    populateNetworkFsFactories();
285 286
    m_thumbnailPath = utils::file::toFolderPath( thumbnailPath );
    if ( createThumbnailFolder( m_thumbnailPath ) == false )
287
    {
288
        LOG_ERROR( "Failed to create thumbnail directory (", m_thumbnailPath,
289 290
                    ": ", strerror( errno ) );
        return InitializeResult::Failed;
291
    }
292
    m_callback = mlCallback;
293
    m_dbConnection = sqlite::Connection::connect( dbPath );
294

295 296 297
    // Give a chance to test overloads to reject the creation of a notifier
    startDeletionNotifier();
    // Which allows us to register hooks, or not, depending on the presence of a notifier
298 299
    registerEntityHooks();

300
    auto res = InitializeResult::Success;
301
    try
302
    {
303
        auto t = m_dbConnection->newTransaction();
304
        createAllTables();
305
        if ( m_settings.load() == false )
306 307
        {
            LOG_ERROR( "Failed to load settings" );
308
            return InitializeResult::Failed;
309
        }
310 311 312
        createAllTriggers();
        t->commit();

313 314
        if ( m_settings.dbModelVersion() != Settings::DbModelVersion )
        {
315 316
            res = updateDatabaseModel( m_settings.dbModelVersion(), dbPath );
            if ( res == InitializeResult::Failed )
317 318
            {
                LOG_ERROR( "Failed to update database model" );
319
                return res;
320 321 322 323 324 325
            }
        }
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Can't initialize medialibrary: ", ex.what() );
326
        return InitializeResult::Failed;
327
    }
328 329
    m_initialized = true;
    LOG_INFO( "Successfuly initialized" );
330
    return res;
331 332 333 334
}

bool MediaLibrary::start()
{
335
    assert( m_initialized == true );
336 337 338
    if ( m_parser != nullptr )
        return false;

339 340
    for ( auto& fsFactory : m_fsFactories )
        refreshDevices( *fsFactory );
341 342 343
    // Now that we know which devices are plugged, check for outdated devices
    // Approximate 6 months for old device precision.
    Device::removeOldDevices( this, std::chrono::seconds{ 3600 * 24 * 30 * 6 } );
344
    Media::removeOldMedia( this, std::chrono::seconds{ 3600 * 24 * 30 * 6 } );
345

346
    startDiscoverer();
347 348
    if ( startParser() == false )
        return false;
349
    startThumbnailer();
350
    return true;
351 352
}

353
void MediaLibrary::setVerbosity( LogLevel v )
354 355 356 357 358
{
    m_verbosity = v;
    Log::setLogLevel( v );
}

359 360 361 362 363
MediaPtr MediaLibrary::media( int64_t mediaId ) const
{
    return Media::fetch( this, mediaId );
}

364 365
MediaPtr MediaLibrary::media( const std::string& mrl ) const
{
366
    LOG_INFO( "Fetching media from mrl: ", mrl );
Hugo Beauzée-Luyssen's avatar
Hugo Beauzée-Luyssen committed
367
    auto file = File::fromExternalMrl( this, mrl );
368 369 370 371 372
    if ( file != nullptr )
    {
        LOG_INFO( "Found external media: ", mrl );
        return file->media();
    }
373
    file = File::fromMrl( this, mrl );
374 375 376 377 378
    if ( file == nullptr )
        return nullptr;
    return file->media();
}

379
MediaPtr MediaLibrary::addExternalMedia( const std::string& mrl, IMedia::Type type )
380
{
381 382
    try
    {
383
        return sqlite::Tools::withRetries( 3, [this, &mrl, type]() -> MediaPtr {
384
            auto t = m_dbConnection->newTransaction();
385
            auto fileName = utils::file::fileName( mrl );
386 387
            auto media = Media::create( this, type, 0, 0,
                                        utils::url::decode( fileName ) );
388 389 390 391 392 393 394 395 396 397 398 399 400
            if ( media == nullptr )
                return nullptr;
            if ( media->addExternalMrl( mrl, IFile::Type::Main ) == nullptr )
                return nullptr;
            t->commit();
            return media;
        });
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create external media: ", ex.what() );
        return nullptr;
    }
401 402
}

403 404 405 406 407 408 409 410 411 412
MediaPtr MediaLibrary::addExternalMedia( const std::string& mrl )
{
    return addExternalMedia( mrl, IMedia::Type::External );
}

MediaPtr MediaLibrary::addStream( const std::string& mrl )
{
    return addExternalMedia( mrl, IMedia::Type::Stream );
}

413 414 415 416 417 418 419 420 421 422 423
bool MediaLibrary::removeExternalMedia(MediaPtr media)
{
    if ( media->type() != Media::Type::External &&
         media->type() != Media::Type::Stream )
    {
        assert( !"Invalid media provided" );
        return false;
    }
    return Media::destroy( this, media->id() );
}

424
Query<IMedia> MediaLibrary::audioFiles( const QueryParameters* params ) const
425
{
426
    return Media::listAll( this, IMedia::Type::Audio, params );
427 428
}

429
Query<IMedia> MediaLibrary::videoFiles( const QueryParameters* params ) const
430
{
431
    return Media::listAll( this, IMedia::Type::Video, params );
432 433
}

434 435 436 437 438 439 440 441
bool MediaLibrary::isExtensionSupported( const char* ext )
{
    return std::binary_search( std::begin( supportedExtensions ),
        std::end( supportedExtensions ), ext, [](const char* l, const char* r) {
            return strcasecmp( l, r ) < 0;
        });
}

442
void MediaLibrary::onDiscoveredFile( std::shared_ptr<fs::IFile> fileFs,
443 444 445 446
                                     std::shared_ptr<Folder> parentFolder,
                                     std::shared_ptr<fs::IDirectory> parentFolderFs,
                                     IFile::Type fileType,
                                     std::pair<std::shared_ptr<Playlist>, unsigned int> parentPlaylist )
447
{
448
    auto mrl = fileFs->mrl();
449 450
    try
    {
451 452 453 454 455
        std::shared_ptr<parser::Task> task;
        if ( parentPlaylist.first == nullptr )
        {
            // Sqlite won't ensure uniqueness for Task with the same (mrl, parent_playlist_id)
            // when parent_playlist_id is null, so we have to ensure of it ourselves
456
            const std::string req = "SELECT * FROM " + parser::Task::Table::Name + " "
457
                    "WHERE mrl = ? AND parent_playlist_id IS NULL";
458
            task = parser::Task::fetch( this, req, mrl );
459 460
            if ( task != nullptr )
            {
461
                LOG_INFO( "Not creating duplicated task for mrl: ", mrl );
462 463 464
                return;
            }
        }
465
        task = parser::Task::create( this, mrl, std::move( fileFs ), std::move( parentFolder ),
466
                                     std::move( parentFolderFs ), fileType,
467
                                     std::move( parentPlaylist ) );
468 469 470 471 472 473 474
        if ( task != nullptr && m_parser != nullptr )
            m_parser->parse( task );
    }
    catch(sqlite::errors::ConstraintViolation& ex)
    {
        // Most likely the file is already scheduled and we restarted the
        // discovery after a crash.
475
        LOG_WARN( "Failed to insert ", mrl, ": ", ex.what(), ". "
476 477
                  "Assuming the file is already scheduled for discovery" );
    }
478 479
}

480 481 482 483 484 485
void MediaLibrary::onUpdatedFile( std::shared_ptr<File> file,
                                  std::shared_ptr<fs::IFile> fileFs )
{
    auto mrl = fileFs->mrl();
    try
    {
486
        auto task = parser::Task::createRefreshTask( this, std::move( file ), std::move( fileFs ) );
487 488 489 490 491 492 493 494 495 496 497 498
        if ( task != nullptr && m_parser != nullptr )
            m_parser->parse( std::move( task ) );
    }
    catch( const sqlite::errors::ConstraintViolation& ex )
    {
        // Most likely the file is already scheduled and we restarted the
        // discovery after a crash.
        LOG_WARN( "Failed to insert ", mrl, ": ", ex.what(), ". "
                  "Assuming the file is already scheduled for discovery" );
    }
}

499
bool MediaLibrary::deleteFolder( const Folder& folder )
500
{
501
    LOG_INFO( "deleting folder ", folder.mrl() );
502
    return Folder::destroy( this, folder.id() );
503 504
}

505 506
LabelPtr MediaLibrary::createLabel( const std::string& label )
{
507 508 509 510 511 512 513 514 515
    try
    {
        return Label::create( this, label );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create a label: ", ex.what() );
        return nullptr;
    }
516
}
517 518 519

bool MediaLibrary::deleteLabel( LabelPtr label )
{
520 521 522 523 524 525 526 527 528
    try
    {
        return Label::destroy( this, label->id() );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to delete label: ", ex.what() );
        return false;
    }
529
}
530

531
AlbumPtr MediaLibrary::album( int64_t id ) const
532
{
533
    return Album::fetch( this, id );
534 535
}

536
std::shared_ptr<Album> MediaLibrary::createAlbum( const std::string& title, int64_t thumbnailId )
537
{
538
    return Album::create( this, title, thumbnailId );
539 540
}

541
Query<IAlbum> MediaLibrary::albums( const QueryParameters* params ) const
542
{
543
    return Album::listAll( this, params );
544 545
}

546
Query<IGenre> MediaLibrary::genres( const QueryParameters* params ) const
547
{
548
    return Genre::listAll( this, params );
549 550
}

551
GenrePtr MediaLibrary::genre( int64_t id ) const
552 553 554 555
{
    return Genre::fetch( this, id );
}

556 557 558 559 560
ShowPtr MediaLibrary::show( int64_t id ) const
{
    return Show::fetch( this, id );
}

561
std::shared_ptr<Show> MediaLibrary::createShow( const std::string& name )
562
{
563
    return Show::create( this, name );
564 565
}

566 567 568 569 570
Query<IShow> MediaLibrary::shows(const QueryParameters* params) const
{
    return Show::listAll( this, params );
}

571 572 573 574 575
MoviePtr MediaLibrary::movie( int64_t id ) const
{
    return Movie::fetch( this, id );
}

576
std::shared_ptr<Movie> MediaLibrary::createMovie( Media& media )
577
{
578
    auto movie = Movie::create( this, media.id() );
579
    media.setMovie( movie );
580
    media.save();
581
    return movie;
582 583
}

584
ArtistPtr MediaLibrary::artist( int64_t id ) const
585
{
586
    return Artist::fetch( this, id );
587 588
}

589
std::shared_ptr<Artist> MediaLibrary::createArtist( const std::string& name )
590
{
591
    return Artist::create( this, name );
592 593
}

594
Query<IArtist> MediaLibrary::artists( bool includeAll, const QueryParameters* params ) const
595
{
596
    return Artist::listAll( this, includeAll, params );
597 598
}

599 600
PlaylistPtr MediaLibrary::createPlaylist( const std::string& name )
{
601 602
    try
    {
603 604 605 606
        auto pl = Playlist::create( this, name );
        if ( pl != nullptr && m_modificationNotifier != nullptr )
            m_modificationNotifier->notifyPlaylistCreation( pl );
        return pl;
607 608 609 610 611 612
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to create a playlist: ", ex.what() );
        return nullptr;
    }
613 614
}

615
Query<IPlaylist> MediaLibrary::playlists( const QueryParameters* params )
616
{
617
    return Playlist::listAll( this, params );
618 619
}

620 621 622 623 624
PlaylistPtr MediaLibrary::playlist( int64_t id ) const
{
    return Playlist::fetch( this, id );
}

625
bool MediaLibrary::deletePlaylist( int64_t playlistId )
626
{
627 628 629 630 631 632 633 634 635
    try
    {
        return Playlist::destroy( this, playlistId );
    }
    catch ( const sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to delete playlist: ", ex.what() );
        return false;
    }
636
}
637

638
Query<IMedia> MediaLibrary::history() const
639
{
640
    return Media::fetchHistory( this );
641 642
}

643
Query<IMedia> MediaLibrary::streamHistory() const
644
{
645
    return Media::fetchStreamHistory( this );
646 647
}

648 649
bool MediaLibrary::clearHistory()
{
650 651 652 653 654 655 656 657 658 659 660 661
    try
    {
        return sqlite::Tools::withRetries( 3, [this]() {
            Media::clearHistory( this );
            return true;
        });
    }
    catch ( sqlite::errors::Generic& ex )
    {
        LOG_ERROR( "Failed to clear history: ", ex.what() );
        return false;
    }
662 663
}

664
Query<IMedia> MediaLibrary::searchMedia( const std::string& title,
665
                                                const QueryParameters* params ) const
666
{
667 668
    if ( validateSearchPattern( title ) == false )
        return {};
669
    return Media::search( this, title, params );
670 671
}

672 673 674 675 676 677 678 679 680 681 682
Query<IMedia> MediaLibrary::searchAudio( const std::string& pattern, const QueryParameters* params ) const
{
    if ( validateSearchPattern( pattern ) == false )
        return {};
    return Media::search( this, pattern, IMedia::Type::Audio, params );
}

Query<IMedia> MediaLibrary::searchVideo( const std::string& pattern, const QueryParameters* params ) const
{
    if ( validateSearchPattern( pattern ) == false )
        return {};
683
    return Media::search( this, pattern, IMedia::Type::Video, params );
684 685
}

686
Query<IPlaylist> MediaLibrary::searchPlaylists( const std::string& name,
687
                                                const QueryParameters* params ) const
688
{
689 690
    if ( validateSearchPattern( name ) == false )
        return {};
691
    return Playlist::search( this, name, params );
692 693
}

694
Query<IAlbum> MediaLibrary::searchAlbums( const std::string& pattern,
695
                                          const QueryParameters* params ) const
696
{
697 698
    if ( validateSearchPattern( pattern ) == false )
        return {};
699
    return Album::search( this, pattern, params );
700 701
}

702 703
Query<IGenre> MediaLibrary::searchGenre( const std::string& genre,
                                         const QueryParameters* params ) const
704
{
705 706
    if ( validateSearchPattern( genre ) == false )
        return {};
707
    return Genre::search( this, genre, params );
708 709
}

710
Query<IArtist> MediaLibrary::searchArtists( const std::string& name, bool includeAll,
711
                                            const QueryParameters* params ) const
712 713 714
{
    if ( validateSearchPattern( name ) == false )
        return {};
715
    return Artist::search( this, name, includeAll, params );
716 717
}

718 719 720 721 722 723
Query<IShow> MediaLibrary::searchShows( const std::string& pattern,
                                        const QueryParameters* params ) const
{
    return Show::search( this, pattern, params );
}

724
SearchAggregate MediaLibrary::search( const std::string& pattern,
725
                                      const QueryParameters* params ) const
726
{
727
    SearchAggregate res;
728
    res.albums = searchAlbums( pattern, params );
729
    res.artists = searchArtists( pattern, true, params );
730 731 732
    res.genres = searchGenre( pattern, params );
    res.media = searchMedia( pattern, params );
    res.playlists = searchPlaylists( pattern, params );
733
    res.shows = searchShows( pattern, params );
734 735 736
    return res;
}

737
bool MediaLibrary::startParser()
738
{
739
    m_parser.reset( new parser::Parser( this ) );
740

741
    if ( m_services.empty() == true )
742
    {
743
#ifdef HAVE_LIBVLC
744
        m_parser->addService( std::make_shared<parser::VLCMetadataService>() );
745 746 747
#else
        return false;
#endif
748 749 750 751 752 753 754
    }
    else
    {
        assert( m_services[0]->targetedStep() == parser::Step::MetadataExtraction );
        m_parser->addService( m_services[0] );
    }
    m_parser->addService( std::make_shared<parser::MetadataAnalyzer>() );
755
    m_parser->start();
756
    return true;
757 758
}

759
void MediaLibrary::startDiscoverer()
760
{
761
    m_discovererWorker.reset( new DiscovererWorker( this ) );
762
    for ( const auto& fsFactory : m_fsFactories )
763
    {
764
        std::unique_ptr<prober::CrawlerProbe> probePtr( new prober::CrawlerProbe{} );
765 766 767
        m_discovererWorker->addDiscoverer( std::unique_ptr<IDiscoverer>( new FsDiscoverer( fsFactory, this, m_callback,
                                                                                           std::move ( probePtr ) ) ) );
    }
768 769
}

770 771
void MediaLibrary::startDeletionNotifier()
{
772 773
    m_modificationNotifier.reset( new ModificationNotifier( this ) );
    m_modificationNotifier->start();
774 775
}

776
void MediaLibrary::startThumbnailer()
777
{
778
#ifdef HAVE_LIBVLC
779 780
    if ( m_thumbnailers.empty() == true )
        m_thumbnailers.push_back( std::make_shared<ThumbnailerType>( this ) );
781
#endif
782 783 784 785 786 787 788 789
    for ( const auto& thumbnailer : m_thumbnailers )
    {
        // For now this make little sense (as we are instantiating the same
        // object in a loop, but at some point we will have multiple thumbnailer,
        // or the thumbnailer worker will handle multiple IThumbnailer implementations
        m_thumbnailer = std::unique_ptr<ThumbnailerWorker>(
                    new ThumbnailerWorker( this, thumbnailer ) );
    }
790 791
}

792
void MediaLibrary::populateNetworkFsFactories()
793 794
{
#ifdef HAVE_LIBVLC
795 796
    m_externalNetworkFsFactories.emplace_back(
        std::make_shared<factory::NetworkFileSystemFactory>( "smb://", "dsm-sd" ) );
797 798 799
#endif
}

800 801
void MediaLibrary::addLocalFsFactory()
{
802
    m_fsFactories.emplace( begin( m_fsFactories ), std::make_shared<factory::FileSystemFactory>( m_deviceLister ) );
803 804
}

805
InitializeResult MediaLibrary::updateDatabaseModel( unsigned int previousVersion,
806
                                        const std::string& dbPath )
807
{
808
    LOG_INFO( "Updating database model from ", previousVersion, " to ", Settings::DbModelVersion );
809
    auto originalPreviousVersion = previousVersion;
810 811
    // Up until model 3, it's safer (and potentially more efficient with index changes) to drop the DB
    // It's also way simpler to implement
812
    // In case of downgrade, just recreate the database
813
    for ( auto i = 0u; i < 3; ++i )
814
    {
815 816
        try
        {
817
            bool needRescan = false;
818 819 820
            // Up until model 3, it's safer (and potentially more efficient with index changes) to drop the DB
            // It's also way simpler to implement
            // In case of downgrade, just recreate the database
821 822
            // We might also have some special cases for failed upgrade (see
            // comments below for per-version details)
823
            if ( previousVersion < 3 ||
824 825
                 previousVersion > Settings::DbModelVersion ||
                 previousVersion == 4 )
826
            {
827
                if( recreateDatabase( dbPath ) == false )
828
                    throw std::runtime_error( "Failed to recreate the database" );
829
                return InitializeResult::DbReset;
830
            }
831 832 833 834 835 836 837 838
            /**
             * Migration from 3 to 4 didn't happen so well and broke a few
             * users DB. So:
             * - Any v4 database will be dropped and recreated in v5
             * - Any v3 database will be upgraded to v5
             * V4 database is only used by VLC-android 2.5.6 // 2.5.8, which are
             * beta versions.
             */
839 840
            if ( previousVersion == 3 )
            {
841
                migrateModel3to5();
842
                previousVersion = 5;
843
            }
844 845
            if ( previousVersion == 5 )
            {
846
                migrateModel5to6();
847 848
                previousVersion = 6;
            }
Alexandre Fernandez's avatar
Alexandre Fernandez committed
849 850
            if ( previousVersion == 6 )
            {
851 852 853
                // Force a rescan to solve metadata analysis problems.
                // The insertion is fixed, but won't edit already inserted data.
                forceRescan();
Alexandre Fernandez's avatar
Alexandre Fernandez committed
854 855
                previousVersion = 7;
            }
856 857 858 859 860 861 862 863 864
            /**
             * V7 introduces artist.nb_tracks and an associated trigger to delete
             * artists when it has no track/album left.
             */
            if ( previousVersion == 7 )
            {
                migrateModel7to8();
                previousVersion = 8;
            }
865 866
            if ( previousVersion == 8 )
            {
867 868 869 870 871 872 873 874 875

                // Multiple changes justify the rescan:
                // - Changes in the way we chose to encode or not MRL, meaning
                //   potentially all MRL are wrong (more precisely, will
                //   mismatch what VLC expects, which makes playlist analysis
                //   break.
                // - Fix in the way we chose album candidates, meaning some
                //   albums were likely to be wrongfully created.
                needRescan = true;
876
                migrateModel8to9();
877 878
                previousVersion = 9;
            }
879 880
            if ( previousVersion == 9 )
            {
881
                needRescan = true;
882 883 884
                migrateModel9to10();
                previousVersion = 10;
            }
885 886
            if ( previousVersion == 10 )
            {
887
                needRescan = true;
888 889 890
                migrateModel10to11();
                previousVersion = 11;
            }
891 892 893 894 895
            if ( previousVersion == 11 )
            {
                parser::Task::recoverUnscannedFiles( this );
                previousVersion = 12;
            }
896 897 898 899 900
            if ( previousVersion == 12 )
            {
                migrateModel12to13();
                previousVersion = 13;
            }
901 902
            if ( previousVersion == 13 )
            {
903 904 905
                // We need to recreate many thumbnail records, and hopefully
                // generate better ones
                needRescan = true;
906
                migrateModel13to14( originalPreviousVersion );
907 908
                previousVersion = 14;
            }
909 910 911 912 913
            if ( previousVersion == 14 )
            {
                migrateModel14to15();
                previousVersion = 15;
            }
914 915
            // To be continued in the future!

916 917 918
            if ( needRescan == true )
                forceRescan();

919 920
            // Safety check: ensure we didn't forget a migration along the way
            assert( previousVersion == Settings::DbModelVersion );
921 922
            assert( previousVersion == m_settings.dbModelVersion() );

923
            return InitializeResult::Success;
924 925 926 927 928 929 930 931 932 933 934
        }
        catch( const std::exception& ex )
        {
            LOG_ERROR( "An error occured during the database upgrade: ",
                       ex.what() );
        }
        catch( ... )
        {
            LOG_ERROR( "An unknown error occured during the database upgrade." );
        }
        LOG_WARN( "Retrying database migration, attempt ", i + 1, " / 3" );
935
    }
936 937
    LOG_ERROR( "Failed to upgrade database, recreating it" );
    for ( auto i = 0u; i < 3; ++i )
938
    {
939 940
        try
        {
941
            if( recreateDatabase( dbPath ) == true )
942
                return InitializeResult::DbReset;
943 944 945 946 947 948 949 950 951 952
        }
        catch( const std::exception& ex )
        {
            LOG_ERROR( "Failed to recreate database: ", ex.what() );
        }
        catch(...)
        {
            LOG_ERROR( "Unknown error while trying to recreate the database." );
        }
        LOG_WARN( "Retrying to recreate the database, attempt ", i + 1, " / 3" );
953
    }
954
    return InitializeResult::Failed;
955 956
}

957
bool MediaLibrary::recreateDatabase( const std::string& dbPath )
958
{
959 960 961 962
    // Close all active connections, flushes all previously run statements.
    m_dbConnection.reset();
    unlink( dbPath.c_str() );
    m_dbConnection = sqlite::Connection::connect( dbPath );
963
    createAllTables();
964
    // We dropped the database, there is no setting to be read anymore
965
    return m_settings.load();
966 967
}

968
void MediaLibrary::migrateModel3to5()
969 970
{
    /*
971 972
     * Disable Foreign Keys & recursive triggers to avoid cascading deletion
     * while remodeling the database into the transaction.
973
     */
974
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
975
    auto t = getConn()->newTransaction();
976 977 978
    // As SQLite do not allow us to remove or add some constraints,
    // we use the method described here https://www.sqlite.org/faq.html#q11
    std::string reqs[] = {
979
#               include "database/migrations/migration3-5.sql"
980 981 982
    };

    for ( const auto& req : reqs )
983
        sqlite::Tools::executeRequest( getConn(), req );
984
    // Re-create triggers removed in the process
985
    Media::createTriggers( getConn(), 5 );
986
    Playlist::createTriggers( getConn() );
987 988
    m_settings.setDbModelVersion( 5 );
    m_settings.save();
989 990 991
    t->commit();
}

992
void MediaLibrary::migrateModel5to6()
993
{
994 995
    // We can't create a transaction here since it would make the weak context
    // creation fail.
996
    std::string req = "DELETE FROM " + Media::Table::Name + " WHERE type = ?";
997 998
    sqlite::Tools::executeRequest( getConn(), req, Media::Type::Unknown );

999
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
1000
    req = "UPDATE " + Media::Table::Name + " SET is_present = 1 WHERE is_present != 0";
1001
    sqlite::Tools::executeRequest( getConn(), req );
1002 1003
    m_settings.setDbModelVersion( 6 );
    m_settings.save();
Alexandre Fernandez's avatar
Alexandre Fernandez committed
1004 1005
}

1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
void MediaLibrary::migrateModel7to8()
{
    sqlite::Connection::WeakDbContext weakConnCtx{ getConn() };
    auto t = getConn()->newTransaction();
    std::string reqs[] = {
#               include "database/migrations/migration7-8.sql"
    };

    for ( const auto& req : reqs )
        sqlite::Tools::executeRequest( getConn(), req );
    // Re-create triggers removed in the process
1017
    Artist::createTriggers( getConn(), 8u );
1018
    Media::createTriggers( getConn(), 5 );
1019
    File::createTriggers( getConn() );
1020 1021
    m_settings.setDbModelVersion( 8 );
    m_settings.save();
1022 1023 1024
    t->commit();
}

1025 1026 1027 1028 1029 1030
void MediaLibrary::migrateModel8to9()
{
    // A bug in a previous migration caused our triggers to be missing for the
    // first application run (after the migration).
    // This could have caused media associated to deleted files not to be
    // deleted as well, so let's do that now.
1031
    auto t = getConn()->newTransaction();
1032
    const std::string req = "DELETE FROM " + Media::Table::Name + " "
1033
            "WHERE id_media IN "
1034 1035
            "(SELECT id_media FROM " + Media::Table::Name + " m LEFT JOIN " +
                File::Table::Name + " f ON f.media_id = m.id_media "
1036 1037 1038 1039 1040
                "WHERE f.media_id IS NULL)";

    // Don't check for the return value, we don't mind if nothing deleted.
    // Quite the opposite actually :)
    sqlite::Tools::executeDelete( getConn(), req );
1041 1042 1043
    m_settings.setDbModelVersion( 9 );
    m_settings.save();
    t->commit();
1044 1045
}

1046 1047
void MediaLibrary::migrateModel9to10()
{
1048
    const std::string req = "SELECT * FROM " + File::Table::Name +
1049 1050 1051 1052 1053
            " WHERE mrl LIKE '%#%%' ESCAPE '#'";
    auto files = File::fetchAll<File>( this, req );
    auto t = getConn()->newTransaction();
    for ( const auto& f : files )
    {
1054 1055 1056 1057
        // We must not call mrl() from here. We might not have all devices yet,
        // and calling mrl would crash for files stored on removable devices.
        auto newMrl = utils::url::encode( utils::url::decode( f->rawMrl() ) );
        LOG_INFO( "Converting ", f->rawMrl(), " to ", newMrl );
1058 1059
        f->setMrl( newMrl );
    }
1060 1061
    m_settings.setDbModelVersion( 10 );
    m_settings.save();
1062 1063 1064
    t->commit();
}

1065 1066
void MediaLibrary::migrateModel10to11()
{
1067
    const std::string req = "SELECT * FROM " + parser::Task::Table::Name +
1068
            " WHERE mrl LIKE '%#%%' ESCAPE '#'";
1069
    const std::string folderReq = "SELECT * FROM " + Folder::Table::Name +
1070
            " WHERE path LIKE '%#%%' ESCAPE '#'";
1071
    auto tasks = parser::Task::fetchAll<parser::Task>( this, req );
1072
    auto folders = Folder::fetchAll<Folder>( this, folderReq );
1073 1074 1075
    auto t = getConn()->newTransaction();
    for ( const auto& t : tasks )
    {
1076 1077
        auto newMrl = utils::url::encode( utils::url::decode( t->item().mrl() ) );
        LOG_INFO( "Converting task mrl: ", t->item().mrl(), " to ", newMrl );
1078 1079
        t->setMrl( std::move( newMrl ) );
    }
1080 1081 1082 1083 1084 1085 1086
    for ( const auto &f : folders )
    {
        // We must not call mrl() from here. We might not have all devices yet,
        // and calling mrl would crash for files stored on removable devices.
        auto newMrl = utils::url::encode( utils::url::decode( f->rawMrl() ) );
        f->setMrl( std::move( newMrl ) );
    }
1087 1088
    m_settings.setDbModelVersion( 11 );
    m_settings.save();
1089 1090 1091
    t->commit();
}

1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
/*
 * - Some is_present related triggers were fixed in model 6 to 7 migration, but
 *   they were not recreated if already existing. The has_file_present trigger
 *   was recreated as part of model 7 to 8 migration, but we need to ensure
 *   has_album_present (Artist) & is_album_present (Album) triggers are
 *   recreated to behave as expected
 * - Due to a typo, is_track_present was named is_track_presentAFTER, and was
 *   executed BEFORE the update took place, thus using the wrong is_present value.
 *   The trigger will be recreated as part of this migration, and the values
 *   will be enforced, causing the entire update chain to be triggered, and
 *   restoring correct is_present values for all AlbumTrack/Album/Artist entries
 */
1104 1105
void MediaLibrary::migrateModel12to13()
{
1106
    auto t = getConn()->newTransaction();
1107 1108 1109 1110 1111 1112 1113 1114 1115
    const std::string reqs[] = {
        "DROP TRIGGER IF EXISTS is_track_presentAFTER",
        "DROP TRIGGER has_album_present",
        "DROP TRIGGER is_album_present",
    };

    for ( const auto& req : reqs )
        sqlite::Tools::executeDelete( getConn(), req );

1116
    AlbumTrack::createTriggers( getConn() );
1117 1118
    Album::createTriggers( getConn() );
    Artist::createTriggers( getConn(), 13 );
1119 1120
    // Leave the weak context as we now need to update is_present fields, which
    // are propagated through recursive triggers
1121 1122
    const std::string migrateData = "UPDATE " + AlbumTrack::Table::Name +
            " SET is_present = (SELECT is_present FROM " + Media::Table::Name +
1123 1124
            " WHERE id_media = media_id)";
    sqlite::Tools::executeUpdate( getConn(), migrateData );
1125 1126
    m_settings.setDbModelVersion( 13 );
    m_settings.save();
1127 1128 1129 1130
    t->commit();
}

/*
1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
 * Movel v13 to v14 migration:
 * - Media:
 *      - Remove
 *          .thumbnail
 *          .
 *      - Add
 *          .nb_playlists
 *          .real_last_played_date
 *          .device_id
 *          .folder_id
 *      - Fix filename being url encoded
 *      - Fix playlist external media type being Unknown
 *  - Playlist:
 *      - Add .file_id
 *  - PlaylistMediaRelation:
 *      - Add .mrl
 *  - Add a playlist FTS table
 *  - Add Thumbnail table
 *  - Task:
 *      - Add .is_refresh
 *  - Device:
 *      - Add last_seen
 *  - Folder:
 *      - Add:
 *          .nb_audio