Task.cpp 17.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2015 - 2017 Hugo Beauzée-Luyssen, Videolabs
 *
 * Authors: Hugo Beauzée-Luyssen<hugo@beauzee.fr>
 *          Alexandre Fernandez <nerf@boboop.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

#if HAVE_CONFIG_H
# include "config.h"
#endif

#include "Task.h"

30 31
#include "medialibrary/filesystem/IFile.h"
#include "medialibrary/filesystem/IDirectory.h"
32
#include "Device.h"
33
#include "File.h"
34 35
#include "Folder.h"
#include "Playlist.h"
36
#include "Media.h"
37 38
#include "parser/Task.h"
#include "utils/Filename.h"
39
#include "utils/Strings.h"
40
#include "utils/Url.h"
41

42 43
#include <algorithm>

44 45 46 47 48 49
namespace medialibrary
{

namespace parser
{

50 51 52 53
const std::string Task::Table::Name = "Task";
const std::string Task::Table::PrimaryKeyColumn = "id_task";
int64_t parser::Task::* const Task::Table::PrimaryKey = &parser::Task::m_id;

54 55 56
Task::Task( MediaLibraryPtr ml, sqlite::Row& row )
    : currentService( 0 )
    , m_ml( ml )
57
{
58
    std::string mrl;
59
    unsigned int parentPlaylistIndex;
60
    IFile::Type fileType;
61
    bool isRefresh;
62 63 64 65
    row >> m_id
        >> m_step
        >> m_retryCount
        >> mrl
66
        >> fileType
67 68 69
        >> m_fileId
        >> m_parentFolderId
        >> m_parentPlaylistId
70 71
        >> parentPlaylistIndex
        >> isRefresh;
72
    m_item = Item{ this, std::move( mrl ), fileType, parentPlaylistIndex, isRefresh };
73 74
}

75
Task::Task( MediaLibraryPtr ml, std::string mrl, std::shared_ptr<fs::IFile> fileFs,
76
            std::shared_ptr<Folder> parentFolder,
77
            std::shared_ptr<fs::IDirectory> parentFolderFs,
78
            IFile::Type fileType,
79
            std::shared_ptr<Playlist> parentPlaylist,
80
            unsigned int parentPlaylistIndex )
81
    : currentService( 0 )
82
    , m_ml( ml )
83
    , m_step( Step::None )
84
    , m_fileId( 0 )
85
    , m_item( this, std::move( mrl ), std::move( fileFs ), std::move( parentFolder ),
86 87
              std::move( parentFolderFs ), fileType,
              std::move( parentPlaylist ), parentPlaylistIndex, false )
88 89 90 91 92 93 94 95 96 97
{
}

Task::Task( MediaLibraryPtr ml, std::shared_ptr<File> file,
            std::shared_ptr<fs::IFile> fileFs )
    : currentService( 0 )
    , m_ml( ml )
    , m_step( Step::None )
    , m_fileId( file->id() )
    , m_item( this, std::move( file ), std::move( fileFs ) )
98 99 100
{
}

101
void Task::markStepCompleted( Step stepCompleted )
102
{
103
    m_step = static_cast<Step>( static_cast<uint8_t>( m_step ) |
104
                                      static_cast<uint8_t>( stepCompleted ) );
105 106
}

107 108
bool Task::saveParserStep()
{
109
    static const std::string req = "UPDATE " + Task::Table::Name + " SET step = ?, "
110
            "retry_count = 0 WHERE id_task = ?";
111
    return sqlite::Tools::executeUpdate( m_ml->getConn(), req, m_step, m_id );
112 113
}

114
bool Task::decrementRetryCount()
115
{
116
    static const std::string req = "UPDATE " + Task::Table::Name + " SET "
117
            "retry_count = retry_count - 1 WHERE id_task = ?";
118 119 120
    return sqlite::Tools::executeUpdate( m_ml->getConn(), req, m_id );
}

121 122
bool Task::isCompleted() const
{
123
    using StepType = typename std::underlying_type<Step>::type;
124
    return ( static_cast<StepType>( m_step ) &
125 126
             static_cast<StepType>( Step::Completed ) ) ==
             static_cast<StepType>( Step::Completed );
127 128
}

129
bool Task::isStepCompleted( Step step ) const
130 131 132 133 134 135
{
    return ( static_cast<uint8_t>( m_step ) & static_cast<uint8_t>( step ) ) != 0;
}

void Task::startParserStep()
{
136
    static const std::string req = "UPDATE " + Task::Table::Name + " SET "
137 138 139 140
            "retry_count = retry_count + 1 WHERE id_task = ?";
    sqlite::Tools::executeUpdate( m_ml->getConn(), req, m_id );
}

141
bool Task::updateFileId( int64_t fileId )
142
{
143 144 145 146 147 148
    // When restoring a task, we will invoke ITaskCb::updateFileId while the
    // task already knows the fileId (since we're using it to restore the file instance)
    // In this case, bail out. Otherwise, it is not expected for the task to change
    // its associated file during the processing.
    if ( m_fileId == fileId && fileId != 0 )
        return true ;
149
    assert( m_fileId == 0 );
150
    assert( fileId != 0 );
151
    static const std::string req = "UPDATE " + Task::Table::Name + " SET "
152
            "file_id = ? WHERE id_task = ?";
153
    if ( sqlite::Tools::executeUpdate( m_ml->getConn(), req, fileId, m_id ) == false )
154
        return false;
155
    m_fileId = fileId;
156 157 158
    return true;
}

159 160 161 162 163
int64_t Task::id() const
{
    return m_id;
}

164 165 166 167 168
Task::Item& Task::item()
{
    return m_item;
}

169 170
Task::Item::Item( ITaskCb* taskCb, std::string mrl, IFile::Type fileType,
                  unsigned int subitemPosition, bool isRefresh )
171 172
    : m_taskCb( taskCb )
    , m_mrl( std::move( mrl ) )
173
    , m_fileType( fileType )
174
    , m_duration( 0 )
175
    , m_parentPlaylistIndex( subitemPosition )
176
    , m_isRefresh( isRefresh )
177 178 179
{
}

180
Task::Item::Item(ITaskCb* taskCb, std::string mrl, std::shared_ptr<fs::IFile> fileFs,
181
                  std::shared_ptr<Folder> parentFolder,
182
                  std::shared_ptr<fs::IDirectory> parentFolderFs, IFile::Type fileType,
183 184
                  std::shared_ptr<Playlist> parentPlaylist, unsigned int parentPlaylistIndex,
                  bool isRefresh )
185
    : m_taskCb( taskCb )
186
    , m_mrl( std::move( mrl ) )
187
    , m_fileType( fileType )
188 189 190 191 192 193
    , m_duration( 0 )
    , m_fileFs( std::move( fileFs ) )
    , m_parentFolder( std::move( parentFolder ) )
    , m_parentFolderFs( std::move( parentFolderFs ) )
    , m_parentPlaylist( std::move( parentPlaylist ) )
    , m_parentPlaylistIndex( parentPlaylistIndex )
194 195 196 197
    , m_isRefresh( isRefresh )
{
}

198 199
Task::Item::Item( ITaskCb* taskCb, std::shared_ptr<File> file,
                  std::shared_ptr<fs::IFile> fileFs )
200 201
    : m_taskCb( taskCb )
    , m_mrl( fileFs->mrl() )
202
    , m_fileType( file->type() )
203 204 205 206
    , m_duration( 0 )
    , m_file( std::move( file ) )
    , m_fileFs( std::move( fileFs ) )
    , m_isRefresh( true )
207 208 209
{
}

210
std::string Task::Item::meta( Task::Item::Metadata type ) const
211 212 213 214 215 216 217
{
    auto it = m_metadata.find( type );
    if ( it == end( m_metadata ) )
        return std::string{};
    return it->second;
}

218
void Task::Item::setMeta( Task::Item::Metadata type, std::string value )
219
{
220
    utils::str::trim( value );
221 222 223
    m_metadata[type] = std::move( value );
}

224 225 226 227 228
const std::string& Task::Item::mrl() const
{
    return m_mrl;
}

229 230 231 232 233
void Task::Item::setMrl( std::string mrl )
{
    m_mrl = std::move( mrl );
}

234 235 236 237 238
IFile::Type Task::Item::fileType() const
{
    return m_fileType;
}

239
size_t Task::Item::nbSubItems() const
240
{
241 242 243
    return m_subItems.size();
}

244
const IItem& Task::Item::subItem( unsigned int index ) const
245 246
{
    return m_subItems[index];
247 248
}

249
IItem& Task::Item::createSubItem( std::string mrl, unsigned int playlistIndex )
250
{
251 252
    m_subItems.emplace_back( nullptr, std::move( mrl ), IFile::Type::Main,
                             playlistIndex, false );
253
    return m_subItems.back();
254 255
}

256 257 258 259 260 261 262 263 264 265
int64_t Task::Item::duration() const
{
    return m_duration;
}

void Task::Item::setDuration( int64_t duration )
{
    m_duration = duration;
}

266 267 268 269 270 271 272 273 274 275
const std::vector<Task::Item::Track>& Task::Item::tracks() const
{
    return m_tracks;
}

void Task::Item::addTrack(Task::Item::Track t)
{
    m_tracks.emplace_back( std::move( t ) );
}

276
MediaPtr Task::Item::media()
277 278 279 280
{
    return m_media;
}

281
void Task::Item::setMedia( MediaPtr media )
282 283 284 285
{
    m_media = std::move( media );
}

286
FilePtr Task::Item::file()
287 288 289 290
{
    return m_file;
}

291
bool Task::Item::setFile( FilePtr file)
292 293
{
    m_file = std::move( file );
294 295
    assert( m_taskCb != nullptr );
    return m_taskCb->updateFileId( m_file->id() );
296 297
}

298
FolderPtr Task::Item::parentFolder()
299 300 301 302 303 304 305 306 307 308 309 310 311 312
{
    return m_parentFolder;
}

std::shared_ptr<fs::IFile> Task::Item::fileFs()
{
    return m_fileFs;
}

std::shared_ptr<fs::IDirectory> Task::Item::parentFolderFs()
{
    return m_parentFolderFs;
}

313
PlaylistPtr Task::Item::parentPlaylist()
314 315 316 317 318 319 320 321 322
{
    return m_parentPlaylist;
}

unsigned int Task::Item::parentPlaylistIndex() const
{
    return m_parentPlaylistIndex;
}

323 324 325 326 327
bool Task::Item::isRefresh() const
{
    return m_isRefresh;
}

328
bool Task::restoreLinkedEntities()
329
{
330
    LOG_INFO("Restoring linked entities of task ", m_id);
331 332
    // MRL will be empty if the task has been resumed from unparsed files
    // parentFolderId == 0 indicates an external file
333 334
    auto mrl = m_item.mrl();
    if ( mrl.empty() == true && m_parentFolderId == 0 )
335 336 337 338 339
    {
        LOG_WARN( "Aborting & removing external file task (#", m_id, ')' );
        destroy( m_ml, m_id );
        return false;
    }
340 341
    // First of all, we need to know if the file has been created already
    // ie. have we run the MetadataParser service, at least partially
342
    std::shared_ptr<File> file;
343
    if ( m_fileId != 0 )
344
        file = File::fetch( m_ml, m_fileId );
345

346 347
    // We might re-create tasks without mrl to ease the handling of files on
    // external storage.
348
    if ( mrl.empty() == true )
349 350 351 352
    {
        // but we expect those to be created from an existing file after a
        // partial/failed migration. If we don't have a file nor an mrl, we
        // can't really process it.
353
        if ( file == nullptr )
354 355 356 357
        {
            assert( !"Can't process a file without a file nor an mrl" );
            return false;
        }
358
        auto folder = Folder::fetch( m_ml, file->folderId() );
359 360 361 362 363 364 365 366 367 368
        if ( folder == nullptr )
        {
            assert( !"Can't file the folder associated with a file" );
            // If the folder can't be found in db while the file can, it looks
            // a lot like a sporadic failure, since file are deleted through
            // triggers when its parent folder gets deleted. Just postpone this.
            return false;
        }
        if ( folder->isPresent() == false )
        {
369
            LOG_WARN( "Postponing rescan of removable file ", file->rawMrl(),
370 371 372
                      " until the device containing it is present again" );
            return false;
        }
373
        setMrl( file->mrl() );
374
    }
375
    auto fsFactory = m_ml->fsFactoryForMrl( mrl );
376 377 378
    if ( fsFactory == nullptr )
        return false;

379
    std::shared_ptr<fs::IDirectory> parentFolderFs;
380 381
    try
    {
382
        parentFolderFs = fsFactory->createDirectory( utils::file::directory( mrl ) );
383 384 385 386
    }
    catch ( const std::system_error& ex )
    {
        LOG_ERROR( "Failed to restore task: ", ex.what() );
387
        return false;
388
    }
389

390
    std::shared_ptr<fs::IFile> fileFs;
391 392
    try
    {
393 394 395
        auto files = parentFolderFs->files();
        auto it = std::find_if( begin( files ), end( files ), [&mrl]( std::shared_ptr<fs::IFile> f ) {
            return f->mrl() == mrl;
396 397 398
        });
        if ( it == end( files ) )
        {
399
            LOG_ERROR( "Failed to restore fs::IFile associated with ", mrl );
400 401
            return false;
        }
402
        fileFs = std::move( *it );
403
    }
404 405 406 407 408
    catch ( const fs::DeviceRemovedException& )
    {
        LOG_WARN( "Failed to restore file on an unmounted device: ", mrl );
        return false;
    }
409
    catch ( const std::system_error& ex )
410
    {
411 412
        // If we never found the file yet, we can delete the task. It will be
        // recreated upon next discovery
413
        if ( file == nullptr )
414
        {
415 416
            LOG_WARN( "Failed to restore file system instances for mrl ", mrl, "(",
                      ex.what(), ").", " Removing the task until it gets detected again." );
417 418 419 420 421 422 423 424
            destroy( m_ml, m_id );
        }
        else
        {
            // Otherwise we need to postpone it, although most likely we will
            // detect that the file is now missing, and we won't try to restore
            // this task until it comes back (since the task restoration request
            // includes the file.is_present flag)
425
            LOG_WARN( "Failed to restore file system instances for mrl ", mrl, "."
426 427
                      " Postponing the task." );
        }
428 429 430
        return false;
    }

431 432 433 434 435 436 437
    std::shared_ptr<Folder> parentFolder = Folder::fetch( m_ml, m_parentFolderId );
    if ( parentFolder == nullptr )
    {
        LOG_ERROR( "Failed to restore parent folder #", m_parentFolderId );
        return false;
    }
    std::shared_ptr<Playlist> parentPlaylist;
438
    if ( m_parentPlaylistId != 0 )
439 440 441 442 443 444 445 446
    {
        parentPlaylist = Playlist::fetch( m_ml, m_parentPlaylistId );
        if ( parentPlaylist == nullptr )
        {
            LOG_ERROR( "Failed to restore parent playlist #", m_parentPlaylistId );
            return false;
        }
    }
447

448
    m_item = Item{ this, m_item.mrl(), std::move( fileFs ), std::move( parentFolder ),
449 450 451
                   std::move( parentFolderFs ), m_item.fileType(),
                   std::move( parentPlaylist ), m_item.parentPlaylistIndex(),
                   m_item.isRefresh() };
452 453 454 455 456
    if ( file != nullptr )
    {
        m_item.setMedia( file->media() );
        m_item.setFile( std::move( file ) );
    }
457 458 459
    return true;
}

460 461
void Task::setMrl( std::string newMrl )
{
462
    if ( m_item.mrl() == newMrl )
463
        return;
464
    static const std::string req = "UPDATE " + Task::Table::Name + " SET "
465 466 467
            "mrl = ? WHERE id_task = ?";
    if ( sqlite::Tools::executeUpdate( m_ml->getConn(), req, newMrl, m_id ) == false )
        return;
468
    m_item.setMrl( std::move( newMrl ) );
469 470
}

471 472
void Task::createTable( sqlite::Connection* dbConnection )
{
473 474 475 476 477
    std::string reqs[] = {
        #include "database/tables/Task_v14.sql"
    };
    for ( const auto& req : reqs )
        sqlite::Tools::executeRequest( dbConnection, req );
478 479 480 481
}

void Task::resetRetryCount( MediaLibraryPtr ml )
{
482
    static const std::string req = "UPDATE " + Task::Table::Name + " SET "
483
            "retry_count = 0 WHERE step & ? != ?";
484 485
    sqlite::Tools::executeUpdate( ml->getConn(), req, Step::Completed,
                                  Step::Completed);
486 487 488 489
}

void Task::resetParsing( MediaLibraryPtr ml )
{
490
    static const std::string req = "UPDATE " + Task::Table::Name + " SET "
491
            "retry_count = 0, step = ?";
492
    sqlite::Tools::executeUpdate( ml->getConn(), req, Step::None );
493 494
}

495
std::vector<std::shared_ptr<Task>> Task::fetchUncompleted( MediaLibraryPtr ml )
496
{
497 498
    static const std::string req = "SELECT * FROM " + Task::Table::Name + " t"
        " LEFT JOIN " + File::Table::Name + " f ON f.id_file = t.file_id"
499 500 501
        " LEFT JOIN " + Folder::Table::Name + " fol ON f.folder_id = fol.id_folder"
        " LEFT JOIN " + Device::Table::Name + " d ON d.id_device = fol.device_id"
        " WHERE step & ? != ? AND retry_count < 3 AND (d.is_present != 0 OR "
502
        " t.file_id IS NULL)";
503 504
    return Task::fetchAll<Task>( ml, req, Step::Completed,
                                 Step::Completed );
505 506 507
}

std::shared_ptr<Task>
508
Task::create( MediaLibraryPtr ml, std::string mrl, std::shared_ptr<fs::IFile> fileFs,
509
              std::shared_ptr<Folder> parentFolder, std::shared_ptr<fs::IDirectory> parentFolderFs,
510
              IFile::Type fileType,
511 512
              std::pair<std::shared_ptr<Playlist>, unsigned int> parentPlaylist )
{
513 514 515 516
    auto parentFolderId = parentFolder->id();
    auto parentPlaylistId = parentPlaylist.first != nullptr ? parentPlaylist.first->id() : 0;
    auto parentPlaylistIndex = parentPlaylist.second;

517
    std::shared_ptr<Task> self = std::make_shared<Task>( ml, std::move( mrl ), std::move( fileFs ),
518
        std::move( parentFolder ), std::move( parentFolderFs ), fileType,
519
        std::move( parentPlaylist.first ), parentPlaylist.second );
520
    const std::string req = "INSERT INTO " + Task::Table::Name +
521 522 523 524
        "(mrl, file_type, parent_folder_id, parent_playlist_id, "
        "parent_playlist_index, is_refresh) "
        "VALUES(?, ?, ?, ?, ?, ?)";
    if ( insert( ml, self, req, self->m_item.mrl(), fileType, parentFolderId,
525
                 sqlite::ForeignKey( parentPlaylistId ),
526
                 parentPlaylistIndex, false ) == false )
527 528
        return nullptr;
    return self;
529 530
}

531
std::shared_ptr<Task>
532
Task::createRefreshTask( MediaLibraryPtr ml, std::shared_ptr<File> file,
533 534 535 536
              std::shared_ptr<fs::IFile> fileFs )
{
    auto self = std::make_shared<Task>( ml, std::move( file ), std::move( fileFs ) );
    const std::string req = "INSERT INTO " + Task::Table::Name +
537 538 539
            "(mrl, file_type, file_id, is_refresh) VALUES(?, ?, ?, ?)";
    if ( insert( ml, self, req, self->m_item.mrl(), self->m_item.file()->type(),
                 self->m_item.file()->id(), true ) == false )
540 541 542 543
        return nullptr;
    return self;
}

544 545
void Task::recoverUnscannedFiles( MediaLibraryPtr ml )
{
546
    static const std::string req = "INSERT INTO " + Task::Table::Name +
547
            "(file_id, parent_folder_id)"
548 549
            " SELECT id_file, folder_id FROM " + File::Table::Name +
            " f LEFT JOIN " + Task::Table::Name + " t"
550 551
            " ON t.file_id = f.id_file WHERE t.file_id IS NULL"
            " AND f.folder_id IS NOT NULL";
552 553 554
    sqlite::Tools::executeInsert( ml->getConn(), req );
}

555 556 557
}

}