Commit 1674fe58 authored by Hugo Beauzée-Luyssen's avatar Hugo Beauzée-Luyssen
Browse files

Task: Split Item in a separate table

parent 915473e9
Pipeline #7852 passed with stage
in 12 minutes and 52 seconds
......@@ -385,6 +385,7 @@ EXTRA_DIST += medialibrary.pc \
src/database/migrations/migration14-15.sql \
src/database/migrations/migration15-16.sql \
src/database/migrations/migration16-17.sql \
src/database/migrations/migration17-18.sql \
src/database/tables/File_v14.sql \
src/database/tables/File_triggers_v14.sql \
src/database/tables/Media_v14.sql \
......
......@@ -138,11 +138,18 @@ MediaLibrary::~MediaLibrary()
void MediaLibrary::createAllTables()
{
auto dbModelVersion = m_settings.dbModelVersion();
// We need to create the tables in order of triggers creation
// Device is the "root of all evil". When a device is modified,
// we will trigger an update on folder, which will trigger
// an update on files, and so on.
// We may specify the current model version when creating the tables, in order
// for the old versions to be created as they were before, only to be migrated
// afterward.
// Individual migrations might take shortcuts, but it will become increasingly
// hard to do, as we have to mainting major changes across versions.
Device::createTable( m_dbConnection.get() );
Folder::createTable( m_dbConnection.get() );
Thumbnail::createTable( m_dbConnection.get() );
......@@ -160,7 +167,8 @@ void MediaLibrary::createAllTables()
AudioTrack::createTable( m_dbConnection.get() );
Artist::createTable( m_dbConnection.get() );
Artist::createDefaultArtists( m_dbConnection.get() );
parser::Task::createTable( m_dbConnection.get() );
parser::Task::createTable( m_dbConnection.get(), dbModelVersion );
parser::Item::createTable( m_dbConnection.get() );
Metadata::createTable( m_dbConnection.get() );
SubtitleTrack::createTable( m_dbConnection.get() );
Chapter::createTable( m_dbConnection.get() );
......@@ -483,6 +491,7 @@ void MediaLibrary::onDiscoveredFile( std::shared_ptr<fs::IFile> fileFs,
auto mrl = fileFs->mrl();
try
{
auto t = getConn()->newTransaction();
if ( parentPlaylist.first == nullptr )
{
// Sqlite won't ensure uniqueness for Task with the same (mrl, parent_playlist_id)
......@@ -493,10 +502,15 @@ void MediaLibrary::onDiscoveredFile( std::shared_ptr<fs::IFile> fileFs,
return;
}
}
auto task = parser::Task::create( this, mrl, std::move( fileFs ), std::move( parentFolder ),
std::move( parentFolderFs ), fileType,
std::move( parentPlaylist ) );
if ( task != nullptr && m_parser != nullptr )
auto task = parser::Task::create( this );
if ( task == nullptr )
return;
if ( task->addItem( mrl, std::move( fileFs ), std::move( parentFolder ),
std::move( parentFolderFs ), fileType,
std::move( parentPlaylist.first ), parentPlaylist.second ) == false )
return;
t->commit();
if ( m_parser != nullptr )
m_parser->parse( task );
}
catch(sqlite::errors::ConstraintViolation& ex)
......@@ -514,8 +528,14 @@ void MediaLibrary::onUpdatedFile( std::shared_ptr<File> file,
auto mrl = fileFs->mrl();
try
{
auto task = parser::Task::createRefreshTask( this, std::move( file ), std::move( fileFs ) );
if ( task != nullptr && m_parser != nullptr )
auto t = getConn()->newTransaction();
auto task = parser::Task::create( this );
if ( task == nullptr )
return;
if ( task->addRefreshItem( std::move( file ), std::move( fileFs ) ) == false )
return;
t->commit();
if ( m_parser != nullptr )
m_parser->parse( std::move( task ) );
}
catch( const sqlite::errors::ConstraintViolation& ex )
......@@ -960,7 +980,7 @@ InitializeResult MediaLibrary::updateDatabaseModel( unsigned int previousVersion
}
if ( previousVersion == 17 )
{
migrateModel17to18();
migrateModel17to18( originalPreviousVersion );
previousVersion = 18;
}
// To be continued in the future!
......@@ -1339,16 +1359,6 @@ void MediaLibrary::migrateModel15to16()
}
}
// Migrate tasks
{
auto tasks = parser::Task::fetchAll<parser::Task>( this );
for ( auto& t : tasks )
{
auto newMrl = utils::url::encode( utils::url::decode( t->item().mrl() ) );
t->setMrl( std::move( newMrl ) );
}
}
m_settings.setDbModelVersion( 16 );
m_settings.save();
t->commit();
......@@ -1407,11 +1417,34 @@ void MediaLibrary::migrateModel16to17( uint32_t originalPreviousVersion )
t->commit();
}
void MediaLibrary::migrateModel17to18()
void MediaLibrary::migrateModel17to18( uint32_t originalPreviousVersion )
{
auto dbConn = getConn();
sqlite::Connection::WeakDbContext weakConnCtx{ dbConn };
auto t = dbConn->newTransaction();
// Task table was introduced in model 8, so no need to migrate from before then
if ( originalPreviousVersion >= 8 )
{
std::string reqs[] = {
# include "database/migrations/migration17-18.sql"
};
for ( const auto& req : reqs )
sqlite::Tools::executeRequest( dbConn, req );
}
if ( originalPreviousVersion <= 15 )
{
// Migrate tasks
auto items = parser::Item::fetchAll<parser::Item>( this );
for ( auto& i : items )
{
auto newMrl = utils::url::encode( utils::url::decode( i->mrl() ) );
i->setMrl( std::move( newMrl ) );
}
}
m_settings.setDbModelVersion( 18 );
m_settings.save();
t->commit();
......
......@@ -219,7 +219,7 @@ private:
void migrateModel14to15();
void migrateModel15to16();
void migrateModel16to17( uint32_t originalPreviousVersion );
void migrateModel17to18();
void migrateModel17to18( uint32_t originalPreviousVersion );
void createAllTables();
void createAllTriggers();
void registerEntityHooks();
......
/******************************************************************
* Split Task into Task & Item *
*****************************************************************/
"CREATE TABLE IF NOT EXISTS " + parser::Task::Table::Name + "_backup"
"("
"id_task INTEGER PRIMARY KEY AUTOINCREMENT,"
"step INTEGER NOT NULL DEFAULT 0,"
"retry_count INTEGER NOT NULL DEFAULT 0,"
"mrl TEXT,"
"file_type INTEGER NOT NULL,"
"file_id UNSIGNED INTEGER,"
"parent_folder_id UNSIGNED INTEGER,"
"parent_playlist_id INTEGER,"
"parent_playlist_index UNSIGNED INTEGER,"
"is_refresh BOOLEAN NOT NULL DEFAULT 0"
")",
"INSERT INTO " + parser::Task::Table::Name + "_backup"
" SELECT * from " + parser::Task::Table::Name,
"DROP TABLE " + parser::Task::Table::Name,
#include "database/tables/Task_v18.sql"
#include "database/tables/Item_v18.sql"
"INSERT INTO " + parser::Task::Table::Name + "(id_task, step, retry_count)"
"SELECT id_task, step, retry_count FROM " + parser::Task::Table::Name + "_backup",
"INSERT INTO " + parser::Item::Table::Name +
"(task_id, mrl, file_type, file_id, parent_folder_id,"
"parent_playlist_id, parent_playlist_index, is_refresh)"
"SELECT "
"id_task, mrl, file_type, file_id, parent_folder_id,"
"parent_playlist_id, parent_playlist_index, is_refresh "
"FROM " + parser::Task::Table::Name + "_backup",
"DROP TABLE " + parser::Task::Table::Name + "_backup",
"CREATE TABLE IF NOT EXISTS " + parser::Item::Table::Name +
"("
"id_item INTEGER PRIMARY KEY AUTOINCREMENT,"
"task_id INTEGER,"
"mrl TEXT,"
"file_type INTEGER NOT NULL,"
"file_id UNSIGNED INTEGER,"
"parent_folder_id UNSIGNED INTEGER,"
"parent_playlist_id INTEGER,"
"parent_playlist_index UNSIGNED INTEGER,"
"is_refresh BOOLEAN NOT NULL DEFAULT 0,"
"UNIQUE(mrl, parent_playlist_id, is_refresh) ON CONFLICT FAIL,"
"FOREIGN KEY (parent_folder_id) REFERENCES " + Folder::Table::Name
+ "(id_folder) ON DELETE CASCADE,"
"FOREIGN KEY (file_id) REFERENCES " + File::Table::Name
+ "(id_file) ON DELETE CASCADE,"
"FOREIGN KEY (parent_playlist_id) REFERENCES " + Playlist::Table::Name
+ "(id_playlist) ON DELETE CASCADE,"
"FOREIGN KEY (task_id) REFERENCES " + parser::Task::Table::Name +
"(id_task) ON DELETE CASCADE"
")",
"CREATE INDEX IF NOT EXISTS task_entities_task_id_idx ON " +
parser::Item::Table::Name + "(task_id)",
......@@ -2,19 +2,5 @@
"("
"id_task INTEGER PRIMARY KEY AUTOINCREMENT,"
"step INTEGER NOT NULL DEFAULT 0,"
"retry_count INTEGER NOT NULL DEFAULT 0,"
"mrl TEXT,"
"file_type INTEGER NOT NULL,"
"file_id UNSIGNED INTEGER,"
"parent_folder_id UNSIGNED INTEGER,"
"parent_playlist_id INTEGER,"
"parent_playlist_index UNSIGNED INTEGER,"
"is_refresh BOOLEAN NOT NULL DEFAULT 0,"
"UNIQUE(mrl, parent_playlist_id, is_refresh) ON CONFLICT FAIL,"
"FOREIGN KEY (parent_folder_id) REFERENCES " + Folder::Table::Name
+ "(id_folder) ON DELETE CASCADE,"
"FOREIGN KEY (file_id) REFERENCES " + File::Table::Name
+ "(id_file) ON DELETE CASCADE,"
"FOREIGN KEY (parent_playlist_id) REFERENCES " + Playlist::Table::Name
+ "(id_playlist) ON DELETE CASCADE"
"retry_count INTEGER NOT NULL DEFAULT 0"
")",
......@@ -28,54 +28,86 @@
#include "Folder.h"
#include "Playlist.h"
#include "File.h"
#include "Media.h"
#include "utils/Strings.h"
#include "utils/Filename.h"
#include <cassert>
#include <algorithm>
namespace medialibrary
{
namespace parser
{
Item::Item( ITaskCb* taskCb, std::string mrl, IFile::Type fileType,
const std::string Item::Table::Name = "ItemTable";
const std::string Item::Table::PrimaryKeyColumn = "id_item";
int64_t Item::* const Item::Table::PrimaryKey = &Item::m_id;
Item::Item( MediaLibraryPtr ml, sqlite::Row& row )
: m_ml( ml )
, m_id( row.extract<decltype(m_id)>() )
, m_taskId( row.extract<decltype(m_taskId)>() )
, m_mrl( row.extract<decltype(m_mrl)>() )
, m_fileType( row.extract<decltype(m_fileType)>() )
, m_fileId( row.extract<decltype(m_fileId)>() )
, m_parentFolderId( row.extract<decltype(m_parentFolderId)>() )
, m_parentPlaylistId( row.extract<decltype(m_parentPlaylistId)>() )
, m_parentPlaylistIndex( row.extract<decltype(m_parentPlaylistIndex)>() )
, m_isRefresh( row.extract<decltype(m_isRefresh)>() )
{
}
Item::Item( std::string mrl, IFile::Type fileType,
unsigned int subitemPosition, bool isRefresh )
: m_taskCb( taskCb )
: m_ml( nullptr )
, m_id( 0 )
, m_taskId( 0 )
, m_mrl( std::move( mrl ) )
, m_fileType( fileType )
, m_duration( 0 )
, m_fileId( 0 )
, m_parentFolderId( 0 )
, m_parentPlaylistId( 0 )
, m_parentPlaylistIndex( subitemPosition )
, m_isRefresh( isRefresh )
, m_duration( 0 )
{
}
Item::Item( ITaskCb* taskCb, std::string mrl, std::shared_ptr<fs::IFile> fileFs,
std::shared_ptr<Folder> parentFolder,
Item::Item( MediaLibraryPtr ml, int64_t taskId, std::string mrl,
std::shared_ptr<fs::IFile> fileFs, std::shared_ptr<Folder> parentFolder,
std::shared_ptr<fs::IDirectory> parentFolderFs, IFile::Type fileType,
std::shared_ptr<Playlist> parentPlaylist, unsigned int parentPlaylistIndex,
bool isRefresh )
: m_taskCb( taskCb )
: m_ml( ml )
, m_id( 0 )
, m_taskId( taskId )
, m_mrl( std::move( mrl ) )
, m_fileType( fileType )
, m_fileId( 0 )
, m_parentFolderId( parentFolder->id() )
, m_parentPlaylistId( parentPlaylist != nullptr ? parentPlaylist->id() : 0 )
, m_parentPlaylistIndex( parentPlaylistIndex )
, m_isRefresh( isRefresh )
, m_duration( 0 )
, m_fileFs( std::move( fileFs ) )
, m_parentFolder( std::move( parentFolder ) )
, m_parentFolderFs( std::move( parentFolderFs ) )
, m_parentPlaylist( std::move( parentPlaylist ) )
, m_parentPlaylistIndex( parentPlaylistIndex )
, m_isRefresh( isRefresh )
{
}
Item::Item( ITaskCb* taskCb, std::shared_ptr<File> file,
std::shared_ptr<fs::IFile> fileFs )
: m_taskCb( taskCb )
Item::Item( MediaLibraryPtr ml, int64_t taskId, std::shared_ptr<File> file,
std::shared_ptr<fs::IFile> fileFs )
: m_ml( ml )
, m_taskId( taskId )
, m_mrl( fileFs->mrl() )
, m_fileType( file->type() )
, m_isRefresh( true )
, m_duration( 0 )
, m_file( std::move( file ) )
, m_fileFs( std::move( fileFs ) )
, m_isRefresh( true )
{
}
......@@ -100,6 +132,9 @@ const std::string& Item::mrl() const
void Item::setMrl( std::string mrl )
{
std::string req{ "UPDATE " + Table::Name + " SET mrl = ? WHERE id_item = ?" };
if ( sqlite::Tools::executeUpdate( m_ml->getConn(), req, mrl, m_id ) == false )
return;
m_mrl = std::move( mrl );
}
......@@ -120,7 +155,7 @@ const IItem& Item::subItem( unsigned int index ) const
IItem& Item::createSubItem( std::string mrl, unsigned int playlistIndex )
{
m_subItems.emplace_back( nullptr, std::move( mrl ), IFile::Type::Main,
m_subItems.emplace_back( std::move( mrl ), IFile::Type::Main,
playlistIndex, false );
return m_subItems.back();
}
......@@ -160,11 +195,22 @@ FilePtr Item::file()
return m_file;
}
bool Item::setFile( FilePtr file)
bool Item::setFile( FilePtr file )
{
m_file = std::move( file );
assert( m_taskCb != nullptr );
return m_taskCb->updateFileId( m_file->id() );
// When restoring a task, we will invoke ITaskCb::updateFileId while the
// task already knows the fileId (since we're using it to restore the file instance)
// In this case, bail out. Otherwise, it is not expected for the task to change
// its associated file during the processing.
if ( m_fileId != 0 && m_fileId == file->id() )
return true;
assert( m_fileId == 0 );
static const std::string req = "UPDATE " + Table::Name + " SET "
"file_id = ? WHERE id_item = ?";
if ( sqlite::Tools::executeUpdate( m_ml->getConn(), req, file->id(), m_id ) == false )
return false;
m_fileId = file->id();
m_file = std::static_pointer_cast<File>( std::move( file ) );
return true;
}
FolderPtr Item::parentFolder()
......@@ -197,5 +243,216 @@ bool Item::isRefresh() const
return m_isRefresh;
}
bool Item::restoreLinkedEntities()
{
LOG_DEBUG("Restoring linked entities of task ", m_id);
// MRL will be empty if the task has been resumed from unparsed files
// (during 11 -> 12 migration)
if ( m_mrl.empty() == true && m_fileId == 0 )
{
LOG_WARN( "Aborting & removing file task without mrl nor file id (#", m_id, ')' );
destroy( m_ml, m_id );
return false;
}
// First of all, we need to know if the file has been created already
// ie. have we run the MetadataParser service, at least partially
if ( m_fileId != 0 )
{
m_file = File::fetch( m_ml, m_fileId );
if ( m_file == nullptr )
{
LOG_WARN( "Failed to restore file associated to the task. Task will "
"be dropped" );
destroy( m_ml, m_id );
return false;
}
m_media = m_file->media();
}
// Now we either have a task with an existing file, and we managed to fetch
// it, or the task was not processed yet, and we don't have a fileId (and
// therefor no file instance)
assert( m_fileId == 0 || m_file != nullptr );
// In case we have a refresh task, old versions didn't provide the parent
// id. That is:
// 0.4.x branch before vlc-android 3.1.5 (excluded)
// 0.5.x branch before 2019 May 17 (unshipped in releases)
// However we must have either a file ID (mandatory for a refresh task) or a
// parent folder ID (mandatory when discovering a file)
assert( m_fileId != 0 || m_parentFolderId != 0 );
// Regardless of the stored mrl, always fetch the file from DB and query its
// mrl. If might have changed in case we're dealing with a removable storage
if ( m_file != nullptr )
{
std::string mrl;
try
{
mrl = m_file->mrl();
}
catch ( const fs::DeviceRemovedException& )
{
LOG_WARN( "Postponing rescan of removable file ", m_file->rawMrl(),
" until the device containing it is present again" );
return false;
}
assert( mrl.empty() == false );
// If we are migrating a task without an mrl, store the mrl for future use
// In case the mrl changed, update it as well, as the later points of the
// parsing process will depend on the mrl stored in the item, not the
// one we now have here.
if ( m_mrl.empty() == true || m_mrl != mrl )
setMrl( mrl );
}
// Now we always have a valid MRL, but we might not have a fileId
// In any case, we need to fetch the corresponding FS entities
auto fsFactory = m_ml->fsFactoryForMrl( m_mrl );
if ( fsFactory == nullptr )
{
LOG_WARN( "No fs factory matched the task mrl (", m_mrl, "). Postponing" );
return false;
}
try
{
m_parentFolderFs = fsFactory->createDirectory( utils::file::directory( m_mrl ) );
}
catch ( const std::system_error& ex )
{
LOG_ERROR( "Failed to restore task: ", ex.what() );
return false;
}
try
{
auto files = m_parentFolderFs->files();
// Don't compare entire mrls, this might yield false negative when a
// device has multiple mountpoints.
auto fileName = utils::file::fileName( m_mrl );
auto it = std::find_if( begin( files ), end( files ), [&fileName]( std::shared_ptr<fs::IFile> f ) {
return f->name() == fileName;
});
if ( it == end( files ) )
{
LOG_ERROR( "Failed to restore fs::IFile associated with ", m_mrl );
return false;
}
m_fileFs = std::move( *it );
}
catch ( const fs::DeviceRemovedException& )
{
LOG_WARN( "Failed to restore file on an unmounted device: ", m_mrl );
return false;
}
catch ( const std::system_error& ex )
{
// If we never found the file yet, we can delete the task. It will be
// recreated upon next discovery
if ( m_file == nullptr )
{
LOG_WARN( "Failed to restore file system instances for mrl ", m_mrl, "(",
ex.what(), ").", " Removing the task until it gets detected again." );
destroy( m_ml, m_id );
}
else
{
// Otherwise we need to postpone it, although most likely we will
// detect that the file is now missing, and we won't try to restore
// this task until it comes back (since the task restoration request
// includes the file.is_present flag)
LOG_WARN( "Failed to restore file system instances for mrl ", m_mrl, "."
" Postponing the task." );
}
return false;
}
// Handle old refresh tasks without a parent folder id
if ( m_parentFolderId == 0 )
{
assert( m_fileId != 0 && m_file != nullptr );
m_parentFolderId = m_file->folderId();
}
m_parentFolder = Folder::fetch( m_ml, m_parentFolderId );
if ( m_parentFolder == nullptr )
{
LOG_ERROR( "Failed to restore parent folder #", m_parentFolderId );
return false;
}
if ( m_parentPlaylistId != 0 )
{
m_parentPlaylist = Playlist::fetch( m_ml, m_parentPlaylistId );
if ( m_parentPlaylist == nullptr )
{
LOG_ERROR( "Failed to restore parent playlist #", m_parentPlaylistId );
return false;
}
}
return true;
}
std::shared_ptr<Item> Item::loadFromTask( MediaLibraryPtr ml, int64_t parentTaskId )
{
std::string req{ "SELECT * FROM " + Table::Name + " WHERE task_id = ?" };
return fetch( ml, req, parentTaskId );
}
void Item::createTable( sqlite::Connection* dbConn )
{
std::string reqs[] = {
#include "database/tables/Item_v18.sql"
};
for ( const auto& req : reqs )
sqlite::Tools::executeRequest( dbConn, req );
}
std::shared_ptr<Item> Item::create( MediaLibraryPtr ml, int64_t taskId, std::string mrl,
std::shared_ptr<fs::IFile> fileFs,
std::shared_ptr<Folder> parentFolder,
std::shared_ptr<fs::IDirectory> parentFolderFs,
IFile::Type fileType,
std::shared_ptr<Playlist> parentPlaylist,
unsigned int parentPlaylistIndex )
{
const std::string req {
"INSERT INTO " + Table::Name + "(task_id, mrl, file_type,"
"parent_folder_id, parent_playlist_id, parent_playlist_index, is_refresh)"
"VALUES(?, ?, ?, ?, ?, ?, ?)"
};
auto parentPlaylistId = parentPlaylist != nullptr ? parentPlaylist->id() : 0;
auto self = std::make_shared<Item>( ml, taskId, std::move( mrl ), std::move( fileFs ),
std::move( parentFolder ),
std::move( parentFolderFs ), fileType,
std::move( parentPlaylist ), parentPlaylistIndex,
false );
if ( insert( ml, self, req, taskId, self->mrl(), self->fileType(),
self->parentFolder()->id(), sqlite::ForeignKey{ parentPlaylistId },
parentPlaylistIndex, false ) == false )
return nullptr;
return self;
}
std::shared_ptr<Item> Item::createRefresh( MediaLibraryPtr ml, int64_t taskId,
std::shared_ptr<File> file,
std::shared_ptr<fs::IFile> fileFs )
{
const std::string req{
"INSERT INTO " + Table::Name + "(task_id, mrl, file_type, file_id,"
"parent_folder_id, is_refresh) VALUES(?, ?, ?, ?, ?, ?)"
};
auto parentFolderId = file->folderId();
auto self = std::make_shared<Item>( ml, taskId, std::move( file ), std::move( fileFs ) );
if ( insert( ml, self, req, taskId, self->m_file->mrl(), self->m_file->type(),
self->m_file->id(), parentFolderId, true ) == false )
return nullptr;
return self;
}
}
}
......@@ -23,6 +23,8 @@
#pragma once
#include "medialibrary/parser/IItem.h"
#include "Types.h"
#include "database/DatabaseHelpers.h"
#include <unordered_map>
......@@ -36,11 +38,16 @@ class File;
namespace parser
{
class ITaskCb;
class Item : public IItem