Commit fc4ad205 authored by Hugo Beauzée-Luyssen's avatar Hugo Beauzée-Luyssen

Delegate FS modification scanning to IDiscoverer

parent 9caf0210
...@@ -5,33 +5,17 @@ ...@@ -5,33 +5,17 @@
#include "Types.h" #include "Types.h"
#include "filesystem/IDirectory.h" #include "filesystem/IDirectory.h"
#include "filesystem/IFile.h" #include "filesystem/IFile.h"
#include "IMediaLibrary.h"
class IDiscovererCb
{
public:
virtual ~IDiscovererCb() = default;
/**
* @brief onNewFolder Called when the discoverer finds a new directory
* @param folderPath The new directory's path
* @param parent The parent folder, or null if this is the root folder
* @return The newly created folder, or nullptr in case of error (or if the
* directory shall not be browsed further)
*/
virtual FolderPtr onNewFolder( const fs::IDirectory* folder, FolderPtr parent ) = 0;
/**
* @brief onNewFile Called when the discoverer finds a new file
* @param filePath The new file's path
* @param parent The parent folder
* @return true if the file was accepted
*/
virtual FilePtr onNewFile( const fs::IFile* file, FolderPtr parent ) = 0;
};
class IDiscoverer class IDiscoverer
{ {
public: public:
virtual ~IDiscoverer() = default; virtual ~IDiscoverer() = default;
virtual bool discover( const std::string& entryPoint ) = 0; // We assume the media library will always outlive the discoverers.
//FIXME: This is currently false since there is no way of interrupting
//a discoverer thread
virtual bool discover( IMediaLibrary* ml, DBConnection dbConn, const std::string& entryPoint ) = 0;
virtual void reload( IMediaLibrary* ml, DBConnection dbConn ) = 0;
}; };
#endif // IDISCOVERER_H #endif // IDISCOVERER_H
...@@ -6,7 +6,6 @@ ...@@ -6,7 +6,6 @@
#include "Types.h" #include "Types.h"
#include "factory/IFileSystem.h" #include "factory/IFileSystem.h"
#include "IDiscoverer.h"
class IMediaLibraryCb class IMediaLibraryCb
{ {
...@@ -49,8 +48,14 @@ class IMediaLibrary ...@@ -49,8 +48,14 @@ class IMediaLibrary
* Calling this after initialize() is not a supported scenario. * Calling this after initialize() is not a supported scenario.
*/ */
virtual void setFsFactory( std::shared_ptr<factory::IFileSystem> fsFactory ) = 0; virtual void setFsFactory( std::shared_ptr<factory::IFileSystem> fsFactory ) = 0;
/// Adds a stand alone file ///
virtual FilePtr addFile( const std::string& path ) = 0; /// \brief addFile Adds a file to the media library.
/// \param path The absolute path to this file
/// \param parentFolder The parent folder, or nullptr to add this file as
/// a stand alone file.
/// \return The newly created file, or nullptr in case of error
///
virtual FilePtr addFile( const std::string& path, FolderPtr parentFolder ) = 0;
virtual FilePtr file( const std::string& path ) = 0; virtual FilePtr file( const std::string& path ) = 0;
virtual bool deleteFile( const std::string& mrl ) = 0; virtual bool deleteFile( const std::string& mrl ) = 0;
virtual bool deleteFile( FilePtr file ) = 0; virtual bool deleteFile( FilePtr file ) = 0;
......
...@@ -107,7 +107,7 @@ bool MediaLibrary::initialize( const std::string& dbPath, const std::string& sna ...@@ -107,7 +107,7 @@ bool MediaLibrary::initialize( const std::string& dbPath, const std::string& sna
addMetadataService( std::move( thumbnailerService ) ); addMetadataService( std::move( thumbnailerService ) );
} }
m_discoverers.emplace_back( new FsDiscoverer( m_fsFactory, this ) ); m_discoverers.emplace_back( new FsDiscoverer( m_fsFactory ) );
sqlite3* dbConnection; sqlite3* dbConnection;
int res = sqlite3_open( dbPath.c_str(), &dbConnection ); int res = sqlite3_open( dbPath.c_str(), &dbConnection );
...@@ -134,7 +134,8 @@ bool MediaLibrary::initialize( const std::string& dbPath, const std::string& sna ...@@ -134,7 +134,8 @@ bool MediaLibrary::initialize( const std::string& dbPath, const std::string& sna
LOG_ERROR( "Failed to create database structure" ); LOG_ERROR( "Failed to create database structure" );
return false; return false;
} }
return loadFolders(); reload();
return true;
} }
std::vector<FilePtr> MediaLibrary::files() std::vector<FilePtr> MediaLibrary::files()
...@@ -162,10 +163,44 @@ FilePtr MediaLibrary::file( const std::string& path ) ...@@ -162,10 +163,44 @@ FilePtr MediaLibrary::file( const std::string& path )
return File::fetch( m_dbConnection, path ); return File::fetch( m_dbConnection, path );
} }
FilePtr MediaLibrary::addFile( const std::string& path ) FilePtr MediaLibrary::addFile( const std::string& path, FolderPtr parentFolder )
{ {
auto fsFile = m_fsFactory->createFile( path ); std::unique_ptr<fs::IFile> file;
return addFile( fsFile.get(), 0 ); try
{
file = m_fsFactory->createFile( path );
}
catch (std::exception& ex)
{
LOG_ERROR( "Failed to create an IFile for ", path, ": ", ex.what() );
return nullptr;
}
auto type = IFile::Type::UnknownType;
if ( std::find( begin( supportedVideoExtensions ), end( supportedVideoExtensions ),
file->extension() ) != end( supportedVideoExtensions ) )
{
type = IFile::Type::VideoType;
}
else if ( std::find( begin( supportedAudioExtensions ), end( supportedAudioExtensions ),
file->extension() ) != end( supportedAudioExtensions ) )
{
type = IFile::Type::AudioType;
}
if ( type == IFile::Type::UnknownType )
return false;
auto fptr = File::create( m_dbConnection, type, file.get(), parentFolder != nullptr ? parentFolder->id() : 0 );
if ( fptr == nullptr )
{
LOG_ERROR( "Failed to add file ", file->fullPath(), " to the media library" );
return nullptr;
}
LOG_INFO( "Adding ", file->name() );
if ( m_callback != nullptr )
m_callback->onFileAdded( fptr );
m_parser->parse( fptr, m_callback );
return fptr;
} }
FolderPtr MediaLibrary::folder( const std::string& path ) FolderPtr MediaLibrary::folder( const std::string& path )
...@@ -276,6 +311,18 @@ void MediaLibrary::addMetadataService(std::unique_ptr<IMetadataService> service) ...@@ -276,6 +311,18 @@ void MediaLibrary::addMetadataService(std::unique_ptr<IMetadataService> service)
m_parser->addService( std::move( service ) ); m_parser->addService( std::move( service ) );
} }
void MediaLibrary::reload()
{
//FIXME: Create a proper wrapper to handle discoverer threading
std::thread t([this] {
//FIXME: This will crash if the media library gets deleted while we
//are discovering.
for ( auto& d : m_discoverers )
d->reload( this, this->m_dbConnection );
});
t.detach();
}
void MediaLibrary::discover( const std::string &entryPoint ) void MediaLibrary::discover( const std::string &entryPoint )
{ {
std::thread t([this, entryPoint] { std::thread t([this, entryPoint] {
...@@ -285,7 +332,7 @@ void MediaLibrary::discover( const std::string &entryPoint ) ...@@ -285,7 +332,7 @@ void MediaLibrary::discover( const std::string &entryPoint )
m_callback->onDiscoveryStarted( entryPoint ); m_callback->onDiscoveryStarted( entryPoint );
for ( auto& d : m_discoverers ) for ( auto& d : m_discoverers )
d->discover( entryPoint ); d->discover( this, this->m_dbConnection, entryPoint );
if ( m_callback != nullptr ) if ( m_callback != nullptr )
m_callback->onDiscoveryCompleted( entryPoint ); m_callback->onDiscoveryCompleted( entryPoint );
...@@ -293,23 +340,6 @@ void MediaLibrary::discover( const std::string &entryPoint ) ...@@ -293,23 +340,6 @@ void MediaLibrary::discover( const std::string &entryPoint )
t.detach(); t.detach();
} }
FolderPtr MediaLibrary::onNewFolder( const fs::IDirectory* directory, FolderPtr parent )
{
//FIXME: Since we insert files/folders with a UNIQUE constraint, maybe we should
//just let sqlite try to insert, throw an exception in case the contraint gets violated
//catch it and return nullptr from here.
//We previously were fetching the folder manually here, but that triggers an eroneous entry
//in the cache. This might also be something to fix...
return Folder::create( m_dbConnection, directory,
parent == nullptr ? 0 : parent->id() );
}
FilePtr MediaLibrary::onNewFile( const fs::IFile *file, FolderPtr parent )
{
//FIXME: Same uniqueness comment as onNewFolder above.
return addFile( file, parent == nullptr ? 0 : parent->id() );
}
const std::string& MediaLibrary::snapshotPath() const const std::string& MediaLibrary::snapshotPath() const
{ {
return m_snapshotPath; return m_snapshotPath;
...@@ -320,132 +350,3 @@ void MediaLibrary::setLogger( ILogger* logger ) ...@@ -320,132 +350,3 @@ void MediaLibrary::setLogger( ILogger* logger )
Log::SetLogger( logger ); Log::SetLogger( logger );
} }
bool MediaLibrary::loadFolders()
{
//FIXME: This should probably be in a sql transaction
//FIXME: This shouldn't be done for "removable"/network files
static const std::string req = "SELECT * FROM " + policy::FolderTable::Name
+ " WHERE id_parent IS NULL";
auto rootFolders = sqlite::Tools::fetchAll<Folder, IFolder>( m_dbConnection, req );
for ( const auto f : rootFolders )
{
auto folder = m_fsFactory->createDirectory( f->path() );
if ( folder->lastModificationDate() == f->lastModificationDate() )
continue;
checkSubfolders( folder.get(), f->id() );
f->setLastModificationDate( folder->lastModificationDate() );
}
return true;
}
bool MediaLibrary::checkSubfolders( fs::IDirectory* folder, unsigned int parentId )
{
// From here we can have:
// - New subfolder(s)
// - Deleted subfolder(s)
// - New file(s)
// - Deleted file(s)
// - Changed file(s)
// ... in this folder, or in all the sub folders.
// Load the folders we already know of:
static const std::string req = "SELECT * FROM " + policy::FolderTable::Name
+ " WHERE id_parent = ?";
auto subFoldersInDB = sqlite::Tools::fetchAll<Folder, IFolder>( m_dbConnection, req, parentId );
for ( const auto& subFolderPath : folder->dirs() )
{
auto it = std::find_if( begin( subFoldersInDB ), end( subFoldersInDB ), [subFolderPath](const std::shared_ptr<IFolder>& f) {
return f->path() == subFolderPath;
});
// We don't know this folder, it's a new one
if ( it == end( subFoldersInDB ) )
{
//FIXME: In order to add the new folder, we need to use the same discoverer.
// This probably means we need to store which discoverer was used to add which file
// and store discoverers as a map instead of a vector
continue;
}
auto subFolder = m_fsFactory->createDirectory( subFolderPath );
if ( subFolder->lastModificationDate() == (*it)->lastModificationDate() )
{
// Remove all folders that still exist in FS. That way, the list of folders that
// will still be in subFoldersInDB when we're done is the list of folders that have
// been deleted from the FS
subFoldersInDB.erase( it );
continue;
}
// This folder was modified, let's recurse
checkSubfolders( subFolder.get(), (*it)->id() );
checkFiles( subFolder.get(), (*it)->id() );
(*it)->setLastModificationDate( subFolder->lastModificationDate() );
subFoldersInDB.erase( it );
}
// Now all folders we had in DB but haven't seen from the FS must have been deleted.
for ( auto f : subFoldersInDB )
{
std::cout << "Folder " << f->path() << " not found in FS, deleting it" << std::endl;
deleteFolder( f );
}
return true;
}
void MediaLibrary::checkFiles( fs::IDirectory* folder, unsigned int parentId )
{
static const std::string req = "SELECT * FROM " + policy::FileTable::Name
+ " WHERE folder_id = ?";
auto files = sqlite::Tools::fetchAll<File, IFile>( m_dbConnection, req, parentId );
for ( const auto& filePath : folder->files() )
{
auto file = m_fsFactory->createFile( filePath );
auto it = std::find_if( begin( files ), end( files ), [filePath](const std::shared_ptr<IFile>& f) {
return f->mrl() == filePath;
});
if ( it == end( files ) )
{
addFile( file.get(), parentId );
continue;
}
if ( file->lastModificationDate() == (*it)->lastModificationDate() )
{
// Unchanged file
files.erase( it );
continue;
}
deleteFile( filePath );
addFile( file.get(), parentId );
files.erase( it );
}
for ( auto file : files )
{
deleteFile( file );
}
}
FilePtr MediaLibrary::addFile( const fs::IFile* file, unsigned int folderId )
{
auto type = IFile::Type::UnknownType;
if ( std::find( begin( supportedVideoExtensions ), end( supportedVideoExtensions ),
file->extension() ) != end( supportedVideoExtensions ) )
{
type = IFile::Type::VideoType;
}
else if ( std::find( begin( supportedAudioExtensions ), end( supportedAudioExtensions ),
file->extension() ) != end( supportedAudioExtensions ) )
{
type = IFile::Type::AudioType;
}
if ( type == IFile::Type::UnknownType )
return false;
auto fptr = File::create( m_dbConnection, type, file, folderId );
if ( fptr == nullptr )
{
LOG_ERROR( "Failed to add file ", file->fullPath(), " to the media library" );
return nullptr;
}
LOG_INFO( "Adding ", file->name() );
m_callback->onFileAdded( fptr );
m_parser->parse( fptr, m_callback );
return fptr;
}
...@@ -10,7 +10,7 @@ class Parser; ...@@ -10,7 +10,7 @@ class Parser;
#include "logging/Logger.h" #include "logging/Logger.h"
#include "vlcpp/vlc.hpp" #include "vlcpp/vlc.hpp"
class MediaLibrary : public IMediaLibrary, public IDiscovererCb class MediaLibrary : public IMediaLibrary
{ {
public: public:
MediaLibrary(); MediaLibrary();
...@@ -22,7 +22,7 @@ class MediaLibrary : public IMediaLibrary, public IDiscovererCb ...@@ -22,7 +22,7 @@ class MediaLibrary : public IMediaLibrary, public IDiscovererCb
virtual std::vector<FilePtr> audioFiles() override; virtual std::vector<FilePtr> audioFiles() override;
virtual std::vector<FilePtr> videoFiles() override; virtual std::vector<FilePtr> videoFiles() override;
virtual FilePtr file( const std::string& path ) override; virtual FilePtr file( const std::string& path ) override;
virtual FilePtr addFile( const std::string& path ) override; virtual FilePtr addFile(const std::string& path , FolderPtr parentFolder) override;
virtual bool deleteFile( const std::string& mrl ) override; virtual bool deleteFile( const std::string& mrl ) override;
virtual bool deleteFile( FilePtr file ) override; virtual bool deleteFile( FilePtr file ) override;
...@@ -48,9 +48,6 @@ class MediaLibrary : public IMediaLibrary, public IDiscovererCb ...@@ -48,9 +48,6 @@ class MediaLibrary : public IMediaLibrary, public IDiscovererCb
virtual std::vector<ArtistPtr> artists() const override; virtual std::vector<ArtistPtr> artists() const override;
virtual void discover( const std::string& entryPoint ) override; virtual void discover( const std::string& entryPoint ) override;
// IDiscovererCb implementation
virtual FolderPtr onNewFolder( const fs::IDirectory* directory, FolderPtr parent ) override;
virtual FilePtr onNewFile(const fs::IFile* file, FolderPtr parent ) override;
virtual const std::string& snapshotPath() const override; virtual const std::string& snapshotPath() const override;
virtual void setLogger( ILogger* logger ) override; virtual void setLogger( ILogger* logger ) override;
...@@ -60,11 +57,8 @@ class MediaLibrary : public IMediaLibrary, public IDiscovererCb ...@@ -60,11 +57,8 @@ class MediaLibrary : public IMediaLibrary, public IDiscovererCb
static const std::vector<std::string> supportedAudioExtensions; static const std::vector<std::string> supportedAudioExtensions;
private: private:
bool loadFolders();
bool checkSubfolders( fs::IDirectory* folder, unsigned int parentId );
void checkFiles( fs::IDirectory* folder, unsigned int parentId );
FilePtr addFile( const fs::IFile* file, unsigned int folderId );
void addMetadataService( std::unique_ptr<IMetadataService> service ); void addMetadataService( std::unique_ptr<IMetadataService> service );
void reload();
private: private:
std::shared_ptr<sqlite3> m_dbConnection; std::shared_ptr<sqlite3> m_dbConnection;
......
#include "FsDiscoverer.h" #include "FsDiscoverer.h"
#include <algorithm>
#include "factory/FileSystem.h" #include "factory/FileSystem.h"
#include "File.h"
#include "Folder.h"
#include <queue> #include <queue>
#include <iostream> #include <iostream>
FsDiscoverer::FsDiscoverer( std::shared_ptr<factory::IFileSystem> fsFactory, IDiscovererCb* discoveryCb ) FsDiscoverer::FsDiscoverer(std::shared_ptr<factory::IFileSystem> fsFactory)
: m_discoveryCb( discoveryCb )
{ {
if ( fsFactory != nullptr ) if ( fsFactory != nullptr )
m_fsFactory = fsFactory; m_fsFactory = fsFactory;
...@@ -13,46 +15,132 @@ FsDiscoverer::FsDiscoverer( std::shared_ptr<factory::IFileSystem> fsFactory, IDi ...@@ -13,46 +15,132 @@ FsDiscoverer::FsDiscoverer( std::shared_ptr<factory::IFileSystem> fsFactory, IDi
m_fsFactory.reset( new factory::FileSystemDefaultFactory ); m_fsFactory.reset( new factory::FileSystemDefaultFactory );
} }
bool FsDiscoverer::discover( const std::string &entryPoint ) bool FsDiscoverer::discover( IMediaLibrary* ml, DBConnection dbConn, const std::string &entryPoint )
{ {
std::queue<std::pair<std::string, FolderPtr>> folders; // Assume :// denotes a scheme that isn't a file path, and refuse to discover it.
if ( entryPoint.find( "://" ) != std::string::npos )
return false;
folders.emplace( entryPoint, nullptr );
while ( folders.empty() == false )
{ {
std::unique_ptr<fs::IDirectory> dir; auto f = Folder::fetch( dbConn, entryPoint );
auto currentFolder = folders.front(); // If the folder exists, we assume it is up to date
auto currentPath = currentFolder.first; if ( f != nullptr )
auto parent = currentFolder.second; return true;
folders.pop(); }
// Otherwise, create a directory, and check it for modifications
std::unique_ptr<fs::IDirectory> fsDir;
try
{
fsDir = m_fsFactory->createDirectory( entryPoint );
}
catch (std::exception& ex)
{
LOG_ERROR("Failed to create an IDirectory for ", entryPoint, ": ", ex.what());
return false;
}
auto f = Folder::create( dbConn, fsDir.get(), 0 );
if ( f == nullptr )
return false;
checkSubfolders( ml, dbConn, fsDir.get(), f );
return true;
}
void FsDiscoverer::reload( IMediaLibrary *ml, DBConnection dbConn )
{
//FIXME: This should probably be in a sql transaction
//FIXME: This shouldn't be done for "removable"/network files
static const std::string req = "SELECT * FROM " + policy::FolderTable::Name
+ " WHERE id_parent IS NULL";
auto rootFolders = sqlite::Tools::fetchAll<Folder, IFolder>( dbConn, req );
for ( const auto f : rootFolders )
{
auto folder = m_fsFactory->createDirectory( f->path() );
if ( folder->lastModificationDate() == f->lastModificationDate() )
continue;
checkSubfolders( ml, dbConn, folder.get(), f );
f->setLastModificationDate( folder->lastModificationDate() );
}
}
bool FsDiscoverer::checkSubfolders( IMediaLibrary* ml, DBConnection dbConn, fs::IDirectory* folder, FolderPtr parentFolder )
{
// From here we can have:
// - New subfolder(s)
// - Deleted subfolder(s)
// - New file(s)
// - Deleted file(s)
// - Changed file(s)
// ... in this folder, or in all the sub folders.
try // Load the folders we already know of:
static const std::string req = "SELECT * FROM " + policy::FolderTable::Name
+ " WHERE id_parent = ?";
auto subFoldersInDB = sqlite::Tools::fetchAll<Folder, IFolder>( dbConn, req, parentFolder->id() );
for ( const auto& subFolderPath : folder->dirs() )
{
auto it = std::find_if( begin( subFoldersInDB ), end( subFoldersInDB ), [subFolderPath](const std::shared_ptr<IFolder>& f) {
return f->path() == subFolderPath;
});
// We don't know this folder, it's a new one
if ( it == end( subFoldersInDB ) )
{ {
dir = m_fsFactory->createDirectory( currentPath ); //FIXME: In order to add the new folder, we need to use the same discoverer.
// This probably means we need to store which discoverer was used to add which file
// and store discoverers as a map instead of a vector
continue;
} }
catch ( std::runtime_error& ex ) auto subFolder = m_fsFactory->createDirectory( subFolderPath );
if ( subFolder->lastModificationDate() == (*it)->lastModificationDate() )
{ {
std::cerr << ex.what() << std::endl; // Remove all folders that still exist in FS. That way, the list of folders that
// If the first directory fails to open, stop now. // will still be in subFoldersInDB when we're done is the list of folders that have
// Otherwise, assume something went wrong in a subdirectory. // been deleted from the FS
if (parent == nullptr) subFoldersInDB.erase( it );
return false;
continue; continue;
} }
// This folder was modified, let's recurse
checkSubfolders( ml, dbConn, subFolder.get(), *it );
checkFiles( ml, dbConn, subFolder.get(), *it );
(*it)->setLastModificationDate( subFolder->lastModificationDate() );
subFoldersInDB.erase( it );
}
// Now all folders we had in DB but haven't seen from the FS must have been deleted.
for ( auto f : subFoldersInDB )
{
std::cout << "Folder " << f->path() << " not found in FS, deleting it" << std::endl;
ml->deleteFolder( f );
}
return true;
}
auto folder = m_discoveryCb->onNewFolder( dir.get(), parent ); void FsDiscoverer::checkFiles( IMediaLibrary *ml, DBConnection dbConn, fs::IDirectory* folder, FolderPtr parentFolder )
if ( folder == nullptr && parent == nullptr ) {
return false; static const std::string req = "SELECT * FROM " + policy::FileTable::Name
if ( folder == nullptr ) + " WHERE folder_id = ?";
auto files = sqlite::Tools::fetchAll<File, IFile>( dbConn, req, parentFolder->id() );
for ( const auto& filePath : folder->files() )
{
auto it = std::find_if( begin( files ), end( files ), [filePath](const std::shared_ptr<IFile>& f) {
return f->mrl() == filePath;
});
if ( it == end( files ) )
{
ml->addFile( filePath, parentFolder );
continue; continue;
}
for ( auto& f : dir->files() ) auto file = m_fsFactory->createFile( filePath );
if ( file->lastModificationDate() == (*it)->lastModificationDate() )
{ {
auto fsFile = m_fsFactory->createFile( f ); // Unchanged file
m_discoveryCb->onNewFile( fsFile.get(), folder ); files.erase( it );
continue;
} }
for ( auto& f : dir->dirs() ) ml->deleteFile( filePath );
folders.emplace( f, folder ); ml->addFile( filePath, parentFolder );
files.erase( it );
}
for ( auto file : files )
{
ml->deleteFile( file );
} }
return true;
} }
...@@ -9,12 +9,16 @@ ...@@ -9,12 +9,16 @@
class FsDiscoverer : public IDiscoverer class FsDiscoverer : public IDiscoverer
{ {
public: public:
FsDiscoverer(std::shared_ptr<factory::IFileSystem> fsFactory , IDiscovererCb *discoveryCb); FsDiscoverer( std::shared_ptr<factory::IFileSystem> fsFactory );
virtual bool discover( const std::string &entryPoint ) override; virtual bool discover(IMediaLibrary *ml, DBConnection dbConn, const std::string &entryPoint ) override;
virtual void reload( IMediaLibrary* ml, DBConnection dbConn ) override;
private:
bool checkSubfolders(IMediaLibrary *ml, DBConnection dbConn, fs::IDirectory *folder, FolderPtr parentFolder );
void checkFiles(IMediaLibrary *ml, DBConnection dbConn, fs::IDirectory *folder, FolderPtr parentFolder );
private: