Delegate FS modification scanning to IDiscoverer

include/IDiscoverer.h

@@ -5,33 +5,17 @@
 #include "Types.h"
 #include "filesystem/IDirectory.h"
 #include "filesystem/IFile.h"
-
-class IDiscovererCb
-{
-public:
-    virtual ~IDiscovererCb() = default;
-    /**
-     * @brief onNewFolder Called when the discoverer finds a new directory
-     * @param folderPath The new directory's path
-     * @param parent The parent folder, or null if this is the root folder
-     * @return The newly created folder, or nullptr in case of error (or if the
-     *         directory shall not be browsed further)
-     */
-    virtual FolderPtr onNewFolder( const fs::IDirectory* folder, FolderPtr parent ) = 0;
-    /**
-     * @brief onNewFile Called when the discoverer finds a new file
-     * @param filePath The new file's path
-     * @param parent The parent folder
-     * @return true if the file was accepted
-     */
-    virtual FilePtr onNewFile( const fs::IFile* file, FolderPtr parent ) = 0;
-};
+#include "IMediaLibrary.h"
 
 class IDiscoverer
 {
 public:
     virtual ~IDiscoverer() = default;
-    virtual bool discover( const std::string& entryPoint ) = 0;
+    // We assume the media library will always outlive the discoverers.
+    //FIXME: This is currently false since there is no way of interrupting
+    //a discoverer thread
+    virtual bool discover( IMediaLibrary* ml, DBConnection dbConn, const std::string& entryPoint ) = 0;
+    virtual void reload( IMediaLibrary* ml, DBConnection dbConn ) = 0;
 };
 
 #endif // IDISCOVERER_H

include/IMediaLibrary.h

@@ -6,7 +6,6 @@
 
 #include "Types.h"
 #include "factory/IFileSystem.h"
-#include "IDiscoverer.h"
 
 class IMediaLibraryCb
 {
@@ -49,8 +48,14 @@ class IMediaLibrary
          * Calling this after initialize() is not a supported scenario.
          */
         virtual void setFsFactory( std::shared_ptr<factory::IFileSystem> fsFactory ) = 0;
-        /// Adds a stand alone file
-        virtual FilePtr addFile( const std::string& path ) = 0;
+        ///
+        /// \brief addFile Adds a file to the media library.
+        /// \param path The absolute path to this file
+        /// \param parentFolder The parent folder, or nullptr to add this file as
+        ///                     a stand alone file.
+        /// \return The newly created file, or nullptr in case of error
+        ///
+        virtual FilePtr addFile( const std::string& path, FolderPtr parentFolder ) = 0;
         virtual FilePtr file( const std::string& path ) = 0;
         virtual bool deleteFile( const std::string& mrl ) = 0;
         virtual bool deleteFile( FilePtr file ) = 0;

src/MediaLibrary.cpp

@@ -107,7 +107,7 @@ bool MediaLibrary::initialize( const std::string& dbPath, const std::string& sna
         addMetadataService( std::move( thumbnailerService ) );
     }
 
-    m_discoverers.emplace_back( new FsDiscoverer( m_fsFactory, this ) );
+    m_discoverers.emplace_back( new FsDiscoverer( m_fsFactory ) );
 
     sqlite3* dbConnection;
     int res = sqlite3_open( dbPath.c_str(), &dbConnection );
@@ -134,7 +134,8 @@ bool MediaLibrary::initialize( const std::string& dbPath, const std::string& sna
         LOG_ERROR( "Failed to create database structure" );
         return false;
     }
-    return loadFolders();
+    reload();
+    return true;
 }
 
 std::vector<FilePtr> MediaLibrary::files()
@@ -162,10 +163,44 @@ FilePtr MediaLibrary::file( const std::string& path )
     return File::fetch( m_dbConnection, path );
 }
 
-FilePtr MediaLibrary::addFile( const std::string& path )
+FilePtr MediaLibrary::addFile( const std::string& path, FolderPtr parentFolder )
 {
-    auto fsFile = m_fsFactory->createFile( path );
-    return addFile( fsFile.get(), 0 );
+    std::unique_ptr<fs::IFile> file;
+    try
+    {
+        file = m_fsFactory->createFile( path );
+    }
+    catch (std::exception& ex)
+    {
+        LOG_ERROR( "Failed to create an IFile for ", path, ": ", ex.what() );
+        return nullptr;
+    }
+
+    auto type = IFile::Type::UnknownType;
+    if ( std::find( begin( supportedVideoExtensions ), end( supportedVideoExtensions ),
+                    file->extension() ) != end( supportedVideoExtensions ) )
+    {
+        type = IFile::Type::VideoType;
+    }
+    else if ( std::find( begin( supportedAudioExtensions ), end( supportedAudioExtensions ),
+                         file->extension() ) != end( supportedAudioExtensions ) )
+    {
+        type = IFile::Type::AudioType;
+    }
+    if ( type == IFile::Type::UnknownType )
+        return false;
+
+    auto fptr = File::create( m_dbConnection, type, file.get(), parentFolder != nullptr ? parentFolder->id() : 0 );
+    if ( fptr == nullptr )
+    {
+        LOG_ERROR( "Failed to add file ", file->fullPath(), " to the media library" );
+        return nullptr;
+    }
+    LOG_INFO( "Adding ", file->name() );
+    if ( m_callback != nullptr )
+        m_callback->onFileAdded( fptr );
+    m_parser->parse( fptr, m_callback );
+    return fptr;
 }
 
 FolderPtr MediaLibrary::folder( const std::string& path )
@@ -276,6 +311,18 @@ void MediaLibrary::addMetadataService(std::unique_ptr<IMetadataService> service)
     m_parser->addService( std::move( service ) );
 }
 
+void MediaLibrary::reload()
+{
+    //FIXME: Create a proper wrapper to handle discoverer threading
+    std::thread t([this] {
+        //FIXME: This will crash if the media library gets deleted while we
+        //are discovering.
+        for ( auto& d : m_discoverers )
+            d->reload( this, this->m_dbConnection );
+    });
+    t.detach();
+}
+
 void MediaLibrary::discover( const std::string &entryPoint )
 {
     std::thread t([this, entryPoint] {
@@ -285,7 +332,7 @@ void MediaLibrary::discover( const std::string &entryPoint )
             m_callback->onDiscoveryStarted( entryPoint );
 
         for ( auto& d : m_discoverers )
-            d->discover( entryPoint );
+            d->discover( this, this->m_dbConnection, entryPoint );
 
         if ( m_callback != nullptr )
             m_callback->onDiscoveryCompleted( entryPoint );
@@ -293,23 +340,6 @@ void MediaLibrary::discover( const std::string &entryPoint )
     t.detach();
 }
 
-FolderPtr MediaLibrary::onNewFolder( const fs::IDirectory* directory, FolderPtr parent )
-{
-    //FIXME: Since we insert files/folders with a UNIQUE constraint, maybe we should
-    //just let sqlite try to insert, throw an exception in case the contraint gets violated
-    //catch it and return nullptr from here.
-    //We previously were fetching the folder manually here, but that triggers an eroneous entry
-    //in the cache. This might also be something to fix...
-    return Folder::create( m_dbConnection, directory,
-                        parent == nullptr ? 0 : parent->id() );
-}
-
-FilePtr MediaLibrary::onNewFile( const fs::IFile *file, FolderPtr parent )
-{
-    //FIXME: Same uniqueness comment as onNewFolder above.
-    return addFile( file, parent == nullptr ? 0 : parent->id() );
-}
-
 const std::string& MediaLibrary::snapshotPath() const
 {
     return m_snapshotPath;
@@ -320,132 +350,3 @@ void MediaLibrary::setLogger( ILogger* logger )
     Log::SetLogger( logger );
 }
 
-bool MediaLibrary::loadFolders()
-{
-    //FIXME: This should probably be in a sql transaction
-    //FIXME: This shouldn't be done for "removable"/network files
-    static const std::string req = "SELECT * FROM " + policy::FolderTable::Name
-            + " WHERE id_parent IS NULL";
-    auto rootFolders = sqlite::Tools::fetchAll<Folder, IFolder>( m_dbConnection, req );
-    for ( const auto f : rootFolders )
-    {
-        auto folder = m_fsFactory->createDirectory( f->path() );
-        if ( folder->lastModificationDate() == f->lastModificationDate() )
-            continue;
-        checkSubfolders( folder.get(), f->id() );
-        f->setLastModificationDate( folder->lastModificationDate() );
-    }
-    return true;
-}
-
-bool MediaLibrary::checkSubfolders( fs::IDirectory* folder, unsigned int parentId )
-{
-    // From here we can have:
-    // - New subfolder(s)
-    // - Deleted subfolder(s)
-    // - New file(s)
-    // - Deleted file(s)
-    // - Changed file(s)
-    // ... in this folder, or in all the sub folders.
-
-    // Load the folders we already know of:
-    static const std::string req = "SELECT * FROM " + policy::FolderTable::Name
-            + " WHERE id_parent = ?";
-    auto subFoldersInDB = sqlite::Tools::fetchAll<Folder, IFolder>( m_dbConnection, req, parentId );
-    for ( const auto& subFolderPath : folder->dirs() )
-    {
-        auto it = std::find_if( begin( subFoldersInDB ), end( subFoldersInDB ), [subFolderPath](const std::shared_ptr<IFolder>& f) {
-            return f->path() == subFolderPath;
-        });
-        // We don't know this folder, it's a new one
-        if ( it == end( subFoldersInDB ) )
-        {
-            //FIXME: In order to add the new folder, we need to use the same discoverer.
-            // This probably means we need to store which discoverer was used to add which file
-            // and store discoverers as a map instead of a vector
-            continue;
-        }
-        auto subFolder = m_fsFactory->createDirectory( subFolderPath );
-        if ( subFolder->lastModificationDate() == (*it)->lastModificationDate() )
-        {
-            // Remove all folders that still exist in FS. That way, the list of folders that
-            // will still be in subFoldersInDB when we're done is the list of folders that have
-            // been deleted from the FS
-            subFoldersInDB.erase( it );
-            continue;
-        }
-        // This folder was modified, let's recurse
-        checkSubfolders( subFolder.get(), (*it)->id() );
-        checkFiles( subFolder.get(), (*it)->id() );
-        (*it)->setLastModificationDate( subFolder->lastModificationDate() );
-        subFoldersInDB.erase( it );
-    }
-    // Now all folders we had in DB but haven't seen from the FS must have been deleted.
-    for ( auto f : subFoldersInDB )
-    {
-        std::cout << "Folder " << f->path() << " not found in FS, deleting it" << std::endl;
-        deleteFolder( f );
-    }
-    return true;
-}
-
-void MediaLibrary::checkFiles( fs::IDirectory* folder, unsigned int parentId )
-{
-    static const std::string req = "SELECT * FROM " + policy::FileTable::Name
-            + " WHERE folder_id = ?";
-    auto files = sqlite::Tools::fetchAll<File, IFile>( m_dbConnection, req, parentId );
-    for ( const auto& filePath : folder->files() )
-    {
-        auto file = m_fsFactory->createFile( filePath );
-        auto it = std::find_if( begin( files ), end( files ), [filePath](const std::shared_ptr<IFile>& f) {
-            return f->mrl() == filePath;
-        });
-        if ( it == end( files ) )
-        {
-            addFile( file.get(), parentId );
-            continue;
-        }
-        if ( file->lastModificationDate() == (*it)->lastModificationDate() )
-        {
-            // Unchanged file
-            files.erase( it );
-            continue;
-        }
-        deleteFile( filePath );
-        addFile( file.get(), parentId );
-        files.erase( it );
-    }
-    for ( auto file : files )
-    {
-        deleteFile( file );
-    }
-}
-
-FilePtr MediaLibrary::addFile( const fs::IFile* file, unsigned int folderId )
-{
-    auto type = IFile::Type::UnknownType;
-    if ( std::find( begin( supportedVideoExtensions ), end( supportedVideoExtensions ),
-                    file->extension() ) != end( supportedVideoExtensions ) )
-    {
-        type = IFile::Type::VideoType;
-    }
-    else if ( std::find( begin( supportedAudioExtensions ), end( supportedAudioExtensions ),
-                         file->extension() ) != end( supportedAudioExtensions ) )
-    {
-        type = IFile::Type::AudioType;
-    }
-    if ( type == IFile::Type::UnknownType )
-        return false;
-
-    auto fptr = File::create( m_dbConnection, type, file, folderId );
-    if ( fptr == nullptr )
-    {
-        LOG_ERROR( "Failed to add file ", file->fullPath(), " to the media library" );
-        return nullptr;
-    }
-    LOG_INFO( "Adding ", file->name() );
-    m_callback->onFileAdded( fptr );
-    m_parser->parse( fptr, m_callback );
-    return fptr;
-}
-

src/MediaLibrary.h

@@ -10,7 +10,7 @@ class Parser;
 #include "logging/Logger.h"
 #include "vlcpp/vlc.hpp"
 
-class MediaLibrary : public IMediaLibrary, public IDiscovererCb
+class MediaLibrary : public IMediaLibrary
 {
     public:
         MediaLibrary();
@@ -22,7 +22,7 @@ class MediaLibrary : public IMediaLibrary, public IDiscovererCb
         virtual std::vector<FilePtr> audioFiles() override;
         virtual std::vector<FilePtr> videoFiles() override;
         virtual FilePtr file( const std::string& path ) override;
-        virtual FilePtr addFile( const std::string& path ) override;
+        virtual FilePtr addFile(const std::string& path , FolderPtr parentFolder) override;
         virtual bool deleteFile( const std::string& mrl ) override;
         virtual bool deleteFile( FilePtr file ) override;
 
@@ -48,9 +48,6 @@ class MediaLibrary : public IMediaLibrary, public IDiscovererCb
         virtual std::vector<ArtistPtr> artists() const override;
 
         virtual void discover( const std::string& entryPoint ) override;
-        // IDiscovererCb implementation
-        virtual FolderPtr onNewFolder( const fs::IDirectory* directory, FolderPtr parent ) override;
-        virtual FilePtr onNewFile(const fs::IFile* file, FolderPtr parent ) override;
 
         virtual const std::string& snapshotPath() const override;
         virtual void setLogger( ILogger* logger ) override;
@@ -60,11 +57,8 @@ class MediaLibrary : public IMediaLibrary, public IDiscovererCb
         static const std::vector<std::string> supportedAudioExtensions;
 
     private:
-        bool loadFolders();
-        bool checkSubfolders( fs::IDirectory* folder, unsigned int parentId );
-        void checkFiles( fs::IDirectory* folder, unsigned int parentId );
-        FilePtr addFile( const fs::IFile* file, unsigned int folderId );
         void addMetadataService( std::unique_ptr<IMetadataService> service );
+        void reload();
 
     private:
         std::shared_ptr<sqlite3> m_dbConnection;

src/discoverer/FsDiscoverer.cpp

 #include "FsDiscoverer.h"
 
+#include <algorithm>
 #include "factory/FileSystem.h"
+#include "File.h"
+#include "Folder.h"
 #include <queue>
 #include <iostream>
 
-FsDiscoverer::FsDiscoverer( std::shared_ptr<factory::IFileSystem> fsFactory, IDiscovererCb* discoveryCb )
-    : m_discoveryCb( discoveryCb )
+FsDiscoverer::FsDiscoverer(std::shared_ptr<factory::IFileSystem> fsFactory)
 {
     if ( fsFactory != nullptr )
         m_fsFactory = fsFactory;
@@ -13,46 +15,132 @@ FsDiscoverer::FsDiscoverer( std::shared_ptr<factory::IFileSystem> fsFactory, IDi
         m_fsFactory.reset( new factory::FileSystemDefaultFactory );
 }
 
-bool FsDiscoverer::discover( const std::string &entryPoint )
+bool FsDiscoverer::discover( IMediaLibrary* ml, DBConnection dbConn, const std::string &entryPoint )
 {
-    std::queue<std::pair<std::string, FolderPtr>> folders;
+    // Assume :// denotes a scheme that isn't a file path, and refuse to discover it.
+    if ( entryPoint.find( "://" ) != std::string::npos )
+        return false;
 
-    folders.emplace( entryPoint, nullptr );
-    while ( folders.empty() == false )
     {
-        std::unique_ptr<fs::IDirectory> dir;
-        auto currentFolder = folders.front();
-        auto currentPath = currentFolder.first;
-        auto parent = currentFolder.second;
-        folders.pop();
-
+        auto f = Folder::fetch( dbConn, entryPoint );
+        // If the folder exists, we assume it is up to date
+        if ( f != nullptr )
+            return true;
+    }
+    // Otherwise, create a directory, and check it for modifications
+    std::unique_ptr<fs::IDirectory> fsDir;
     try
     {
-            dir = m_fsFactory->createDirectory( currentPath );
+        fsDir = m_fsFactory->createDirectory( entryPoint );
     }
-        catch ( std::runtime_error& ex )
+    catch (std::exception& ex)
     {
-            std::cerr << ex.what() << std::endl;
-            // If the first directory fails to open, stop now.
-            // Otherwise, assume something went wrong in a subdirectory.
-            if (parent == nullptr)
+        LOG_ERROR("Failed to create an IDirectory for ", entryPoint, ": ", ex.what());
         return false;
-            continue;
     }
-
-        auto folder = m_discoveryCb->onNewFolder( dir.get(), parent );
-        if ( folder == nullptr && parent == nullptr )
+    auto f = Folder::create( dbConn, fsDir.get(), 0 );
+    if ( f == nullptr )
         return false;
-        if ( folder == nullptr )
+    checkSubfolders( ml, dbConn, fsDir.get(), f );
+    return true;
+}
+
+void FsDiscoverer::reload( IMediaLibrary *ml, DBConnection dbConn )
+{
+    //FIXME: This should probably be in a sql transaction
+    //FIXME: This shouldn't be done for "removable"/network files
+    static const std::string req = "SELECT * FROM " + policy::FolderTable::Name
+            + " WHERE id_parent IS NULL";
+    auto rootFolders = sqlite::Tools::fetchAll<Folder, IFolder>( dbConn, req );
+    for ( const auto f : rootFolders )
+    {
+        auto folder = m_fsFactory->createDirectory( f->path() );
+        if ( folder->lastModificationDate() == f->lastModificationDate() )
             continue;
+        checkSubfolders( ml, dbConn, folder.get(), f );
+        f->setLastModificationDate( folder->lastModificationDate() );
+    }
+}
 
-        for ( auto& f : dir->files() )
+bool FsDiscoverer::checkSubfolders( IMediaLibrary* ml, DBConnection dbConn, fs::IDirectory* folder, FolderPtr parentFolder )
+{
+    // From here we can have:
+    // - New subfolder(s)
+    // - Deleted subfolder(s)
+    // - New file(s)
+    // - Deleted file(s)
+    // - Changed file(s)
+    // ... in this folder, or in all the sub folders.
+
+    // Load the folders we already know of:
+    static const std::string req = "SELECT * FROM " + policy::FolderTable::Name
+            + " WHERE id_parent = ?";
+    auto subFoldersInDB = sqlite::Tools::fetchAll<Folder, IFolder>( dbConn, req, parentFolder->id() );
+    for ( const auto& subFolderPath : folder->dirs() )
+    {
+        auto it = std::find_if( begin( subFoldersInDB ), end( subFoldersInDB ), [subFolderPath](const std::shared_ptr<IFolder>& f) {
+            return f->path() == subFolderPath;
+        });
+        // We don't know this folder, it's a new one
+        if ( it == end( subFoldersInDB ) )
         {
-            auto fsFile = m_fsFactory->createFile( f );
-            m_discoveryCb->onNewFile( fsFile.get(), folder );
+            //FIXME: In order to add the new folder, we need to use the same discoverer.
+            // This probably means we need to store which discoverer was used to add which file
+            // and store discoverers as a map instead of a vector
+            continue;
         }
-        for ( auto& f : dir->dirs() )
-            folders.emplace( f, folder );
+        auto subFolder = m_fsFactory->createDirectory( subFolderPath );
+        if ( subFolder->lastModificationDate() == (*it)->lastModificationDate() )
+        {
+            // Remove all folders that still exist in FS. That way, the list of folders that
+            // will still be in subFoldersInDB when we're done is the list of folders that have
+            // been deleted from the FS
+            subFoldersInDB.erase( it );
+            continue;
+        }
+        // This folder was modified, let's recurse
+        checkSubfolders( ml, dbConn, subFolder.get(), *it );
+        checkFiles( ml, dbConn, subFolder.get(), *it );
+        (*it)->setLastModificationDate( subFolder->lastModificationDate() );
+        subFoldersInDB.erase( it );
+    }
+    // Now all folders we had in DB but haven't seen from the FS must have been deleted.
+    for ( auto f : subFoldersInDB )
+    {
+        std::cout << "Folder " << f->path() << " not found in FS, deleting it" << std::endl;
+        ml->deleteFolder( f );
     }
     return true;
 }
+
+void FsDiscoverer::checkFiles( IMediaLibrary *ml, DBConnection dbConn, fs::IDirectory* folder, FolderPtr parentFolder )
+{
+    static const std::string req = "SELECT * FROM " + policy::FileTable::Name
+            + " WHERE folder_id = ?";
+    auto files = sqlite::Tools::fetchAll<File, IFile>( dbConn, req, parentFolder->id() );
+    for ( const auto& filePath : folder->files() )
+    {        
+        auto it = std::find_if( begin( files ), end( files ), [filePath](const std::shared_ptr<IFile>& f) {
+            return f->mrl() == filePath;
+        });
+        if ( it == end( files ) )
+        {
+            ml->addFile( filePath, parentFolder );
+            continue;
+        }
+        auto file = m_fsFactory->createFile( filePath );
+        if ( file->lastModificationDate() == (*it)->lastModificationDate() )
+        {
+            // Unchanged file
+            files.erase( it );
+            continue;
+        }
+        ml->deleteFile( filePath );
+        ml->addFile( filePath, parentFolder );
+        files.erase( it );
+    }
+    for ( auto file : files )
+    {
+        ml->deleteFile( file );
+    }
+}

src/discoverer/FsDiscoverer.h

@@ -9,12 +9,16 @@
 class FsDiscoverer : public IDiscoverer
 {
 public:
-    FsDiscoverer(std::shared_ptr<factory::IFileSystem> fsFactory , IDiscovererCb *discoveryCb);
-    virtual bool discover( const std::string &entryPoint ) override;
+    FsDiscoverer( std::shared_ptr<factory::IFileSystem> fsFactory );
+    virtual bool discover(IMediaLibrary *ml, DBConnection dbConn, const std::string &entryPoint ) override;
+    virtual void reload( IMediaLibrary* ml, DBConnection dbConn ) override;
+
+private:
+    bool checkSubfolders(IMediaLibrary *ml, DBConnection dbConn, fs::IDirectory *folder, FolderPtr parentFolder );
+    void checkFiles(IMediaLibrary *ml, DBConnection dbConn, fs::IDirectory *folder, FolderPtr parentFolder );