fs: Add a linux Device implementation

src/discoverer/FsDiscoverer.cpp

@@ -195,6 +195,7 @@ std::vector<std::shared_ptr<Folder> > FsDiscoverer::blacklist() const
 bool FsDiscoverer::isBlacklisted( const fs::IDirectory& directory, const std::vector<std::shared_ptr<Folder>>& blacklist ) const
 {
     auto deviceFs = directory.device();
+    //FIXME: We could avoid fetching the device if the directory is non removable.
     auto device = Device::fromUuid( m_dbConn, deviceFs->uuid() );
     // When blacklisting, we would insert the device if we haven't encoutered it yet.
     // So when reading, a missing device means a non-blacklisted device.

src/factory/FileSystem.cpp

@@ -39,6 +39,13 @@
 namespace factory
 {
 
+FileSystemFactory::FileSystemFactory()
+{
+    static bool isCachePopulated = fs::Device::populateCache();
+    if ( isCachePopulated == false )
+        throw std::runtime_error( "Failed to populate device cache" );
+}
+
 std::shared_ptr<fs::IDirectory> FileSystemFactory::createDirectory( const std::string& path )
 {
     std::lock_guard<std::mutex> lock( m_mutex );

src/factory/FileSystem.h

@@ -33,6 +33,7 @@ namespace factory
     class FileSystemFactory : public IFileSystem
     {
     public:
+        FileSystemFactory();
         virtual std::shared_ptr<fs::IDirectory> createDirectory( const std::string& path ) override;
         virtual std::unique_ptr<fs::IFile> createFile( const std::string& fileName ) override;
         virtual std::shared_ptr<fs::IDevice> createDevice( const std::string& uuid ) override;

src/filesystem/unix/Device.cpp

@@ -21,29 +21,41 @@
  *****************************************************************************/
 
 #include "Device.h"
+#include "Directory.h"
+#include "utils/Filename.h"
+#include "logging/Logger.h"
 
-#include <mntent.h>
+#include <algorithm>
 #include <cstring>
+#include <cstdio>
+#include <dirent.h>
+#include <limits.h>
+#include <mntent.h>
+#include <unistd.h>
+#include <sys/types.h>
 
 namespace
 {
     // Allow private ctors to be used from make_shared
-    struct DeviceBuilder : fs::Device
+    struct DeviceBuilder : public fs::Device
     {
-        DeviceBuilder( const std::string& path ) : Device( path ) {}
+        template <typename... Args>
+        DeviceBuilder( Args&&... args ) : Device( std::forward<Args>( args )... ) {}
     };
 }
 
 namespace fs
 {
 
-const Device::DeviceMap Device::Cache = Device::listDevices();
+Device::DeviceMap Device::Devices;
+Device::MountpointMap Device::Mountpoints;
+Device::DeviceCacheMap Device::DeviceCache;
 
-Device::Device( const std::string& devicePath )
-    : m_device( devicePath )
-    , m_uuid( "fake uuid" )
-    , m_mountpoint( "/mnt/fixme" )
+Device::Device( const std::string& uuid, const std::string& mountpoint, bool isRemovable )
+    : m_uuid( uuid )
+    , m_mountpoint( mountpoint )
     , m_present( true )
+    , m_removable( isRemovable )
 {
 }
 
@@ -54,7 +66,7 @@ const std::string& Device::uuid() const
 
 bool Device::isRemovable() const
 {
-    return false;
+    return m_removable;
 }
 
 bool Device::isPresent() const
@@ -69,55 +81,186 @@ const std::string&Device::mountpoint() const
 
 std::shared_ptr<IDevice> Device::fromPath( const std::string& path )
 {
-    for ( const auto& p : Cache )
+    std::shared_ptr<IDevice> res;
+    for ( const auto& p : DeviceCache )
     {
-        if ( path.find( p.first ) == 0 )
-            return p.second;
+        if ( path.find( p.second->mountpoint() ) == 0 )
+        {
+            if ( res == nullptr || res->mountpoint().length() < p.second->mountpoint().length() )
+                res = p.second;
+        }
     }
-    return nullptr;
+    return res;
 }
 
 std::shared_ptr<IDevice> Device::fromUuid( const std::string& uuid )
 {
-    for ( const auto& p : Cache )
-    {
-        const auto& d = p.second;
-        if ( d->uuid() == uuid )
-            return d;
-    }
+    auto it = DeviceCache.find( uuid );
+    if ( it != end( DeviceCache ) )
+        return it->second;
     return nullptr;
 }
 
+bool Device::populateCache()
+{
+    Devices = listDevices();
+    Mountpoints = listMountpoints();
+    DeviceCache = populateDeviceCache();
+    return true;
+}
+
 Device::DeviceMap Device::listDevices()
 {
+
+    static const std::vector<std::string> deviceBlacklist = { "loop", "dm-" };
+    const std::string devPath = "/dev/disk/by-uuid/";
+    // Don't use fs::Directory to iterate, as it resolves the symbolic links automatically.
+    // We need the link name & what it points to.
+    std::unique_ptr<DIR, int(*)(DIR*)> dir( opendir( devPath.c_str() ), &closedir );
+    if ( dir == nullptr )
+    {
+        std::stringstream err;
+        err << "Failed to open /dev/disk/by-uuid: " << strerror(errno);
+        throw std::runtime_error( err.str() );
+    }
     DeviceMap res;
+    dirent* result = nullptr;
+
+    while ( ( result = readdir( dir.get() ) ) != nullptr )
+    {
+        if ( strcmp( result->d_name, "." ) == 0 ||
+             strcmp( result->d_name, ".." ) == 0 )
+        {
+            continue;
+        }
+        std::string path = devPath + result->d_name;
+        char linkPath[PATH_MAX] = {};
+        if ( readlink( path.c_str(), linkPath, PATH_MAX ) < 0 )
+        {
+            std::stringstream err;
+            err << "Failed to resolve uuid -> device link: "
+                << result->d_name << " (" << strerror(errno) << ')';
+            throw std::runtime_error( err.str() );
+        }
+        auto deviceName = utils::file::fileName( linkPath );
+        if ( std::find_if( begin( deviceBlacklist ), end( deviceBlacklist ), [&deviceName]( const std::string& pattern ) {
+                return deviceName.length() >= pattern.length() && deviceName.find( pattern ) == 0;
+            }) != end( deviceBlacklist ) )
+            continue;
+        auto uuid = result->d_name;
+        LOG_INFO( "Discovered device ", deviceName, " -> {", uuid, '}' );
+        res[deviceName] = uuid;
+    }
+    return res;
+}
+
+Device::MountpointMap Device::listMountpoints()
+{
+    static const std::vector<std::string> allowedFsType = { "vfat", "exfat", "sdcardfs", "fuse",
+                                                            "ntfs", "fat32", "ext3", "ext4", "esdfs" };
+    MountpointMap res;
     FILE* f = setmntent("/etc/mtab", "r");
     if ( f == nullptr )
         throw std::runtime_error( "Failed to read /etc/mtab" );
-    std::unique_ptr<FILE, int(*)(FILE*)>( f, &endmntent );
+    std::unique_ptr<FILE, int(*)(FILE*)> fPtr( f, &endmntent );
     char buff[512];
     mntent s;
+    errno = 0;
     while ( getmntent_r( f, &s, buff, sizeof(buff) ) != nullptr )
     {
-        // Ugly work around for mountpoints we don't care
-        if ( strcmp( s.mnt_type, "proc" ) == 0
-                || strcmp( s.mnt_type, "devtmpfs" ) == 0
-                || strcmp( s.mnt_type, "devpts" ) == 0
-                || strcmp( s.mnt_type, "sysfs" ) == 0
-                || strcmp( s.mnt_type, "cgroup" ) == 0
-                || strcmp( s.mnt_type, "debugfs" ) == 0
-                || strcmp( s.mnt_type, "hugetlbfs" ) == 0
-                || strcmp( s.mnt_type, "efivarfs" ) == 0
-                || strcmp( s.mnt_type, "securityfs" ) == 0
-                || strcmp( s.mnt_type, "mqueue" ) == 0
-                || strcmp( s.mnt_type, "pstore" ) == 0
-                || strcmp( s.mnt_type, "autofs" ) == 0
-                || strcmp( s.mnt_type, "binfmt_misc" ) == 0
-                || strcmp( s.mnt_type, "tmpfs" ) == 0 )
+        if ( std::find( begin( allowedFsType ), end( allowedFsType ), s.mnt_type ) == end( allowedFsType ) )
             continue;
-        res[s.mnt_dir] = std::make_shared<DeviceBuilder>( s.mnt_fsname );
+        auto deviceName = s.mnt_fsname;
+        LOG_INFO( "Discovered device ", deviceName, " mounted on ", s.mnt_dir );
+        res[deviceName] = s.mnt_dir;
+        errno = 0;
     }
+    if ( errno != 0 )
+    {
+        LOG_ERROR( "Failed to read mountpoints: ", strerror( errno ) );
+    }
+    return res;
+}
+
+Device::DeviceCacheMap Device::populateDeviceCache()
+{
+    Device::DeviceCacheMap res;
+    for ( const auto& p : Mountpoints )
+    {
+        const auto& devicePath = p.first;
+        auto deviceName = utils::file::fileName( devicePath );
+        const auto& mountpoint = p.second;
+        auto it = Devices.find( deviceName );
+        std::string uuid;
+        if ( it != end( Devices ) )
+            uuid = it->second;
+        else
+        {
+            deviceName = deviceFromDeviceMapper( devicePath );
+            it = Devices.find( deviceName );
+            if ( it != end( Devices ) )
+                uuid = it->second;
+            else
+            {
+                LOG_ERROR( "Failed to resolve mountpoint ", mountpoint, " to any known device" );
+                continue;
+            }
+        }
+        auto removable = isRemovable( deviceName );
+        LOG_INFO( "Adding device to cache: {", uuid, "} mounted on ", mountpoint, " Removable: ", removable );
+        res[uuid] = std::make_shared<DeviceBuilder>( uuid, mountpoint, removable );
+    }
+    return res;
+}
+
+std::string Device::deviceFromDeviceMapper( const std::string& devicePath )
+{
+    if ( devicePath.find( "/dev/mapper" ) != 0 )
+        return {};
+    char linkPath[PATH_MAX];
+    if ( readlink( devicePath.c_str(), linkPath, PATH_MAX ) < 0 )
+    {
+        std::stringstream err;
+        err << "Failed to resolve device -> mapper link: "
+            << devicePath << " (" << strerror(errno) << ')';
+        throw std::runtime_error( err.str() );
+    }
+    const auto dmName = utils::file::fileName( linkPath );
+    std::string dmSlavePath = "/sys/block/" + dmName + "/slaves";
+    std::unique_ptr<DIR, int(*)(DIR*)> dir( opendir( dmSlavePath.c_str() ), &closedir );
+    std::string res;
+    if ( dir == nullptr )
+        return {};
+    dirent* result;
+    while ( ( result = readdir( dir.get() ) ) != nullptr )
+    {
+        if ( strcmp( result->d_name, "." ) == 0 ||
+             strcmp( result->d_name, ".." ) == 0 )
+        {
+            continue;
+        }
+        if ( res.empty() == true )
+            res = result->d_name;
+        else
+            LOG_WARN( "More than one slave for device mapper ", linkPath );
+    }
+    LOG_INFO( "Device mapper ", dmName, " maps to ", res );
     return res;
 }
 
+bool Device::isRemovable( const std::string& deviceName )
+{
+    std::stringstream removableFilePath;
+    removableFilePath << "/sys/block/" << deviceName << "/removable";
+    std::unique_ptr<FILE, int(*)(FILE*)> removableFile( fopen( removableFilePath.str().c_str(), "r" ), &fclose );
+    // Assume the file isn't removable by default
+    if ( removableFile != nullptr )
+    {
+        char buff;
+        fread(&buff, sizeof(buff), 1, removableFile.get() );
+        return buff == '1';
+    }
+    return false;
+}
+
 }

src/filesystem/unix/Device.h

@@ -32,7 +32,12 @@ namespace fs
 class Device : public IDevice
 {
 public:
-    using DeviceMap = std::unordered_map<std::string, std::shared_ptr<IDevice>>;
+    // Device name / UUID map
+    using DeviceMap = std::unordered_map<std::string, std::string>;
+    // Device path / Mountpoints map
+    using MountpointMap = std::unordered_map<std::string, std::string>;
+    // UUID -> Device instance map
+    using DeviceCacheMap = std::unordered_map<std::string, std::shared_ptr<IDevice>>;
 
     virtual const std::string& uuid() const override;
     virtual bool isRemovable() const override;
@@ -44,21 +49,28 @@ public:
     ///
     static std::shared_ptr<IDevice> fromPath( const std::string& path );
     static std::shared_ptr<IDevice> fromUuid( const std::string& uuid );
+    static bool populateCache();
 
 protected:
-    Device( const std::string& devicePath );
+    Device( const std::string& uuid, const std::string& mountpoint, bool isRemovable );
 
 private:
     static DeviceMap listDevices();
+    static MountpointMap listMountpoints();
+    static DeviceCacheMap populateDeviceCache();
+    static std::string deviceFromDeviceMapper( const std::string& devicePath );
+    static bool isRemovable( const std::string& deviceName );
 
 private:
-    static const DeviceMap Cache;
+    static DeviceMap Devices;
+    static MountpointMap Mountpoints;
+    static DeviceCacheMap DeviceCache;
 
 private:
-    std::string m_device;
     std::string m_uuid;
     std::string m_mountpoint;
     bool m_present;
+    bool m_removable;
 };
 
 }

src/filesystem/unix/Directory.cpp

@@ -66,8 +66,10 @@ const std::vector<std::string>&Directory::dirs()
 
 std::shared_ptr<IDevice> Directory::device() const
 {
-    //FIXME: Cache this?
-    return Device::fromPath( m_path );
+    auto lock = m_device.lock();
+    if ( m_device.isCached() == false )
+        m_device = Device::fromPath( m_path );
+    return m_device.get();
 }
 
 std::string Directory::toAbsolute(const std::string& path)

src/filesystem/unix/Directory.h

@@ -23,6 +23,7 @@
 #pragma once
 
 #include "filesystem/IDirectory.h"
+#include "utils/Cache.h"
 #include <string>
 
 namespace fs
@@ -47,6 +48,7 @@ private:
     const std::string m_path;
     std::vector<std::string> m_files;
     std::vector<std::string> m_dirs;
+    mutable Cache<std::shared_ptr<IDevice>> m_device;
 };
 
 }