MetadataParser.cpp 17.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2015 Hugo Beauzée-Luyssen, Videolabs
 *
 * Authors: Hugo Beauzée-Luyssen<hugo@beauzee.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

23
24
25
26
#if HAVE_CONFIG_H
# include "config.h"
#endif

27
28
29
30
31
#include "MetadataParser.h"
#include "Album.h"
#include "AlbumTrack.h"
#include "Artist.h"
#include "File.h"
32
#include "Genre.h"
33
34
35
#include "Media.h"
#include "Show.h"
#include "utils/Filename.h"
36
#include "utils/ModificationsNotifier.h"
37
#include <cstdlib>
38

39
40
41
namespace medialibrary
{

42
43
bool MetadataParser::initialize()
{
44
    m_unknownArtist = Artist::fetch( m_ml, UnknownArtistID );
45
46
47
48
49
    if ( m_unknownArtist == nullptr )
        LOG_ERROR( "Failed to cache unknown artist" );
    return m_unknownArtist != nullptr;
}

50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
int MetadataParser::toInt( VLC::Media& vlcMedia, libvlc_meta_t meta, const char* name )
{
    auto str = vlcMedia.meta( meta );
    if ( str.empty() == false )
    {
        try
        {
            return std::stoi( str );
        }
        catch( std::logic_error& ex)
        {
            LOG_WARN( "Invalid ", name, " provided (", str, "): ", ex.what() );
        }
    }
    return 0;
}

67
68
69
parser::Task::Status MetadataParser::run( parser::Task& task )
{
    auto& media = task.media;
70
    const auto& tracks = task.vlcMedia.tracks();
71
72
73
74
75

    // If we failed to extract any tracks, don't make any assumption and forward to the
    // thumbnailer. Since it starts an actual playback, it will have more information.
    // Since the metadata steps won't be marked, it will run again once the thumbnailer has completed.
    if ( tracks.empty() == true )
76
77
    {
        // However, if the file is not unknown anymore, it means the thumbnailer has already processed it
78
        if ( task.media->type() == Media::Type::Unknown )
79
80
        {
            LOG_INFO( "Skipping metadata parsing for file with unknown type: ", task.file->mrl() );
81
            return parser::Task::Status::Success;
82
        }
83
84
85
        // In that case, stop trying to do something with this file.
        return parser::Task::Status::Fatal;
    }
86

87
    bool isAudio = true;
88
    {
89
        auto t = m_ml->getConn()->newTransaction();
90
        for ( const auto& t : tracks )
91
        {
92
93
94
95
96
97
98
99
100
101
102
103
104
105
            auto codec = t.codec();
            std::string fcc( reinterpret_cast<const char*>( &codec ), 4 );
            if ( t.type() == VLC::MediaTrack::Type::Video )
            {
                media->addVideoTrack( fcc, t.width(), t.height(),
                                      static_cast<float>( t.fpsNum() ) / static_cast<float>( t.fpsDen() ),
                                      t.language(), t.description() );
                isAudio = false;
            }
            else if ( t.type() == VLC::MediaTrack::Type::Audio )
            {
                media->addAudioTrack( fcc, t.bitrate(), t.rate(), t.channels(),
                                      t.language(), t.description() );
            }
106
107
        }
        t->commit();
108
109
110
    }
    if ( isAudio == true )
    {
111
        if ( parseAudioFile( task ) == false )
112
113
114
115
            return parser::Task::Status::Fatal;
    }
    else
    {
116
        if (parseVideoFile( task ) == false )
117
118
            return parser::Task::Status::Fatal;
    }
119
    media->setDuration( task.vlcMedia.duration() );
120
121

    auto t = m_ml->getConn()->newTransaction();
122
    if ( media->save() == false )
123
        return parser::Task::Status::Fatal;
124
    task.file->markStepCompleted( File::ParserStep::MetadataAnalysis );
125
126
127
128
    // Save ourselves from the useless processing of a thumbnail later if
    // we're analyzing an audio file
    if ( isAudio == true )
        task.file->markStepCompleted( File::ParserStep::Thumbnailer );
129
    if ( task.file->saveParserStep() == false )
130
        return parser::Task::Status::Fatal;
131
    t->commit();
132
    m_notifier->notifyMediaCreation( media );
133
134
135
136
137
    return parser::Task::Status::Success;
}

/* Video files */

138
bool MetadataParser::parseVideoFile( parser::Task& task ) const
139
{
140
    auto media = task.media.get();
141
    media->setType( IMedia::Type::Video );
142
143
    const auto& title = task.vlcMedia.meta( libvlc_meta_Title );
    if ( title.length() == 0 )
144
        return true;
145

146
147
    const auto& showName = task.vlcMedia.meta( libvlc_meta_ShowName );
    if ( showName.length() == 0 )
148
    {
149
        auto show = m_ml->show( showName );
150
151
        if ( show == nullptr )
        {
152
            show = m_ml->createShow( showName );
153
154
155
            if ( show == nullptr )
                return false;
        }
156
157
        auto episode = toInt( task.vlcMedia, libvlc_meta_Episode, "episode number" );
        if ( episode != 0 )
158
159
        {
            std::shared_ptr<Show> s = std::static_pointer_cast<Show>( show );
160
            s->addEpisode( *media, title, episode );
161
162
163
164
165
166
167
168
169
170
171
        }
    }
    else
    {
        // How do we know if it's a movie or a random video?
    }
    return true;
}

/* Audio files */

172
bool MetadataParser::parseAudioFile( parser::Task& task ) const
173
{
174
    task.media->setType( IMedia::Type::Audio );
175

176
177
178
    const auto artworkMrl = task.vlcMedia.meta( libvlc_meta_ArtworkURL );
    if ( artworkMrl.empty() == false )
        task.media->setThumbnail( artworkMrl );
179

180
    auto genre = handleGenre( task );
181
    auto artists = findOrCreateArtist( task );
182
183
    auto album = findAlbum( task, artists.first, artists.second );
    auto t = m_ml->getConn()->newTransaction();
184
185
    if ( album == nullptr )
    {
186
187
        const auto& albumName = task.vlcMedia.meta( libvlc_meta_Album );
        album = m_ml->createAlbum( albumName, artworkMrl );
188
189
190
        if ( album == nullptr )
            return false;
        m_notifier->notifyAlbumCreation( album );
191
    }
192
193
194
195
    // If we know a track artist, specify it, otherwise, fallback to the album/unknown artist
    auto track = handleTrack( album, task, artists.second ? artists.second :
                            (artists.first ? artists.first : m_unknownArtist), genre.get() );

196
    auto res = link( *task.media, album, artists.first, artists.second );
197
198
199
200
    t->commit();
    return res;
}

201
202
std::shared_ptr<Genre> MetadataParser::handleGenre( parser::Task& task ) const
{
203
204
    const auto& genreStr = task.vlcMedia.meta( libvlc_meta_Genre );
    if ( genreStr.length() == 0 )
205
        return nullptr;
206
    auto genre = Genre::fromName( m_ml, genreStr );
207
208
    if ( genre == nullptr )
    {
209
        genre = Genre::create( m_ml, genreStr );
210
        if ( genre == nullptr )
211
            LOG_ERROR( "Failed to get/create Genre", genreStr );
212
213
214
215
    }
    return genre;
}

216
/* Album handling */
217
218
219

std::shared_ptr<Album> MetadataParser::findAlbum( parser::Task& task, std::shared_ptr<Artist> albumArtist,
                                                    std::shared_ptr<Artist> trackArtist ) const
220
{
221
222
    const auto& albumName = task.vlcMedia.meta( libvlc_meta_Album );
    if ( albumName.empty() == true )
223
    {
224
225
226
227
228
229
        std::shared_ptr<Artist> artist = albumArtist;
        if ( albumArtist != nullptr )
            return albumArtist->unknownAlbum();
        else if ( trackArtist != nullptr )
            return trackArtist->unknownAlbum();
        return m_unknownArtist->unknownAlbum();
230
231
232
233
    }

    // Album matching depends on the difference between artist & album artist.
    // Specificaly pass the albumArtist here.
234
235
    static const std::string req = "SELECT * FROM " + policy::AlbumTable::Name +
            " WHERE title = ?";
236
    auto albums = Album::fetchAll<Album>( m_ml, req, albumName );
237
238
239
240

    if ( albums.size() == 0 )
        return nullptr;

241
242
    const auto discTotal = toInt( task.vlcMedia, libvlc_meta_DiscTotal, "disc total" );
    const auto discNumber = toInt( task.vlcMedia, libvlc_meta_DiscNumber, "dist number" );
243
244
245
246
247
    /*
     * Even if we get only 1 album, we need to filter out invalid matches.
     * For instance, if we have already inserted an album "A" by an artist "john"
     * but we are now trying to handle an album "A" by an artist "doe", not filtering
     * candidates would yield the only "A" album we know, while we should return
248
     * nullptr, so the link() method can create a new one.
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
     */
    for ( auto it = begin( albums ); it != end( albums ); )
    {
        auto a = (*it).get();
        if ( albumArtist != nullptr )
        {
            // We assume that an album without album artist is a positive match.
            // At the end of the day, without proper tags, there's only so much we can do.
            auto candidateAlbumArtist = a->albumArtist();
            if ( candidateAlbumArtist != nullptr && candidateAlbumArtist->id() != albumArtist->id() )
            {
                it = albums.erase( it );
                continue;
            }
        }
        // If this is a multidisc album, assume it could be in a multiple amount of folders.
        // Since folders can come in any order, we can't assume the first album will be the
        // first media we see. If the discTotal or discNumber meta are provided, that's easy. If not,
        // we assume that another CD with the same name & artists, and a disc number > 1
        // denotes a multi disc album
        // Check the first case early to avoid fetching tracks if unrequired.
270
        if ( discTotal > 1 || discNumber > 1 )
271
272
273
274
        {
            ++it;
            continue;
        }
275
        const auto tracks = a->cachedTracks();
276
277
278
279
280
281
282
        // If there is no tracks to compare with, we just have to hope this will be the only valid
        // album match
        if ( tracks.size() == 0 )
        {
            ++it;
            continue;
        }
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301

        auto multiDisc = false;
        for ( auto& t : tracks )
        {
            auto at = t->albumTrack();
            assert( at != nullptr );
            if ( at != nullptr && at->discNumber() > 1 )
            {
                multiDisc = true;
                break;
            }
        }
        if ( multiDisc )
        {
            ++it;
            continue;
        }

        // Assume album files will be in the same folder.
302
        auto newFileFolder = utils::file::directory( task.file->mrl() );
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
        auto trackFiles = tracks[0]->files();
        bool excluded = false;
        for ( auto& f : trackFiles )
        {
            auto candidateFolder = utils::file::directory( f->mrl() );
            if ( candidateFolder != newFileFolder )
            {
                excluded = true;
                break;
            }
        }
        if ( excluded == true )
        {
            it = albums.erase( it );
            continue;
        }
        ++it;
    }
    if ( albums.size() == 0 )
        return nullptr;
    if ( albums.size() > 1 )
    {
325
        LOG_WARN( "Multiple candidates for album ", albumName, ". Selecting first one out of luck" );
326
327
328
329
330
331
    }
    return std::static_pointer_cast<Album>( albums[0] );
}

///
/// \brief MetadataParser::handleArtists Returns Artist's involved on a track
332
/// \param task The current parser task
333
334
335
336
/// \return A pair containing:
/// The album artist as a first element
/// The track artist as a second element, or nullptr if it is the same as album artist
///
337
std::pair<std::shared_ptr<Artist>, std::shared_ptr<Artist>> MetadataParser::findOrCreateArtist( parser::Task& task ) const
338
339
340
{
    std::shared_ptr<Artist> albumArtist;
    std::shared_ptr<Artist> artist;
341
    static const std::string req = "SELECT * FROM " + policy::ArtistTable::Name + " WHERE name = ?";
342

343
344
345
    const auto& albumArtistStr = task.vlcMedia.meta( libvlc_meta_AlbumArtist );
    const auto& artistStr = task.vlcMedia.meta( libvlc_meta_Artist );
    if ( albumArtistStr.empty() == true && artistStr.empty() == true )
346
    {
347
        return {m_unknownArtist, m_unknownArtist};
348
349
    }

350
    if ( albumArtistStr.empty() == false )
351
    {
352
        albumArtist = Artist::fetch( m_ml, req, albumArtistStr );
353
354
        if ( albumArtist == nullptr )
        {
355
            albumArtist = m_ml->createArtist( albumArtistStr );
356
357
            if ( albumArtist == nullptr )
            {
358
                LOG_ERROR( "Failed to create new artist ", albumArtistStr );
359
360
                return {nullptr, nullptr};
            }
361
            m_notifier->notifyArtistCreation( albumArtist );
362
363
        }
    }
364
    if ( artistStr.empty() == false && artistStr != albumArtistStr )
365
    {
366
        artist = Artist::fetch( m_ml, req, artistStr );
367
368
        if ( artist == nullptr )
        {
369
            artist = m_ml->createArtist( artistStr );
370
371
            if ( artist == nullptr )
            {
372
                LOG_ERROR( "Failed to create new artist ", artistStr );
373
374
                return {nullptr, nullptr};
            }
375
            m_notifier->notifyArtistCreation( albumArtist );
376
377
378
379
380
381
382
        }
    }
    return {albumArtist, artist};
}

/* Tracks handling */

383
384
std::shared_ptr<AlbumTrack> MetadataParser::handleTrack( std::shared_ptr<Album> album, parser::Task& task,
                                                         std::shared_ptr<Artist> artist, Genre* genre ) const
385
{
386
387
388
    auto title = task.vlcMedia.meta( libvlc_meta_Title );
    const auto trackNumber = toInt( task.vlcMedia, libvlc_meta_TrackNumber, "track number" );
    const auto discNumber = toInt( task.vlcMedia, libvlc_meta_DiscNumber, "disc number" );
389
390
391
    if ( title.empty() == true )
    {
        LOG_WARN( "Failed to get track title" );
392
        if ( trackNumber != 0 )
393
394
        {
            title = "Track #";
395
            title += std::to_string( trackNumber );
396
397
398
        }
    }
    if ( title.empty() == false )
399
        task.media->setTitle( title );
400

401
402
    auto track = std::static_pointer_cast<AlbumTrack>( album->addTrack( task.media, trackNumber,
                                                                        discNumber, artist->id(),
403
                                                                        genre != nullptr ? genre->id() : 0 ) );
404
405
406
407
408
    if ( track == nullptr )
    {
        LOG_ERROR( "Failed to create album track" );
        return nullptr;
    }
409

410
411
    const auto& releaseDate = task.vlcMedia.meta( libvlc_meta_Date );
    if ( releaseDate.empty() == false )
412
    {
413
        auto releaseYear = atoi( releaseDate.c_str() );
414
        task.media->setReleaseDate( releaseYear );
415
416
417
418
419
        // Let the album handle multiple dates. In order to do this properly, we need
        // to know if the date has been changed before, which can be known only by
        // using Album class internals.
        album->setReleaseYear( releaseYear, false );
    }
420
    m_notifier->notifyAlbumTrackCreation( track );
421
422
423
424
425
426
427
428
    return track;
}

/* Misc */

bool MetadataParser::link( Media& media, std::shared_ptr<Album> album,
                               std::shared_ptr<Artist> albumArtist, std::shared_ptr<Artist> artist ) const
{
429
430
    if ( albumArtist == nullptr )
        albumArtist = artist;
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452

    // We might modify albumArtist later, hence handle thumbnails before.
    // If we have an albumArtist (meaning the track was properly tagged, we
    // can assume this artist is a correct match. We can use the thumbnail from
    // the current album for the albumArtist, if none has been set before.
    if ( albumArtist != nullptr && albumArtist->artworkMrl().empty() == true &&
         album != nullptr && album->artworkMrl().empty() == false )
        albumArtist->setArtworkMrl( album->artworkMrl() );

    if ( albumArtist != nullptr )
        albumArtist->addMedia( media );
    if ( artist != nullptr && ( albumArtist == nullptr || albumArtist->id() != artist->id() ) )
        artist->addMedia( media );

    auto currentAlbumArtist = album->albumArtist();

    // If we have no main artist yet, that's easy, we need to assign one.
    if ( currentAlbumArtist == nullptr )
    {
        // We don't know if the artist was tagged as artist or albumartist, however, we simply add it
        // as the albumartist until proven we were wrong (ie. until one of the next tracks
        // has a different artist)
453
        album->setAlbumArtist( albumArtist );
454
455
456
457
458
459
460
461
462
463
        // Always add the album artist as an artist
        album->addArtist( albumArtist );
        if ( artist != nullptr )
            album->addArtist( artist );
    }
    else
    {
        if ( albumArtist->id() != currentAlbumArtist->id() )
        {
            // We have more than a single artist on this album, fallback to various artists
464
            auto variousArtists = Artist::fetch( m_ml, VariousArtistID );
465
            album->setAlbumArtist( variousArtists );
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
            // Add those two artists as "featuring".
            album->addArtist( albumArtist );
        }
        if ( artist != nullptr && artist->id() != albumArtist->id() )
        {
            if ( albumArtist->id() != artist->id() )
               album->addArtist( artist );
        }
    }

    return true;
}

const char* MetadataParser::name() const
{
    return "Metadata";
}
483
484
485
486
487
488
489
490
491
492

uint8_t MetadataParser::nbThreads() const
{
//    auto nbProcs = std::thread::hardware_concurrency();
//    if ( nbProcs == 0 )
//        return 1;
//    return nbProcs;
    // Let's make this code thread-safe first :)
    return 1;
}
493

494
bool MetadataParser::isCompleted( const parser::Task& task ) const
495
{
496
497
498
    // We always need to run this task if the metadata extraction isn't completed
    return ( static_cast<uint8_t>( task.file->parserStep() ) &
            static_cast<uint8_t>( File::ParserStep::MetadataAnalysis ) ) != 0;
499
500
}

501
}