Commit eea04696 authored by Geoffroy Lacarriere's avatar Geoffroy Lacarriere

first version of threading the audioSpectrum generation

parent 7ea27bd3
......@@ -189,37 +189,6 @@ Media::FileType Media::getFileType() const
return m_fileType;
}
void Media::initAudioData( void* datas, unsigned int* freq, unsigned int* nbChannels, unsigned int* fourCCFormat, unsigned int* frameSize )
{
m_audioData.freq = freq;
m_audioData.nbChannels = nbChannels;
m_audioData.frameSize = frameSize;
m_audioData.fourCCFormat = fourCCFormat;
m_audioData.datas = datas;
}
void Media::addAudioFrame( void* datas, unsigned char* buffer, size_t buffSize, unsigned int nbSample )
{
Q_UNUSED( datas );
m_audioData.nbSample = nbSample;
m_audioData.buffSize = buffSize;
int* frame = new int[ m_audioData.buffSize ];
for (unsigned int i = 0, u = 0; u < m_audioData.nbSample; i += 4, u++)
{
int value = buffer[i];
value <<= 8;
value += buffer[i + 1];
value <<= 8;
value += buffer[i + 2];
value <<= 8;
value += buffer[i + 3];
frame[u] = value;
}
m_audioData.frameList.append( frame );
// qDebug() << m_audioData.frameList.size();
}
void Media::emitMetaDataComputed( bool hasMetadata )
{
if ( hasMetadata == true )
......@@ -301,3 +270,4 @@ Media::MetadataState Media::getMetadata() const
{
return m_metadataState;
}
......@@ -40,19 +40,6 @@
#include "VLCMedia.h"
#include "Clip.h"
struct audioData
{
void* datas;
unsigned int* freq;
unsigned int* nbChannels;
unsigned int* fourCCFormat;
unsigned int* frameSize;
unsigned int nbSample;
unsigned char* buffer;
size_t buffSize;
QVector<int*> frameList;
};
class Clip;
/**
......@@ -140,13 +127,6 @@ public:
InputType getInputType() const;
static const QString streamPrefix;
void initAudioData( void* datas, unsigned int* freq, unsigned int* nbChannels, unsigned int* fourCCFormat, unsigned int* frameSize );
void addAudioFrame( void* datas, unsigned char* buffer, size_t buffSize, unsigned int nbSample );
audioData* getAudioData() { return &m_audioData; }
QVector<int*> getAudioFrameList() { return m_audioData.frameList; }
unsigned int getAudioNbSample() { return m_audioData.nbSample; }
const QStringList& getMetaTags() const;
void setMetaTags( const QStringList& tags );
bool matchMetaTag( const QString& tag ) const;
......@@ -162,6 +142,9 @@ public:
Clip* clip( const QUuid& uuid ) const { return m_clips[uuid]; }
const QHash<QUuid, Clip*>* clips() const { return &m_clips; }
QPixmap* getPixmap() { return m_audioPixmap; }
void setAudioPixmap( QPixmap* audioPixmap ) { m_audioPixmap = audioPixmap; }
private:
void setFileType();
......@@ -179,14 +162,13 @@ protected:
unsigned int m_width;
unsigned int m_height;
float m_fps;
int* m_audioSpectrum;
audioData m_audioData;
FileType m_fileType;
InputType m_inputType;
MetadataState m_metadataState;
QString m_fileName;
QStringList m_metaTags;
QHash<QUuid, Clip*> m_clips;
QPixmap* m_audioPixmap;
signals:
void metaDataComputed( Media* );
......
......@@ -103,11 +103,13 @@ void MetaDataManager::checkMediasToCompute()
//qDebug() << "checking media to compute" << m_mediasToComputeMetaData << m_mediasToComputeSnapshot;
m_mediasToComputeMetaDataMutex.lock();
m_mediasToComputeSnapshotMutex.lock();
m_mediasToComputeAudioSpectrumMutex.lock();
m_mediaPlayersMutex.lock();
QMap<MediaPlayerState, LibVLCpp::MediaPlayer*>::iterator it;
if ( m_mediasToComputeMetaData.count() > 0 &&
( it = m_mediaPlayers.find( Idle ) ) != m_mediaPlayers.end() )
{
m_mediasToComputeAudioSpectrumMutex.unlock();
Media* media;
media = m_mediasToComputeMetaData.dequeue();
m_mediasToComputeSnapshot.enqueue( media );
......@@ -128,7 +130,9 @@ void MetaDataManager::checkMediasToCompute()
m_mediasToComputeMetaDataMutex.unlock();
Media* media;
media = m_mediasToComputeSnapshot.dequeue();
m_mediasToComputeAudioSpectrum.enqueue( media );
m_mediasToComputeSnapshotMutex.unlock();
m_mediasToComputeAudioSpectrumMutex.unlock();
LibVLCpp::MediaPlayer* mediaPlayer = it.value();
m_mediaPlayers.erase( it );
m_mediaPlayers.insert( Running, mediaPlayer );
......@@ -139,10 +143,27 @@ void MetaDataManager::checkMediasToCompute()
//connect( media, SIGNAL( snapshotComputed( Media* ) ), this, SLOT( checkMediasToCompute() ) );
worker->compute();
}
else if (m_mediasToComputeAudioSpectrum.count() > 0 &&
(it = m_mediaPlayers.find( Idle ) ) != m_mediaPlayers.end() )
{
m_mediasToComputeMetaDataMutex.unlock();
m_mediasToComputeSnapshotMutex.unlock();
Media* media;
media = m_mediasToComputeAudioSpectrum.dequeue();
m_mediasToComputeAudioSpectrumMutex.unlock();
LibVLCpp::MediaPlayer* mediaPlayer = it.value();
m_mediaPlayers.erase( it );
m_mediaPlayers.insert( Running, mediaPlayer );
m_mediaPlayersMutex.unlock();
MetaDataWorker* worker = new MetaDataWorker( mediaPlayer, media, MetaDataWorker::AudioSpectrum );
connect( worker, SIGNAL( mediaPlayerIdle( LibVLCpp::MediaPlayer* ) ), this, SLOT( mediaPlayerIdle( LibVLCpp::MediaPlayer* ) ), Qt::DirectConnection );
worker->compute();
}
else
{
m_mediasToComputeSnapshotMutex.unlock();
m_mediasToComputeMetaDataMutex.unlock();
m_mediasToComputeAudioSpectrumMutex.unlock();
m_mediaPlayersMutex.unlock();
}
return;
......
......@@ -33,7 +33,7 @@
#include "Media.h"
#include "Singleton.hpp"
#define DEFAULT_MAX_MEDIA_PLAYER 2
#define DEFAULT_MAX_MEDIA_PLAYER 3
class MetaDataManager : public QObject, public Singleton<MetaDataManager>
{
......@@ -62,9 +62,11 @@ class MetaDataManager : public QObject, public Singleton<MetaDataManager>
private:
QQueue<Media*> m_mediasToComputeMetaData;
QQueue<Media*> m_mediasToComputeSnapshot;
QQueue<Media*> m_mediasToComputeAudioSpectrum;
QMultiMap<MediaPlayerState, LibVLCpp::MediaPlayer*> m_mediaPlayers;
QMutex m_mediasToComputeMetaDataMutex;
QMutex m_mediasToComputeSnapshotMutex;
QMutex m_mediasToComputeAudioSpectrumMutex;
QMutex m_mediaPlayersMutex;
int m_mediaPlayersMaxCount;
int m_mediaPlayersToRemove;
......
......@@ -21,11 +21,47 @@
*****************************************************************************/
#include <QtDebug>
#include <QPainter>
#include <QLabel>
#include <QImage>
#include "vlmc.h"
#include "MetaDataWorker.h"
#include "Library.h"
#include <QThreadPool>
#include <QRunnable>
class Help : public QRunnable
{
private:
QList<int>* m_audioValueList;
QImage* m_image;
QPainter* m_painter;
QPainterPath m_path;
public:
Help(QList<int>* audioValueList) : m_audioValueList( audioValueList )
{
m_image = new QImage( m_audioValueList->count(), 800, QImage::Format_RGB32);
m_image->fill( 0 );
m_painter = new QPainter( m_image );
m_painter->setRenderHint( QPainter::Antialiasing, true );
m_painter->setPen( QPen( QColor( 79, 106, 25 ), 1, Qt::SolidLine, Qt::FlatCap, Qt::MiterJoin ) );
}
void run()
{
for( int x = 0; x < m_audioValueList->count(); x++ )
{
int y = m_audioValueList->at(x);
//qDebug() << y;
m_path.lineTo( x, ( y / 30 ) - 700 );
}
m_painter->drawPath( m_path );
}
};
MetaDataWorker::MetaDataWorker( LibVLCpp::MediaPlayer* mediaPlayer, Media* media, MetaDataWorker::MetaDataType type ) :
m_mediaPlayer( mediaPlayer ),
m_type( type ),
......@@ -33,6 +69,7 @@ MetaDataWorker::MetaDataWorker( LibVLCpp::MediaPlayer* mediaPlayer, Media* media
m_mediaIsPlaying( false),
m_lengthHasChanged( false )
{
m_audioValueList = new QList<int>();
}
MetaDataWorker::~MetaDataWorker()
......@@ -45,12 +82,16 @@ void MetaDataWorker::compute()
{
if ( m_media->getFileType() == Media::Video )
{
computeVideoMetaData();
if ( m_type == AudioSpectrum )
computeAudioMetaData();
else
computeVideoMetaData();
}
else if ( m_media->getFileType() == Media::Image )
{
computeImageMetaData();
}
m_media->addConstantParam( ":vout=dummy" );
m_mediaPlayer->setMedia( m_media->getVLCMedia() );
connect( m_mediaPlayer, SIGNAL( playing() ), this, SLOT( entrypointPlaying() ) );
......@@ -73,6 +114,18 @@ void MetaDataWorker::computeImageMetaData()
m_lengthHasChanged = true;
}
void MetaDataWorker::computeAudioMetaData()
{
m_media->getVLCMedia()->addOption( ":no-sout-video" );
m_media->getVLCMedia()->addOption( ":sout=#transcode{}:smem" );
m_media->getVLCMedia()->setAudioDataCtx( this );
m_media->getVLCMedia()->setAudioLockCallback( reinterpret_cast<void*>( lock ) );
m_media->getVLCMedia()->setAudioUnlockCallback( reinterpret_cast<void*>( unlock ) );
m_media->getVLCMedia()->addOption( ":sout-transcode-acodec=s16l" );
m_media->getVLCMedia()->addOption( ":no-sout-smem-time-sync" );
connect( m_mediaPlayer, SIGNAL( endReached() ), this, SLOT( generateAudioSpectrum() ) );
}
void MetaDataWorker::getMetaData()
{
m_mediaIsPlaying = false;
......@@ -103,10 +156,7 @@ void MetaDataWorker::getMetaData()
// connect( m_mediaPlayer, SIGNAL( stopped () ), this, SLOT( mediaPlayerStopped() ), Qt::QueuedConnection );
m_mediaPlayer->stop();
emit mediaPlayerIdle( m_mediaPlayer );
if ( m_type == Snapshot )
m_media->emitSnapshotComputed();
else
m_media->emitMetaDataComputed( true );
m_media->emitMetaDataComputed( true );
delete this;
return;
}
......@@ -160,12 +210,13 @@ void MetaDataWorker::setSnapshot()
// connect( m_mediaPlayer, SIGNAL( stopped () ), this, SLOT( mediaPlayerStopped() ), Qt::QueuedConnection );
m_mediaPlayer->stop();
emit mediaPlayerIdle( m_mediaPlayer );
if ( m_type == Snapshot )
m_media->emitSnapshotComputed();
else
m_media->emitMetaDataComputed( true );
delete this;
//startAudioDataParsing();
}
void MetaDataWorker::mediaPlayerStopped()
......@@ -179,82 +230,109 @@ void MetaDataWorker::mediaPlayerStopped()
delete this;
}
void MetaDataWorker::startAudioDataParsing()
void MetaDataWorker::entrypointLengthChanged()
{
qDebug() << "Starting audio parsing";
char osb[64], psb[64], csb[64], iph[64], data[64];
// disconnect( m_mediaPlayer, SIGNAL( stopped() ), this, SLOT( startAudioDataParsing() ) );
sprintf( osb, ":amem-opensb=%lld", (long long int)(intptr_t) &MetaDataWorker::openSoundBuffer);
sprintf( psb, ":amem-playsb=%lld", (long long int)(intptr_t) &MetaDataWorker::playSoundBuffer);
sprintf( csb, ":amem-closesb=%lld", (long long int)(intptr_t) &MetaDataWorker::closeSoundBuffer);
sprintf( iph, ":amem-iph=%lld", (long long int)(intptr_t) &MetaDataWorker::instanceParameterHandler);
sprintf( data, ":amem-data=%lld", (long long int)(intptr_t) this);
m_media->addVolatileParam( ":no-video", ":video" );
m_media->addConstantParam( ":audio" );
m_media->addVolatileParam( ":aout=amem", ":aout=''" ); //I'm really not sure about this one...
m_media->addConstantParam( osb );
m_media->addConstantParam( psb );
m_media->addConstantParam( csb );
m_media->addConstantParam( iph );
m_media->addConstantParam( data );
m_mediaPlayer->setMedia( m_media->getVLCMedia() );
m_media->flushVolatileParameters();
connect( m_mediaPlayer, SIGNAL( endReached() ), this, SLOT( stopAudioDataParsing() ) );
qDebug() << "Starting playback again";
m_mediaPlayer->play();
disconnect( m_mediaPlayer, SIGNAL( lengthChanged() ), this, SLOT( entrypointLengthChanged() ) );
m_lengthHasChanged = true;
if ( m_mediaIsPlaying == true )
getMetaData();
}
void MetaDataWorker::stopAudioDataParsing()
void MetaDataWorker::entrypointPlaying()
{
qDebug() << "Stopping AudioDataParsing";
m_mediaPlayer->stop();
disconnect( m_mediaPlayer, SIGNAL( playing() ), this, SLOT( entrypointPlaying() ) );
m_mediaIsPlaying = true;
if ( m_lengthHasChanged == true )
getMetaData();
}
void MetaDataWorker::openSoundBuffer( void* datas, unsigned int* freq, unsigned int* nbChannels, unsigned int* fourCCFormat, unsigned int* frameSize )
{
//qDebug() << "Opening sound buffer with freq =" << *freq << "nbChannels =" << *nbChannels << "frameSize =" << *frameSize;
MetaDataWorker* self = reinterpret_cast<MetaDataWorker*>( datas );
self->m_media->initAudioData( datas, freq, nbChannels, fourCCFormat, frameSize );
}
void MetaDataWorker::playSoundBuffer( void* datas, unsigned char* buffer, size_t buffSize, unsigned int nbSample )
void MetaDataWorker::lock( MetaDataWorker* metaDataWorker, uint8_t** pcm_buffer , unsigned int size )
{
//qDebug() << "Playing sound buffer with nbSample =" << nbSample << "buffSize =" << buffSize;
// qDebug() << "Buff[0] = " << (unsigned int)buffer[0];
//if (MetaDataWorker::getInstance()->getCurrentMedia()->getAudioData()->frameList.size() < 500 )
MetaDataWorker* self = reinterpret_cast<MetaDataWorker*>( datas );
self->m_media->addAudioFrame( datas, buffer, buffSize, nbSample );
//else
// MetaDataWorker::getInstance()->getMediaPlayer()->stop();
if ( metaDataWorker->m_audioBuffer == NULL )
metaDataWorker->m_audioBuffer = new unsigned char[size];
*pcm_buffer = metaDataWorker->m_audioBuffer;
}
void MetaDataWorker::closeSoundBuffer( void* datas )
void MetaDataWorker::unlock( MetaDataWorker* metaDataWorker, uint8_t* pcm_buffer,
unsigned int channels, unsigned int rate,
unsigned int nb_samples, unsigned int bits_per_sample,
unsigned int size, int pts )
{
qDebug() << "Closing sound buffer";
MetaDataWorker* self = reinterpret_cast<MetaDataWorker*>( datas );
Q_UNUSED( rate );
Q_UNUSED( size );
Q_UNUSED( pts );
int bytePerChannelPerSample = bits_per_sample / 8;
int bytePerSample = bytePerChannelPerSample * channels;
int leftAverage = 0;
int rightAverage = 0;
self->m_mediaPlayer->stop();
int it = 0;
for ( int i = 0; i < nb_samples; i++)
{
int left = 0;
int right = 0;
for ( int u = 0 ; u < bytePerChannelPerSample; u++, it++ )
{
int increment = 0;
if ( channels == 2 )
increment = bytePerChannelPerSample;
left <<= 8;
left += pcm_buffer[ it ];
right <<= 8;
right += pcm_buffer[ it + increment ];
}
leftAverage += left;
rightAverage += right;
}
leftAverage /= nb_samples;
metaDataWorker->addAudioValue( leftAverage );
}
void MetaDataWorker::instanceParameterHandler( void*, char*, char* )
void MetaDataWorker::generateAudioSpectrum()
{
if ( m_mediaIsPlaying == true )
{
qDebug() << "generateAudioSpectrum";
disconnect( m_mediaPlayer, SIGNAL(endReached()), this, SLOT(generateAudioSpectrum()));
m_mediaPlayer->stop();
emit mediaPlayerIdle( m_mediaPlayer );
m_media->emitMetaDataComputed( true );
//generateAudioPixmap();
Help* h = new Help(m_audioValueList);
QThreadPool::globalInstance()->start(h);
delete this;
}
}
void MetaDataWorker::entrypointLengthChanged()
void MetaDataWorker::addAudioValue( int value )
{
disconnect( m_mediaPlayer, SIGNAL( lengthChanged() ), this, SLOT( entrypointLengthChanged() ) );
m_lengthHasChanged = true;
if ( m_mediaIsPlaying == true )
getMetaData();
m_audioValueList->append( value );
}
void MetaDataWorker::entrypointPlaying()
void MetaDataWorker::generateAudioPixmap()
{
disconnect( m_mediaPlayer, SIGNAL( playing() ), this, SLOT( entrypointPlaying() ) );
m_mediaIsPlaying = true;
if ( m_lengthHasChanged == true )
getMetaData();
m_audioDebugWidget = new QLabel();
m_audioDebugWidget->setFixedSize( m_audioValueList->count(), 800 );
m_audioDebugWidget->show();
//QImage* image = new QImage( 300, 300, QImage::Format_RGB32);
QPixmap image(m_audioValueList->count(), 800);
QPainter painter(&image);
painter.setRenderHint(QPainter::Antialiasing, true);
painter.setPen( QPen( QColor( 79, 106, 25 ), 1, Qt::SolidLine, Qt::FlatCap, Qt::MiterJoin ) );
QPainterPath path;
for( int x = 0; x < m_audioValueList->count(); x++ )
{
// qDebug() << (m_audioValueList[x] / 30) - 700;
int y = m_audioValueList->at(x);
path.lineTo( x, ( y / 30) - 700);
}
painter.drawPath(path);
m_audioDebugWidget->setPixmap(image);
}
......@@ -26,9 +26,7 @@
#include <QList>
#include <QTemporaryFile>
#include <QThreadPool>
#include <QRunnable>
#include <QWidget>
#include <QLabel>
#include "Media.h"
#include "VLCMediaPlayer.h"
......@@ -41,7 +39,8 @@ class MetaDataWorker : public QObject
enum MetaDataType
{
MetaData,
Snapshot
Snapshot,
AudioSpectrum
};
public:
......@@ -52,18 +51,19 @@ class MetaDataWorker : public QObject
private:
void computeVideoMetaData();
void computeImageMetaData();
void computeAudioMetaData();
void addAudioValue( int value );
//AMEM part :
static void openSoundBuffer( void* datas, unsigned int* freq,
unsigned int* nbChannels, unsigned int* fourCCFormat,
unsigned int* frameSize );
static void playSoundBuffer( void* datas, unsigned char* buffer,
size_t buffSize, unsigned int nbSample );
static void closeSoundBuffer( void* datas );
static void instanceParameterHandler( void*, char*, char* );
void generateAudioPixmap();
private:
void getMetaData();
void initVlcOutput();
static void lock( MetaDataWorker* metaDataWorker, uint8_t** pcm_buffer , unsigned int size );
static void unlock( MetaDataWorker* metaDataWorker, uint8_t* pcm_buffer,
unsigned int channels, unsigned int rate,
unsigned int nb_samples, unsigned int bits_per_sample,
unsigned int size, int pts );
private:
LibVLCpp::MediaPlayer* m_mediaPlayer;
......@@ -75,6 +75,12 @@ class MetaDataWorker : public QObject
bool m_mediaIsPlaying;
bool m_lengthHasChanged;
QWidget* m_widgetAudioDebug;
unsigned char* m_audioBuffer;
QLabel* m_audioDebugWidget;
QList<int>* m_audioValueList;
friend class SnapshotHelper;
signals:
void snapshotRequested();
......@@ -83,11 +89,10 @@ class MetaDataWorker : public QObject
private slots:
void renderSnapshot();
void setSnapshot();
void startAudioDataParsing();
void stopAudioDataParsing();
void entrypointPlaying();
void entrypointLengthChanged();
void mediaPlayerStopped();
void generateAudioSpectrum();
};
#endif // METADATAWORKER_H
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment