opencv_wrapper.c 17.4 KB
Newer Older
1 2 3
/*****************************************************************************
 * opencv_wrapper.c : OpenCV wrapper video filter
 *****************************************************************************
Jean-Baptiste Kempf's avatar
Jean-Baptiste Kempf committed
4
 * Copyright (C) 2006-2012 VLC authors and VideoLAN
5
 * Copyright (C) 2012 Edward Wang
6 7
 *
 * Authors: Dugal Harris <dugalh@protoclea.co.za>
8
 *          Edward Wang <edward.c.wang@compdigitec.com>
9
 *
Jean-Baptiste Kempf's avatar
Jean-Baptiste Kempf committed
10 11 12
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
13 14 15 16
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
Jean-Baptiste Kempf's avatar
Jean-Baptiste Kempf committed
17 18
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
19
 *
Jean-Baptiste Kempf's avatar
Jean-Baptiste Kempf committed
20 21 22
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
23 24 25 26 27 28
 *****************************************************************************/

/*****************************************************************************
 * Preamble
 *****************************************************************************/

29 30 31 32
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

33
#include <vlc_common.h>
34
#include <vlc_plugin.h>
35
#include <vlc_vout.h>
36
#include <vlc_modules.h>
37
#include <vlc_picture.h>
Rémi Duraffort's avatar
Rémi Duraffort committed
38 39
#include <vlc_filter.h>
#include <vlc_image.h>
40
#include "filter_picture.h"
41

42 43
#include <opencv2/core/core_c.h>
#include <opencv2/core/types_c.h>
44

45 46 47 48 49 50
/*****************************************************************************
 * Local prototypes
 *****************************************************************************/
static int  Create    ( vlc_object_t * );
static void Destroy   ( vlc_object_t * );

51
static picture_t* Filter( filter_t*, picture_t* );
52

53 54
static void ReleaseImages( filter_t* p_filter );
static void VlcPictureToIplImage( filter_t* p_filter, picture_t* p_in );
55 56 57 58 59

/*****************************************************************************
 * Module descriptor
 *****************************************************************************/

60 61
static const char *const chroma_list[] = { "input", "I420", "RGB32"};
static const char *const chroma_list_text[] = { N_("Use input chroma unaltered"),
62
  N_("I420 - first plane is grayscale"), N_("RGB32")};
63

64 65
static const char *const output_list[] = { "none", "input", "processed"};
static const char *const output_list_text[] = { N_("Don't display any video"),
66 67
  N_("Display the input video"), N_("Display the processed video")};

68 69
static const char *const verbosity_list[] = { "error", "warning", "debug"};
static const char *const verbosity_list_text[] = { N_("Show only errors"),
70 71
  N_("Show errors and warnings"), N_("Show everything including debug messages")};

72 73 74 75 76
vlc_module_begin ()
    set_description( N_("OpenCV video filter wrapper") )
    set_shortname( N_("OpenCV" ))
    set_category( CAT_VIDEO )
    set_subcategory( SUBCAT_VIDEO_VFILTER )
77
    set_capability( "video filter", 0 )
78 79
    add_shortcut( "opencv_wrapper" )
    set_callbacks( Create, Destroy )
80
    add_float_with_range( "opencv-scale", 1.0, 0.1, 2.0,
81
                          N_("Scale factor (0.1-2.0)"),
Michał's avatar
Michał committed
82
                          N_("Amount by which to scale the picture before sending it to the internal OpenCV filter"),
Rémi Denis-Courmont's avatar
Rémi Denis-Courmont committed
83
                          false )
84
    add_string( "opencv-chroma", "input",
85
                          N_("OpenCV filter chroma"),
86
                          N_("Chroma to convert picture to before sending it to the internal OpenCV filter"), false);
87
        change_string_list( chroma_list, chroma_list_text )
88
    add_string( "opencv-output", "input",
89
                          N_("Wrapper filter output"),
90
                          N_("Determines what (if any) video is displayed by the wrapper filter"), false);
91
        change_string_list( output_list, output_list_text )
92
    add_string( "opencv-filter-name", "none",
93
                          N_("OpenCV internal filter name"),
94
                          N_("Name of internal OpenCV plugin filter to use"), false);
95
vlc_module_end ()
96 97 98 99 100 101 102


/*****************************************************************************
 * wrapper_output_t: what video is output
 *****************************************************************************/
enum wrapper_output_t
{
103
   NONE,
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
   VINPUT,
   PROCESSED
};

/*****************************************************************************
 * internal_chroma_t: what chroma is sent to the internal opencv filter
 *****************************************************************************/
enum internal_chroma_t
{
   CINPUT,
   GREY,
   RGB
};

/*****************************************************************************
119
 * filter_sys_t: opencv_wrapper video output method descriptor
120 121 122 123
 *****************************************************************************
 * This structure is part of the video output thread descriptor.
 * It describes the opencv_wrapper specific properties of an output thread.
 *****************************************************************************/
124
struct filter_sys_t
125 126
{
    image_handler_t *p_image;
127

128
    int i_cv_image_size;
129

130 131
    picture_t *p_proc_image;
    picture_t *p_to_be_freed;
132

133
    float f_scale;
134

135 136 137 138
    int i_wrapper_output;
    int i_internal_chroma;

    IplImage *p_cv_image[VOUT_MAX_PLANES];
139

140 141
    filter_t *p_opencv;
    char* psz_inner_name;
142

143 144 145 146 147 148 149 150 151 152
    picture_t hacked_pic;
};

/*****************************************************************************
 * Create: allocates opencv_wrapper video thread output method
 *****************************************************************************
 * This function allocates and initializes a opencv_wrapper vout method.
 *****************************************************************************/
static int Create( vlc_object_t *p_this )
{
153
    filter_t* p_filter = (filter_t*)p_this;
154
    char *psz_chroma, *psz_output;
155 156

    /* Allocate structure */
157 158
    p_filter->p_sys = malloc( sizeof( filter_sys_t ) );
    if( p_filter->p_sys == NULL )
159 160
        return VLC_ENOMEM;

161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
    /* Load the internal OpenCV filter.
     *
     * This filter object is needed to call the internal OpenCV filter
     * for processing, the wrapper just converts into an IplImage* for
     * the other filter.
     *
     * We don't need to set up video formats for this filter as it not
     * actually using a picture_t.
     */
    p_filter->p_sys->p_opencv = vlc_object_create( p_filter, sizeof(filter_t) );
    if( !p_filter->p_sys->p_opencv ) {
        free( p_filter->p_sys );
        return VLC_ENOMEM;
    }

    p_filter->p_sys->psz_inner_name = var_InheritString( p_filter, "opencv-filter-name" );
    if( p_filter->p_sys->psz_inner_name )
        p_filter->p_sys->p_opencv->p_module =
            module_need( p_filter->p_sys->p_opencv,
                         "opencv internal filter",
                         p_filter->p_sys->psz_inner_name,
                         true );

    if( !p_filter->p_sys->p_opencv->p_module )
    {
        msg_Err( p_filter, "can't open internal opencv filter: %s", p_filter->p_sys->psz_inner_name );
        free( p_filter->p_sys->psz_inner_name );
        p_filter->p_sys->psz_inner_name = NULL;
        vlc_object_release( p_filter->p_sys->p_opencv );
        free( p_filter->p_sys );

        return VLC_ENOMOD;
    }


196
    /* Init structure */
197 198 199 200 201 202
    p_filter->p_sys->p_image = image_HandlerCreate( p_filter );
    for( int i = 0; i < VOUT_MAX_PLANES; i++ )
        p_filter->p_sys->p_cv_image[i] = NULL;
    p_filter->p_sys->p_proc_image = NULL;
    p_filter->p_sys->p_to_be_freed = NULL;
    p_filter->p_sys->i_cv_image_size = 0;
203 204

    /* Retrieve and apply config */
205
    psz_chroma = var_InheritString( p_filter, "opencv-chroma" );
206
    if( psz_chroma == NULL )
207
    {
208
        msg_Err( p_filter, "configuration variable %s empty, using 'grey'",
209
                         "opencv-chroma" );
210 211 212 213 214 215 216 217 218 219
        p_filter->p_sys->i_internal_chroma = GREY;
    } else if( !strcmp( psz_chroma, "input" ) )
        p_filter->p_sys->i_internal_chroma = CINPUT;
    else if( !strcmp( psz_chroma, "I420" ) )
        p_filter->p_sys->i_internal_chroma = GREY;
    else if( !strcmp( psz_chroma, "RGB32" ) )
        p_filter->p_sys->i_internal_chroma = RGB;
    else {
        msg_Err( p_filter, "no valid opencv-chroma provided, using 'grey'" );
        p_filter->p_sys->i_internal_chroma = GREY;
220 221
    }

222 223 224
    free( psz_chroma );

    psz_output = var_InheritString( p_filter, "opencv-output" );
225
    if( psz_output == NULL )
226
    {
227
        msg_Err( p_filter, "configuration variable %s empty, using 'input'",
228
                         "opencv-output" );
229 230 231 232 233 234 235 236 237 238
        p_filter->p_sys->i_wrapper_output = VINPUT;
    } else if( !strcmp( psz_output, "none" ) )
        p_filter->p_sys->i_wrapper_output = NONE;
    else if( !strcmp( psz_output, "input" ) )
        p_filter->p_sys->i_wrapper_output = VINPUT;
    else if( !strcmp( psz_output, "processed" ) )
        p_filter->p_sys->i_wrapper_output = PROCESSED;
    else {
        msg_Err( p_filter, "no valid opencv-output provided, using 'input'" );
        p_filter->p_sys->i_wrapper_output = VINPUT;
239
    }
240
    free( psz_output );
241

242 243
    p_filter->p_sys->f_scale =
        var_InheritFloat( p_filter, "opencv-scale" );
244

245 246 247 248 249 250
    msg_Info(p_filter, "Configuration: opencv-scale: %f, opencv-chroma: %d, "
        "opencv-output: %d, opencv-filter %s",
        p_filter->p_sys->f_scale,
        p_filter->p_sys->i_internal_chroma,
        p_filter->p_sys->i_wrapper_output,
        p_filter->p_sys->psz_inner_name);
251

252 253 254
#ifndef NDEBUG
    msg_Dbg( p_filter, "opencv_wrapper successfully started" );
#endif
255

256
    p_filter->pf_video_filter = Filter;
257 258 259 260 261 262 263 264 265 266 267

    return VLC_SUCCESS;
}

/*****************************************************************************
 * Destroy: destroy opencv_wrapper video thread output method
 *****************************************************************************
 * Terminate an output method created by opencv_wrapperCreateOutputMethod
 *****************************************************************************/
static void Destroy( vlc_object_t *p_this )
{
268 269
    filter_t* p_filter = (filter_t*)p_this;
    ReleaseImages( p_filter );
270

271 272 273 274
    // Release the internal OpenCV filter.
    module_unneed( p_filter->p_sys->p_opencv, p_filter->p_sys->p_opencv->p_module );
    vlc_object_release( p_filter->p_sys->p_opencv );
    p_filter->p_sys->p_opencv = NULL;
275

276
    free( p_filter->p_sys );
277 278 279
}

/*****************************************************************************
280
 * ReleaseImages: Release OpenCV images in filter_sys_t.
281
 *****************************************************************************/
282
static void ReleaseImages( filter_t* p_filter )
283
{
284 285
    filter_sys_t* p_sys = p_filter->p_sys;

286
    for( int i = 0; i < VOUT_MAX_PLANES; i++ )
287
    {
288
        if (p_sys->p_cv_image[i] != NULL)
289
        {
290 291
            cvReleaseImageHeader(&(p_sys->p_cv_image[i]));
            p_sys->p_cv_image[i] = NULL;
292 293
        }
    }
294
    p_sys->i_cv_image_size = 0;
295

296
    /* Release temp picture_t if it exists */
297
    if (p_sys->p_to_be_freed)
298
    {
299 300
        picture_Release( p_sys->p_to_be_freed );
        p_sys->p_to_be_freed = NULL;
301
    }
302 303 304 305

#ifndef NDEBUG
    msg_Dbg( p_filter, "images released" );
#endif
306 307 308 309 310 311 312 313
}

/*****************************************************************************
 * VlcPictureToIplImage: Convert picture_t to IplImage
 *****************************************************************************
 * Converts given picture_t into IplImage(s) according to module config.
 * IplImage(s) are stored in vout_sys_t.
 *****************************************************************************/
314
static void VlcPictureToIplImage( filter_t* p_filter, picture_t* p_in )
315 316
{
    int planes = p_in->i_planes;    //num input video planes
317
    // input video size
318
    CvSize sz = cvSize(abs(p_in->format.i_width), abs(p_in->format.i_height));
319
    video_format_t fmt_out;
320
    filter_sys_t* p_sys = p_filter->p_sys;
321

322 323
    memset( &fmt_out, 0, sizeof(video_format_t) );

324 325 326 327
    //do scale / color conversion according to p_sys config
    if ((p_sys->f_scale != 1) || (p_sys->i_internal_chroma != CINPUT))
    {
        fmt_out = p_in->format;
328

329 330
        //calc the scaled video size
        fmt_out.i_width = p_in->format.i_width * p_sys->f_scale;
331
        fmt_out.i_height = p_in->format.i_height * p_sys->f_scale;
332 333 334 335

        if (p_sys->i_internal_chroma == RGB)
        {
            //rgb2 gives 3 separate planes, this gives 1 interleaved plane
336
            //rv24 gives is about 20% faster but gives r&b the wrong way round
337
            //and I can't think of an easy way to fix this
338
            fmt_out.i_chroma = VLC_CODEC_RGB24;
339 340 341
        }
        else if (p_sys->i_internal_chroma == GREY)
        {
342
            //take the I (gray) plane (video seems to commonly be in this fmt so usually the
343
            //conversion does nothing)
344
            fmt_out.i_chroma = VLC_CODEC_I420;
345 346 347 348 349
        }

        //convert from the input image
        p_sys->p_proc_image = image_Convert( p_sys->p_image, p_in,
                                     &(p_in->format), &fmt_out );
350

351 352
        if (!p_sys->p_proc_image)
        {
353
            msg_Err(p_filter, "can't convert (unsupported formats?), aborting...");
354
            return;
355
        }
356 357

        p_sys->p_to_be_freed = p_sys->p_proc_image;    //remember this so we can free it later
358

359 360 361
    }
    else    //((p_sys->f_scale != 1) || (p_sys->i_internal_chroma != CINPUT))
    {
362 363 364 365 366 367
        // In theory, you could use the input image without conversion,
        // but it seems to cause weird picture effects (like repeated
        // image filtering) and picture leaking.
        p_sys->p_proc_image = filter_NewPicture( p_filter ); //p_in
        picture_Copy( p_sys->p_proc_image, p_in );
        p_sys->p_to_be_freed = p_sys->p_proc_image;
368 369 370 371 372
    }

    //Convert to the IplImage array that is to be processed.
    //If there are multiple planes in p_sys->p_proc_image, then 1 IplImage
    //is created for each plane.
373
    planes = p_sys->p_proc_image->i_planes;
374
    p_sys->i_cv_image_size = planes;
375
    for( int i = 0; i < planes; i++ )
376
    {
377 378
        sz = cvSize(abs(p_sys->p_proc_image->p[i].i_visible_pitch /
            p_sys->p_proc_image->p[i].i_pixel_pitch),
379 380
            abs(p_sys->p_proc_image->p[i].i_visible_lines));

381
        p_sys->p_cv_image[i] = cvCreateImageHeader(sz, IPL_DEPTH_8U,
382 383
            p_sys->p_proc_image->p[i].i_pixel_pitch);

384
        cvSetData( p_sys->p_cv_image[i],
385 386 387
            (char*)(p_sys->p_proc_image->p[i].p_pixels), p_sys->p_proc_image->p[i].i_pitch );
    }

388
    //Hack the above opencv image array into a picture_t so that it can be sent to
389 390 391 392
    //another video filter
    p_sys->hacked_pic.i_planes = planes;
    p_sys->hacked_pic.format.i_chroma = fmt_out.i_chroma;

393 394 395
#ifndef NDEBUG
    msg_Dbg( p_filter, "VlcPictureToIplImageRgb() completed" );
#endif
396 397 398
}

/*****************************************************************************
399
 * Filter: displays previously rendered output
400 401 402 403
 *****************************************************************************
 * This function send the currently rendered image to the internal opencv
 * filter for processing.
 *****************************************************************************/
404
static picture_t* Filter( filter_t* p_filter, picture_t* p_pic )
405
{
406
    picture_t* p_outpic = filter_NewPicture( p_filter );
407 408 409 410 411
    if( p_outpic == NULL ) {
        msg_Err( p_filter, "couldn't get a p_outpic!" );
        picture_Release( p_pic );
        return NULL;
    }
412

413 414
    video_format_t fmt_out;

415 416 417
    // Make a copy if we want to show the original input
    if (p_filter->p_sys->i_wrapper_output == VINPUT)
        picture_Copy( p_outpic, p_pic );
418

419 420 421
    VlcPictureToIplImage( p_filter, p_pic );
    // Pass the image (as a pointer to the first IplImage*) to the
    // internal OpenCV filter for processing.
422
    p_filter->p_sys->p_opencv->pf_video_filter( p_filter->p_sys->p_opencv, (picture_t*)&(p_filter->p_sys->p_cv_image[0]) );
423

424 425
    if(p_filter->p_sys->i_wrapper_output == PROCESSED) {
        // Processed video
426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448
        if( (p_filter->p_sys->p_proc_image) &&
            (p_filter->p_sys->p_proc_image->i_planes > 0) &&
            (p_filter->p_sys->i_internal_chroma != CINPUT) ) {
            //p_filter->p_sys->p_proc_image->format.i_chroma = VLC_CODEC_RGB24;

            memset( &fmt_out, 0, sizeof(video_format_t) );
            fmt_out = p_pic->format;
            //picture_Release( p_outpic );

            /*
             * We have to copy out the image from image_Convert(), otherwise
             * you leak pictures for some reason:
             * main video output error: pictures leaked, trying to workaround
             */
            picture_t* p_outpic_tmp = image_Convert(
                        p_filter->p_sys->p_image,
                        p_filter->p_sys->p_proc_image,
                        &(p_filter->p_sys->p_proc_image->format),
                        &fmt_out );

            picture_CopyPixels( p_outpic, p_outpic_tmp );
            CopyInfoAndRelease( p_outpic, p_outpic_tmp );
        } else if( p_filter->p_sys->i_internal_chroma == CINPUT ) {
449
            picture_CopyPixels( p_outpic, p_filter->p_sys->p_proc_image );
450
            picture_CopyProperties( p_outpic, p_filter->p_sys->p_proc_image );
451
        }
452
    }
453

454
    ReleaseImages( p_filter );
455
    picture_Release( p_pic );
456

457 458 459
#ifndef NDEBUG
    msg_Dbg( p_filter, "Filter() done" );
#endif
460

461
    if( p_filter->p_sys->i_wrapper_output != NONE ) {
462 463 464 465 466
        return p_outpic;
    } else { // NONE
        picture_Release( p_outpic );
        return NULL;
    }
467 468
}