modules/access/avcapture.m

@@ -72,11 +72,20 @@ vlc_module_begin ()
    set_callbacks(Open, Close)
 vlc_module_end ()
 
+static vlc_tick_t vlc_CMTime_to_tick(CMTime timestamp)
+{
+    CMTime scaled = CMTimeConvertScale(
+            timestamp, CLOCK_FREQ,
+            kCMTimeRoundingMethod_Default);
+
+    return VLC_TICK_0 + scaled.value;
+}
 
 /*****************************************************************************
 * AVFoundation Bridge
 *****************************************************************************/
-@interface VLCAVDecompressedVideoOutput : AVCaptureVideoDataOutput
+@interface VLCAVDecompressedVideoOutput :
+    AVCaptureVideoDataOutput <AVCaptureVideoDataOutputSampleBufferDelegate>
 {
     demux_t             *p_avcapture;
 
@@ -157,23 +166,22 @@ vlc_module_end ()
         self.videoDimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
         bytesPerRow = CVPixelBufferGetBytesPerRow(CMSampleBufferGetImageBuffer(sampleBuffer));
         videoDimensionsReady = YES;
-        msg_Dbg(p_avcapture, "Dimensionns obtained height:%i width:%i bytesPerRow:%lu", [self height], [self width], bytesPerRow);
+        msg_Dbg(p_avcapture, "Dimensions obtained height:%i width:%i bytesPerRow:%lu", [self height], [self width], bytesPerRow);
     }
 }
 
--(vlc_tick_t)currentPts
+- (vlc_tick_t)currentPts
 {
     vlc_tick_t pts;
 
-    if ( !currentImageBuffer || currentPts == previousPts )
-        return 0;
-
     @synchronized (self)
     {
+       if ( !currentImageBuffer || currentPts == previousPts )
+           return VLC_TICK_INVALID;
         pts = previousPts = currentPts;
     }
 
-    return currentPts;
+    return pts;
 }
 
 - (void)captureOutput:(AVCaptureOutput *)captureOutput
@@ -187,13 +195,14 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
         CVBufferRetain(videoFrame);
         [self getVideoDimensions:sampleBuffer];
 
+
         @synchronized (self) {
             imageBufferToRelease = currentImageBuffer;
             currentImageBuffer = videoFrame;
-            currentPts = (vlc_tick_t)presentationtimestamp.value;
+            currentPts = vlc_CMTime_to_tick(presentationtimestamp);
             timeScale = (long)presentationtimestamp.timescale;
         }
-        
+
         CVBufferRelease(imageBufferToRelease);
     }
 }
@@ -237,16 +246,21 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
 * Struct
 *****************************************************************************/
 
-typedef struct demux_sys_t
-{
-    CFTypeRef _Nullable             session;       // AVCaptureSession
-    CFTypeRef _Nullable             device;        // AVCaptureDevice
-    CFTypeRef _Nullable             output;        // VLCAVDecompressedVideoOutput
-    es_out_id_t                     *p_es_video;
-    es_format_t                     fmt;
-    int                             height, width;
-    BOOL                            b_es_setup;
-} demux_sys_t;
+@interface VLCAVCaptureDemux : NSObject {
+    demux_t                         *_demux;
+    AVCaptureSession                *_session;
+    AVCaptureDevice                 *_device;
+    VLCAVDecompressedVideoOutput    *_output;
+    es_out_id_t                     *_es_video;
+    es_format_t                     _fmt;
+    int                             _height, _width;
+}
+
+- (VLCAVCaptureDemux*)init:(demux_t *)demux;
+- (int)demux;
+- (vlc_tick_t)pts;
+- (void)dealloc;
+@end
 
 /*****************************************************************************
 * Open:
@@ -254,8 +268,82 @@ typedef struct demux_sys_t
 static int Open(vlc_object_t *p_this)
 {
     demux_t                 *p_demux = (demux_t*)p_this;
-    demux_sys_t             *p_sys = NULL;
 
+    if (p_demux->out == NULL)
+        return VLC_EGENERIC;
+
+    @autoreleasepool {
+        VLCAVCaptureDemux *demux = [[VLCAVCaptureDemux alloc] init:p_demux];
+        if (demux == nil)
+            return VLC_EGENERIC;
+        p_demux->p_sys = (__bridge_retained void*)demux;
+    }
+    p_demux->pf_demux = Demux;
+    p_demux->pf_control = Control;
+
+    return VLC_SUCCESS;
+}
+
+/*****************************************************************************
+* Close:
+*****************************************************************************/
+static void Close(vlc_object_t *p_this)
+{
+    demux_t *p_demux = (demux_t*)p_this;
+    VLCAVCaptureDemux *demux =
+        (__bridge_transfer VLCAVCaptureDemux*)p_demux->p_sys;
+
+    /* Signal ARC we won't use those references anymore. */
+    p_demux->p_sys = nil;
+    demux = nil;
+}
+
+/*****************************************************************************
+* Demux:
+*****************************************************************************/
+static int Demux(demux_t *p_demux)
+{
+    VLCAVCaptureDemux *demux = (__bridge VLCAVCaptureDemux *)p_demux->p_sys;
+    return [demux demux];
+}
+
+/*****************************************************************************
+* Control:
+*****************************************************************************/
+static int Control(demux_t *p_demux, int i_query, va_list args)
+{
+    VLCAVCaptureDemux *demux = (__bridge VLCAVCaptureDemux *)p_demux->p_sys;
+    bool        *pb;
+
+    switch( i_query )
+    {
+        /* Special for access_demux */
+        case DEMUX_CAN_PAUSE:
+        case DEMUX_CAN_SEEK:
+        case DEMUX_SET_PAUSE_STATE:
+        case DEMUX_CAN_CONTROL_PACE:
+           pb = va_arg(args, bool *);
+           *pb = false;
+           return VLC_SUCCESS;
+
+        case DEMUX_GET_PTS_DELAY:
+           *va_arg(args, vlc_tick_t *) =
+               VLC_TICK_FROM_MS(var_InheritInteger(p_demux, "live-caching"));
+           return VLC_SUCCESS;
+
+        case DEMUX_GET_TIME:
+            *va_arg(args, vlc_tick_t *) = [demux pts];
+            return VLC_SUCCESS;
+
+        default:
+           return VLC_EGENERIC;
+    }
+    return VLC_EGENERIC;
+}
+
+@implementation VLCAVCaptureDemux
+- (VLCAVCaptureDemux *)init:(demux_t *)p_demux
+{
     NSString                *avf_currdevice_uid;
     NSArray                 *myVideoDevices;
     NSError                 *o_returnedError;
@@ -266,174 +354,135 @@ static int Open(vlc_object_t *p_this)
 
     char                    *psz_uid = NULL;
 
-    if (p_demux->out == NULL)
-        return VLC_EGENERIC;
-
-    @autoreleasepool {
-        if (p_demux->psz_location && *p_demux->psz_location)
-            psz_uid = strdup(p_demux->psz_location);
+    _demux = p_demux;
 
-        msg_Dbg(p_demux, "avcapture uid = %s", psz_uid);
-        avf_currdevice_uid = [[NSString alloc] initWithFormat:@"%s", psz_uid];
+    if (_demux->psz_location && *_demux->psz_location)
+        psz_uid = strdup(_demux->psz_location);
 
-        /* Set up p_demux */
-        p_demux->pf_demux = Demux;
-        p_demux->pf_control = Control;
+    msg_Dbg(_demux, "avcapture uid = %s", psz_uid);
+    avf_currdevice_uid = [[NSString alloc] initWithFormat:@"%s", psz_uid];
 
-        p_demux->p_sys = p_sys = calloc(1, sizeof(demux_sys_t));
-        if ( !p_sys )
-            return VLC_ENOMEM;
-
-        myVideoDevices = [[AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]
-                           arrayByAddingObjectsFromArray:[AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed]];
-        if ( [myVideoDevices count] == 0 )
-        {
-            vlc_dialog_display_error(p_demux, _("No video devices found"),
-                _("Your Mac does not seem to be equipped with a suitable video input device. "
-                "Please check your connectors and drivers."));
-            msg_Err(p_demux, "Can't find any suitable video device");
-            goto error;
-        }
+    myVideoDevices = [[AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]
+                       arrayByAddingObjectsFromArray:[AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed]];
+    if ( [myVideoDevices count] == 0 )
+    {
+        vlc_dialog_display_error(_demux, _("No video devices found"),
+            _("Your Mac does not seem to be equipped with a suitable video input device. "
+            "Please check your connectors and drivers."));
+        msg_Err(_demux, "Can't find any suitable video device");
+        return nil;
+    }
 
-        deviceCount = [myVideoDevices count];
-        for ( ivideo = 0; ivideo < deviceCount; ivideo++ )
-        {
-            AVCaptureDevice *avf_device;
-            avf_device = [myVideoDevices objectAtIndex:ivideo];
-            msg_Dbg(p_demux, "avcapture %i/%i %s %s", ivideo, deviceCount, [[avf_device modelID] UTF8String], [[avf_device uniqueID] UTF8String]);
-            if ([[[avf_device uniqueID]stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]] isEqualToString:avf_currdevice_uid]) {
-                break;
-            }
+    deviceCount = [myVideoDevices count];
+    for ( ivideo = 0; ivideo < deviceCount; ivideo++ )
+    {
+        AVCaptureDevice *avf_device;
+        avf_device = [myVideoDevices objectAtIndex:ivideo];
+        msg_Dbg(_demux, "avcapture %i/%i %s %s", ivideo, deviceCount, [[avf_device modelID] UTF8String], [[avf_device uniqueID] UTF8String]);
+        if ([[[avf_device uniqueID]stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]] isEqualToString:avf_currdevice_uid]) {
+            break;
         }
+    }
 
-        if ( ivideo < [myVideoDevices count] )
-        {
-            p_sys->device = CFBridgingRetain([myVideoDevices objectAtIndex:ivideo]);
-        }
-        else
-        {
-            msg_Dbg(p_demux, "Cannot find designated device as %s, falling back to default.", [avf_currdevice_uid UTF8String]);
-            p_sys->device = CFBridgingRetain([AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]);
-        }
-        if ( !p_sys->device )
-        {
-            vlc_dialog_display_error(p_demux, _("No video devices found"),
-                _("Your Mac does not seem to be equipped with a suitable input device. "
-                "Please check your connectors and drivers."));
-            msg_Err(p_demux, "Can't find any suitable video device");
-            goto error;
-        }
+    if ( ivideo < [myVideoDevices count] )
+    {
+        _device = [myVideoDevices objectAtIndex:ivideo];
+    }
+    else
+    {
+        msg_Dbg(_demux, "Cannot find designated device as %s, falling back to default.", [avf_currdevice_uid UTF8String]);
+        _device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
+    }
 
-        if ( [(__bridge AVCaptureDevice *)p_sys->device isInUseByAnotherApplication] == YES )
-        {
-            msg_Err(p_demux, "default capture device is exclusively in use by another application");
-            goto error;
-        }
+    if ( !_device )
+    {
+        vlc_dialog_display_error(_demux, _("No video devices found"),
+            _("Your Mac does not seem to be equipped with a suitable input device. "
+            "Please check your connectors and drivers."));
+        msg_Err(_demux, "Can't find any suitable video device");
+        return nil;
+    }
 
-        if (@available(macOS 10.14, *)) {
-            msg_Dbg(p_demux, "Check user consent for access to the video device");
-
-            dispatch_semaphore_t sema = dispatch_semaphore_create(0);
-            __block bool accessGranted = NO;
-            [AVCaptureDevice requestAccessForMediaType: AVMediaTypeVideo completionHandler:^(BOOL granted) {
-                accessGranted = granted;
-                dispatch_semaphore_signal(sema);
-            } ];
-            dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER);
-            dispatch_release(sema);
-            if (!accessGranted) {
-                msg_Err(p_demux, "Can't use the video device as access has not been granted by the user");
-                vlc_dialog_display_error(p_demux, _("Problem accessing a system resource"),
-                    _("Please open \"System Preferences\" -> \"Security & Privacy\" "
-                      "and allow VLC to access your camera."));
-
-                goto error;
-            }
-        }
+    AVCaptureDevice *device = _device;
 
-        input = [AVCaptureDeviceInput deviceInputWithDevice:(__bridge AVCaptureDevice *)p_sys->device error:&o_returnedError];
+    if ( [device isInUseByAnotherApplication] == YES )
+    {
+        msg_Err(_demux, "default capture device is exclusively in use by another application");
+        return nil;
+    }
 
-        if ( !input )
-        {
-            msg_Err(p_demux, "can't create a valid capture input facility: %s (%ld)",[[o_returnedError localizedDescription] UTF8String], [o_returnedError code]);
-            goto error;
+    if (@available(macOS 10.14, *)) {
+        msg_Dbg(_demux, "Check user consent for access to the video device");
+
+        dispatch_semaphore_t sema = dispatch_semaphore_create(0);
+        __block bool accessGranted = NO;
+        [AVCaptureDevice requestAccessForMediaType: AVMediaTypeVideo completionHandler:^(BOOL granted) {
+            accessGranted = granted;
+            dispatch_semaphore_signal(sema);
+        } ];
+        dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER);
+        dispatch_release(sema);
+        if (!accessGranted) {
+            msg_Err(_demux, "Can't use the video device as access has not been granted by the user");
+            vlc_dialog_display_error(_demux, _("Problem accessing a system resource"),
+                _("Please open \"System Preferences\" -> \"Security & Privacy\" "
+                  "and allow VLC to access your camera."));
+
+            return nil;
         }
+    }
 
+    input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&o_returnedError];
 
-        int chroma = VLC_CODEC_RGB32;
+    if ( !input )
+    {
+        msg_Err(_demux, "can't create a valid capture input facility: %s (%ld)",[[o_returnedError localizedDescription] UTF8String], [o_returnedError code]);
+        return nil;
+    }
 
-        memset(&p_sys->fmt, 0, sizeof(es_format_t));
-        es_format_Init(&p_sys->fmt, VIDEO_ES, chroma);
+    int chroma = VLC_CODEC_BGRA;
 
-        p_sys->session = CFBridgingRetain([[AVCaptureSession alloc] init]);
-        [(__bridge AVCaptureSession *)p_sys->session addInput:input];
+    memset(&_fmt, 0, sizeof(es_format_t));
+    es_format_Init(&_fmt, VIDEO_ES, chroma);
 
-        p_sys->output = CFBridgingRetain([[VLCAVDecompressedVideoOutput alloc] initWithDemux:p_demux]);
-        [(__bridge AVCaptureSession *)p_sys->session addOutput:(__bridge VLCAVDecompressedVideoOutput *)p_sys->output];
+    _session = [[AVCaptureSession alloc] init];
 
-        dispatch_queue_t queue = dispatch_queue_create("avCaptureQueue", NULL);
-        [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output setSampleBufferDelegate:(__bridge id)p_sys->output queue:queue];
-        dispatch_release(queue);
+    [_session addInput:input];
 
-        [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_32BGRA] forKey:(id)kCVPixelBufferPixelFormatTypeKey]];
-        [(__bridge AVCaptureSession *)p_sys->session startRunning];
+    _output = [[VLCAVDecompressedVideoOutput alloc] initWithDemux:_demux];
 
-        input = nil;
+    [_session addOutput:_output];
 
-        msg_Dbg(p_demux, "AVCapture: Video device ready!");
+    dispatch_queue_t queue = dispatch_queue_create("avCaptureQueue", NULL);
+    [_output setSampleBufferDelegate:_output queue:queue];
+    dispatch_release(queue);
 
-        return VLC_SUCCESS;
-    error:
-        msg_Err(p_demux, "Error");
-        input = nil;
+    [_output setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_32BGRA] forKey:(id)kCVPixelBufferPixelFormatTypeKey]];
+    [_session startRunning];
 
-        free(p_sys);
+    input = nil;
 
-        return VLC_EGENERIC;
-    }
-}
+    msg_Dbg(_demux, "AVCapture: Video device ready!");
 
-/*****************************************************************************
-* Close:
-*****************************************************************************/
-static void Close(vlc_object_t *p_this)
-{
-    demux_t             *p_demux = (demux_t*)p_this;
-    demux_sys_t         *p_sys = p_demux->p_sys;
-
-    @autoreleasepool {
-        msg_Dbg(p_demux,"Close AVCapture");
-
-        // Perform this on main thread, as the framework itself will sometimes try to synchronously
-        // work on main thread. And this will create a dead lock.
-        [(__bridge AVCaptureSession *)p_sys->session performSelectorOnMainThread:@selector(stopRunning) withObject:nil waitUntilDone:NO];
-        CFBridgingRelease(p_sys->output);
-        CFBridgingRelease(p_sys->session);
-
-        free(p_sys);
-    }
+    return self;
 }
 
-/*****************************************************************************
-* Demux:
-*****************************************************************************/
-static int Demux(demux_t *p_demux)
+- (int)demux
 {
-    demux_sys_t *p_sys = p_demux->p_sys;
     block_t     *p_block;
 
     @autoreleasepool {
-        @synchronized ( p_sys->output )
+        @synchronized(_output)
         {
-            p_block = block_Alloc([(__bridge VLCAVDecompressedVideoOutput *)p_sys->output width] * [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output bytesPerRow]);
+            p_block = block_Alloc([_output width] * [_output bytesPerRow]);
 
             if ( !p_block )
             {
-                msg_Err(p_demux, "cannot get block");
+                msg_Err(_demux, "cannot get block");
                 return 0;
             }
 
-            p_block->i_pts = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output copyCurrentFrameToBuffer: p_block->p_buffer];
+            p_block->i_pts = [_output copyCurrentFrameToBuffer: p_block->p_buffer];
 
             if ( !p_block->i_pts )
             {
@@ -442,54 +491,41 @@ static int Demux(demux_t *p_demux)
                 vlc_tick_sleep(VLC_HARD_MIN_SLEEP);
                 return 1;
             }
-            else if ( !p_sys->b_es_setup )
+            else if ( !_es_video )
             {
-                p_sys->fmt.video.i_frame_rate_base = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output timeScale];
-                msg_Dbg(p_demux, "using frame rate base: %i", p_sys->fmt.video.i_frame_rate_base);
-                p_sys->width = p_sys->fmt.video.i_width = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output width];
-                p_sys->height = p_sys->fmt.video.i_height = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output height];
-                p_sys->p_es_video = es_out_Add(p_demux->out, &p_sys->fmt);
-                msg_Dbg(p_demux, "added new video es %4.4s %dx%d", (char*)&p_sys->fmt.i_codec, p_sys->width, p_sys->height);
-                p_sys->b_es_setup = YES;
+                _fmt.video.i_frame_rate = 1;
+                _fmt.video.i_frame_rate_base = [_output timeScale];
+                msg_Dbg(_demux, "using frame rate base: %i", _fmt.video.i_frame_rate_base);
+                _width
+                    = _fmt.video.i_width
+                    = _fmt.video.i_visible_width
+                    = [_output width];
+                _height
+                    = _fmt.video.i_height
+                    = _fmt.video.i_visible_height
+                    = [_output height];
+                _fmt.video.i_chroma = _fmt.i_codec;
+
+                _es_video = es_out_Add(_demux->out, &_fmt);
+                video_format_Print(&_demux->obj, "added new video es", &_fmt.video);
             }
         }
-        
-        es_out_SetPCR(p_demux->out, p_block->i_pts);
-        es_out_Send(p_demux->out, p_sys->p_es_video, p_block);
-        
+
+        es_out_SetPCR(_demux->out, p_block->i_pts);
+        es_out_Send(_demux->out, _es_video, p_block);
     }
     return 1;
 }
 
-/*****************************************************************************
-* Control:
-*****************************************************************************/
-static int Control(demux_t *p_demux, int i_query, va_list args)
+- (vlc_tick_t)pts
 {
-    bool        *pb;
-
-    switch( i_query )
-    {
-        /* Special for access_demux */
-        case DEMUX_CAN_PAUSE:
-        case DEMUX_CAN_SEEK:
-        case DEMUX_SET_PAUSE_STATE:
-        case DEMUX_CAN_CONTROL_PACE:
-           pb = va_arg(args, bool *);
-           *pb = false;
-           return VLC_SUCCESS;
-
-        case DEMUX_GET_PTS_DELAY:
-           *va_arg(args, vlc_tick_t *) =
-               VLC_TICK_FROM_MS(var_InheritInteger(p_demux, "live-caching"));
-           return VLC_SUCCESS;
-
-        case DEMUX_GET_TIME:
-            *va_arg(args, vlc_tick_t *) = vlc_tick_now();
-            return VLC_SUCCESS;
+    return [_output currentPts];
+}
 
-        default:
-           return VLC_EGENERIC;
-    }
-    return VLC_EGENERIC;
+- (void)dealloc
+{
+    // Perform this on main thread, as the framework itself will sometimes try to synchronously
+    // work on main thread. And this will create a dead lock.
+    [_session performSelectorOnMainThread:@selector(stopRunning) withObject:nil waitUntilDone:NO];
 }
+@end