From 1606d0115800b99609e7a7585a8bd9ac8a857b48 Mon Sep 17 00:00:00 2001 From: cnhenry Date: Thu, 20 Aug 2015 14:23:01 -0500 Subject: [PATCH] Implemented video calls for OSX --- cfg/systems/Darwin.mk | 2 +- src/audio_call.c | 1 + src/osx_video.h | 14 ++-- src/osx_video.m | 151 ++++++++++++++++++++++++++++-------------- src/video_device.c | 74 +++++++++++---------- 5 files changed, 151 insertions(+), 91 deletions(-) diff --git a/cfg/systems/Darwin.mk b/cfg/systems/Darwin.mk index bb28f26..d228b6a 100644 --- a/cfg/systems/Darwin.mk +++ b/cfg/systems/Darwin.mk @@ -11,7 +11,7 @@ CFLAGS += -I/usr/local/opt/freealut/include/AL -I/usr/local/opt/glib/include/gli OSX_LIBRARIES = -lobjc -lresolv OSX_FRAMEWORKS = -framework Foundation -framework CoreFoundation -framework AVFoundation \ - -framework QuartzCore + -framework QuartzCore -framework CoreMedia OSX_VIDEO = osx_video.m LDFLAGS += $(OSX_LIBRARIES) $(OSX_FRAMEWORKS) diff --git a/src/audio_call.c b/src/audio_call.c index e207b7a..8cd6d9b 100644 --- a/src/audio_call.c +++ b/src/audio_call.c @@ -408,6 +408,7 @@ void callback_peer_timeout(uint32_t friend_number) CB_BODY(friend_number, onPeerTimeout); callback_video_end(friend_number); + callback_recv_video_end(friend_number); stop_transmission(&CallControl.calls[friend_number], friend_number); /* Call is stopped manually since there might be some other * actions that one can possibly take on timeout diff --git a/src/osx_video.h b/src/osx_video.h index 09d0b81..fce78fe 100644 --- a/src/osx_video.h +++ b/src/osx_video.h @@ -25,26 +25,30 @@ #include -void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t *rgb, uint16_t width, uint16_t height); - #ifdef __OBJC__ #import #import +#endif /* __OBJC__ */ +#define RELEASE_CHK(func, obj) if ((obj))\ + func((obj)); + +void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t *rgb, uint16_t width, uint16_t height); + +#ifdef __OBJC__ @interface OSXVideo : NSObject - (instancetype)initWithDeviceNames:(char **)device_names AmtDevices:(int *)size; @end - #endif /* __OBJC__ */ int osx_video_init(char **device_names, int *size); void osx_video_release(); /* Start device */ -int osx_video_open_device(uint32_t device_idx, uint16_t *width, uint16_t *height); +int osx_video_open_device(uint32_t selection, uint16_t *width, uint16_t *height); /* Stop device */ void osx_video_close_device(uint32_t device_idx); /* Read data from device */ -void osx_video_read_device(uint8_t *y, uint8_t *u, uint8_t *v, uint16_t *width, uint16_t *height); +int osx_video_read_device(uint8_t *y, uint8_t *u, uint8_t *v, uint16_t *width, uint16_t *height); #endif /* OSX_VIDEO_H */ \ No newline at end of file diff --git a/src/osx_video.m b/src/osx_video.m index f183c18..befbc4c 100644 --- a/src/osx_video.m +++ b/src/osx_video.m @@ -59,7 +59,7 @@ static uint8_t rgb_to_v(int r, int g, int b) return v>255? 255 : v<0 ? 0 : v; } -void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t *rgb, uint16_t width, uint16_t height) +void bgrxtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t *rgb, uint16_t width, uint16_t height) { uint16_t x, y; uint8_t *p; @@ -71,6 +71,8 @@ void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t * b = *rgb++; g = *rgb++; r = *rgb++; + rgb++; + *plane_y++ = rgb_to_y(r, g, b); } @@ -78,21 +80,26 @@ void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t * b = *rgb++; g = *rgb++; r = *rgb++; + rgb++; + *plane_y++ = rgb_to_y(r, g, b); b = *rgb++; g = *rgb++; r = *rgb++; + rgb++; + *plane_y++ = rgb_to_y(r, g, b); - b = ((int)b + (int)*(rgb - 6) + (int)*p + (int)*(p + 3) + 2) / 4; p++; - g = ((int)g + (int)*(rgb - 5) + (int)*p + (int)*(p + 3) + 2) / 4; p++; - r = ((int)r + (int)*(rgb - 4) + (int)*p + (int)*(p + 3) + 2) / 4; p++; + b = ((int)b + (int)*(rgb - 8) + (int)*p + (int)*(p + 4) + 2) / 4; p++; + g = ((int)g + (int)*(rgb - 7) + (int)*p + (int)*(p + 4) + 2) / 4; p++; + r = ((int)r + (int)*(rgb - 6) + (int)*p + (int)*(p + 4) + 2) / 4; p++; + p++; *plane_u++ = rgb_to_u(r, g, b); *plane_v++ = rgb_to_v(r, g, b); - p += 3; + p += 4; } } } @@ -112,21 +119,23 @@ void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t * CVImageBufferRef _currentFrame; pthread_mutex_t _frameLock; + BOOL _shouldMangleDimensions; } - (instancetype)initWithDeviceNames: (char **)device_names AmtDevices: (int *)size { _session = [[AVCaptureSession alloc] init]; NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; - int i = 0; - for (AVCaptureDevice *device in devices) { + int i; + for (i = 0; i < [devices count]; ++i) { + AVCaptureDevice *device = [devices objectAtIndex:i]; char *video_input_name; NSString *localizedName = [device localizedName]; video_input_name = (char*)malloc(strlen([localizedName cStringUsingEncoding:NSUTF8StringEncoding]) + 1); strcpy(video_input_name, (char*)[localizedName cStringUsingEncoding:NSUTF8StringEncoding]); device_names[i] = video_input_name; - ++i; } + if ( i <= 0 ) return nil; *size = i; @@ -134,80 +143,115 @@ void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t * return self; } -- (void)release { +- (void)dealloc { + pthread_mutex_destroy(&_frameLock); [_session release]; - [super release]; + [_linkerVideo release]; + dispatch_release(_processingQueue); + [super dealloc]; } - (int)openVideoDeviceIndex: (uint32_t)device_idx Width: (uint16_t *)width Height: (uint16_t *)height { pthread_mutex_init(&_frameLock, NULL); + pthread_mutex_lock(&_frameLock); _processingQueue = dispatch_queue_create("Toxic processing queue", DISPATCH_QUEUE_SERIAL); NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; - int i = 0; - for (AVCaptureDevice *device in devices) { - if ( i == device_idx ) { - NSError *error = NULL; - AVCaptureInput *input = [[AVCaptureDeviceInput alloc] initWithDevice:device error:&error]; + AVCaptureDevice *device = [devices objectAtIndex:device_idx]; + NSError *error = NULL; + AVCaptureInput *input = [[AVCaptureDeviceInput alloc] initWithDevice:device error:&error]; - if ( error != NULL ) - return -1; + if ( error != NULL ) { + [input release]; + return -1; + } - /* Obtain device resolution */ - /*AVCaptureInputPort *port = [input.ports objectAtIndex:0]; - CMFormatDescriptionRef format_description = port.formatDescription; - if ( format_description ) { - CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format_description); - *width = dimensions.width; - *height = dimensions.height; - } else { - *width = 0; - *height = 0; - }*/ + [_session beginConfiguration]; + [_session addInput:input]; + //_session.sessionPreset = AVCaptureSessionPreset640x480; + //*width = 640; + //*height = 480; + _shouldMangleDimensions = YES; + [_session commitConfiguration]; + [input release]; + [device release]; - [_session beginConfiguration]; - [_session addInput:input]; - //session.sessionPreset = AVCaptureSessionPreset640x480; - [_session commitConfiguration]; - [input release]; - - break; - } else { - ++i; - } + /* Obtain device resolution */ + AVCaptureInputPort *port = [input.ports objectAtIndex:0]; + CMFormatDescriptionRef format_description = port.formatDescription; + if ( format_description ) { + CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format_description); + *width = dimensions.width; + *height = dimensions.height; + } else { + *width = 0; + *height = 0; } _linkerVideo = [[AVCaptureVideoDataOutput alloc] init]; [_linkerVideo setSampleBufferDelegate:self queue:_processingQueue]; // TODO possibly get a better pixel format - [_linkerVideo setVideoSettings:@{(id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)}]; + if (_shouldMangleDimensions) { + [_linkerVideo setVideoSettings:@{(id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), + (id)kCVPixelBufferWidthKey: @640, + (id)kCVPixelBufferHeightKey: @480}]; + } else { + [_linkerVideo setVideoSettings:@{(id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)}]; + } [_session addOutput:_linkerVideo]; - [_session startRunning]; + pthread_mutex_unlock(&_frameLock); return 0; } - (void)closeVideoDeviceIndex: (uint32_t)device_idx { + NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; + AVCaptureDevice *device = [devices objectAtIndex:device_idx]; + NSError *error = NULL; + AVCaptureInput *input = [[AVCaptureDeviceInput alloc] initWithDevice:device error:&error]; [_session stopRunning]; + [_session removeOutput:_linkerVideo]; + [_session removeInput:input]; [_linkerVideo release]; } -- (void)getVideoFrameY: (uint8_t *)y U: (uint8_t *)u V: (uint8_t *)v Width: (uint16_t *)width Height: (uint16_t *)height { - /*CVImageBufferRef currentFrame = NULL; +- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { + pthread_mutex_lock(&_frameLock); + CVImageBufferRef img = CMSampleBufferGetImageBuffer(sampleBuffer); + if (!img) { + NSLog(@"Toxic WARNING: Bad sampleBuffer from AVfoundation!"); + } else { + CVPixelBufferUnlockBaseAddress(_currentFrame, kCVPixelBufferLock_ReadOnly); + RELEASE_CHK(CFRelease, _currentFrame); - CFRetain(currentFrame); + _currentFrame = (CVImageBufferRef)CFRetain(img); + // we're not going to do anything to it, so it's safe to lock it always + CVPixelBufferLockBaseAddress(_currentFrame, kCVPixelBufferLock_ReadOnly); + } + pthread_mutex_unlock(&_frameLock); +} - CFTypeID imageType = CFGetTypeID(currentFrame); +- (int)getVideoFrameY: (uint8_t *)y U: (uint8_t *)u V: (uint8_t *)v Width: (uint16_t *)width Height: (uint16_t *)height { + if (!_currentFrame) { + return -1; + } + + pthread_mutex_lock(&_frameLock); + CFRetain(_currentFrame); + + CFTypeID imageType = CFGetTypeID(_currentFrame); if (imageType == CVPixelBufferGetTypeID()) { // TODO maybe handle other formats - bgrxtoyuv420(y, u, v, CVPixelBufferGetBaseAddress(currentFrame), width, height); + bgrxtoyuv420(y, u, v, CVPixelBufferGetBaseAddress(_currentFrame), *width, *height); } else if (imageType == CVOpenGLBufferGetTypeID()) { // OpenGL pbuffer } else if (imageType == CVOpenGLTextureGetTypeID()) { // OpenGL Texture (Do we need to handle these?) } - CVPixelBufferRelease(currentFrame);*/ + CVPixelBufferRelease(_currentFrame); + pthread_mutex_unlock(&_frameLock); + return 0; } @end @@ -234,11 +278,15 @@ int osx_video_init(char **device_names, int *size) void osx_video_release() { [_OSXVideo release]; + _OSXVideo = nil; } -int osx_video_open_device(uint32_t device_idx, uint16_t *width, uint16_t *height) +int osx_video_open_device(uint32_t selection, uint16_t *width, uint16_t *height) { - return [_OSXVideo openVideoDeviceIndex: device_idx Width: width Height: height]; + if ( _OSXVideo == nil ) + return -1; + + return [_OSXVideo openVideoDeviceIndex: selection Width: width Height: height]; } void osx_video_close_device(uint32_t device_idx) @@ -246,9 +294,12 @@ void osx_video_close_device(uint32_t device_idx) [_OSXVideo closeVideoDeviceIndex: device_idx]; } -void osx_video_read_device(uint8_t *y, uint8_t *u, uint8_t *v, uint16_t *width, uint16_t *height) +int osx_video_read_device(uint8_t *y, uint8_t *u, uint8_t *v, uint16_t *width, uint16_t *height) { - [_OSXVideo getVideoFrameY: y U: u V: v Width: width Height: height]; + if ( _OSXVideo == nil ) + return -1; + + return [_OSXVideo getVideoFrameY: y U: u V: v Width: width Height: height]; } /* * End of C-interface for OSXVideo diff --git a/src/video_device.c b/src/video_device.c index 600a470..c0217d4 100644 --- a/src/video_device.c +++ b/src/video_device.c @@ -71,8 +71,6 @@ typedef struct VideoDevice { struct v4l2_format fmt; struct VideoBuffer *buffers; uint32_t n_buffers; -#else /* __OSX__ */ - #endif uint32_t ref_count; @@ -109,17 +107,6 @@ bool video_thread_running = true, void* video_thread_poll(void*); -static int xioctl(int fh, unsigned long request, void *arg) -{ - int r; - - do { - r = ioctl(fh, request, arg); - } while (-1 == r && EINTR == errno); - - return r; -} - static void yuv420tobgr(uint16_t width, uint16_t height, const uint8_t *y, const uint8_t *u, const uint8_t *v, unsigned int ystride, unsigned int ustride, unsigned int vstride, uint8_t *out) @@ -145,6 +132,7 @@ static void yuv420tobgr(uint16_t width, uint16_t height, const uint8_t *y, } } +#ifdef __linux__ static void yuv422to420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t *input, uint16_t width, uint16_t height) { @@ -168,6 +156,19 @@ static void yuv422to420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, } } +static int xioctl(int fh, unsigned long request, void *arg) +{ + int r; + + do { + r = ioctl(fh, request, arg); + } while (-1 == r && EINTR == errno); + + return r; +} + +#endif /* __linux__ */ + /* Meet devices */ #ifdef VIDEO VideoDeviceError init_video_devices(ToxAV* av_) @@ -210,11 +211,10 @@ VideoDeviceError init_video_devices() } #else /* __OSX__ */ - if( osx_video_init(video_devices_names[vdt_input], &size[vdt_input]) != 0 ) + if( osx_video_init((char**)video_devices_names[vdt_input], &size[vdt_input]) != 0 ) return vde_InternalError; #endif - size[vdt_output] = 1; char* video_output_name = "Toxic Video Receiver"; video_devices_names[vdt_output][0] = video_output_name; @@ -249,7 +249,7 @@ VideoDeviceError terminate_video_devices() return (VideoDeviceError) vde_InternalError; #ifdef __OSX__ - void osx_video_release(); + osx_video_release(); #endif /* __OSX__ */ return (VideoDeviceError) vde_None; @@ -449,7 +449,11 @@ VideoDeviceError open_video_device(VideoDeviceType type, int32_t selection, uint } #else /* __OSX__ */ - osx_video_open_device(*device_idx, device->video_width, device->video_height); + if ( osx_video_open_device(selection, &device->video_width, &device->video_height) != 0 ) { + free(device); + + return vde_FailedStart; + } #endif /* Create X11 window associated to device */ @@ -612,10 +616,11 @@ void* video_thread_poll (void* arg) // TODO: maybe use thread for every input so { /* Obtain frame image data from device buffers */ VideoDevice* device = video_devices_running[vdt_input][i]; - void *data; - uint16_t video_width; - uint16_t video_height; - uint8_t *y, *u, *v; + uint16_t video_width = device->video_width; + uint16_t video_height = device->video_height; + uint8_t *y = device->input.planes[0]; + uint8_t *u = device->input.planes[1]; + uint8_t *v = device->input.planes[2]; #ifdef __linux__ struct v4l2_buffer buf; @@ -629,19 +634,18 @@ void* video_thread_poll (void* arg) // TODO: maybe use thread for every input so continue; } - data = (void*)device->buffers[buf.index].start; - video_width = device->video_width; - video_height = device->video_height; - y = device->input.planes[0]; - u = device->input.planes[1]; - v = device->input.planes[2]; -#else /* __OSX__*/ - -#endif + void *data = (void*)device->buffers[buf.index].start; /* Convert frame image data to YUV420 for ToxAV */ yuv422to420(y, u, v, data, video_width, video_height); +#else /* __OSX__*/ + if ( osx_video_read_device(y, u, v, &video_width, &video_height) != 0 ) { + unlock; + continue; + } +#endif + /* Send frame data to friend through ToxAV */ if ( device->cb ) device->cb(video_width, video_height, y, u, v, device->cb_data); @@ -681,8 +685,8 @@ void* video_thread_poll (void* arg) // TODO: maybe use thread for every input so unlock; continue; } -#else /* __OSX__ */ -#endif +#endif /* __linux__ */ + } unlock; } @@ -725,7 +729,7 @@ VideoDeviceError close_video_device(VideoDeviceType type, uint32_t device_idx) #else /* __OSX__ */ osx_video_close_device(device_idx); #endif - + vpx_img_free(&device->input); XDestroyWindow(device->x_display, device->x_window); XFlush(device->x_display); XCloseDisplay(device->x_display); @@ -733,8 +737,8 @@ VideoDeviceError close_video_device(VideoDeviceType type, uint32_t device_idx) #ifdef __linux__ free(device->buffers); -#else /* __OSX__ */ -#endif +#endif /* __linux__ */ + free(device); } else { vpx_img_free(&device->input);