1
0
mirror of https://github.com/Tha14/toxic.git synced 2024-12-23 04:03:25 +01:00

Implemented video calls for OSX

This commit is contained in:
cnhenry 2015-08-20 14:23:01 -05:00
parent c8a9ac21f3
commit 1606d01158
5 changed files with 151 additions and 91 deletions

View File

@ -11,7 +11,7 @@ CFLAGS += -I/usr/local/opt/freealut/include/AL -I/usr/local/opt/glib/include/gli
OSX_LIBRARIES = -lobjc -lresolv
OSX_FRAMEWORKS = -framework Foundation -framework CoreFoundation -framework AVFoundation \
-framework QuartzCore
-framework QuartzCore -framework CoreMedia
OSX_VIDEO = osx_video.m
LDFLAGS += $(OSX_LIBRARIES) $(OSX_FRAMEWORKS)

View File

@ -408,6 +408,7 @@ void callback_peer_timeout(uint32_t friend_number)
CB_BODY(friend_number, onPeerTimeout);
callback_video_end(friend_number);
callback_recv_video_end(friend_number);
stop_transmission(&CallControl.calls[friend_number], friend_number);
/* Call is stopped manually since there might be some other
* actions that one can possibly take on timeout

View File

@ -25,26 +25,30 @@
#include <netinet/in.h>
void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t *rgb, uint16_t width, uint16_t height);
#ifdef __OBJC__
#import <Foundation/Foundation.h>
#import <AVFoundation/AVFoundation.h>
#endif /* __OBJC__ */
#define RELEASE_CHK(func, obj) if ((obj))\
func((obj));
void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t *rgb, uint16_t width, uint16_t height);
#ifdef __OBJC__
@interface OSXVideo : NSObject <AVCaptureVideoDataOutputSampleBufferDelegate>
- (instancetype)initWithDeviceNames:(char **)device_names AmtDevices:(int *)size;
@end
#endif /* __OBJC__ */
int osx_video_init(char **device_names, int *size);
void osx_video_release();
/* Start device */
int osx_video_open_device(uint32_t device_idx, uint16_t *width, uint16_t *height);
int osx_video_open_device(uint32_t selection, uint16_t *width, uint16_t *height);
/* Stop device */
void osx_video_close_device(uint32_t device_idx);
/* Read data from device */
void osx_video_read_device(uint8_t *y, uint8_t *u, uint8_t *v, uint16_t *width, uint16_t *height);
int osx_video_read_device(uint8_t *y, uint8_t *u, uint8_t *v, uint16_t *width, uint16_t *height);
#endif /* OSX_VIDEO_H */

View File

@ -59,7 +59,7 @@ static uint8_t rgb_to_v(int r, int g, int b)
return v>255? 255 : v<0 ? 0 : v;
}
void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t *rgb, uint16_t width, uint16_t height)
void bgrxtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t *rgb, uint16_t width, uint16_t height)
{
uint16_t x, y;
uint8_t *p;
@ -71,6 +71,8 @@ void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t *
b = *rgb++;
g = *rgb++;
r = *rgb++;
rgb++;
*plane_y++ = rgb_to_y(r, g, b);
}
@ -78,21 +80,26 @@ void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t *
b = *rgb++;
g = *rgb++;
r = *rgb++;
rgb++;
*plane_y++ = rgb_to_y(r, g, b);
b = *rgb++;
g = *rgb++;
r = *rgb++;
rgb++;
*plane_y++ = rgb_to_y(r, g, b);
b = ((int)b + (int)*(rgb - 6) + (int)*p + (int)*(p + 3) + 2) / 4; p++;
g = ((int)g + (int)*(rgb - 5) + (int)*p + (int)*(p + 3) + 2) / 4; p++;
r = ((int)r + (int)*(rgb - 4) + (int)*p + (int)*(p + 3) + 2) / 4; p++;
b = ((int)b + (int)*(rgb - 8) + (int)*p + (int)*(p + 4) + 2) / 4; p++;
g = ((int)g + (int)*(rgb - 7) + (int)*p + (int)*(p + 4) + 2) / 4; p++;
r = ((int)r + (int)*(rgb - 6) + (int)*p + (int)*(p + 4) + 2) / 4; p++;
p++;
*plane_u++ = rgb_to_u(r, g, b);
*plane_v++ = rgb_to_v(r, g, b);
p += 3;
p += 4;
}
}
}
@ -112,21 +119,23 @@ void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t *
CVImageBufferRef _currentFrame;
pthread_mutex_t _frameLock;
BOOL _shouldMangleDimensions;
}
- (instancetype)initWithDeviceNames: (char **)device_names AmtDevices: (int *)size {
_session = [[AVCaptureSession alloc] init];
NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
int i = 0;
for (AVCaptureDevice *device in devices) {
int i;
for (i = 0; i < [devices count]; ++i) {
AVCaptureDevice *device = [devices objectAtIndex:i];
char *video_input_name;
NSString *localizedName = [device localizedName];
video_input_name = (char*)malloc(strlen([localizedName cStringUsingEncoding:NSUTF8StringEncoding]) + 1);
strcpy(video_input_name, (char*)[localizedName cStringUsingEncoding:NSUTF8StringEncoding]);
device_names[i] = video_input_name;
++i;
}
if ( i <= 0 )
return nil;
*size = i;
@ -134,80 +143,115 @@ void bgrtoyuv420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v, uint8_t *
return self;
}
- (void)release {
- (void)dealloc {
pthread_mutex_destroy(&_frameLock);
[_session release];
[super release];
[_linkerVideo release];
dispatch_release(_processingQueue);
[super dealloc];
}
- (int)openVideoDeviceIndex: (uint32_t)device_idx Width: (uint16_t *)width Height: (uint16_t *)height {
pthread_mutex_init(&_frameLock, NULL);
pthread_mutex_lock(&_frameLock);
_processingQueue = dispatch_queue_create("Toxic processing queue", DISPATCH_QUEUE_SERIAL);
NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
int i = 0;
for (AVCaptureDevice *device in devices) {
if ( i == device_idx ) {
NSError *error = NULL;
AVCaptureInput *input = [[AVCaptureDeviceInput alloc] initWithDevice:device error:&error];
AVCaptureDevice *device = [devices objectAtIndex:device_idx];
NSError *error = NULL;
AVCaptureInput *input = [[AVCaptureDeviceInput alloc] initWithDevice:device error:&error];
if ( error != NULL )
return -1;
if ( error != NULL ) {
[input release];
return -1;
}
/* Obtain device resolution */
/*AVCaptureInputPort *port = [input.ports objectAtIndex:0];
CMFormatDescriptionRef format_description = port.formatDescription;
if ( format_description ) {
CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format_description);
*width = dimensions.width;
*height = dimensions.height;
} else {
*width = 0;
*height = 0;
}*/
[_session beginConfiguration];
[_session addInput:input];
//_session.sessionPreset = AVCaptureSessionPreset640x480;
//*width = 640;
//*height = 480;
_shouldMangleDimensions = YES;
[_session commitConfiguration];
[input release];
[device release];
[_session beginConfiguration];
[_session addInput:input];
//session.sessionPreset = AVCaptureSessionPreset640x480;
[_session commitConfiguration];
[input release];
break;
} else {
++i;
}
/* Obtain device resolution */
AVCaptureInputPort *port = [input.ports objectAtIndex:0];
CMFormatDescriptionRef format_description = port.formatDescription;
if ( format_description ) {
CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format_description);
*width = dimensions.width;
*height = dimensions.height;
} else {
*width = 0;
*height = 0;
}
_linkerVideo = [[AVCaptureVideoDataOutput alloc] init];
[_linkerVideo setSampleBufferDelegate:self queue:_processingQueue];
// TODO possibly get a better pixel format
[_linkerVideo setVideoSettings:@{(id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)}];
if (_shouldMangleDimensions) {
[_linkerVideo setVideoSettings:@{(id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA),
(id)kCVPixelBufferWidthKey: @640,
(id)kCVPixelBufferHeightKey: @480}];
} else {
[_linkerVideo setVideoSettings:@{(id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)}];
}
[_session addOutput:_linkerVideo];
[_session startRunning];
pthread_mutex_unlock(&_frameLock);
return 0;
}
- (void)closeVideoDeviceIndex: (uint32_t)device_idx {
NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
AVCaptureDevice *device = [devices objectAtIndex:device_idx];
NSError *error = NULL;
AVCaptureInput *input = [[AVCaptureDeviceInput alloc] initWithDevice:device error:&error];
[_session stopRunning];
[_session removeOutput:_linkerVideo];
[_session removeInput:input];
[_linkerVideo release];
}
- (void)getVideoFrameY: (uint8_t *)y U: (uint8_t *)u V: (uint8_t *)v Width: (uint16_t *)width Height: (uint16_t *)height {
/*CVImageBufferRef currentFrame = NULL;
- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection {
pthread_mutex_lock(&_frameLock);
CVImageBufferRef img = CMSampleBufferGetImageBuffer(sampleBuffer);
if (!img) {
NSLog(@"Toxic WARNING: Bad sampleBuffer from AVfoundation!");
} else {
CVPixelBufferUnlockBaseAddress(_currentFrame, kCVPixelBufferLock_ReadOnly);
RELEASE_CHK(CFRelease, _currentFrame);
CFRetain(currentFrame);
_currentFrame = (CVImageBufferRef)CFRetain(img);
// we're not going to do anything to it, so it's safe to lock it always
CVPixelBufferLockBaseAddress(_currentFrame, kCVPixelBufferLock_ReadOnly);
}
pthread_mutex_unlock(&_frameLock);
}
CFTypeID imageType = CFGetTypeID(currentFrame);
- (int)getVideoFrameY: (uint8_t *)y U: (uint8_t *)u V: (uint8_t *)v Width: (uint16_t *)width Height: (uint16_t *)height {
if (!_currentFrame) {
return -1;
}
pthread_mutex_lock(&_frameLock);
CFRetain(_currentFrame);
CFTypeID imageType = CFGetTypeID(_currentFrame);
if (imageType == CVPixelBufferGetTypeID()) {
// TODO maybe handle other formats
bgrxtoyuv420(y, u, v, CVPixelBufferGetBaseAddress(currentFrame), width, height);
bgrxtoyuv420(y, u, v, CVPixelBufferGetBaseAddress(_currentFrame), *width, *height);
} else if (imageType == CVOpenGLBufferGetTypeID()) {
// OpenGL pbuffer
} else if (imageType == CVOpenGLTextureGetTypeID()) {
// OpenGL Texture (Do we need to handle these?)
}
CVPixelBufferRelease(currentFrame);*/
CVPixelBufferRelease(_currentFrame);
pthread_mutex_unlock(&_frameLock);
return 0;
}
@end
@ -234,11 +278,15 @@ int osx_video_init(char **device_names, int *size)
void osx_video_release()
{
[_OSXVideo release];
_OSXVideo = nil;
}
int osx_video_open_device(uint32_t device_idx, uint16_t *width, uint16_t *height)
int osx_video_open_device(uint32_t selection, uint16_t *width, uint16_t *height)
{
return [_OSXVideo openVideoDeviceIndex: device_idx Width: width Height: height];
if ( _OSXVideo == nil )
return -1;
return [_OSXVideo openVideoDeviceIndex: selection Width: width Height: height];
}
void osx_video_close_device(uint32_t device_idx)
@ -246,9 +294,12 @@ void osx_video_close_device(uint32_t device_idx)
[_OSXVideo closeVideoDeviceIndex: device_idx];
}
void osx_video_read_device(uint8_t *y, uint8_t *u, uint8_t *v, uint16_t *width, uint16_t *height)
int osx_video_read_device(uint8_t *y, uint8_t *u, uint8_t *v, uint16_t *width, uint16_t *height)
{
[_OSXVideo getVideoFrameY: y U: u V: v Width: width Height: height];
if ( _OSXVideo == nil )
return -1;
return [_OSXVideo getVideoFrameY: y U: u V: v Width: width Height: height];
}
/*
* End of C-interface for OSXVideo

View File

@ -71,8 +71,6 @@ typedef struct VideoDevice {
struct v4l2_format fmt;
struct VideoBuffer *buffers;
uint32_t n_buffers;
#else /* __OSX__ */
#endif
uint32_t ref_count;
@ -109,17 +107,6 @@ bool video_thread_running = true,
void* video_thread_poll(void*);
static int xioctl(int fh, unsigned long request, void *arg)
{
int r;
do {
r = ioctl(fh, request, arg);
} while (-1 == r && EINTR == errno);
return r;
}
static void yuv420tobgr(uint16_t width, uint16_t height, const uint8_t *y,
const uint8_t *u, const uint8_t *v, unsigned int ystride,
unsigned int ustride, unsigned int vstride, uint8_t *out)
@ -145,6 +132,7 @@ static void yuv420tobgr(uint16_t width, uint16_t height, const uint8_t *y,
}
}
#ifdef __linux__
static void yuv422to420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v,
uint8_t *input, uint16_t width, uint16_t height)
{
@ -168,6 +156,19 @@ static void yuv422to420(uint8_t *plane_y, uint8_t *plane_u, uint8_t *plane_v,
}
}
static int xioctl(int fh, unsigned long request, void *arg)
{
int r;
do {
r = ioctl(fh, request, arg);
} while (-1 == r && EINTR == errno);
return r;
}
#endif /* __linux__ */
/* Meet devices */
#ifdef VIDEO
VideoDeviceError init_video_devices(ToxAV* av_)
@ -210,11 +211,10 @@ VideoDeviceError init_video_devices()
}
#else /* __OSX__ */
if( osx_video_init(video_devices_names[vdt_input], &size[vdt_input]) != 0 )
if( osx_video_init((char**)video_devices_names[vdt_input], &size[vdt_input]) != 0 )
return vde_InternalError;
#endif
size[vdt_output] = 1;
char* video_output_name = "Toxic Video Receiver";
video_devices_names[vdt_output][0] = video_output_name;
@ -249,7 +249,7 @@ VideoDeviceError terminate_video_devices()
return (VideoDeviceError) vde_InternalError;
#ifdef __OSX__
void osx_video_release();
osx_video_release();
#endif /* __OSX__ */
return (VideoDeviceError) vde_None;
@ -449,7 +449,11 @@ VideoDeviceError open_video_device(VideoDeviceType type, int32_t selection, uint
}
#else /* __OSX__ */
osx_video_open_device(*device_idx, device->video_width, device->video_height);
if ( osx_video_open_device(selection, &device->video_width, &device->video_height) != 0 ) {
free(device);
return vde_FailedStart;
}
#endif
/* Create X11 window associated to device */
@ -612,10 +616,11 @@ void* video_thread_poll (void* arg) // TODO: maybe use thread for every input so
{
/* Obtain frame image data from device buffers */
VideoDevice* device = video_devices_running[vdt_input][i];
void *data;
uint16_t video_width;
uint16_t video_height;
uint8_t *y, *u, *v;
uint16_t video_width = device->video_width;
uint16_t video_height = device->video_height;
uint8_t *y = device->input.planes[0];
uint8_t *u = device->input.planes[1];
uint8_t *v = device->input.planes[2];
#ifdef __linux__
struct v4l2_buffer buf;
@ -629,19 +634,18 @@ void* video_thread_poll (void* arg) // TODO: maybe use thread for every input so
continue;
}
data = (void*)device->buffers[buf.index].start;
video_width = device->video_width;
video_height = device->video_height;
y = device->input.planes[0];
u = device->input.planes[1];
v = device->input.planes[2];
#else /* __OSX__*/
#endif
void *data = (void*)device->buffers[buf.index].start;
/* Convert frame image data to YUV420 for ToxAV */
yuv422to420(y, u, v, data, video_width, video_height);
#else /* __OSX__*/
if ( osx_video_read_device(y, u, v, &video_width, &video_height) != 0 ) {
unlock;
continue;
}
#endif
/* Send frame data to friend through ToxAV */
if ( device->cb )
device->cb(video_width, video_height, y, u, v, device->cb_data);
@ -681,8 +685,8 @@ void* video_thread_poll (void* arg) // TODO: maybe use thread for every input so
unlock;
continue;
}
#else /* __OSX__ */
#endif
#endif /* __linux__ */
}
unlock;
}
@ -725,7 +729,7 @@ VideoDeviceError close_video_device(VideoDeviceType type, uint32_t device_idx)
#else /* __OSX__ */
osx_video_close_device(device_idx);
#endif
vpx_img_free(&device->input);
XDestroyWindow(device->x_display, device->x_window);
XFlush(device->x_display);
XCloseDisplay(device->x_display);
@ -733,8 +737,8 @@ VideoDeviceError close_video_device(VideoDeviceType type, uint32_t device_idx)
#ifdef __linux__
free(device->buffers);
#else /* __OSX__ */
#endif
#endif /* __linux__ */
free(device);
} else {
vpx_img_free(&device->input);