282 lines
10 KiB
Python
282 lines
10 KiB
Python
import pyaudio
|
|
import time
|
|
import threading
|
|
from wrapper.toxav_enums import *
|
|
import cv2
|
|
import itertools
|
|
import numpy as np
|
|
from av import screen_sharing
|
|
from av.call import Call
|
|
import common.tox_save
|
|
|
|
|
|
class AV(common.tox_save.ToxAvSave):
|
|
|
|
def __init__(self, toxav, settings):
|
|
super().__init__(toxav)
|
|
self._settings = settings
|
|
self._running = True
|
|
|
|
self._calls = {} # dict: key - friend number, value - Call instance
|
|
|
|
self._audio = None
|
|
self._audio_stream = None
|
|
self._audio_thread = None
|
|
self._audio_running = False
|
|
self._out_stream = None
|
|
|
|
self._audio_rate = 8000
|
|
self._audio_channels = 1
|
|
self._audio_duration = 60
|
|
self._audio_sample_count = self._audio_rate * self._audio_channels * self._audio_duration // 1000
|
|
|
|
self._video = None
|
|
self._video_thread = None
|
|
self._video_running = False
|
|
|
|
self._video_width = 640
|
|
self._video_height = 480
|
|
|
|
def stop(self):
|
|
self._running = False
|
|
self.stop_audio_thread()
|
|
self.stop_video_thread()
|
|
|
|
def __contains__(self, friend_number):
|
|
return friend_number in self._calls
|
|
|
|
# -----------------------------------------------------------------------------------------------------------------
|
|
# Calls
|
|
# -----------------------------------------------------------------------------------------------------------------
|
|
|
|
def __call__(self, friend_number, audio, video):
|
|
"""Call friend with specified number"""
|
|
self._toxav.call(friend_number, 32 if audio else 0, 5000 if video else 0)
|
|
self._calls[friend_number] = Call(audio, video)
|
|
threading.Timer(30.0, lambda: self.finish_not_started_call(friend_number)).start()
|
|
|
|
def accept_call(self, friend_number, audio_enabled, video_enabled):
|
|
if self._running:
|
|
self._calls[friend_number] = Call(audio_enabled, video_enabled)
|
|
self._toxav.answer(friend_number, 32 if audio_enabled else 0, 5000 if video_enabled else 0)
|
|
if audio_enabled:
|
|
self.start_audio_thread()
|
|
if video_enabled:
|
|
self.start_video_thread()
|
|
|
|
def finish_call(self, friend_number, by_friend=False):
|
|
if not by_friend:
|
|
self._toxav.call_control(friend_number, TOXAV_CALL_CONTROL['CANCEL'])
|
|
if friend_number in self._calls:
|
|
del self._calls[friend_number]
|
|
if not len(list(filter(lambda c: c.out_audio, self._calls))):
|
|
self.stop_audio_thread()
|
|
if not len(list(filter(lambda c: c.out_video, self._calls))):
|
|
self.stop_video_thread()
|
|
|
|
def finish_not_started_call(self, friend_number):
|
|
if friend_number in self:
|
|
call = self._calls[friend_number]
|
|
if not call.is_active:
|
|
self.finish_call(friend_number)
|
|
|
|
def toxav_call_state_cb(self, friend_number, state):
|
|
"""
|
|
New call state
|
|
"""
|
|
call = self._calls[friend_number]
|
|
call.is_active = True
|
|
|
|
call.in_audio = state | TOXAV_FRIEND_CALL_STATE['SENDING_A'] > 0
|
|
call.in_video = state | TOXAV_FRIEND_CALL_STATE['SENDING_V'] > 0
|
|
|
|
if state | TOXAV_FRIEND_CALL_STATE['ACCEPTING_A'] and call.out_audio:
|
|
self.start_audio_thread()
|
|
|
|
if state | TOXAV_FRIEND_CALL_STATE['ACCEPTING_V'] and call.out_video:
|
|
self.start_video_thread()
|
|
|
|
def is_video_call(self, number):
|
|
return number in self and self._calls[number].in_video
|
|
|
|
# -----------------------------------------------------------------------------------------------------------------
|
|
# Threads
|
|
# -----------------------------------------------------------------------------------------------------------------
|
|
|
|
def start_audio_thread(self):
|
|
"""
|
|
Start audio sending
|
|
"""
|
|
if self._audio_thread is not None:
|
|
return
|
|
|
|
self._audio_running = True
|
|
|
|
self._audio = pyaudio.PyAudio()
|
|
self._audio_stream = self._audio.open(format=pyaudio.paInt16,
|
|
rate=self._audio_rate,
|
|
channels=self._audio_channels,
|
|
input=True,
|
|
input_device_index=self._settings.audio['input'],
|
|
frames_per_buffer=self._audio_sample_count * 10)
|
|
|
|
self._audio_thread = threading.Thread(target=self.send_audio)
|
|
self._audio_thread.start()
|
|
|
|
def stop_audio_thread(self):
|
|
|
|
if self._audio_thread is None:
|
|
return
|
|
|
|
self._audio_running = False
|
|
|
|
self._audio_thread.join()
|
|
|
|
self._audio_thread = None
|
|
self._audio_stream = None
|
|
self._audio = None
|
|
|
|
if self._out_stream is not None:
|
|
self._out_stream.stop_stream()
|
|
self._out_stream.close()
|
|
self._out_stream = None
|
|
|
|
def start_video_thread(self):
|
|
if self._video_thread is not None:
|
|
return
|
|
|
|
self._video_running = True
|
|
self._video_width = s.video['width']
|
|
self._video_height = s.video['height']
|
|
|
|
if s.video['device'] == -1:
|
|
self._video = screen_sharing.DesktopGrabber(self._settings.video['x'], self._settings.video['y'],
|
|
self._settings.video['width'], self._settings.video['height'])
|
|
else:
|
|
self._video = cv2.VideoCapture(self._settings.video['device'])
|
|
self._video.set(cv2.CAP_PROP_FPS, 25)
|
|
self._video.set(cv2.CAP_PROP_FRAME_WIDTH, self._video_width)
|
|
self._video.set(cv2.CAP_PROP_FRAME_HEIGHT, self._video_height)
|
|
|
|
self._video_thread = threading.Thread(target=self.send_video)
|
|
self._video_thread.start()
|
|
|
|
def stop_video_thread(self):
|
|
if self._video_thread is None:
|
|
return
|
|
|
|
self._video_running = False
|
|
self._video_thread.join()
|
|
self._video_thread = None
|
|
self._video = None
|
|
|
|
# -----------------------------------------------------------------------------------------------------------------
|
|
# Incoming chunks
|
|
# -----------------------------------------------------------------------------------------------------------------
|
|
|
|
def audio_chunk(self, samples, channels_count, rate):
|
|
"""
|
|
Incoming chunk
|
|
"""
|
|
|
|
if self._out_stream is None:
|
|
self._out_stream = self._audio.open(format=pyaudio.paInt16,
|
|
channels=channels_count,
|
|
rate=rate,
|
|
output_device_index=self._settings.audio['output'],
|
|
output=True)
|
|
self._out_stream.write(samples)
|
|
|
|
# -----------------------------------------------------------------------------------------------------------------
|
|
# AV sending
|
|
# -----------------------------------------------------------------------------------------------------------------
|
|
|
|
def send_audio(self):
|
|
"""
|
|
This method sends audio to friends
|
|
"""
|
|
|
|
while self._audio_running:
|
|
try:
|
|
pcm = self._audio_stream.read(self._audio_sample_count)
|
|
if pcm:
|
|
for friend_num in self._calls:
|
|
if self._calls[friend_num].out_audio:
|
|
try:
|
|
self._toxav.audio_send_frame(friend_num, pcm, self._audio_sample_count,
|
|
self._audio_channels, self._audio_rate)
|
|
except:
|
|
pass
|
|
except:
|
|
pass
|
|
|
|
time.sleep(0.01)
|
|
|
|
def send_video(self):
|
|
"""
|
|
This method sends video to friends
|
|
"""
|
|
while self._video_running:
|
|
try:
|
|
result, frame = self._video.read()
|
|
if result:
|
|
height, width, channels = frame.shape
|
|
for friend_num in self._calls:
|
|
if self._calls[friend_num].out_video:
|
|
try:
|
|
y, u, v = self.convert_bgr_to_yuv(frame)
|
|
self._toxav.video_send_frame(friend_num, width, height, y, u, v)
|
|
except:
|
|
pass
|
|
except:
|
|
pass
|
|
|
|
time.sleep(0.01)
|
|
|
|
def convert_bgr_to_yuv(self, frame):
|
|
"""
|
|
:param frame: input bgr frame
|
|
:return y, u, v: y, u, v values of frame
|
|
|
|
How this function works:
|
|
OpenCV creates YUV420 frame from BGR
|
|
This frame has following structure and size:
|
|
width, height - dim of input frame
|
|
width, height * 1.5 - dim of output frame
|
|
|
|
width
|
|
-------------------------
|
|
| |
|
|
| Y | height
|
|
| |
|
|
-------------------------
|
|
| | |
|
|
| U even | U odd | height // 4
|
|
| | |
|
|
-------------------------
|
|
| | |
|
|
| V even | V odd | height // 4
|
|
| | |
|
|
-------------------------
|
|
|
|
width // 2 width // 2
|
|
|
|
Y, U, V can be extracted using slices and joined in one list using itertools.chain.from_iterable()
|
|
Function returns bytes(y), bytes(u), bytes(v), because it is required for ctypes
|
|
"""
|
|
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV_I420)
|
|
|
|
y = frame[:self._video_height, :]
|
|
y = list(itertools.chain.from_iterable(y))
|
|
|
|
u = np.zeros((self._video_height // 2, self._video_width // 2), dtype=np.int)
|
|
u[::2, :] = frame[self._video_height:self._video_height * 5 // 4, :self._video_width // 2]
|
|
u[1::2, :] = frame[self._video_height:self._video_height * 5 // 4, self._video_width // 2:]
|
|
u = list(itertools.chain.from_iterable(u))
|
|
v = np.zeros((self._video_height // 2, self._video_width // 2), dtype=np.int)
|
|
v[::2, :] = frame[self._video_height * 5 // 4:, :self._video_width // 2]
|
|
v[1::2, :] = frame[self._video_height * 5 // 4:, self._video_width // 2:]
|
|
v = list(itertools.chain.from_iterable(v))
|
|
|
|
return bytes(y), bytes(u), bytes(v)
|