# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- import pyaudio import time import threading import itertools from wrapper.toxav_enums import * from av import screen_sharing from av.call import Call import common.tox_save from utils import ui as util_ui import tests.support_testing as ts from middleware.threads import invoke_in_main_thread from main import sleep from middleware.threads import BaseThread global LOG import logging LOG = logging.getLogger('app.'+__name__) TIMER_TIMEOUT = 30.0 bSTREAM_CALLBACK = False class AV(common.tox_save.ToxAvSave): def __init__(self, toxav, settings): super().__init__(toxav) self._settings = settings self._running = True s = settings if 'video' not in s: LOG.warn("AV.__init__ 'video' not in s" ) LOG.debug(f"AV.__init__ {s!r}" ) elif 'device' not in s['video']: LOG.warn("AV.__init__ 'device' not in s.video" ) LOG.debug(f"AV.__init__ {s['video']!r}" ) self._calls = {} # dict: key - friend number, value - Call instance self._audio = None self._audio_stream = None self._audio_thread = None self._audio_running = False self._out_stream = None self._audio_channels = 1 self._audio_duration = 60 self._audio_rate_pa = 48000 self._audio_rate_tox = 48000 self._audio_rate_pa = 48000 self._audio_krate_tox_audio = self._audio_rate_tox // 1000 self._audio_krate_tox_video = 5000 self._audio_sample_count_pa = self._audio_rate_pa * self._audio_channels * self._audio_duration // 1000 self._audio_sample_count_tox = self._audio_rate_tox * self._audio_channels * self._audio_duration // 1000 self._video = None self._video_thread = None self._video_running = False self._video_width = 320 self._video_height = 240 iOutput = self._settings._args.audio['output'] self.lPaSampleratesO = ts.lSdSamplerates(iOutput) global oPYA oPYA = self._audio = pyaudio.PyAudio() def stop(self): self._running = False self.stop_audio_thread() self.stop_video_thread() def __contains__(self, friend_number): return friend_number in self._calls # ----------------------------------------------------------------------------------------------------------------- # Calls # ----------------------------------------------------------------------------------------------------------------- def __call__(self, friend_number, audio, video): """Call friend with specified number""" if friend_number in self._calls: LOG.warn(f"__call__ already has {friend_number}") return if self._audio_krate_tox_audio not in ts.lToxSampleratesK: LOG.warn(f"__call__ {self._audio_krate_tox_audio} not in {ts.lToxSampleratesK}") try: self._toxav.call(friend_number, self._audio_krate_tox_audio if audio else 0, self._audio_krate_tox_video if video else 0) except ArgumentError as e: LOG.warn(f"_toxav.call already has {friend_number}") return self._calls[friend_number] = Call(audio, video) threading.Timer(TIMER_TIMEOUT, lambda: self.finish_not_started_call(friend_number)).start() def accept_call(self, friend_number, audio_enabled, video_enabled): # obsolete return call_accept_call(self, friend_number, audio_enabled, video_enabled) def call_accept_call(self, friend_number, audio_enabled, video_enabled): LOG.debug(f"call_accept_call from {friend_number} {self._running}" + f"{audio_enabled} {video_enabled}") # import pdb; pdb.set_trace() - gets into q Qt exec_ problem # ts.trepan_handler() if self._audio_krate_tox_audio not in ts.lToxSampleratesK: LOG.warn(f"__call__ {self._audio_krate_tox_audio} not in {ts.lToxSampleratesK}") if self._running: self._calls[friend_number] = Call(audio_enabled, video_enabled) # audio_bit_rate: Audio bit rate in Kb/sec. Set this to 0 to disable audio sending. # video_bit_rate: Video bit rate in Kb/sec. Set this to 0 to disable video sending. try: self._toxav.answer(friend_number, self._audio_krate_tox_audio if audio_enabled else 0, self._audio_krate_tox_video if video_enabled else 0) except ArgumentError as e: LOG.debug(f"AV accept_call error from {friend_number} {self._running}" + f"{e}") raise if audio_enabled: # may raise self.start_audio_thread() if video_enabled: # may raise self.start_video_thread() def finish_call(self, friend_number, by_friend=False): LOG.debug(f"finish_call {friend_number}") if not by_friend: self._toxav.call_control(friend_number, TOXAV_CALL_CONTROL['CANCEL']) if friend_number in self._calls: del self._calls[friend_number] try: # AttributeError: 'int' object has no attribute 'out_audio' if not len(list(filter(lambda c: c.out_audio, self._calls))): self.stop_audio_thread() if not len(list(filter(lambda c: c.out_video, self._calls))): self.stop_video_thread() except Exception as e: LOG.error(f"finish_call FixMe: {e}") # dunno self.stop_audio_thread() self.stop_video_thread() def finish_not_started_call(self, friend_number): if friend_number in self: call = self._calls[friend_number] if not call.is_active: self.finish_call(friend_number) def toxav_call_state_cb(self, friend_number, state): """ New call state """ LOG.debug(f"toxav_call_state_cb {friend_number}") call = self._calls[friend_number] call.is_active = True call.in_audio = state | TOXAV_FRIEND_CALL_STATE['SENDING_A'] > 0 call.in_video = state | TOXAV_FRIEND_CALL_STATE['SENDING_V'] > 0 if state | TOXAV_FRIEND_CALL_STATE['ACCEPTING_A'] and call.out_audio: self.start_audio_thread() if state | TOXAV_FRIEND_CALL_STATE['ACCEPTING_V'] and call.out_video: self.start_video_thread() def is_video_call(self, number): return number in self and self._calls[number].in_video # ----------------------------------------------------------------------------------------------------------------- # Threads # ----------------------------------------------------------------------------------------------------------------- def start_audio_thread(self): """ Start audio sending """ global oPYA iInput = self._settings._args.audio['input'] if self._audio_thread is not None: LOG.warn(f"start_audio_thread device={iInput}") return iInput = self._settings._args.audio['input'] LOG.debug(f"start_audio_thread device={iInput}") lPaSamplerates = ts.lSdSamplerates(iInput) if not(len(lPaSamplerates)): e = f"No supported sample rates for device: audio[input]={iInput!r}" LOG.error(f"No supported sample rates {e}") raise RuntimeError(e) if not self._audio_rate_pa in lPaSamplerates: LOG.warn(f"{self._audio_rate_pa} not in {lPaSamplerates!r}") if False: self._audio_rate_pa = oPYA.get_device_info_by_index(iInput)['defaultSampleRate'] else: LOG.warn(f"Setting audio_rate to: {lPaSamplerates[0]}") self._audio_rate_pa = lPaSamplerates[0] try: LOG.debug( f"start_audio_thread framerate: {self._audio_rate_pa}" \ +f" device: {iInput}" +f" supported: {lPaSamplerates!r}") if self._audio_rate_pa not in lPaSamplerates: LOG.warn(f"PAudio sampling rate was {self._audio_rate_pa} changed to {lPaSamplerates[0]}") self._audio_rate_pa = lPaSamplerates[0] if bSTREAM_CALLBACK: self._audio_stream = oPYA.open(format=pyaudio.paInt16, rate=self._audio_rate_pa, channels=self._audio_channels, input=True, input_device_index=iInput, frames_per_buffer=self._audio_sample_count_pa * 10, stream_callback=self.send_audio_data) self._audio_running = True self._audio_stream.start_stream() while self._audio_stream.is_active(): sleep(0.1) self._audio_stream.stop_stream() self._audio_stream.close() else: self._audio_stream = oPYA.open(format=pyaudio.paInt16, rate=self._audio_rate_pa, channels=self._audio_channels, input=True, input_device_index=iInput, frames_per_buffer=self._audio_sample_count_pa * 10) self._audio_running = True self._audio_thread = BaseThread(target=self.send_audio, name='_audio_thread') self._audio_thread.start() except Exception as e: LOG.error(f"Starting self._audio.open {e}") LOG.debug(repr(dict(format=pyaudio.paInt16, rate=self._audio_rate_pa, channels=self._audio_channels, input=True, input_device_index=iInput, frames_per_buffer=self._audio_sample_count_pa * 10))) # catcher in place in calls_manager raise RuntimeError(e) else: LOG.debug(f"start_audio_thread {self._audio_stream!r}") def stop_audio_thread(self): if self._audio_thread is None: return self._audio_running = False self._audio_thread = None self._audio_stream = None self._audio = None if self._out_stream is not None: self._out_stream.stop_stream() self._out_stream.close() self._out_stream = None def start_video_thread(self): if self._video_thread is not None: return s = self._settings if 'video' not in s: LOG.warn("AV.__init__ 'video' not in s" ) LOG.debug(f"start_video_thread {s!r}" ) raise RuntimeError("start_video_thread not 'video' in s)" ) elif 'device' not in s['video']: LOG.error("start_video_thread not 'device' in s['video']" ) LOG.debug(f"start_video_thread {s['video']!r}" ) raise RuntimeError("start_video_thread not 'device' ins s['video']" ) self._video_width = s['video']['width'] self._video_height = s['video']['height'] LOG.info("start_video_thread " \ +f" device: {s['video']['device']}" \ +f" supported: {s['video']['width']} {s['video']['height']}") s['video']['device'] = -1 if s['video']['device'] == -1: self._video = screen_sharing.DesktopGrabber(s['video']['x'], s['video']['y'], s['video']['width'], s['video']['height']) else: with ts.ignoreStdout(): import cv2 self._video = cv2.VideoCapture(s['video']['device']) self._video.set(cv2.CAP_PROP_FPS, 25) self._video.set(cv2.CAP_PROP_FRAME_WIDTH, self._video_width) self._video.set(cv2.CAP_PROP_FRAME_HEIGHT, self._video_height) self._video_running = True self._video_thread = BaseThread(target=self.send_video, name='_video_thread') self._video_thread.start() def stop_video_thread(self): if self._video_thread is None: return self._video_running = False i = 0 while i < ts.iTHREAD_JOINS: self._video_thread.join(ts.iTHREAD_TIMEOUT) try: if not self._video_thread.is_alive(): break except: # AttributeError: 'NoneType' object has no attribute 'join' break i = i + 1 else: LOG.warn("self._video_thread.is_alive BLOCKED") self._video_thread = None self._video = None # ----------------------------------------------------------------------------------------------------------------- # Incoming chunks # ----------------------------------------------------------------------------------------------------------------- def audio_chunk(self, samples, channels_count, rate): """ Incoming chunk """ if self._out_stream is None: iOutput = self._settings._args.audio['output'] if not rate in self.lPaSampleratesO: LOG.warn(f"{rate} not in {self.lPaSampleratesO!r}") if False: rate = oPYA.get_device_info_by_index(iOutput)['defaultSampleRate'] LOG.warn(f"Setting audio_rate to: {self.lPaSampleratesO[0]}") rate = self.lPaSampleratesO[0] try: with ts.ignoreStderr(): self._out_stream = oPYA.open(format=pyaudio.paInt16, channels=channels_count, rate=rate, output_device_index=iOutput, output=True) except Exception as e: LOG.error(f"Error playing audio_chunk creating self._out_stream {e}") LOG.debug(f"audio_chunk output_device_index={self._settings._args.audio['input']} rate={rate} channels={channels_count}") invoke_in_main_thread(util_ui.message_box, str(e), util_ui.tr("Error Chunking audio")) # dunno self.stop() return self._out_stream.write(samples) # ----------------------------------------------------------------------------------------------------------------- # AV sending # ----------------------------------------------------------------------------------------------------------------- def send_audio_data(self, data, count, *largs, **kwargs): pcm = data # :param sampling_rate: Audio sampling rate used in this frame. if self._toxav is None: raise RuntimeError("_toxav not initialized") if self._audio_rate_tox not in ts.lToxSamplerates: LOG.warn(f"ToxAudio sampling rate was {self._audio_rate_tox} changed to {ts.lToxSamplerates[0]}") self._audio_rate_tox = ts.lToxSamplerates[0] for friend_num in self._calls: if self._calls[friend_num].out_audio: try: # app.av.calls ERROR Error send_audio: One of the frame parameters was invalid. E.g. the resolution may be too small or too large, or the audio sampling rate may be unsupported self._toxav.audio_send_frame(friend_num, pcm, count, self._audio_channels, self._audio_rate_tox) except Exception as e: LOG.error(f"Error send_audio audio_send_frame: {e}") LOG.debug(f"send_audio self._audio_rate_tox={self._audio_rate_tox} self._audio_channels={self._audio_channels}") invoke_in_main_thread(util_ui.message_box, str(e), util_ui.tr("Error send_audio audio_send_frame")) pass def send_audio(self): """ This method sends audio to friends """ i=0 count = self._audio_sample_count_tox LOG.debug(f"send_audio stream={self._audio_stream}") while self._audio_running: try: pcm = self._audio_stream.read(count, exception_on_overflow=False) if not pcm: sleep(0.1) else: self.send_audio_data(pcm, count) except: pass i += 1 LOG.debug(f"send_audio {i}") sleep(0.01) def send_video(self): """ This method sends video to friends """ LOG.debug(f"send_video thread={threading.current_thread()}" +f" self._video_running={self._video_running}" +f" device: {self._settings['video']['device']}" ) while self._video_running: try: result, frame = self._video.read() if result: LOG.warn(f"send_video video_send_frame _video.read") else: height, width, channels = frame.shape for friend_num in self._calls: if self._calls[friend_num].out_video: try: y, u, v = self.convert_bgr_to_yuv(frame) self._toxav.video_send_frame(friend_num, width, height, y, u, v) except Exception as e: LOG.debug(f"send_video video_send_frame ERROR {e}") pass except: pass sleep(0.1) def convert_bgr_to_yuv(self, frame): """ :param frame: input bgr frame :return y, u, v: y, u, v values of frame How this function works: OpenCV creates YUV420 frame from BGR This frame has following structure and size: width, height - dim of input frame width, height * 1.5 - dim of output frame width ------------------------- | | | Y | height | | ------------------------- | | | | U even | U odd | height // 4 | | | ------------------------- | | | | V even | V odd | height // 4 | | | ------------------------- width // 2 width // 2 Y, U, V can be extracted using slices and joined in one list using itertools.chain.from_iterable() Function returns bytes(y), bytes(u), bytes(v), because it is required for ctypes """ with ts.ignoreStdout(): import cv2 frame = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV_I420) y = frame[:self._video_height, :] y = list(itertools.chain.from_iterable(y)) import numpy as np u = np.zeros((self._video_height // 2, self._video_width // 2), dtype=np.int) u[::2, :] = frame[self._video_height:self._video_height * 5 // 4, :self._video_width // 2] u[1::2, :] = frame[self._video_height:self._video_height * 5 // 4, self._video_width // 2:] u = list(itertools.chain.from_iterable(u)) v = np.zeros((self._video_height // 2, self._video_width // 2), dtype=np.int) v[::2, :] = frame[self._video_height * 5 // 4:, :self._video_width // 2] v[1::2, :] = frame[self._video_height * 5 // 4:, self._video_width // 2:] v = list(itertools.chain.from_iterable(v)) return bytes(y), bytes(u), bytes(v)