From 9ec25ed109c5748d6119603268448c68db53c26d Mon Sep 17 00:00:00 2001 From: bkfox Date: Wed, 25 Jan 2023 13:02:21 +0100 Subject: [PATCH] sound check --- aircox/management/commands/sounds_monitor.py | 220 +----------------- .../commands/sounds_quality_check.py | 114 +-------- aircox/management/sound_file.py | 216 +++++++++++++++++ aircox/management/sound_monitor.py | 163 +++++++++++++ aircox/management/sound_stats.py | 115 +++++++++ aircox/models/sound.py | 17 +- aircox/models/station.py | 2 +- aircox/tests/__init__.py | 2 + aircox/tests/management/__init__.py | 2 + aircox/tests/management/sound_file.py | 73 ++++++ aircox/tests/management/sound_monitor.py | 76 ++++++ aircox/{tests.py => tests/old.py} | 0 requirements.txt | 1 + 13 files changed, 671 insertions(+), 330 deletions(-) create mode 100644 aircox/management/sound_file.py create mode 100644 aircox/management/sound_monitor.py create mode 100644 aircox/management/sound_stats.py create mode 100644 aircox/tests/__init__.py create mode 100644 aircox/tests/management/__init__.py create mode 100644 aircox/tests/management/sound_file.py create mode 100644 aircox/tests/management/sound_monitor.py rename aircox/{tests.py => tests/old.py} (100%) diff --git a/aircox/management/commands/sounds_monitor.py b/aircox/management/commands/sounds_monitor.py index f7eb834..01a987a 100755 --- a/aircox/management/commands/sounds_monitor.py +++ b/aircox/management/commands/sounds_monitor.py @@ -25,233 +25,23 @@ Sox (and soxi). """ from argparse import RawTextHelpFormatter import concurrent.futures as futures -import datetime import atexit import logging import os -import re import time -import mutagen from watchdog.observers import Observer -from watchdog.events import PatternMatchingEventHandler, FileModifiedEvent -from django.conf import settings as conf -from django.core.management.base import BaseCommand, CommandError -from django.utils import timezone as tz -from django.utils.translation import gettext as _ +from django.core.management.base import BaseCommand -from aircox import settings, utils -from aircox.models import Diffusion, Program, Sound, Track -from .import_playlist import PlaylistImport +from aircox import settings +from aircox.models import Program, Sound +from aircox.management.sound_file import SoundFile +from aircox.management.sound_monitor import MonitorHandler logger = logging.getLogger('aircox.commands') -sound_path_re = re.compile( - '^(?P[0-9]{4})(?P[0-9]{2})(?P[0-9]{2})' - '(_(?P[0-9]{2})h(?P[0-9]{2}))?' - '(_(?P[0-9]+))?' - '_?[ -]*(?P.*)$' -) - - -class SoundFile: - path = None - info = None - path_info = None - sound = None - - def __init__(self, path): - self.path = path - - @property - def sound_path(self): - """ Relative path name """ - return self.path.replace(conf.MEDIA_ROOT + '/', '') - - def sync(self, sound=None, program=None, deleted=False, **kwargs): - """ - Update related sound model and save it. - """ - if deleted: - sound = Sound.objects.path(self.path).first() - if sound: - sound.type = sound.TYPE_REMOVED - sound.check_on_file() - sound.save() - return sound - - # FIXME: sound.program as not null - if not program: - program = Program.get_from_path(self.path) - logger.info('program from path "%s" -> %s', self.path, program) - kwargs['program_id'] = program.pk - - sound, created = Sound.objects.get_or_create(file=self.sound_path, defaults=kwargs) \ - if not sound else (sound, False) - self.sound = sound - sound.program = program - if created or sound.check_on_file(): - logger.info('sound is new or have been modified -> %s', self.sound_path) - self.read_path() - sound.name = self.path_info.get('name') - - self.read_file_info() - if self.info is not None: - sound.duration = utils.seconds_to_time(self.info.info.length) - - # check for episode - if sound.episode is None and self.read_path(): - self.find_episode(program) - - sound.save() - - # check for playlist - self.find_playlist(sound) - return sound - - def read_path(self): - """ - Parse file name to get info on the assumption it has the correct - format (given in Command.help). Return True if path contains informations. - """ - if self.path_info: - return 'year' in self.path_info - - name = os.path.splitext(os.path.basename(self.path))[0] - match = sound_path_re.search(name) - if match: - path_info = match.groupdict() - for k in ('year', 'month', 'day', 'hour', 'minute'): - if path_info.get(k) is not None: - path_info[k] = int(path_info[k]) - self.path_info = path_info - return True - else: - self.path_info = {'name': name} - return False - - def read_file_info(self): - """ Read file information and metadata. """ - if os.path.exists(self.path): - self.info = mutagen.File(self.path) - else: - self.info = None - - def find_episode(self, program): - """ - For a given program, check if there is an initial diffusion - to associate to, using the date info we have. Update self.sound - and save it consequently. - - We only allow initial diffusion since there should be no - rerun. - """ - pi = self.path_info - if 'year' not in pi or not self.sound or self.sound.episode: - return None - - if pi.get('hour') is not None: - date = tz.datetime(pi.get('year'), pi.get('month'), pi.get('day'), - pi.get('hour') or 0, pi.get('minute') or 0) - date = tz.get_current_timezone().localize(date) - else: - date = datetime.date(pi.get('year'), pi.get('month'), pi.get('day')) - - diffusion = program.diffusion_set.at(date).first() - if not diffusion: - return None - - logger.info('%s <--> %s', self.sound.file.name, str(diffusion.episode)) - self.sound.episode = diffusion.episode - return diffusion - - def find_playlist(self, sound=None, use_meta=True): - """ - Find a playlist file corresponding to the sound path, such as: - my_sound.ogg => my_sound.csv - - Use sound's file metadata if no corresponding playlist has been - found and `use_meta` is True. - """ - if sound is None: - sound = self.sound - if sound.track_set.count() > 1: - return - - # import playlist - path_noext, ext = os.path.splitext(self.sound.file.path) - path = path_noext + '.csv' - if os.path.exists(path): - PlaylistImport(path, sound=sound).run() - # use metadata - elif use_meta: - if self.info is None: - self.read_file_info() - if self.info and self.info.tags: - tags = self.info.tags - title, artist, album, year = tuple( - t and ', '.join(t) for t in ( - tags.get(k) for k in ('title', 'artist', 'album', 'year')) - ) - title = title or (self.path_info and self.path_info.get('name')) or \ - os.path.basename(path_noext) - info = '{} ({})'.format(album, year) if album and year else \ - album or year or '' - track = Track(sound=sound, - position=int(tags.get('tracknumber', 0)), - title=title, - artist=artist or _('unknown'), - info=info) - track.save() - - -class MonitorHandler(PatternMatchingEventHandler): - """ - Event handler for watchdog, in order to be used in monitoring. - """ - pool = None - - def __init__(self, subdir, pool): - """ - subdir: AIRCOX_SOUND_ARCHIVES_SUBDIR or AIRCOX_SOUND_EXCERPTS_SUBDIR - """ - self.subdir = subdir - self.pool = pool - if self.subdir == settings.AIRCOX_SOUND_ARCHIVES_SUBDIR: - self.sound_kwargs = {'type': Sound.TYPE_ARCHIVE} - else: - self.sound_kwargs = {'type': Sound.TYPE_EXCERPT} - - patterns = ['*/{}/*{}'.format(self.subdir, ext) - for ext in settings.AIRCOX_SOUND_FILE_EXT] - super().__init__(patterns=patterns, ignore_directories=True) - - def on_created(self, event): - self.on_modified(event) - - def on_modified(self, event): - logger.info('sound modified: %s', event.src_path) - def updated(event, sound_kwargs): - SoundFile(event.src_path).sync(**sound_kwargs) - self.pool.submit(updated, event, self.sound_kwargs) - - def on_moved(self, event): - logger.info('sound moved: %s -> %s', event.src_path, event.dest_path) - def moved(event, sound_kwargs): - sound = Sound.objects.filter(file=event.src_path) - sound_file = SoundFile(event.dest_path) if not sound else sound - sound_file.sync(**sound_kwargs) - self.pool.submit(moved, event, self.sound_kwargs) - - def on_deleted(self, event): - logger.info('sound deleted: %s', event.src_path) - def deleted(event): - SoundFile(event.src_path).sync(deleted=True) - self.pool.submit(deleted, event) - - class Command(BaseCommand): help = __doc__ diff --git a/aircox/management/commands/sounds_quality_check.py b/aircox/management/commands/sounds_quality_check.py index 88d0df9..bce6e80 100755 --- a/aircox/management/commands/sounds_quality_check.py +++ b/aircox/management/commands/sounds_quality_check.py @@ -1,118 +1,16 @@ """ Analyse and check files using Sox, prints good and bad files. """ -import sys import logging -import re -import subprocess from argparse import RawTextHelpFormatter from django.core.management.base import BaseCommand, CommandError +from aircox.management.sound_stats import SoxStats, SoundStats + logger = logging.getLogger('aircox.commands') -class Stats: - attributes = [ - 'DC offset', 'Min level', 'Max level', - 'Pk lev dB', 'RMS lev dB', 'RMS Pk dB', - 'RMS Tr dB', 'Flat factor', 'Length s', - ] - - def __init__(self, path, **kwargs): - """ - If path is given, call analyse with path and kwargs - """ - self.values = {} - if path: - self.analyse(path, **kwargs) - - def get(self, attr): - return self.values.get(attr) - - def parse(self, output): - for attr in Stats.attributes: - value = re.search(attr + r'\s+(?P\S+)', output) - value = value and value.groupdict() - if value: - try: - value = float(value.get('value')) - except ValueError: - value = None - self.values[attr] = value - self.values['length'] = self.values['Length s'] - - def analyse(self, path, at=None, length=None): - """ - If at and length are given use them as excerpt to analyse. - """ - args = ['sox', path, '-n'] - - if at is not None and length is not None: - args += ['trim', str(at), str(length)] - - args.append('stats') - - p = subprocess.Popen(args, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - # sox outputs to stderr (my god WHYYYY) - out_, out = p.communicate() - self.parse(str(out, encoding='utf-8')) - - -class SoundStats: - path = None # file path - sample_length = 120 # default sample length in seconds - stats = None # list of samples statistics - bad = None # list of bad samples - good = None # list of good samples - - def __init__(self, path, sample_length=None): - self.path = path - self.sample_length = sample_length if sample_length is not None \ - else self.sample_length - - def get_file_stats(self): - return self.stats and self.stats[0] - - def analyse(self): - logger.info('complete file analysis') - self.stats = [Stats(self.path)] - position = 0 - length = self.stats[0].get('length') - - if not self.sample_length: - return - - logger.info('start samples analysis...') - while position < length: - stats = Stats(self.path, at=position, length=self.sample_length) - self.stats.append(stats) - position += self.sample_length - - def check(self, name, min_val, max_val): - self.good = [index for index, stats in enumerate(self.stats) - if min_val <= stats.get(name) <= max_val] - self.bad = [index for index, stats in enumerate(self.stats) - if index not in self.good] - self.resume() - - def resume(self): - def view(array): return [ - 'file' if index is 0 else - 'sample {} (at {} seconds)'.format( - index, (index-1) * self.sample_length) - for index in array - ] - - if self.good: - logger.info(self.path + ' -> good: \033[92m%s\033[0m', - ', '.join(view(self.good))) - if self.bad: - logger.info(self.path + ' -> bad: \033[91m%s\033[0m', - ', '.join(view(self.bad))) - - class Command (BaseCommand): help = __doc__ sounds = None @@ -132,7 +30,7 @@ class Command (BaseCommand): parser.add_argument( '-a', '--attribute', type=str, help='attribute name to use to check, that can be:\n' + - ', '.join(['"{}"'.format(attr) for attr in Stats.attributes]) + ', '.join(['"{}"'.format(attr) for attr in SoxStats.attributes]) ) parser.add_argument( '-r', '--range', type=float, nargs=2, @@ -160,7 +58,7 @@ class Command (BaseCommand): self.bad = [] self.good = [] for sound in self.sounds: - logger.info('analyse ' + sound.file.name) + logger.info('analyse ' + sound.path) sound.analyse() sound.check(attr, minmax[0], minmax[1]) if sound.bad: @@ -171,6 +69,6 @@ class Command (BaseCommand): # resume if options.get('resume'): for sound in self.good: - logger.info('\033[92m+ %s\033[0m', sound.file.name) + logger.info('\033[92m+ %s\033[0m', sound.path) for sound in self.bad: - logger.info('\033[91m+ %s\033[0m', sound.file.name) + logger.info('\033[91m+ %s\033[0m', sound.path) diff --git a/aircox/management/sound_file.py b/aircox/management/sound_file.py new file mode 100644 index 0000000..23c9275 --- /dev/null +++ b/aircox/management/sound_file.py @@ -0,0 +1,216 @@ +#! /usr/bin/env python3 +""" +Provide SoundFile which is used to link between database and file system. + + +File name +========= +It tries to parse the file name to get the date of the diffusion of an +episode and associate the file with it; We use the following format: + yyyymmdd[_n][_][name] + +Where: + 'yyyy' the year of the episode's diffusion; + 'mm' the month of the episode's diffusion; + 'dd' the day of the episode's diffusion; + 'n' the number of the episode (if multiple episodes); + 'name' the title of the sound; + +Sound Quality +============= +To check quality of files, call the command sound_quality_check using the +parameters given by the setting AIRCOX_SOUND_QUALITY. This script requires +Sox (and soxi). +""" +from datetime import date +import logging +import os +import re + +import mutagen + +from django.conf import settings as conf +from django.utils import timezone as tz +from django.utils.translation import gettext as _ + +from aircox import utils +from aircox.models import Program, Sound, Track +from .commands.import_playlist import PlaylistImport + +logger = logging.getLogger('aircox.commands') + + +class SoundFile: + """ + Handle synchronisation between sounds on files and database. + """ + path = None + info = None + path_info = None + sound = None + + def __init__(self, path): + self.path = path + + @property + def sound_path(self): + """ Relative path name """ + return self.path.replace(conf.MEDIA_ROOT + '/', '') + + @property + def episode(self): + return self.sound and self.sound.episode + + def sync(self, sound=None, program=None, deleted=False, **kwargs): + """ + Update related sound model and save it. + """ + if deleted: + return self._on_delete(self.path) + + # FIXME: sound.program as not null + if not program: + program = Program.get_from_path(self.path) + logger.debug('program from path "%s" -> %s', self.path, program) + kwargs['program_id'] = program.pk + + if sound: + created = False + else: + sound, created = Sound.objects.get_or_create( + file=self.sound_path, defaults=kwargs) + + self.sound = sound + self.path_info = self.read_path(self.path) + + sound.program = program + if created or sound.check_on_file(): + sound.name = self.path_info.get('name') + self.info = self.read_file_info() + if self.info is not None: + sound.duration = utils.seconds_to_time(self.info.info.length) + + # check for episode + if sound.episode is None and 'year' in self.path_info: + sound.episode = self.find_episode(sound, self.path_info) + sound.save() + + # check for playlist + self.find_playlist(sound) + return sound + + def _on_delete(self, path): + # TODO: remove from db on delete + sound = Sound.objects.path(self.path).first() + if sound: + sound.type = sound.TYPE_REMOVED + sound.check_on_file() + sound.save() + return sound + + def read_path(self, path): + """ + Parse path name returning dictionary of extracted info. It can contain: + - `year`, `month`, `day`: diffusion date + - `hour`, `minute`: diffusion time + - `n`: sound arbitrary number (used for sound ordering) + - `name`: cleaned name extracted or file name (without extension) + """ + basename = os.path.basename(path) + basename = os.path.splitext(basename)[0] + reg_match = self._path_re.search(basename) + if reg_match: + info = reg_match.groupdict() + for k in ('year', 'month', 'day', 'hour', 'minute', 'n'): + if info.get(k) is not None: + info[k] = int(info[k]) + + name = info.get('name') + info['name'] = name and self._into_name(name) or basename + else: + info = {'name': basename} + return info + + _path_re = re.compile( + '^(?P[0-9]{4})(?P[0-9]{2})(?P[0-9]{2})' + '(_(?P[0-9]{2})h(?P[0-9]{2}))?' + '(_(?P[0-9]+))?' + '_?[ -]*(?P.*)$' + ) + + def _into_name(self, name): + name = name.replace('_', ' ') + return ' '.join(r.capitalize() for r in name.split(' ')) + + def read_file_info(self): + """ Read file information and metadata. """ + return mutagen.File(self.path) if os.path.exists(self.path) else None + + def find_episode(self, sound, path_info): + """ + For a given program, check if there is an initial diffusion + to associate to, using the date info we have. Update self.sound + and save it consequently. + + We only allow initial diffusion since there should be no + rerun. + """ + program, pi = sound.program, path_info + if 'year' not in pi or not sound or sound.episode: + return None + + year, month, day = pi.get('year'), pi.get('month'), pi.get('day') + if pi.get('hour') is not None: + at = tz.datetime(year, month, day, pi.get('hour', 0), + pi.get('minute', 0)) + at = tz.get_current_timezone().localize(at) + else: + at = date(year, month, day) + + diffusion = program.diffusion_set.at(at).first() + if not diffusion: + return None + + logger.debug('%s <--> %s', sound.file.name, str(diffusion.episode)) + return diffusion.episode + + def find_playlist(self, sound=None, use_meta=True): + """ + Find a playlist file corresponding to the sound path, such as: + my_sound.ogg => my_sound.csv + + Use sound's file metadata if no corresponding playlist has been + found and `use_meta` is True. + """ + if sound is None: + sound = self.sound + if sound.track_set.count() > 1: + return + + # import playlist + path_noext, ext = os.path.splitext(self.sound.file.path) + path = path_noext + '.csv' + if os.path.exists(path): + PlaylistImport(path, sound=sound).run() + # use metadata + elif use_meta: + if self.info is None: + self.read_file_info() + if self.info and self.info.tags: + tags = self.info.tags + title, artist, album, year = tuple( + t and ', '.join(t) for t in ( + tags.get(k) for k in ('title', 'artist', 'album', + 'year')) + ) + title = title or (self.path_info and + self.path_info.get('name')) or \ + os.path.basename(path_noext) + info = '{} ({})'.format(album, year) if album and year else \ + album or year or '' + track = Track(sound=sound, + position=int(tags.get('tracknumber', 0)), + title=title, + artist=artist or _('unknown'), + info=info) + track.save() diff --git a/aircox/management/sound_monitor.py b/aircox/management/sound_monitor.py new file mode 100644 index 0000000..4beaffb --- /dev/null +++ b/aircox/management/sound_monitor.py @@ -0,0 +1,163 @@ +#! /usr/bin/env python3 + +""" +Monitor sound files; For each program, check for: +- new files; +- deleted files; +- differences between files and sound; +- quality of the files; + +It tries to parse the file name to get the date of the diffusion of an +episode and associate the file with it; WNotifye the following format: + yyyymmdd[_n][_][name] + +Where: + 'yyyy' the year Notifyhe episode's diffusion; + 'mm' the month of the episode's difNotifyon; + 'dd' the day of the episode's diffusion; + 'n' the number of the episode (if multiple episodes); + 'name' the title of the sNotify; + + +To check quality of files, call the command sound_quality_check using the +parameters given by the setting AIRCOX_SOUND_QUALITY. This script requires +Sox (and soxi). +""" +from datetime import datetime, timedelta +import logging +import time + +from watchdog.events import PatternMatchingEventHandler + +from aircox import settings +from aircox.models import Sound + +from .sound_file import SoundFile + + +logger = logging.getLogger('aircox.commands') + + +__all__ = ('NotifyHandler', 'CreateHandler', 'DeleteHandler', + 'MoveHandler', 'ModifiedHandler', 'MonitorHandler',) + + +class NotifyHandler: + future = None + log_msg = None + + def __init__(self): + self.timestamp = datetime.now() + + def ping(self): + self.timestamp = datetime.now() + + def __call__(self, event, path=None, **kw): + sound_file = SoundFile(path or event.src_path) + if self.log_msg: + msg = self.log_msg.format(event=event, sound_file=sound_file) + logger.info(msg) + + sound_file.sync(**kw) + return sound_file + + +class CreateHandler(NotifyHandler): + log_msg = 'Sound file created: {sound_file.path}' + + +class DeleteHandler(NotifyHandler): + log_msg = 'Sound file deleted: {sound_file.path}' + + def __call__(self, *args, **kwargs): + kwargs['deleted'] = True + return super().__call__(*args, **kwargs) + + +class MoveHandler(NotifyHandler): + log_msg = 'Sound file moved: {event.src_path} -> {event.dest_path}' + + def __call__(self, event, **kw): + sound = Sound.objects.filter(file=event.src_path) + # FIXME: this is wrong + if sound: + kw['sound'] = sound + kw['path'] = event.src_path + else: + kw['path'] = event.dest_path + return super().__call__(event, **kw) + + +class ModifiedHandler(NotifyHandler): + timeout_delta = timedelta(seconds=30) + log_msg = 'Sound file updated: {sound_file.path}' + + def wait(self): + # multiple call of this handler can be done consecutively, we block + # its thread using timeout + # Note: this method may be subject to some race conflicts, but this + # should not be big a real issue. + timeout = self.timestamp + self.timeout_delta + while datetime.now() < timeout: + time.sleep(self.timeout_delta.total_seconds()) + timeout = self.timestamp + self.timeout_delta + + def __call__(self, event, **kw): + self.wait() + return super().__call__(event, **kw) + + +class MonitorHandler(PatternMatchingEventHandler): + """ + Event handler for watchdog, in order to be used in monitoring. + """ + pool = None + jobs = {} + + def __init__(self, subdir, pool): + """ + subdir: AIRCOX_SOUND_ARCHIVES_SUBDIR or AIRCOX_SOUND_EXCERPTS_SUBDIR + """ + self.subdir = subdir + self.pool = pool + if self.subdir == settings.AIRCOX_SOUND_ARCHIVES_SUBDIR: + self.sync_kw = {'type': Sound.TYPE_ARCHIVE} + else: + self.sync_kw = {'type': Sound.TYPE_EXCERPT} + + patterns = ['*/{}/*{}'.format(self.subdir, ext) + for ext in settings.AIRCOX_SOUND_FILE_EXT] + super().__init__(patterns=patterns, ignore_directories=True) + + def on_created(self, event): + self._submit(CreateHandler(), event, 'new', **self.sync_kw) + + def on_deleted(self, event): + self._submit(DeleteHandler(), event, 'del') + + def on_moved(self, event): + self._submit(MoveHandler(), event, 'mv', **self.sync_kw) + + def on_modified(self, event): + self._submit(ModifiedHandler(), event, 'up', **self.sync_kw) + + def _submit(self, handler, event, job_key_prefix, **kwargs): + """ + Send handler job to pool if not already running. + Return tuple with running job and boolean indicating if its a new one. + """ + key = job_key_prefix + ':' + event.src_path + job = self.jobs.get(key) + if job and not job.future.done(): + job.ping() + return job, False + + handler.future = self.pool.submit(handler, event, **kwargs) + self.jobs[key] = handler + + def done(r): + print(':::: job done', key) + if self.jobs.get(key) is handler: + del self.jobs[key] + handler.future.add_done_callback(done) + return handler, True diff --git a/aircox/management/sound_stats.py b/aircox/management/sound_stats.py new file mode 100644 index 0000000..a98205b --- /dev/null +++ b/aircox/management/sound_stats.py @@ -0,0 +1,115 @@ +""" +Provide sound analysis class using Sox. +""" +import logging +import re +import subprocess + +logger = logging.getLogger('aircox.commands') + + +__all__ = ('SoxStats', 'SoundStats') + + +class SoxStats: + """ + Run Sox process and parse output + """ + attributes = [ + 'DC offset', 'Min level', 'Max level', + 'Pk lev dB', 'RMS lev dB', 'RMS Pk dB', + 'RMS Tr dB', 'Flat factor', 'Length s', + ] + + def __init__(self, path, **kwargs): + """ + If path is given, call analyse with path and kwargs + """ + self.values = {} + if path: + self.analyse(path, **kwargs) + + def get(self, attr): + return self.values.get(attr) + + def parse(self, output): + for attr in self.attributes: + value = re.search(attr + r'\s+(?P\S+)', output) + value = value and value.groupdict() + if value: + try: + value = float(value.get('value')) + except ValueError: + value = None + self.values[attr] = value + self.values['length'] = self.values['Length s'] + + def analyse(self, path, at=None, length=None): + """ + If at and length are given use them as excerpt to analyse. + """ + args = ['sox', path, '-n'] + + if at is not None and length is not None: + args += ['trim', str(at), str(length)] + + args.append('stats') + + p = subprocess.Popen(args, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # sox outputs to stderr (my god WHYYYY) + out_, out = p.communicate() + self.parse(str(out, encoding='utf-8')) + + +class SoundStats: + path = None # file path + sample_length = 120 # default sample length in seconds + stats = None # list of samples statistics + bad = None # list of bad samples + good = None # list of good samples + + def __init__(self, path, sample_length=None): + self.path = path + self.sample_length = sample_length if sample_length is not None \ + else self.sample_length + + def get_file_stats(self): + return self.stats and self.stats[0] + + def analyse(self): + logger.debug('complete file analysis') + self.stats = [SoxStats(self.path)] + position = 0 + length = self.stats[0].get('length') + + if not self.sample_length: + return + + logger.debug('start samples analysis...') + while position < length: + stats = SoxStats(self.path, at=position, length=self.sample_length) + self.stats.append(stats) + position += self.sample_length + + def check(self, name, min_val, max_val): + self.good = [index for index, stats in enumerate(self.stats) + if min_val <= stats.get(name) <= max_val] + self.bad = [index for index, stats in enumerate(self.stats) + if index not in self.good] + self.resume() + + def resume(self): + def view(array): return [ + 'file' if index == 0 else + 'sample {} (at {} seconds)'.format( + index, (index-1) * self.sample_length) + for index in array + ] + + if self.good: + logger.debug(self.path + ' -> good: \033[92m%s\033[0m', + ', '.join(view(self.good))) + if self.bad: + logger.debug(self.path + ' -> bad: \033[91m%s\033[0m', + ', '.join(view(self.bad))) diff --git a/aircox/models/sound.py b/aircox/models/sound.py index eb9583b..5eb40be 100644 --- a/aircox/models/sound.py +++ b/aircox/models/sound.py @@ -9,6 +9,7 @@ from django.utils.translation import gettext_lazy as _ from taggit.managers import TaggableManager +from aircox import settings from .program import Program from .episode import Episode @@ -73,6 +74,8 @@ class SoundQuerySet(models.QuerySet): ) +# TODO: +# - provide a default name based on program and episode class Sound(models.Model): """ A Sound is the representation of a sound file that can be either an excerpt @@ -105,13 +108,14 @@ class Sound(models.Model): ) def _upload_to(self, filename): - subdir = AIRCOX_SOUND_ARCHIVES_SUBDIR if self.type == self.TYPE_ARCHIVE else \ - AIRCOX_SOUND_EXCERPTS_SUBDIR + subdir = settings.AIRCOX_SOUND_ARCHIVES_SUBDIR \ + if self.type == self.TYPE_ARCHIVE else \ + settings.AIRCOX_SOUND_EXCERPTS_SUBDIR return os.path.join(self.program.path, subdir, filename) file = models.FileField( _('file'), upload_to=_upload_to, max_length=256, - db_index=True, + db_index=True, unique=True, ) duration = models.TimeField( _('duration'), @@ -175,6 +179,7 @@ class Sound(models.Model): return os.path.exists(self.file.path) + # TODO: rename to sync_fs() def check_on_file(self): """ Check sound file info again'st self, and update informations if @@ -183,7 +188,7 @@ class Sound(models.Model): if not self.file_exists(): if self.type == self.TYPE_REMOVED: return - logger.info('sound %s: has been removed', self.file.name) + logger.debug('sound %s: has been removed', self.file.name) self.type = self.TYPE_REMOVED return True @@ -202,8 +207,8 @@ class Sound(models.Model): if self.mtime != mtime: self.mtime = mtime self.is_good_quality = None - logger.info('sound %s: m_time has changed. Reset quality info', - self.file.name) + logger.debug('sound %s: m_time has changed. Reset quality info', + self.file.name) return True return changed diff --git a/aircox/models/station.py b/aircox/models/station.py index aaf6ae3..3528abd 100644 --- a/aircox/models/station.py +++ b/aircox/models/station.py @@ -46,7 +46,7 @@ class Station(models.Model): ) default = models.BooleanField( _('default station'), - default=True, + default=False, help_text=_('use this station as the main one.') ) active = models.BooleanField( diff --git a/aircox/tests/__init__.py b/aircox/tests/__init__.py new file mode 100644 index 0000000..7265899 --- /dev/null +++ b/aircox/tests/__init__.py @@ -0,0 +1,2 @@ +from .management import * + diff --git a/aircox/tests/management/__init__.py b/aircox/tests/management/__init__.py new file mode 100644 index 0000000..424e77d --- /dev/null +++ b/aircox/tests/management/__init__.py @@ -0,0 +1,2 @@ +from .sound_file import * +from .sound_monitor import * diff --git a/aircox/tests/management/sound_file.py b/aircox/tests/management/sound_file.py new file mode 100644 index 0000000..e414897 --- /dev/null +++ b/aircox/tests/management/sound_file.py @@ -0,0 +1,73 @@ +from datetime import timedelta + +from django.conf import settings as conf +from django.test import TestCase +from django.utils import timezone as tz + +from aircox import models +from aircox.management.sound_file import SoundFile + + +__all__ = ('SoundFileTestCase',) + + +class SoundFileTestCase(TestCase): + path_infos = { + 'test/20220101_10h13_1_sample_1.mp3': { + 'year': 2022, 'month': 1, 'day': 1, 'hour': 10, 'minute': 13, + 'n': 1, 'name': 'Sample 1'}, + 'test/20220102_10h13_sample_2.mp3': { + 'year': 2022, 'month': 1, 'day': 2, 'hour': 10, 'minute': 13, + 'name': 'Sample 2'}, + 'test/20220103_1_sample_3.mp3': { + 'year': 2022, 'month': 1, 'day': 3, 'n': 1, 'name': 'Sample 3'}, + 'test/20220104_sample_4.mp3': { + 'year': 2022, 'month': 1, 'day': 4, 'name': 'Sample 4'}, + 'test/20220105.mp3': { + 'year': 2022, 'month': 1, 'day': 5, 'name': '20220105'}, + } + subdir_prefix = 'test' + sound_files = {k: r for k, r in ( + (path, SoundFile(conf.MEDIA_ROOT + '/' + path)) + for path in path_infos.keys() + )} + + def test_sound_path(self): + for path, sound_file in self.sound_files.items(): + self.assertEqual(path, sound_file.sound_path) + + def test_read_path(self): + for path, sound_file in self.sound_files.items(): + expected = self.path_infos[path] + result = sound_file.read_path(path) + # remove None values + result = {k: v for k, v in result.items() if v is not None} + self.assertEqual(expected, result, "path: {}".format(path)) + + def _setup_diff(self, program, info): + episode = models.Episode(program=program, title='test-episode') + at = tz.datetime(**{ + k: info[k] for k in ('year', 'month', 'day', 'hour', 'minute') + if info.get(k) + }) + at = tz.make_aware(at) + diff = models.Diffusion(episode=episode, start=at, + end=at+timedelta(hours=1)) + episode.save() + diff.save() + return diff + + def test_find_episode(self): + station = models.Station(name='test-station') + program = models.Program(station=station, title='test') + station.save() + program.save() + + for path, sound_file in self.sound_files.items(): + infos = sound_file.read_path(path) + diff = self._setup_diff(program, infos) + sound = models.Sound(program=diff.program, file=path) + result = sound_file.find_episode(sound, infos) + self.assertEquals(diff.episode, result) + + # TODO: find_playlist, sync diff --git a/aircox/tests/management/sound_monitor.py b/aircox/tests/management/sound_monitor.py new file mode 100644 index 0000000..1ba2f07 --- /dev/null +++ b/aircox/tests/management/sound_monitor.py @@ -0,0 +1,76 @@ +import concurrent.futures as futures +from datetime import datetime, timedelta +import time + +from django.test import TestCase + +from aircox.management.sound_monitor import \ + NotifyHandler, MoveHandler, ModifiedHandler, MonitorHandler + + +__all__ = ('NotifyHandlerTestCase', 'MoveHandlerTestCase', + 'ModifiedHandlerTestCase', 'MonitorHandlerTestCase',) + + +class FakeEvent: + def __init__(self, **kwargs): + self.__dict__.update(**kwargs) + + +class WaitHandler(NotifyHandler): + def __call__(self, timeout=0.5, *args, **kwargs): + # using time.sleep make the future done directly, don't know why + start = datetime.now() + while datetime.now() - start < timedelta(seconds=4): + pass + + +class NotifyHandlerTestCase(TestCase): + pass + + +class MoveHandlerTestCase(TestCase): + pass + + +class ModifiedHandlerTestCase(TestCase): + def test_wait(self): + handler = ModifiedHandler() + handler.timeout_delta = timedelta(seconds=0.1) + start = datetime.now() + handler.wait() + delta = datetime.now() - start + self.assertTrue(delta < handler.timeout_delta + timedelta(seconds=0.1)) + + def test_wait_ping(self): + pool = futures.ThreadPoolExecutor(1) + handler = ModifiedHandler() + handler.timeout_delta = timedelta(seconds=0.5) + + future = pool.submit(handler.wait) + time.sleep(0.3) + handler.ping() + time.sleep(0.3) + self.assertTrue(future.running()) + + +class MonitorHandlerTestCase(TestCase): + def setUp(self): + pool = futures.ThreadPoolExecutor(2) + self.monitor = MonitorHandler('archives', pool) + + def test_submit_new_job(self): + event = FakeEvent(src_path='dummy_src') + handler = NotifyHandler() + result = self.monitor._submit(handler, event, 'up') + self.assertIs(handler, result) + self.assertIsInstance(handler.future, futures.Future) + self.monitor.pool.shutdown() + + def test_submit_job_exists(self): + event = FakeEvent(src_path='dummy_src') + + job_1 = self.monitor._submit(WaitHandler(), event, 'up') + job_2 = self.monitor._submit(NotifyHandler(), event, 'up') + self.assertIs(job_1, job_2) + self.monitor.pool.shutdown() diff --git a/aircox/tests.py b/aircox/tests/old.py similarity index 100% rename from aircox/tests.py rename to aircox/tests/old.py diff --git a/requirements.txt b/requirements.txt index 8b446a9..e0fa1b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ bleach~=5.0 easy-thumbnails~=2.8 tzlocal~=4.2 +dateutils~=0.6 mutagen~=1.45 Pillow~=9.0 psutil~=5.9