From d70593a461b8b3732eb596d199a1a9f3b3caad8f Mon Sep 17 00:00:00 2001 From: bkfox Date: Tue, 29 Dec 2015 12:05:59 +0100 Subject: [PATCH] sound_monitor: filesystem monitoring using watchdog --- liquidsoap/management/commands/liquidsoap.py | 26 +- liquidsoap/utils.py | 2 +- .../management/commands/sounds_monitor.py | 300 +++++++++++++----- .../commands/sounds_quality_check.py | 4 +- programs/models.py | 26 +- programs/settings.py | 2 +- requirements.txt | 1 + 7 files changed, 257 insertions(+), 104 deletions(-) diff --git a/liquidsoap/management/commands/liquidsoap.py b/liquidsoap/management/commands/liquidsoap.py index d384f45..b917749 100644 --- a/liquidsoap/management/commands/liquidsoap.py +++ b/liquidsoap/management/commands/liquidsoap.py @@ -44,7 +44,6 @@ class StationConfig: log_script = os.path.join(log_script, 'manage.py') + \ ' liquidsoap_log' - context = { 'controller': self.controller, 'settings': settings, @@ -161,6 +160,7 @@ class Monitor: Keep trace of played sounds on the given source. For the moment we only keep track of known sounds. """ + # TODO: repetition of the same sound out of an interval of time last_log = programs.Log.objects.filter( source = source.id, ).prefetch_related('related_object').order_by('-date') @@ -170,10 +170,13 @@ class Monitor: return if last_log: + now = tz.datetime.now() last_log = last_log[0] - if type(last_log.related_object) == programs.Sound and \ - on_air == last_log.related_object.path: - return + last_obj = last_log.related_object + if type(last_obj) == programs.Sound and on_air == last_obj.path: + if not last_obj.duration or + now < log.date + programs_utils.to_timedelta(last_obj.duration) + return sound = programs.Sound.objects.filter(path = on_air) if not sound: @@ -194,6 +197,10 @@ class Command (BaseCommand): def add_arguments (self, parser): parser.formatter_class=RawTextHelpFormatter + parser.add_argument( + '-e', '--exec', action='store_true', + help='run liquidsoap on exit' + ) group = parser.add_argument_group('monitor') group.add_argument( @@ -211,24 +218,19 @@ class Command (BaseCommand): ) group = parser.add_argument_group('configuration') - parser.add_argument( + group.add_argument( '-s', '--station', type=int, help='generate files for the given station (if not set, do it for' ' all available stations)' ) - parser.add_argument( + group.add_argument( '-c', '--config', action='store_true', help='generate liquidsoap config file' ) - parser.add_argument( + group.add_argument( '-S', '--streams', action='store_true', help='generate all stream playlists' ) - parser.add_argument( - '-a', '--all', action='store_true', - help='shortcut for -cS' - ) - def handle (self, *args, **options): if options.get('station'): diff --git a/liquidsoap/utils.py b/liquidsoap/utils.py index b00745c..e9349d5 100644 --- a/liquidsoap/utils.py +++ b/liquidsoap/utils.py @@ -186,7 +186,7 @@ class Source: return { 'begin': stream.begin.strftime('%Hh%M') if stream.begin else None, 'end': stream.end.strftime('%Hh%M') if stream.end else None, - 'delay': to_seconds(stream.delay) if stream.delay else None + 'delay': to_seconds(stream.delay) if stream.delay else 0 } def skip (self): diff --git a/programs/management/commands/sounds_monitor.py b/programs/management/commands/sounds_monitor.py index e239f24..194b9bb 100644 --- a/programs/management/commands/sounds_monitor.py +++ b/programs/management/commands/sounds_monitor.py @@ -22,10 +22,15 @@ parameters given by the setting AIRCOX_SOUND_QUALITY. This script requires Sox (and soxi). """ import os +import time import re import logging import subprocess from argparse import RawTextHelpFormatter +import atexit + +from watchdog.observers import Observer +from watchdog.events import PatternMatchingEventHandler, FileModifiedEvent from django.core.management.base import BaseCommand, CommandError @@ -35,6 +40,172 @@ import aircox.programs.utils as utils logger = logging.getLogger('aircox.tools') +class SoundInfo: + name = '' + sound = None + + year = None + month = None + day = None + n = None + duration = None + + @property + def path (self): + return self._path + + @path.setter + def path (self, value): + """ + Parse file name to get info on the assumption it has the correct + format (given in Command.help) + """ + file_name = os.path.basename(value) + file_name = os.path.splitext(file_name)[0] + r = re.search('^(?P[0-9]{4})' + '(?P[0-9]{2})' + '(?P[0-9]{2})' + '(_(?P[0-9]+))?' + '_?(?P.*)$', + file_name) + + if not (r and r.groupdict()): + r = { 'name': file_name } + logger.info('file name can not be parsed -> %s', value) + else: + r = r.groupdict() + + self._path = value + self.name = r['name'].replace('_', ' ').capitalize() + self.year = int(r.get('year')) if 'year' in r else None + self.month = int(r.get('month')) if 'month' in r else None + self.day = int(r.get('day')) if 'day' in r else None + self.n = r.get('n') + return r + + def __init__ (self, path = ''): + self.path = path + + def get_duration (self): + p = subprocess.Popen(['soxi', '-D', self.path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate() + if not err: + duration = utils.seconds_to_time(int(float(out))) + self.duration = duration + return duration + + def get_sound (self, kwargs = None, save = True): + """ + Get or create a sound using self info. + + If the sound is created/modified, get its duration and update it + (if save is True, sync to DB). + """ + sound, created = Sound.objects.get_or_create( + path = self.path, + defaults = kwargs + ) + if created or sound.check_on_file(): + logger.info('sound is new or have been modified -> %s', self.path) + sound.duration = self.get_duration() + sound.name = self.name + if save: + sound.save() + self.sound = sound + return sound + + def find_diffusion (self, program, attach = False): + """ + For a given program, check if there is an initial diffusion + to associate to, using the date info we have. + + We only allow initial diffusion since there should be no + rerun. + + If attach is True and we have self.sound, we add self.sound to + the diffusion and update the DB. + """ + if self.year == None: + return; + + # check on episodes + diffusion = Diffusion.objects.filter( + program = program, + date__year = self.year, + date__month = self.month, + date__day = self.day, + initial = None, + ) + if not diffusion: + return + + diffusion = diffusion[0] + if attach and self.sound: + qs = diffusion.sounds.get_queryset().filter(path = sound.path) + if not qs: + logger.info('diffusion %s mathes to sound -> %s', str(diffusion), + sound.path) + diffusion.sounds.add(sound.pk) + diffusion.save() + return diffusion + + +class MonitorHandler (PatternMatchingEventHandler): + """ + Event handler for watchdog, in order to be used in monitoring. + """ + def __init__ (self, subdir): + """ + subdir: AIRCOX_SOUND_ARCHIVES_SUBDIR or AIRCOX_SOUND_EXCERPTS_SUBDIR + """ + self.subdir = subdir + if self.subdir == settings.AIRCOX_SOUND_ARCHIVES_SUBDIR: + self.sound_kwargs = { 'type': Sound.Type['archive'] } + else: + self.sound_kwargs = { 'type': Sound.Type['excerpt'] } + + patterns = ['*/{}/*{}'.format(self.subdir, ext) + for ext in settings.AIRCOX_SOUND_FILE_EXT ] + super().__init__(patterns=patterns, ignore_directories=True) + + def on_created (self, event): + self.on_modified(event) + + def on_modified (self, event): + logger.info('sound modified: %s', event.src_path) + program = Program.get_from_path(event.src_path) + if not program: + return + + si = SoundInfo(event.src_path) + si.get_sound(self.sound_kwargs, True) + if si.year != None: + si.find_diffusion(program, True) + + def on_deleted (self, event): + logger.info('sound deleted: %s', event.src_path) + sound = Sound.objects.filter(path = event.src_path) + if sound: + sound = sound[0] + sound.removed = True + sound.save() + + def on_moved (self, event): + logger.info('sound moved: %s -> %s', event.src_path, event.dest_path) + sound = Sound.objects.filter(path = event.src_path) + if not sound: + self.on_modified( + FileModifiedEvent(event.dest_path) + ) + return + + sound = sound[0] + sound.path = event.dest_path + sound.save() + + class Command (BaseCommand): help= __doc__ @@ -57,66 +228,20 @@ class Command (BaseCommand): help='Scan programs directories for changes, plus check for a ' ' matching episode on sounds that have not been yet assigned' ) + parser.add_argument( + '-m', '--monitor', action='store_true', + help='Run in monitor mode, watch for modification in the filesystem ' + 'and react in consequence' + ) + def handle (self, *args, **options): if options.get('scan'): self.scan() if options.get('quality_check'): self.check_quality(check = (not options.get('scan')) ) - - def _get_duration (self, path): - p = subprocess.Popen(['soxi', '-D', path], stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = p.communicate() - if not err: - return utils.seconds_to_time(int(float(out))) - - def _get_sound_info (self, program, path): - """ - Parse file name to get info on the assumption it has the correct - format (given in Command.help) - """ - file_name = os.path.basename(path) - file_name = os.path.splitext(file_name)[0] - r = re.search('^(?P[0-9]{4})' - '(?P[0-9]{2})' - '(?P[0-9]{2})' - '(_(?P[0-9]+))?' - '_?(?P.*)$', - file_name) - - if not (r and r.groupdict()): - self.report(program, path, "file path is not correct, use defaults") - r = { - 'name': file_name - } - else: - r = r.groupdict() - - r['name'] = r['name'].replace('_', ' ').capitalize() - r['path'] = path - return r - - def find_initial (self, program, sound_info): - """ - For a given program, and sound path check if there is an initial - diffusion to associate to, using the diffusion's date. - - If there is no matching episode, return None. - """ - # check on episodes - diffusion = Diffusion.objects.filter( - program = program, - date__year = int(sound_info['year']), - date__month = int(sound_info['month']), - date__day = int(sound_info['day']) - ) - - if not diffusion.count(): - self.report(program, sound_info['path'], - 'no diffusion found for the given date') - return - return diffusion[0] + if options.get('monitor'): + self.monitor() @staticmethod def check_sounds (qs): @@ -156,43 +281,23 @@ class Command (BaseCommand): return subdir = os.path.join(program.path, subdir) + new_sounds = [] - # new/existing sounds + # sounds in directory for path in os.listdir(subdir): path = os.path.join(subdir, path) if not path.endswith(settings.AIRCOX_SOUND_FILE_EXT): continue - sound, created = Sound.objects.get_or_create( - path = path, - defaults = sound_kwargs, - ) + si = SoundInfo(path) + si.get_sound(sound_kwargs, True) + if si.year != None: + si.find_diffusion(program, True) + new_sounds = [si.sound.pk] - sound_info = self._get_sound_info(program, path) - - if created or sound.check_on_file(): - sound_info['duration'] = self._get_duration() - sound.__dict__.update(sound_info) - sound.save(check = False) - - # initial diffusion association - if 'year' in sound_info: - initial = self.find_initial(program, sound_info) - if initial: - if initial.initial: - # FIXME: allow user to overwrite rerun info? - self.report(program, path, - 'the diffusion must be an initial diffusion') - else: - sound_ = initial.sounds.get_queryset() \ - .filter(path = sound.path) - if not sound_: - self.report(program, path, - 'add sound to diffusion ', initial.id) - initial.sounds.add(sound.pk) - initial.save() - - self.check_sounds(Sound.objects.filter(path__startswith = subdir)) + # sounds in db + self.check_sounds(Sound.objects.filter(path__startswith = subdir) \ + .exclude(pk__in = new_sounds )) def check_quality (self, check = False): """ @@ -233,3 +338,30 @@ class Command (BaseCommand): update_stats(sound_info, sound) sound.save(check = False) + def monitor (self): + """ + Run in monitor mode + """ + archives_handler = MonitorHandler( + subdir = settings.AIRCOX_SOUND_ARCHIVES_SUBDIR + ) + excerpts_handler = MonitorHandler( + subdir = settings.AIRCOX_SOUND_EXCERPTS_SUBDIR + ) + + observer = Observer() + observer.schedule(archives_handler, settings.AIRCOX_PROGRAMS_DIR, + recursive=True) + observer.schedule(excerpts_handler, settings.AIRCOX_PROGRAMS_DIR, + recursive=True) + observer.start() + + def leave(): + observer.stop() + observer.join() + atexit.register(leave) + + while True: + time.sleep(1) + + diff --git a/programs/management/commands/sounds_quality_check.py b/programs/management/commands/sounds_quality_check.py index 846a51c..d512a36 100644 --- a/programs/management/commands/sounds_quality_check.py +++ b/programs/management/commands/sounds_quality_check.py @@ -104,10 +104,10 @@ class Sound: ] if self.good: - logger.info(self.path + ': good samples:\033[92m%s\033[0m', + logger.info(self.path + ' -> good: \033[92m%s\033[0m', ', '.join(view(self.good))) if self.bad: - logger.info(self.path + ': good samples:\033[91m%s\033[0m', + logger.info(self.path + ' -> bad: \033[91m%s\033[0m', ', '.join(view(self.bad))) class Command (BaseCommand): diff --git a/programs/models.py b/programs/models.py index 9a8859c..ff46a6a 100755 --- a/programs/models.py +++ b/programs/models.py @@ -192,9 +192,9 @@ class Sound (Nameable): self.check_on_file() if not self.name and self.path: - self.name = os.path.basename(self.path) \ - .splitext() \ - .replace('_', ' ') + self.name = os.path.basename(self.path) + self.name = os.path.splitext(self.name)[0] + self.name = self.name.replace('_', ' ') super().save(*args, **kwargs) def __str__ (self): @@ -221,7 +221,7 @@ class Stream (models.Model): delay = models.TimeField( _('delay'), blank = True, null = True, - help_text = _('plays this playlist at least every delay') + help_text = _('delay between two sound plays') ) begin = models.TimeField( _('begin'), @@ -516,6 +516,24 @@ class Program (Nameable): sound.path.replace(self.__original_path, self.path) sound.save() + @classmethod + def get_from_path (cl, path): + """ + Return a Program from the given path. We assume the path has been + given in a previous time by this model (Program.path getter). + """ + path = path.replace(settings.AIRCOX_PROGRAMS_DIR, '') + while path[0] == '/': path = path[1:] + while path[-1] == '/': path = path[:-2] + if '/' in path: + path = path[:path.index('/')] + + path = path.split('_') + path = path[-1] + qs = cl.objects.filter(id = int(path)) + return qs[0] if qs else None + + class Diffusion (models.Model): """ A Diffusion is an occurrence of a Program that is scheduled on the diff --git a/programs/settings.py b/programs/settings.py index 36abc2a..c1e7f9a 100755 --- a/programs/settings.py +++ b/programs/settings.py @@ -12,7 +12,7 @@ ensure('AIRCOX_PROGRAMS_DIR', # Default directory for the sounds that not linked to a program ensure('AIRCOX_SOUND_DEFAULT_DIR', - os.path.join(AIRCOX_PROGRAMS_DIR, 'defaults')) + os.path.join(AIRCOX_PROGRAMS_DIR, 'defaults')), # Sub directory used for the complete episode sounds ensure('AIRCOX_SOUND_ARCHIVES_SUBDIR', 'archives') # Sub directory used for the excerpts of the episode diff --git a/requirements.txt b/requirements.txt index 544b85f..a7d5ac1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ django-taggit>=0.12.1 django-suit>=0.2.15 django-autocomplete-light>=2.2.8 easy-thumbnails>=2.2 +watchdog>=0.8.3