sound_monitor: filesystem monitoring using watchdog

liquidsoap/management/commands/liquidsoap.py

@@ -44,7 +44,6 @@ class StationConfig:
         log_script = os.path.join(log_script, 'manage.py') + \
                         ' liquidsoap_log'
 
-
         context = {
             'controller': self.controller,
             'settings': settings,
@@ -161,6 +160,7 @@ class Monitor:
         Keep trace of played sounds on the given source. For the moment we only
         keep track of known sounds.
         """
+        # TODO: repetition of the same sound out of an interval of time
         last_log = programs.Log.objects.filter(
             source = source.id,
         ).prefetch_related('related_object').order_by('-date')
@@ -170,9 +170,12 @@ class Monitor:
             return
 
         if last_log:
+            now = tz.datetime.now()
             last_log = last_log[0]
-            if type(last_log.related_object) == programs.Sound and \
+            last_obj = last_log.related_object
-                    on_air == last_log.related_object.path:
+            if type(last_obj) == programs.Sound and on_air == last_obj.path:
+                if not last_obj.duration or
+                    now < log.date + programs_utils.to_timedelta(last_obj.duration)
                     return
 
         sound = programs.Sound.objects.filter(path = on_air)
@@ -194,6 +197,10 @@ class Command (BaseCommand):
 
     def add_arguments (self, parser):
         parser.formatter_class=RawTextHelpFormatter
+        parser.add_argument(
+            '-e', '--exec', action='store_true',
+            help='run liquidsoap on exit'
+        )
 
         group = parser.add_argument_group('monitor')
         group.add_argument(
@@ -211,24 +218,19 @@ class Command (BaseCommand):
         )
 
         group = parser.add_argument_group('configuration')
-        parser.add_argument(
+        group.add_argument(
             '-s', '--station', type=int,
             help='generate files for the given station (if not set, do it for'
                  ' all available stations)'
         )
-        parser.add_argument(
+        group.add_argument(
             '-c', '--config', action='store_true',
             help='generate liquidsoap config file'
         )
-        parser.add_argument(
+        group.add_argument(
             '-S', '--streams', action='store_true',
             help='generate all stream playlists'
         )
-        parser.add_argument(
-            '-a', '--all', action='store_true',
-            help='shortcut for -cS'
-        )
-
 
     def handle (self, *args, **options):
         if options.get('station'):

liquidsoap/utils.py

@@ -186,7 +186,7 @@ class Source:
         return {
             'begin': stream.begin.strftime('%Hh%M') if stream.begin else None,
             'end': stream.end.strftime('%Hh%M') if stream.end else None,
-            'delay': to_seconds(stream.delay) if stream.delay else None
+            'delay': to_seconds(stream.delay) if stream.delay else 0
         }
 
     def skip (self):

programs/management/commands/sounds_monitor.py

@@ -22,10 +22,15 @@ parameters given by the setting AIRCOX_SOUND_QUALITY. This script requires
 Sox (and soxi).
 """
 import os
+import time
 import re
 import logging
 import subprocess
 from argparse import RawTextHelpFormatter
+import atexit
+
+from watchdog.observers import Observer
+from watchdog.events import PatternMatchingEventHandler, FileModifiedEvent
 
 from django.core.management.base import BaseCommand, CommandError
 
@@ -35,6 +40,172 @@ import aircox.programs.utils as utils
 
 logger = logging.getLogger('aircox.tools')
 
+class SoundInfo:
+    name = ''
+    sound = None
+
+    year = None
+    month = None
+    day = None
+    n = None
+    duration = None
+
+    @property
+    def path (self):
+        return self._path
+
+    @path.setter
+    def path (self, value):
+        """
+        Parse file name to get info on the assumption it has the correct
+        format (given in Command.help)
+        """
+        file_name = os.path.basename(value)
+        file_name = os.path.splitext(file_name)[0]
+        r = re.search('^(?P<year>[0-9]{4})'
+                      '(?P<month>[0-9]{2})'
+                      '(?P<day>[0-9]{2})'
+                      '(_(?P<n>[0-9]+))?'
+                      '_?(?P<name>.*)$',
+                      file_name)
+
+        if not (r and r.groupdict()):
+            r = { 'name': file_name }
+            logger.info('file name can not be parsed -> %s', value)
+        else:
+            r = r.groupdict()
+
+        self._path = value
+        self.name = r['name'].replace('_', ' ').capitalize()
+        self.year = int(r.get('year')) if 'year' in r else None
+        self.month = int(r.get('month')) if 'month' in r else None
+        self.day = int(r.get('day')) if 'day' in r else None
+        self.n = r.get('n')
+        return r
+
+    def __init__ (self, path = ''):
+        self.path = path
+
+    def get_duration (self):
+        p = subprocess.Popen(['soxi', '-D', self.path],
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        out, err = p.communicate()
+        if not err:
+            duration = utils.seconds_to_time(int(float(out)))
+            self.duration = duration
+            return duration
+
+    def get_sound (self, kwargs = None, save = True):
+        """
+        Get or create a sound using self info.
+
+        If the sound is created/modified, get its duration and update it
+        (if save is True, sync to DB).
+        """
+        sound, created = Sound.objects.get_or_create(
+            path = self.path,
+            defaults = kwargs
+        )
+        if created or sound.check_on_file():
+            logger.info('sound is new or have been modified -> %s', self.path)
+            sound.duration = self.get_duration()
+            sound.name = self.name
+            if save:
+                sound.save()
+        self.sound = sound
+        return sound
+
+    def find_diffusion (self, program, attach = False):
+        """
+        For a given program, check if there is an initial diffusion
+        to associate to, using the date info we have.
+
+        We only allow initial diffusion since there should be no
+        rerun.
+
+        If attach is True and we have self.sound, we add self.sound to
+        the diffusion and update the DB.
+        """
+        if self.year == None:
+            return;
+
+        # check on episodes
+        diffusion = Diffusion.objects.filter(
+            program = program,
+            date__year = self.year,
+            date__month = self.month,
+            date__day = self.day,
+            initial = None,
+        )
+        if not diffusion:
+            return
+
+        diffusion = diffusion[0]
+        if attach and self.sound:
+            qs = diffusion.sounds.get_queryset().filter(path = sound.path)
+            if not qs:
+                logger.info('diffusion %s mathes to sound -> %s', str(diffusion),
+                            sound.path)
+                diffusion.sounds.add(sound.pk)
+                diffusion.save()
+        return diffusion
+
+
+class MonitorHandler (PatternMatchingEventHandler):
+    """
+    Event handler for watchdog, in order to be used in monitoring.
+    """
+    def __init__ (self, subdir):
+        """
+        subdir: AIRCOX_SOUND_ARCHIVES_SUBDIR or AIRCOX_SOUND_EXCERPTS_SUBDIR
+        """
+        self.subdir = subdir
+        if self.subdir == settings.AIRCOX_SOUND_ARCHIVES_SUBDIR:
+            self.sound_kwargs = { 'type': Sound.Type['archive'] }
+        else:
+            self.sound_kwargs = { 'type': Sound.Type['excerpt'] }
+
+        patterns = ['*/{}/*{}'.format(self.subdir, ext)
+                    for ext in settings.AIRCOX_SOUND_FILE_EXT ]
+        super().__init__(patterns=patterns, ignore_directories=True)
+
+    def on_created (self, event):
+        self.on_modified(event)
+
+    def on_modified (self, event):
+        logger.info('sound modified: %s', event.src_path)
+        program = Program.get_from_path(event.src_path)
+        if not program:
+            return
+
+        si = SoundInfo(event.src_path)
+        si.get_sound(self.sound_kwargs, True)
+        if si.year != None:
+            si.find_diffusion(program, True)
+
+    def on_deleted (self, event):
+        logger.info('sound deleted: %s', event.src_path)
+        sound = Sound.objects.filter(path = event.src_path)
+        if sound:
+            sound = sound[0]
+            sound.removed = True
+            sound.save()
+
+    def on_moved (self, event):
+        logger.info('sound moved: %s -> %s', event.src_path, event.dest_path)
+        sound = Sound.objects.filter(path = event.src_path)
+        if not sound:
+            self.on_modified(
+                FileModifiedEvent(event.dest_path)
+            )
+            return
+
+        sound = sound[0]
+        sound.path = event.dest_path
+        sound.save()
+
+
 class Command (BaseCommand):
     help= __doc__
 
@@ -57,66 +228,20 @@ class Command (BaseCommand):
             help='Scan programs directories for changes, plus check for a '
                  ' matching episode on sounds that have not been yet assigned'
         )
+        parser.add_argument(
+            '-m', '--monitor', action='store_true',
+            help='Run in monitor mode, watch for modification in the filesystem '
+                 'and react in consequence'
+        )
+
 
     def handle (self, *args, **options):
         if options.get('scan'):
             self.scan()
         if options.get('quality_check'):
             self.check_quality(check = (not options.get('scan')) )
-
+        if options.get('monitor'):
-    def _get_duration (self, path):
+            self.monitor()
-        p = subprocess.Popen(['soxi', '-D', path], stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
-        out, err = p.communicate()
-        if not err:
-            return utils.seconds_to_time(int(float(out)))
-
-    def _get_sound_info (self, program, path):
-        """
-        Parse file name to get info on the assumption it has the correct
-        format (given in Command.help)
-        """
-        file_name = os.path.basename(path)
-        file_name = os.path.splitext(file_name)[0]
-        r = re.search('^(?P<year>[0-9]{4})'
-                      '(?P<month>[0-9]{2})'
-                      '(?P<day>[0-9]{2})'
-                      '(_(?P<n>[0-9]+))?'
-                      '_?(?P<name>.*)$',
-                      file_name)
-
-        if not (r and r.groupdict()):
-            self.report(program, path, "file path is not correct, use defaults")
-            r = {
-                'name': file_name
-            }
-        else:
-            r = r.groupdict()
-
-        r['name'] = r['name'].replace('_', ' ').capitalize()
-        r['path'] = path
-        return r
-
-    def find_initial (self, program, sound_info):
-        """
-        For a given program, and sound path check if there is an initial
-        diffusion to associate to, using the diffusion's date.
-
-        If there is no matching episode, return None.
-        """
-        # check on episodes
-        diffusion = Diffusion.objects.filter(
-            program = program,
-            date__year = int(sound_info['year']),
-            date__month = int(sound_info['month']),
-            date__day = int(sound_info['day'])
-        )
-
-        if not diffusion.count():
-            self.report(program, sound_info['path'],
-                        'no diffusion found for the given date')
-            return
-        return diffusion[0]
 
     @staticmethod
     def check_sounds (qs):
@@ -156,43 +281,23 @@ class Command (BaseCommand):
             return
 
         subdir = os.path.join(program.path, subdir)
+        new_sounds = []
 
-        # new/existing sounds
+        # sounds in directory
         for path in os.listdir(subdir):
             path = os.path.join(subdir, path)
             if not path.endswith(settings.AIRCOX_SOUND_FILE_EXT):
                 continue
 
-            sound, created = Sound.objects.get_or_create(
+            si = SoundInfo(path)
-                path = path,
+            si.get_sound(sound_kwargs, True)
-                defaults = sound_kwargs,
+            if si.year != None:
-            )
+                si.find_diffusion(program, True)
+            new_sounds = [si.sound.pk]
 
-            sound_info = self._get_sound_info(program, path)
+        # sounds in db
-
+        self.check_sounds(Sound.objects.filter(path__startswith = subdir) \
-            if created or sound.check_on_file():
+                                       .exclude(pk__in = new_sounds ))
-                sound_info['duration'] = self._get_duration()
-                sound.__dict__.update(sound_info)
-                sound.save(check = False)
-
-            # initial diffusion association
-            if 'year' in sound_info:
-                initial = self.find_initial(program, sound_info)
-                if initial:
-                    if initial.initial:
-                        # FIXME: allow user to overwrite rerun info?
-                        self.report(program, path,
-                                    'the diffusion must be an initial diffusion')
-                    else:
-                        sound_ = initial.sounds.get_queryset() \
-                                    .filter(path = sound.path)
-                        if not sound_:
-                            self.report(program, path,
-                                        'add sound to diffusion ', initial.id)
-                            initial.sounds.add(sound.pk)
-                            initial.save()
-
-        self.check_sounds(Sound.objects.filter(path__startswith = subdir))
 
     def check_quality (self, check = False):
         """
@@ -233,3 +338,30 @@ class Command (BaseCommand):
             update_stats(sound_info, sound)
             sound.save(check = False)
 
+    def monitor (self):
+        """
+        Run in monitor mode
+        """
+        archives_handler = MonitorHandler(
+            subdir = settings.AIRCOX_SOUND_ARCHIVES_SUBDIR
+        )
+        excerpts_handler = MonitorHandler(
+            subdir = settings.AIRCOX_SOUND_EXCERPTS_SUBDIR
+        )
+
+        observer = Observer()
+        observer.schedule(archives_handler, settings.AIRCOX_PROGRAMS_DIR,
+                          recursive=True)
+        observer.schedule(excerpts_handler, settings.AIRCOX_PROGRAMS_DIR,
+                          recursive=True)
+        observer.start()
+
+        def leave():
+            observer.stop()
+            observer.join()
+        atexit.register(leave)
+
+        while True:
+            time.sleep(1)
+
+

programs/management/commands/sounds_quality_check.py

@@ -104,10 +104,10 @@ class Sound:
         ]
 
         if self.good:
-            logger.info(self.path + ': good samples:\033[92m%s\033[0m',
+            logger.info(self.path + ' -> good: \033[92m%s\033[0m',
                         ', '.join(view(self.good)))
         if self.bad:
-            logger.info(self.path + ': good samples:\033[91m%s\033[0m',
+            logger.info(self.path + ' -> bad: \033[91m%s\033[0m',
                         ', '.join(view(self.bad)))
 
 class Command (BaseCommand):

programs/models.py

@@ -192,9 +192,9 @@ class Sound (Nameable):
             self.check_on_file()
 
         if not self.name and self.path:
-            self.name = os.path.basename(self.path) \
+            self.name = os.path.basename(self.path)
-                            .splitext() \
+            self.name = os.path.splitext(self.name)[0]
-                            .replace('_', ' ')
+            self.name = self.name.replace('_', ' ')
         super().save(*args, **kwargs)
 
     def __str__ (self):
@@ -221,7 +221,7 @@ class Stream (models.Model):
     delay = models.TimeField(
         _('delay'),
         blank = True, null = True,
-        help_text = _('plays this playlist at least every delay')
+        help_text = _('delay between two sound plays')
     )
     begin = models.TimeField(
         _('begin'),
@@ -516,6 +516,24 @@ class Program (Nameable):
                 sound.path.replace(self.__original_path, self.path)
                 sound.save()
 
+    @classmethod
+    def get_from_path (cl, path):
+        """
+        Return a Program from the given path. We assume the path has been
+        given in a previous time by this model (Program.path getter).
+        """
+        path = path.replace(settings.AIRCOX_PROGRAMS_DIR, '')
+        while path[0] == '/': path = path[1:]
+        while path[-1] == '/': path = path[:-2]
+        if '/' in path:
+            path = path[:path.index('/')]
+
+        path = path.split('_')
+        path = path[-1]
+        qs = cl.objects.filter(id = int(path))
+        return qs[0] if qs else None
+
+
 class Diffusion (models.Model):
     """
     A Diffusion is an occurrence of a Program that is scheduled on the

programs/settings.py

@@ -12,7 +12,7 @@ ensure('AIRCOX_PROGRAMS_DIR',
 
 # Default directory for the sounds that not linked to a program
 ensure('AIRCOX_SOUND_DEFAULT_DIR',
-       os.path.join(AIRCOX_PROGRAMS_DIR, 'defaults'))
+       os.path.join(AIRCOX_PROGRAMS_DIR, 'defaults')),
 # Sub directory used for the complete episode sounds
 ensure('AIRCOX_SOUND_ARCHIVES_SUBDIR', 'archives')
 # Sub directory used for the excerpts of the episode

requirements.txt

@@ -3,4 +3,5 @@ django-taggit>=0.12.1
 django-suit>=0.2.15
 django-autocomplete-light>=2.2.8
 easy-thumbnails>=2.2
+watchdog>=0.8.3