Merge pull request 'Fix sound monitor issues' (#82) from dev-1.0-sound-monitor-fixes into develop-1.0

Reviewed-on: #82

aircox/management/commands/sounds_monitor.py

@@ -25,233 +25,23 @@ Sox (and soxi).
 """
 from argparse import RawTextHelpFormatter
 import concurrent.futures as futures
-import datetime
 import atexit
 import logging
 import os
-import re
 import time
 
-import mutagen
 from watchdog.observers import Observer
-from watchdog.events import PatternMatchingEventHandler, FileModifiedEvent
 
-from django.conf import settings as conf
+from django.core.management.base import BaseCommand
-from django.core.management.base import BaseCommand, CommandError
-from django.utils import timezone as tz
-from django.utils.translation import gettext as _
 
-from aircox import settings, utils
+from aircox import settings
-from aircox.models import Diffusion, Program, Sound, Track
+from aircox.models import Program, Sound
-from .import_playlist import PlaylistImport
+from aircox.management.sound_file import SoundFile
+from aircox.management.sound_monitor import MonitorHandler
 
 logger = logging.getLogger('aircox.commands')
 
 
-sound_path_re = re.compile(
-    '^(?P<year>[0-9]{4})(?P<month>[0-9]{2})(?P<day>[0-9]{2})'
-    '(_(?P<hour>[0-9]{2})h(?P<minute>[0-9]{2}))?'
-    '(_(?P<n>[0-9]+))?'
-    '_?[ -]*(?P<name>.*)$'
-)
-
-
-class SoundFile:
-    path = None
-    info = None
-    path_info = None
-    sound = None
-
-    def __init__(self, path):
-        self.path = path
-
-    @property
-    def sound_path(self):
-        """ Relative path name """
-        return self.path.replace(conf.MEDIA_ROOT + '/', '')
-
-    def sync(self, sound=None, program=None, deleted=False, **kwargs):
-        """
-        Update related sound model and save it.
-        """
-        if deleted:
-            sound = Sound.objects.path(self.path).first()
-            if sound:
-                sound.type = sound.TYPE_REMOVED
-                sound.check_on_file()
-                sound.save()
-                return sound
-
-        # FIXME: sound.program as not null
-        if not program:
-            program = Program.get_from_path(self.path)
-            logger.info('program from path "%s" -> %s', self.path, program)
-        kwargs['program_id'] = program.pk
-
-        sound, created = Sound.objects.get_or_create(file=self.sound_path, defaults=kwargs) \
-                         if not sound else (sound, False)
-        self.sound = sound
-        sound.program = program
-        if created or sound.check_on_file():
-            logger.info('sound is new or have been modified -> %s', self.sound_path)
-            self.read_path()
-            sound.name = self.path_info.get('name')
-
-            self.read_file_info()
-            if self.info is not None:
-                sound.duration = utils.seconds_to_time(self.info.info.length)
-
-        # check for episode
-        if sound.episode is None and self.read_path():
-            self.find_episode(program)
-
-        sound.save()
-
-        # check for playlist
-        self.find_playlist(sound)
-        return sound
-
-    def read_path(self):
-        """
-        Parse file name to get info on the assumption it has the correct
-        format (given in Command.help). Return True if path contains informations.
-        """
-        if self.path_info:
-            return 'year' in self.path_info
-
-        name = os.path.splitext(os.path.basename(self.path))[0]
-        match = sound_path_re.search(name)
-        if match:
-            path_info = match.groupdict()
-            for k in ('year', 'month', 'day', 'hour', 'minute'):
-                if path_info.get(k) is not None:
-                    path_info[k] = int(path_info[k])
-            self.path_info = path_info
-            return True
-        else:
-            self.path_info = {'name': name}
-            return False
-
-    def read_file_info(self):
-        """ Read file information and metadata. """
-        if os.path.exists(self.path):
-            self.info = mutagen.File(self.path)
-        else:
-            self.info = None
-
-    def find_episode(self, program):
-        """
-        For a given program, check if there is an initial diffusion
-        to associate to, using the date info we have. Update self.sound
-        and save it consequently.
-
-        We only allow initial diffusion since there should be no
-        rerun.
-        """
-        pi = self.path_info
-        if 'year' not in pi or not self.sound or self.sound.episode:
-            return None
-
-        if pi.get('hour') is not None:
-            date = tz.datetime(pi.get('year'), pi.get('month'), pi.get('day'),
-                               pi.get('hour') or 0, pi.get('minute') or 0)
-            date = tz.get_current_timezone().localize(date)
-        else:
-            date = datetime.date(pi.get('year'), pi.get('month'), pi.get('day'))
-
-        diffusion = program.diffusion_set.at(date).first()
-        if not diffusion:
-            return None
-
-        logger.info('%s <--> %s', self.sound.file.name, str(diffusion.episode))
-        self.sound.episode = diffusion.episode
-        return diffusion
-
-    def find_playlist(self, sound=None, use_meta=True):
-        """
-        Find a playlist file corresponding to the sound path, such as:
-            my_sound.ogg => my_sound.csv
-
-        Use sound's file metadata if no corresponding playlist has been
-        found and `use_meta` is True.
-        """
-        if sound is None:
-            sound = self.sound
-        if sound.track_set.count() > 1:
-            return
-
-        # import playlist
-        path_noext, ext = os.path.splitext(self.sound.file.path)
-        path = path_noext + '.csv'
-        if os.path.exists(path):
-            PlaylistImport(path, sound=sound).run()
-        # use metadata
-        elif use_meta:
-            if self.info is None:
-                self.read_file_info()
-            if self.info and self.info.tags:
-                tags = self.info.tags
-                title, artist, album, year = tuple(
-                    t and ', '.join(t) for t in (
-                        tags.get(k) for k in ('title', 'artist', 'album', 'year'))
-                )
-                title = title or (self.path_info and self.path_info.get('name')) or \
-                            os.path.basename(path_noext)
-                info = '{} ({})'.format(album, year) if album and year else \
-                       album or year or ''
-                track = Track(sound=sound,
-                              position=int(tags.get('tracknumber', 0)),
-                              title=title,
-                              artist=artist or _('unknown'),
-                              info=info)
-                track.save()
-
-
-class MonitorHandler(PatternMatchingEventHandler):
-    """
-    Event handler for watchdog, in order to be used in monitoring.
-    """
-    pool = None
-
-    def __init__(self, subdir, pool):
-        """
-        subdir: AIRCOX_SOUND_ARCHIVES_SUBDIR or AIRCOX_SOUND_EXCERPTS_SUBDIR
-        """
-        self.subdir = subdir
-        self.pool = pool
-        if self.subdir == settings.AIRCOX_SOUND_ARCHIVES_SUBDIR:
-            self.sound_kwargs = {'type': Sound.TYPE_ARCHIVE}
-        else:
-            self.sound_kwargs = {'type': Sound.TYPE_EXCERPT}
-
-        patterns = ['*/{}/*{}'.format(self.subdir, ext)
-                    for ext in settings.AIRCOX_SOUND_FILE_EXT]
-        super().__init__(patterns=patterns, ignore_directories=True)
-
-    def on_created(self, event):
-        self.on_modified(event)
-
-    def on_modified(self, event):
-        logger.info('sound modified: %s', event.src_path)
-        def updated(event, sound_kwargs):
-            SoundFile(event.src_path).sync(**sound_kwargs)
-        self.pool.submit(updated, event, self.sound_kwargs)
-
-    def on_moved(self, event):
-        logger.info('sound moved: %s -> %s', event.src_path, event.dest_path)
-        def moved(event, sound_kwargs):
-            sound = Sound.objects.filter(file=event.src_path)
-            sound_file = SoundFile(event.dest_path) if not sound else sound
-            sound_file.sync(**sound_kwargs)
-        self.pool.submit(moved, event, self.sound_kwargs)
-
-    def on_deleted(self, event):
-        logger.info('sound deleted: %s', event.src_path)
-        def deleted(event):
-            SoundFile(event.src_path).sync(deleted=True)
-        self.pool.submit(deleted, event)
-
-
 class Command(BaseCommand):
     help = __doc__
 

aircox/management/commands/sounds_quality_check.py

@@ -1,118 +1,16 @@
 """
 Analyse and check files using Sox, prints good and bad files.
 """
-import sys
 import logging
-import re
-import subprocess
 from argparse import RawTextHelpFormatter
 
 from django.core.management.base import BaseCommand, CommandError
 
+from aircox.management.sound_stats import SoxStats, SoundStats
+
 logger = logging.getLogger('aircox.commands')
 
 
-class Stats:
-    attributes = [
-        'DC offset', 'Min level', 'Max level',
-        'Pk lev dB', 'RMS lev dB', 'RMS Pk dB',
-        'RMS Tr dB', 'Flat factor', 'Length s',
-    ]
-
-    def __init__(self, path, **kwargs):
-        """
-        If path is given, call analyse with path and kwargs
-        """
-        self.values = {}
-        if path:
-            self.analyse(path, **kwargs)
-
-    def get(self, attr):
-        return self.values.get(attr)
-
-    def parse(self, output):
-        for attr in Stats.attributes:
-            value = re.search(attr + r'\s+(?P<value>\S+)', output)
-            value = value and value.groupdict()
-            if value:
-                try:
-                    value = float(value.get('value'))
-                except ValueError:
-                    value = None
-                self.values[attr] = value
-        self.values['length'] = self.values['Length s']
-
-    def analyse(self, path, at=None, length=None):
-        """
-        If at and length are given use them as excerpt to analyse.
-        """
-        args = ['sox', path, '-n']
-
-        if at is not None and length is not None:
-            args += ['trim', str(at), str(length)]
-
-        args.append('stats')
-
-        p = subprocess.Popen(args, stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
-        # sox outputs to stderr (my god WHYYYY)
-        out_, out = p.communicate()
-        self.parse(str(out, encoding='utf-8'))
-
-
-class SoundStats:
-    path = None             # file path
-    sample_length = 120     # default sample length in seconds
-    stats = None            # list of samples statistics
-    bad = None              # list of bad samples
-    good = None             # list of good samples
-
-    def __init__(self, path, sample_length=None):
-        self.path = path
-        self.sample_length = sample_length if sample_length is not None \
-            else self.sample_length
-
-    def get_file_stats(self):
-        return self.stats and self.stats[0]
-
-    def analyse(self):
-        logger.info('complete file analysis')
-        self.stats = [Stats(self.path)]
-        position = 0
-        length = self.stats[0].get('length')
-
-        if not self.sample_length:
-            return
-
-        logger.info('start samples analysis...')
-        while position < length:
-            stats = Stats(self.path, at=position, length=self.sample_length)
-            self.stats.append(stats)
-            position += self.sample_length
-
-    def check(self, name, min_val, max_val):
-        self.good = [index for index, stats in enumerate(self.stats)
-                     if min_val <= stats.get(name) <= max_val]
-        self.bad = [index for index, stats in enumerate(self.stats)
-                    if index not in self.good]
-        self.resume()
-
-    def resume(self):
-        def view(array): return [
-            'file' if index is 0 else
-            'sample {} (at {} seconds)'.format(
-                index, (index-1) * self.sample_length)
-            for index in array
-        ]
-
-        if self.good:
-            logger.info(self.path + ' -> good: \033[92m%s\033[0m',
-                        ', '.join(view(self.good)))
-        if self.bad:
-            logger.info(self.path + ' -> bad: \033[91m%s\033[0m',
-                        ', '.join(view(self.bad)))
-
-
 class Command (BaseCommand):
     help = __doc__
     sounds = None
@@ -132,7 +30,7 @@ class Command (BaseCommand):
         parser.add_argument(
             '-a', '--attribute', type=str,
             help='attribute name to use to check, that can be:\n' +
-                 ', '.join(['"{}"'.format(attr) for attr in Stats.attributes])
+                 ', '.join(['"{}"'.format(attr) for attr in SoxStats.attributes])
         )
         parser.add_argument(
             '-r', '--range', type=float, nargs=2,
@@ -160,7 +58,7 @@ class Command (BaseCommand):
         self.bad = []
         self.good = []
         for sound in self.sounds:
-            logger.info('analyse ' + sound.file.name)
+            logger.info('analyse ' + sound.path)
             sound.analyse()
             sound.check(attr, minmax[0], minmax[1])
             if sound.bad:
@@ -171,6 +69,6 @@ class Command (BaseCommand):
         # resume
         if options.get('resume'):
             for sound in self.good:
-                logger.info('\033[92m+ %s\033[0m', sound.file.name)
+                logger.info('\033[92m+ %s\033[0m', sound.path)
             for sound in self.bad:
-                logger.info('\033[91m+ %s\033[0m', sound.file.name)
+                logger.info('\033[91m+ %s\033[0m', sound.path)

aircox/management/sound_file.py

@@ -0,0 +1,216 @@
+#! /usr/bin/env python3
+"""
+Provide SoundFile which is used to link between database and file system.
+
+
+File name
+=========
+It tries to parse the file name to get the date of the diffusion of an
+episode and associate the file with it; We use the following format:
+    yyyymmdd[_n][_][name]
+
+Where:
+    'yyyy' the year of the episode's diffusion;
+    'mm' the month of the episode's diffusion;
+    'dd' the day of the episode's diffusion;
+    'n' the number of the episode (if multiple episodes);
+    'name' the title of the sound;
+
+Sound Quality
+=============
+To check quality of files, call the command sound_quality_check using the
+parameters given by the setting AIRCOX_SOUND_QUALITY. This script requires
+Sox (and soxi).
+"""
+from datetime import date
+import logging
+import os
+import re
+
+import mutagen
+
+from django.conf import settings as conf
+from django.utils import timezone as tz
+from django.utils.translation import gettext as _
+
+from aircox import utils
+from aircox.models import Program, Sound, Track
+from .commands.import_playlist import PlaylistImport
+
+logger = logging.getLogger('aircox.commands')
+
+
+class SoundFile:
+    """
+    Handle synchronisation between sounds on files and database.
+    """
+    path = None
+    info = None
+    path_info = None
+    sound = None
+
+    def __init__(self, path):
+        self.path = path
+
+    @property
+    def sound_path(self):
+        """ Relative path name """
+        return self.path.replace(conf.MEDIA_ROOT + '/', '')
+
+    @property
+    def episode(self):
+        return self.sound and self.sound.episode
+
+    def sync(self, sound=None, program=None, deleted=False, **kwargs):
+        """
+        Update related sound model and save it.
+        """
+        if deleted:
+            return self._on_delete(self.path)
+
+        # FIXME: sound.program as not null
+        if not program:
+            program = Program.get_from_path(self.path)
+            logger.debug('program from path "%s" -> %s', self.path, program)
+        kwargs['program_id'] = program.pk
+
+        if sound:
+            created = False
+        else:
+            sound, created = Sound.objects.get_or_create(
+                file=self.sound_path, defaults=kwargs)
+
+        self.sound = sound
+        self.path_info = self.read_path(self.path)
+
+        sound.program = program
+        if created or sound.check_on_file():
+            sound.name = self.path_info.get('name')
+            self.info = self.read_file_info()
+            if self.info is not None:
+                sound.duration = utils.seconds_to_time(self.info.info.length)
+
+        # check for episode
+        if sound.episode is None and 'year' in self.path_info:
+            sound.episode = self.find_episode(sound, self.path_info)
+        sound.save()
+
+        # check for playlist
+        self.find_playlist(sound)
+        return sound
+
+    def _on_delete(self, path):
+        # TODO: remove from db on delete
+        sound = Sound.objects.path(self.path).first()
+        if sound:
+            sound.type = sound.TYPE_REMOVED
+            sound.check_on_file()
+            sound.save()
+        return sound
+
+    def read_path(self, path):
+        """
+        Parse path name returning dictionary of extracted info. It can contain:
+        - `year`, `month`, `day`: diffusion date
+        - `hour`, `minute`: diffusion time
+        - `n`: sound arbitrary number (used for sound ordering)
+        - `name`: cleaned name extracted or file name (without extension)
+        """
+        basename = os.path.basename(path)
+        basename = os.path.splitext(basename)[0]
+        reg_match = self._path_re.search(basename)
+        if reg_match:
+            info = reg_match.groupdict()
+            for k in ('year', 'month', 'day', 'hour', 'minute', 'n'):
+                if info.get(k) is not None:
+                    info[k] = int(info[k])
+
+            name = info.get('name')
+            info['name'] = name and self._into_name(name) or basename
+        else:
+            info = {'name': basename}
+        return info
+
+    _path_re = re.compile(
+        '^(?P<year>[0-9]{4})(?P<month>[0-9]{2})(?P<day>[0-9]{2})'
+        '(_(?P<hour>[0-9]{2})h(?P<minute>[0-9]{2}))?'
+        '(_(?P<n>[0-9]+))?'
+        '_?[ -]*(?P<name>.*)$'
+    )
+
+    def _into_name(self, name):
+        name = name.replace('_', ' ')
+        return ' '.join(r.capitalize() for r in name.split(' '))
+
+    def read_file_info(self):
+        """ Read file information and metadata. """
+        return mutagen.File(self.path) if os.path.exists(self.path) else None
+
+    def find_episode(self, sound, path_info):
+        """
+        For a given program, check if there is an initial diffusion
+        to associate to, using the date info we have. Update self.sound
+        and save it consequently.
+
+        We only allow initial diffusion since there should be no
+        rerun.
+        """
+        program, pi = sound.program, path_info
+        if 'year' not in pi or not sound or sound.episode:
+            return None
+
+        year, month, day = pi.get('year'), pi.get('month'), pi.get('day')
+        if pi.get('hour') is not None:
+            at = tz.datetime(year, month, day, pi.get('hour', 0),
+                             pi.get('minute', 0))
+            at = tz.get_current_timezone().localize(at)
+        else:
+            at = date(year, month, day)
+
+        diffusion = program.diffusion_set.at(at).first()
+        if not diffusion:
+            return None
+
+        logger.debug('%s <--> %s', sound.file.name, str(diffusion.episode))
+        return diffusion.episode
+
+    def find_playlist(self, sound=None, use_meta=True):
+        """
+        Find a playlist file corresponding to the sound path, such as:
+            my_sound.ogg => my_sound.csv
+
+        Use sound's file metadata if no corresponding playlist has been
+        found and `use_meta` is True.
+        """
+        if sound is None:
+            sound = self.sound
+        if sound.track_set.count() > 1:
+            return
+
+        # import playlist
+        path_noext, ext = os.path.splitext(self.sound.file.path)
+        path = path_noext + '.csv'
+        if os.path.exists(path):
+            PlaylistImport(path, sound=sound).run()
+        # use metadata
+        elif use_meta:
+            if self.info is None:
+                self.read_file_info()
+            if self.info and self.info.tags:
+                tags = self.info.tags
+                title, artist, album, year = tuple(
+                    t and ', '.join(t) for t in (
+                        tags.get(k) for k in ('title', 'artist', 'album',
+                                              'year'))
+                )
+                title = title or (self.path_info and
+                                  self.path_info.get('name')) or \
+                    os.path.basename(path_noext)
+                info = '{} ({})'.format(album, year) if album and year else \
+                    album or year or ''
+                track = Track(sound=sound,
+                              position=int(tags.get('tracknumber', 0)),
+                              title=title,
+                              artist=artist or _('unknown'),
+                              info=info)
+                track.save()

aircox/management/sound_monitor.py

@@ -0,0 +1,163 @@
+#! /usr/bin/env python3
+
+"""
+Monitor sound files; For each program, check for:
+- new files;
+- deleted files;
+- differences between files and sound;
+- quality of the files;
+
+It tries to parse the file name to get the date of the diffusion of an
+episode and associate the file with it; WNotifye the following format:
+    yyyymmdd[_n][_][name]
+
+Where:
+    'yyyy' the year Notifyhe episode's diffusion;
+    'mm' the month of the episode's difNotifyon;
+    'dd' the day of the episode's diffusion;
+    'n' the number of the episode (if multiple episodes);
+    'name' the title of the sNotify;
+
+
+To check quality of files, call the command sound_quality_check using the
+parameters given by the setting AIRCOX_SOUND_QUALITY. This script requires
+Sox (and soxi).
+"""
+from datetime import datetime, timedelta
+import logging
+import time
+
+from watchdog.events import PatternMatchingEventHandler
+
+from aircox import settings
+from aircox.models import Sound
+
+from .sound_file import SoundFile
+
+
+logger = logging.getLogger('aircox.commands')
+
+
+__all__ = ('NotifyHandler', 'CreateHandler', 'DeleteHandler',
+           'MoveHandler', 'ModifiedHandler', 'MonitorHandler',)
+
+
+class NotifyHandler:
+    future = None
+    log_msg = None
+
+    def __init__(self):
+        self.timestamp = datetime.now()
+
+    def ping(self):
+        self.timestamp = datetime.now()
+
+    def __call__(self, event, path=None, **kw):
+        sound_file = SoundFile(path or event.src_path)
+        if self.log_msg:
+            msg = self.log_msg.format(event=event, sound_file=sound_file)
+            logger.info(msg)
+
+        sound_file.sync(**kw)
+        return sound_file
+
+
+class CreateHandler(NotifyHandler):
+    log_msg = 'Sound file created: {sound_file.path}'
+
+
+class DeleteHandler(NotifyHandler):
+    log_msg = 'Sound file deleted: {sound_file.path}'
+
+    def __call__(self, *args, **kwargs):
+        kwargs['deleted'] = True
+        return super().__call__(*args, **kwargs)
+
+
+class MoveHandler(NotifyHandler):
+    log_msg = 'Sound file moved: {event.src_path} -> {event.dest_path}'
+
+    def __call__(self, event, **kw):
+        sound = Sound.objects.filter(file=event.src_path)
+        # FIXME: this is wrong
+        if sound:
+            kw['sound'] = sound
+            kw['path'] = event.src_path
+        else:
+            kw['path'] = event.dest_path
+        return super().__call__(event, **kw)
+
+
+class ModifiedHandler(NotifyHandler):
+    timeout_delta = timedelta(seconds=30)
+    log_msg = 'Sound file updated: {sound_file.path}'
+
+    def wait(self):
+        # multiple call of this handler can be done consecutively, we block
+        # its thread using timeout
+        # Note: this method may be subject to some race conflicts, but this
+        #       should not be big a real issue.
+        timeout = self.timestamp + self.timeout_delta
+        while datetime.now() < timeout:
+            time.sleep(self.timeout_delta.total_seconds())
+            timeout = self.timestamp + self.timeout_delta
+
+    def __call__(self, event, **kw):
+        self.wait()
+        return super().__call__(event, **kw)
+
+
+class MonitorHandler(PatternMatchingEventHandler):
+    """
+    Event handler for watchdog, in order to be used in monitoring.
+    """
+    pool = None
+    jobs = {}
+
+    def __init__(self, subdir, pool):
+        """
+        subdir: AIRCOX_SOUND_ARCHIVES_SUBDIR or AIRCOX_SOUND_EXCERPTS_SUBDIR
+        """
+        self.subdir = subdir
+        self.pool = pool
+        if self.subdir == settings.AIRCOX_SOUND_ARCHIVES_SUBDIR:
+            self.sync_kw = {'type': Sound.TYPE_ARCHIVE}
+        else:
+            self.sync_kw = {'type': Sound.TYPE_EXCERPT}
+
+        patterns = ['*/{}/*{}'.format(self.subdir, ext)
+                    for ext in settings.AIRCOX_SOUND_FILE_EXT]
+        super().__init__(patterns=patterns, ignore_directories=True)
+
+    def on_created(self, event):
+        self._submit(CreateHandler(), event, 'new', **self.sync_kw)
+
+    def on_deleted(self, event):
+        self._submit(DeleteHandler(), event, 'del')
+
+    def on_moved(self, event):
+        self._submit(MoveHandler(), event, 'mv', **self.sync_kw)
+
+    def on_modified(self, event):
+        self._submit(ModifiedHandler(), event, 'up', **self.sync_kw)
+
+    def _submit(self, handler, event, job_key_prefix, **kwargs):
+        """
+        Send handler job to pool if not already running.
+        Return tuple with running job and boolean indicating if its a new one.
+        """
+        key = job_key_prefix + ':' + event.src_path
+        job = self.jobs.get(key)
+        if job and not job.future.done():
+            job.ping()
+            return job, False
+
+        handler.future = self.pool.submit(handler, event, **kwargs)
+        self.jobs[key] = handler
+
+        def done(r):
+            print(':::: job done', key)
+            if self.jobs.get(key) is handler:
+                del self.jobs[key]
+        handler.future.add_done_callback(done)
+        return handler, True

aircox/management/sound_stats.py

@@ -0,0 +1,115 @@
+"""
+Provide sound analysis class using Sox.
+"""
+import logging
+import re
+import subprocess
+
+logger = logging.getLogger('aircox.commands')
+
+
+__all__ = ('SoxStats', 'SoundStats')
+
+
+class SoxStats:
+    """
+    Run Sox process and parse output 
+    """
+    attributes = [
+        'DC offset', 'Min level', 'Max level',
+        'Pk lev dB', 'RMS lev dB', 'RMS Pk dB',
+        'RMS Tr dB', 'Flat factor', 'Length s',
+    ]
+
+    def __init__(self, path, **kwargs):
+        """
+        If path is given, call analyse with path and kwargs
+        """
+        self.values = {}
+        if path:
+            self.analyse(path, **kwargs)
+
+    def get(self, attr):
+        return self.values.get(attr)
+
+    def parse(self, output):
+        for attr in self.attributes:
+            value = re.search(attr + r'\s+(?P<value>\S+)', output)
+            value = value and value.groupdict()
+            if value:
+                try:
+                    value = float(value.get('value'))
+                except ValueError:
+                    value = None
+                self.values[attr] = value
+        self.values['length'] = self.values['Length s']
+
+    def analyse(self, path, at=None, length=None):
+        """
+        If at and length are given use them as excerpt to analyse.
+        """
+        args = ['sox', path, '-n']
+
+        if at is not None and length is not None:
+            args += ['trim', str(at), str(length)]
+
+        args.append('stats')
+
+        p = subprocess.Popen(args, stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        # sox outputs to stderr (my god WHYYYY)
+        out_, out = p.communicate()
+        self.parse(str(out, encoding='utf-8'))
+
+
+class SoundStats:
+    path = None             # file path
+    sample_length = 120     # default sample length in seconds
+    stats = None            # list of samples statistics
+    bad = None              # list of bad samples
+    good = None             # list of good samples
+
+    def __init__(self, path, sample_length=None):
+        self.path = path
+        self.sample_length = sample_length if sample_length is not None \
+            else self.sample_length
+
+    def get_file_stats(self):
+        return self.stats and self.stats[0]
+
+    def analyse(self):
+        logger.debug('complete file analysis')
+        self.stats = [SoxStats(self.path)]
+        position = 0
+        length = self.stats[0].get('length')
+
+        if not self.sample_length:
+            return
+
+        logger.debug('start samples analysis...')
+        while position < length:
+            stats = SoxStats(self.path, at=position, length=self.sample_length)
+            self.stats.append(stats)
+            position += self.sample_length
+
+    def check(self, name, min_val, max_val):
+        self.good = [index for index, stats in enumerate(self.stats)
+                     if min_val <= stats.get(name) <= max_val]
+        self.bad = [index for index, stats in enumerate(self.stats)
+                    if index not in self.good]
+        self.resume()
+
+    def resume(self):
+        def view(array): return [
+            'file' if index == 0 else
+            'sample {} (at {} seconds)'.format(
+                index, (index-1) * self.sample_length)
+            for index in array
+        ]
+
+        if self.good:
+            logger.debug(self.path + ' -> good: \033[92m%s\033[0m',
+                         ', '.join(view(self.good)))
+        if self.bad:
+            logger.debug(self.path + ' -> bad: \033[91m%s\033[0m',
+                         ', '.join(view(self.bad)))

aircox/models/sound.py

@@ -9,6 +9,7 @@ from django.utils.translation import gettext_lazy as _
 
 from taggit.managers import TaggableManager
 
+from aircox import settings
 from .program import Program
 from .episode import Episode
 
@@ -73,6 +74,8 @@ class SoundQuerySet(models.QuerySet):
         )
 
 
+# TODO:
+# - provide a default name based on program and episode
 class Sound(models.Model):
     """
     A Sound is the representation of a sound file that can be either an excerpt
@@ -105,13 +108,14 @@ class Sound(models.Model):
     )
 
     def _upload_to(self, filename):
-        subdir = AIRCOX_SOUND_ARCHIVES_SUBDIR if self.type == self.TYPE_ARCHIVE else \
+        subdir = settings.AIRCOX_SOUND_ARCHIVES_SUBDIR \
-                 AIRCOX_SOUND_EXCERPTS_SUBDIR
+                    if self.type == self.TYPE_ARCHIVE else \
+                    settings.AIRCOX_SOUND_EXCERPTS_SUBDIR
         return os.path.join(self.program.path, subdir, filename)
 
     file = models.FileField(
         _('file'), upload_to=_upload_to, max_length=256,
-        db_index=True,
+        db_index=True, unique=True,
     )
     duration = models.TimeField(
         _('duration'),
@@ -175,6 +179,7 @@ class Sound(models.Model):
 
         return os.path.exists(self.file.path)
 
+    # TODO: rename to sync_fs()
     def check_on_file(self):
         """
         Check sound file info again'st self, and update informations if
@@ -183,7 +188,7 @@ class Sound(models.Model):
         if not self.file_exists():
             if self.type == self.TYPE_REMOVED:
                 return
-            logger.info('sound %s: has been removed', self.file.name)
+            logger.debug('sound %s: has been removed', self.file.name)
             self.type = self.TYPE_REMOVED
             return True
 
@@ -202,7 +207,7 @@ class Sound(models.Model):
         if self.mtime != mtime:
             self.mtime = mtime
             self.is_good_quality = None
-            logger.info('sound %s: m_time has changed. Reset quality info',
+            logger.debug('sound %s: m_time has changed. Reset quality info',
                          self.file.name)
             return True
 

aircox/models/station.py

@@ -46,7 +46,7 @@ class Station(models.Model):
     )
     default = models.BooleanField(
         _('default station'),
-        default=True,
+        default=False,
         help_text=_('use this station as the main one.')
     )
     active = models.BooleanField(

aircox/tests/__init__.py

@@ -0,0 +1,2 @@
+from .management import *
+

aircox/tests/management/__init__.py

@@ -0,0 +1,2 @@
+from .sound_file import *
+from .sound_monitor import *

aircox/tests/management/sound_file.py

@@ -0,0 +1,73 @@
+from datetime import timedelta
+
+from django.conf import settings as conf
+from django.test import TestCase
+from django.utils import timezone as tz
+
+from aircox import models
+from aircox.management.sound_file import SoundFile
+
+
+__all__ = ('SoundFileTestCase',)
+
+
+class SoundFileTestCase(TestCase):
+    path_infos = {
+        'test/20220101_10h13_1_sample_1.mp3': {
+            'year': 2022, 'month': 1, 'day': 1, 'hour': 10, 'minute': 13,
+            'n': 1, 'name': 'Sample 1'},
+        'test/20220102_10h13_sample_2.mp3': {
+            'year': 2022, 'month': 1, 'day': 2, 'hour': 10, 'minute': 13,
+            'name': 'Sample 2'},
+        'test/20220103_1_sample_3.mp3': {
+            'year': 2022, 'month': 1, 'day': 3, 'n': 1, 'name': 'Sample 3'},
+        'test/20220104_sample_4.mp3': {
+            'year': 2022, 'month': 1, 'day': 4, 'name': 'Sample 4'},
+        'test/20220105.mp3': {
+            'year': 2022, 'month': 1, 'day': 5, 'name': '20220105'},
+    }
+    subdir_prefix = 'test'
+    sound_files = {k: r for k, r in (
+        (path, SoundFile(conf.MEDIA_ROOT + '/' + path))
+        for path in path_infos.keys()
+    )}
+
+    def test_sound_path(self):
+        for path, sound_file in self.sound_files.items():
+            self.assertEqual(path, sound_file.sound_path)
+
+    def test_read_path(self):
+        for path, sound_file in self.sound_files.items():
+            expected = self.path_infos[path]
+            result = sound_file.read_path(path)
+            # remove None values
+            result = {k: v for k, v in result.items() if v is not None}
+            self.assertEqual(expected, result, "path: {}".format(path))
+
+    def _setup_diff(self, program, info):
+        episode = models.Episode(program=program, title='test-episode')
+        at = tz.datetime(**{
+            k: info[k] for k in ('year', 'month', 'day', 'hour', 'minute')
+            if info.get(k)
+        })
+        at = tz.make_aware(at)
+        diff = models.Diffusion(episode=episode, start=at,
+                                end=at+timedelta(hours=1))
+        episode.save()
+        diff.save()
+        return diff
+
+    def test_find_episode(self):
+        station = models.Station(name='test-station')
+        program = models.Program(station=station, title='test')
+        station.save()
+        program.save()
+
+        for path, sound_file in self.sound_files.items():
+            infos = sound_file.read_path(path)
+            diff = self._setup_diff(program, infos)
+            sound = models.Sound(program=diff.program, file=path)
+            result = sound_file.find_episode(sound, infos)
+            self.assertEquals(diff.episode, result)
+
+    # TODO: find_playlist, sync

aircox/tests/management/sound_monitor.py

@@ -0,0 +1,76 @@
+import concurrent.futures as futures
+from datetime import datetime, timedelta
+import time
+
+from django.test import TestCase
+
+from aircox.management.sound_monitor import \
+    NotifyHandler, MoveHandler, ModifiedHandler, MonitorHandler
+
+
+__all__ = ('NotifyHandlerTestCase', 'MoveHandlerTestCase',
+           'ModifiedHandlerTestCase', 'MonitorHandlerTestCase',)
+
+
+class FakeEvent:
+    def __init__(self, **kwargs):
+        self.__dict__.update(**kwargs)
+
+
+class WaitHandler(NotifyHandler):
+    def __call__(self, timeout=0.5, *args, **kwargs):
+        # using time.sleep make the future done directly, don't know why
+        start = datetime.now()
+        while datetime.now() - start < timedelta(seconds=4):
+            pass
+
+
+class NotifyHandlerTestCase(TestCase):
+    pass
+
+
+class MoveHandlerTestCase(TestCase):
+    pass
+
+
+class ModifiedHandlerTestCase(TestCase):
+    def test_wait(self):
+        handler = ModifiedHandler()
+        handler.timeout_delta = timedelta(seconds=0.1)
+        start = datetime.now()
+        handler.wait()
+        delta = datetime.now() - start
+        self.assertTrue(delta < handler.timeout_delta + timedelta(seconds=0.1))
+
+    def test_wait_ping(self):
+        pool = futures.ThreadPoolExecutor(1)
+        handler = ModifiedHandler()
+        handler.timeout_delta = timedelta(seconds=0.5)
+
+        future = pool.submit(handler.wait)
+        time.sleep(0.3)
+        handler.ping()
+        time.sleep(0.3)
+        self.assertTrue(future.running())
+
+
+class MonitorHandlerTestCase(TestCase):
+    def setUp(self):
+        pool = futures.ThreadPoolExecutor(2)
+        self.monitor = MonitorHandler('archives', pool)
+
+    def test_submit_new_job(self):
+        event = FakeEvent(src_path='dummy_src')
+        handler = NotifyHandler()
+        result = self.monitor._submit(handler, event, 'up')
+        self.assertIs(handler, result)
+        self.assertIsInstance(handler.future, futures.Future)
+        self.monitor.pool.shutdown()
+
+    def test_submit_job_exists(self):
+        event = FakeEvent(src_path='dummy_src')
+
+        job_1 = self.monitor._submit(WaitHandler(), event, 'up')
+        job_2 = self.monitor._submit(NotifyHandler(), event, 'up')
+        self.assertIs(job_1, job_2)
+        self.monitor.pool.shutdown()

requirements.txt

@@ -12,6 +12,7 @@ bleach~=5.0
 easy-thumbnails~=2.8
 tzlocal~=4.2
 
+dateutils~=0.6
 mutagen~=1.45
 Pillow~=9.0
 psutil~=5.9