move files

programs/management/commands/diffusions_monitor.py

@@ -0,0 +1,158 @@
+"""
+Manage diffusions using schedules, to update, clean up or check diffusions.
+
+A generated diffusion can be unconfirmed, that means that the user must confirm
+it by changing its type to "normal". The behaviour is controlled using
+--approval.
+
+Different actions are available:
+- "update" is the process that is used to generated them using programs
+schedules for the (given) month.
+
+- "clean" will remove all diffusions that are still unconfirmed and have been
+planified before the (given) month.
+
+- "check" will remove all diffusions that are unconfirmed and have been planified
+from the (given) month and later.
+"""
+from argparse import RawTextHelpFormatter
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone as tz
+from aircox.programs.models import *
+
+
+class Actions:
+    @staticmethod
+    def __check_conflicts (item, saved_items):
+        """
+        Check for conflicts, and update conflictual
+        items if they have been generated during this
+        update.
+
+        Return the number of conflicts
+        """
+        conflicts = item.get_conflicts()
+        if not conflicts:
+            item.type = Diffusion.Type['normal']
+            return 0
+
+        item.type = Diffusion.Type['unconfirmed']
+        for conflict in conflicts:
+            if conflict.pk in saved_items and \
+                    conflict.type != Diffusion.Type['unconfirmed']:
+                conflict.type = Diffusion.Type['unconfirmed']
+                conflict.save()
+        return len(conflicts)
+
+    @classmethod
+    def update (cl, date, mode):
+        manual = (mode == 'manual')
+        if not manual:
+            saved_items = set()
+
+        count = [0, 0]
+        for schedule in Schedule.objects.filter(program__active = True) \
+                .order_by('initial'):
+            # in order to allow rerun links between diffusions, we save items
+            # by schedule;
+            items = schedule.diffusions_of_month(date, exclude_saved = True)
+            count[0] += len(items)
+
+            if manual:
+                Diffusion.objects.bulk_create(items)
+            else:
+                for item in items:
+                    count[1] += cl.__check_conflicts(item, saved_items)
+                    item.save()
+                    saved_items.add(item)
+
+            print('> {} new diffusions for schedule #{} ({})'.format(
+                    len(items), schedule.id, str(schedule)
+                 ))
+
+        print('total of {} diffusions have been created,'.format(count[0]),
+              'do not forget manual approval' if manual else
+                '{} conflicts found'.format(count[1]))
+
+    @staticmethod
+    def clean (date):
+        qs = Diffusion.objects.filter(type = Diffusion.Type['unconfirmed'],
+                                      date__lt = date)
+        print('{} diffusions will be removed'.format(qs.count()))
+        qs.delete()
+
+    @staticmethod
+    def check (date):
+        qs = Diffusion.objects.filter(type = Diffusion.Type['unconfirmed'],
+                                      date__gt = date)
+        items = []
+        for diffusion in qs:
+            schedules = Schedule.objects.filter(program = diffusion.program)
+            for schedule in schedules:
+                if schedule.match(diffusion.date):
+                    break
+            else:
+                print('> #{}: {}'.format(diffusion.pk, str(diffusion)))
+                items.append(diffusion.id)
+
+        print('{} diffusions will be removed'.format(len(items)))
+        if len(items):
+            Diffusion.objects.filter(id__in = items).delete()
+
+
+class Command (BaseCommand):
+    help= __doc__
+
+    def add_arguments (self, parser):
+        parser.formatter_class=RawTextHelpFormatter
+        now = tz.datetime.today()
+
+        group = parser.add_argument_group('action')
+        group.add_argument(
+            '--update', action='store_true',
+            help='generate (unconfirmed) diffusions for the given month. '
+                 'These diffusions must be confirmed manually by changing '
+                 'their type to "normal"')
+        group.add_argument(
+            '--clean', action='store_true',
+            help='remove unconfirmed diffusions older than the given month')
+
+        group.add_argument(
+            '--check', action='store_true',
+            help='check future unconfirmed diffusions from the given date '
+                 'agains\'t schedules and remove it if that do not match any '
+                 'schedule')
+
+        group = parser.add_argument_group('date')
+        group.add_argument(
+            '--year', type=int, default=now.year,
+            help='used by update, default is today\'s year')
+        group.add_argument(
+            '--month', type=int, default=now.month,
+            help='used by update, default is today\'s month')
+
+        group = parser.add_argument_group('mode')
+        group.add_argument(
+            '--approval', type=str, choices=['manual', 'auto'],
+            default='auto',
+            help='manual means that all generated diffusions are unconfirmed, '
+                 'thus must be approved manually; auto confirmes all '
+                 'diffusions except those that conflicts with others'
+        )
+
+
+    def handle (self, *args, **options):
+        date = tz.datetime(year = options.get('year'),
+                                 month = options.get('month'),
+                                 day = 1)
+        date = tz.make_aware(date)
+
+        if options.get('update'):
+            Actions.update(date, mode = options.get('mode'))
+        elif options.get('clean'):
+            Actions.clean(date)
+        elif options.get('check'):
+            Actions.check(date)
+        else:
+            raise CommandError('no action has been given')
+

programs/management/commands/sounds_monitor.py

@@ -0,0 +1,228 @@
+"""
+Monitor sound files; For each program, check for:
+- new files;
+- deleted files;
+- differences between files and sound;
+- quality of the files;
+
+It tries to parse the file name to get the date of the diffusion of an
+episode and associate the file with it; We use the following format:
+    yyyymmdd[_n][_][name]
+
+Where:
+    'yyyy' the year of the episode's diffusion;
+    'mm' the month of the episode's diffusion;
+    'dd' the day of the episode's diffusion;
+    'n' the number of the episode (if multiple episodes);
+    'name' the title of the sound;
+
+
+To check quality of files, call the command sound_quality_check using the
+parameters given by the setting AIRCOX_SOUND_QUALITY. This script requires
+Sox (and soxi).
+"""
+import os
+import re
+import subprocess
+from argparse import RawTextHelpFormatter
+
+from django.core.management.base import BaseCommand, CommandError
+
+from aircox.programs.models import *
+import aircox.programs.settings as settings
+import aircox.programs.utils as utils
+
+
+class Command (BaseCommand):
+    help= __doc__
+
+    def report (self, program = None, component = None, *content):
+        if not component:
+            print('{}: '.format(program), *content)
+        else:
+            print('{}, {}: '.format(program, component), *content)
+
+    def add_arguments (self, parser):
+        parser.formatter_class=RawTextHelpFormatter
+        parser.add_argument(
+            '-q', '--quality_check', action='store_true',
+            help='Enable quality check using sound_quality_check on all ' \
+                 'sounds marqued as not good'
+        )
+        parser.add_argument(
+            '-s', '--scan', action='store_true',
+            help='Scan programs directories for changes, plus check for a '
+                 ' matching episode on sounds that have not been yet assigned'
+        )
+
+    def handle (self, *args, **options):
+        if options.get('scan'):
+            self.scan()
+        if options.get('quality_check'):
+            self.check_quality(check = (not options.get('scan')) )
+
+    def _get_duration (self, path):
+        p = subprocess.Popen(['soxi', '-D', path], stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        out, err = p.communicate()
+        if not err:
+            return utils.seconds_to_time(int(float(out)))
+
+    def get_sound_info (self, program, path):
+        """
+        Parse file name to get info on the assumption it has the correct
+        format (given in Command.help)
+        """
+        file_name = os.path.basename(path)
+        file_name = os.path.splitext(file_name)[0]
+        r = re.search('^(?P<year>[0-9]{4})'
+                      '(?P<month>[0-9]{2})'
+                      '(?P<day>[0-9]{2})'
+                      '(_(?P<n>[0-9]+))?'
+                      '_?(?P<name>.*)$',
+                      file_name)
+
+        if not (r and r.groupdict()):
+            self.report(program, path, "file path is not correct, use defaults")
+            r = {
+                'name': file_name
+            }
+        else:
+            r = r.groupdict()
+
+        r['duration'] = self._get_duration(path)
+        r['name'] = r['name'].replace('_', ' ').capitalize()
+        r['path'] = path
+        return r
+
+    def find_initial (self, program, sound_info):
+        """
+        For a given program, and sound path check if there is an initial
+        diffusion to associate to, using the diffusion's date.
+
+        If there is no matching episode, return None.
+        """
+        # check on episodes
+        diffusion = Diffusion.objects.filter(
+            program = program,
+            date__year = int(sound_info['year']),
+            date__month = int(sound_info['month']),
+            date__day = int(sound_info['day'])
+        )
+
+        if not diffusion.count():
+            self.report(program, sound_info['path'],
+                        'no diffusion found for the given date')
+            return
+        return diffusion[0]
+
+    @staticmethod
+    def check_sounds (qs):
+        """
+        Only check for the sound existence or update
+        """
+        # check files
+        for sound in qs:
+            if sound.check_on_file():
+                sound.save(check = False)
+
+    def scan (self):
+        """
+        For all programs, scan dirs
+        """
+        print('scan files for all programs...')
+        programs = Program.objects.filter()
+
+        for program in programs:
+            print('- program', program.name)
+            self.scan_for_program(
+                program, settings.AIRCOX_SOUND_ARCHIVES_SUBDIR,
+                type = Sound.Type['archive'],
+            )
+            self.scan_for_program(
+                program, settings.AIRCOX_SOUND_EXCERPTS_SUBDIR,
+                type = Sound.Type['excerpt'],
+            )
+
+    def scan_for_program (self, program, subdir, **sound_kwargs):
+        """
+        Scan a given directory that is associated to the given program, and
+        update sounds information.
+        """
+        print(' - scan files in', subdir)
+        if not program.ensure_dir(subdir):
+            return
+
+        subdir = os.path.join(program.path, subdir)
+
+        # new/existing sounds
+        for path in os.listdir(subdir):
+            path = os.path.join(subdir, path)
+            if not path.endswith(settings.AIRCOX_SOUND_FILE_EXT):
+                continue
+
+            sound_info = self.get_sound_info(program, path)
+            sound = Sound.objects.get_or_create(
+                path = path,
+                defaults = { 'name': sound_info['name'],
+                             'duration': sound_info['duration'] or None }
+            )[0]
+            sound.__dict__.update(sound_kwargs)
+            sound.save(check = False)
+
+            # initial diffusion association
+            if 'year' in sound_info:
+                initial = self.find_initial(program, sound_info)
+                if initial:
+                    if initial.initial:
+                        # FIXME: allow user to overwrite rerun info?
+                        self.report(program, path,
+                                    'the diffusion must be an initial diffusion')
+                    else:
+                        sound = initial.sounds.get_queryset() \
+                                    .filter(path == sound.path)
+                        if not sound:
+                            self.report(program, path,
+                                        'add sound to diffusion ', initial.id)
+                            initial.sounds.add(sound)
+                            initial.save()
+
+        self.check_sounds(Sound.objects.filter(path__startswith = subdir))
+
+    def check_quality (self, check = False):
+        """
+        Check all files where quality has been set to bad
+        """
+        import aircox.programs.management.commands.sounds_quality_check \
+                as quality_check
+
+        sounds = Sound.objects.filter(good_quality = False)
+        if check:
+            self.check_sounds(sounds)
+            files = [ sound.path for sound in sounds if not sound.removed ]
+        else:
+            files = [ sound.path for sound in sounds.filter(removed = False) ]
+
+        print('start quality check...')
+        cmd = quality_check.Command()
+        cmd.handle( files = files,
+                    **settings.AIRCOX_SOUND_QUALITY )
+
+        print('- update sounds in database')
+        def update_stats(sound_info, sound):
+            stats = sound_info.get_file_stats()
+            if stats:
+                duration = int(stats.get('length'))
+                sound.duration = utils.seconds_to_time(duration)
+
+        for sound_info in cmd.good:
+            sound = Sound.objects.get(path = sound_info.path)
+            sound.good_quality = True
+            update_stats(sound_info, sound)
+            sound.save(check = False)
+
+        for sound_info in cmd.bad:
+            sound = Sound.objects.get(path = sound_info.path)
+            update_stats(sound_info, sound)
+            sound.save(check = False)
+

programs/management/commands/sounds_quality_check.py

@@ -0,0 +1,175 @@
+"""
+Analyse and check files using Sox, prints good and bad files.
+"""
+import sys
+import re
+import subprocess
+
+from argparse import RawTextHelpFormatter
+from django.core.management.base import BaseCommand, CommandError
+
+class Stats:
+    attributes = [
+        'DC offset', 'Min level', 'Max level',
+        'Pk lev dB', 'RMS lev dB', 'RMS Pk dB',
+        'RMS Tr dB', 'Flat factor', 'Length s',
+    ]
+
+    def __init__ (self, path, **kwargs):
+        """
+        If path is given, call analyse with path and kwargs
+        """
+        self.values = {}
+        if path:
+            self.analyse(path, **kwargs)
+
+    def get (self, attr):
+        return self.values.get(attr)
+
+    def parse (self, output):
+        for attr in Stats.attributes:
+            value = re.search(attr + r'\s+(?P<value>\S+)', output)
+            value = value and value.groupdict()
+            if value:
+                try:
+                    value = float(value.get('value'))
+                except ValueError:
+                    value = None
+                self.values[attr] = value
+        self.values['length'] = self.values['Length s']
+
+    def analyse (self, path, at = None, length = None):
+        """
+        If at and length are given use them as excerpt to analyse.
+        """
+        args = ['sox', path, '-n']
+
+        if at is not None and length is not None:
+            args += ['trim', str(at), str(length) ]
+
+        args.append('stats')
+
+        p = subprocess.Popen(args, stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        # sox outputs to stderr (my god WHYYYY)
+        out_, out = p.communicate()
+        self.parse(str(out, encoding='utf-8'))
+
+
+class Sound:
+    path = None             # file path
+    sample_length = 120     # default sample length in seconds
+    stats = None            # list of samples statistics
+    bad = None              # list of bad samples
+    good = None             # list of good samples
+
+    def __init__ (self, path, sample_length = None):
+        self.path = path
+        self.sample_length = sample_length if sample_length is not None \
+                                else self.sample_length
+
+    def get_file_stats (self):
+        return self.stats and self.stats[0]
+
+    def analyse (self):
+        print('- complete file analysis')
+        self.stats = [ Stats(self.path) ]
+        position = 0
+        length = self.stats[0].get('length')
+
+        if not self.sample_length:
+            return
+
+        print('- samples analysis: ', end=' ')
+        while position < length:
+            print(len(self.stats), end=' ')
+            stats = Stats(self.path, at = position, length = self.sample_length)
+            self.stats.append(stats)
+            position += self.sample_length
+        print()
+
+    def check (self, name, min_val, max_val):
+        self.good = [ index for index, stats in enumerate(self.stats)
+                      if min_val <= stats.get(name) <= max_val ]
+        self.bad = [ index for index, stats in enumerate(self.stats)
+                      if index not in self.good ]
+        self.resume()
+
+    def resume (self):
+        view = lambda array: [
+            'file' if index is 0 else
+            'sample {} (at {} seconds)'.format(index, (index-1) * self.sample_length)
+            for index in array
+        ]
+
+        if self.good:
+            print('- good:\033[92m', ', '.join( view(self.good) ), '\033[0m')
+        if self.bad:
+            print('- bad:\033[91m', ', '.join( view(self.bad) ), '\033[0m')
+
+class Command (BaseCommand):
+    help = __doc__
+    sounds = None
+
+    def add_arguments (self, parser):
+        parser.formatter_class=RawTextHelpFormatter
+
+        parser.add_argument(
+            'files', metavar='FILE', type=str, nargs='+',
+            help='file(s) to analyse'
+        )
+        parser.add_argument(
+            '-s', '--sample_length', type=int, default=120,
+            help='size of sample to analyse in seconds. If not set (or 0), does'
+                 ' not analyse by sample',
+        )
+        parser.add_argument(
+            '-a', '--attribute', type=str,
+            help='attribute name to use to check, that can be:\n' + \
+                 ', '.join([ '"{}"'.format(attr) for attr in Stats.attributes ])
+        )
+        parser.add_argument(
+            '-r', '--range', type=float, nargs=2,
+            help='range of minimal and maximal accepted value such as: ' \
+                 '--range min max'
+        )
+        parser.add_argument(
+            '-i', '--resume', action='store_true',
+            help='print a resume of good and bad files'
+        )
+
+    def handle (self, *args, **options):
+        # parameters
+        minmax = options.get('range')
+        if not minmax:
+            raise CommandError('no range specified')
+
+        attr = options.get('attribute')
+        if not attr:
+            raise CommandError('no attribute specified')
+
+        # sound analyse and checks
+        self.sounds = [ Sound(path, options.get('sample_length'))
+                        for path in options.get('files') ]
+        self.bad = []
+        self.good = []
+        for sound in self.sounds:
+            print(sound.path)
+            sound.analyse()
+            sound.check(attr, minmax[0], minmax[1])
+            print()
+            if sound.bad:
+                self.bad.append(sound)
+            else:
+                self.good.append(sound)
+
+        # resume
+        if options.get('resume'):
+            if self.good:
+                print('files that did not failed the test:\033[92m\n   ',
+                      '\n    '.join([sound.path for sound in self.good]), '\033[0m')
+            if self.bad:
+                # bad at the end for ergonomy
+                print('files that failed the test:\033[91m\n   ',
+                      '\n    '.join([sound.path for sound in self.bad]),'\033[0m')
+