forked from rc/aircox
merge aircox and aircox_instance
This commit is contained in:
bkfox
0
aircox/management/commands/__init__.py
Normal file
0
aircox/management/commands/__init__.py
Normal file
184
aircox/management/commands/diffusions_monitor.py
Normal file
184
aircox/management/commands/diffusions_monitor.py
Normal file
@ -0,0 +1,184 @@
|
||||
"""
|
||||
Manage diffusions using schedules, to update, clean up or check diffusions.
|
||||
|
||||
A generated diffusion can be unconfirmed, that means that the user must confirm
|
||||
it by changing its type to "normal". The behaviour is controlled using
|
||||
--approval.
|
||||
|
||||
Different actions are available:
|
||||
- "update" is the process that is used to generated them using programs
|
||||
schedules for the (given) month.
|
||||
|
||||
- "clean" will remove all diffusions that are still unconfirmed and have been
|
||||
planified before the (given) month.
|
||||
|
||||
- "check" will remove all diffusions that are unconfirmed and have been planified
|
||||
from the (given) month and later.
|
||||
"""
|
||||
import logging
|
||||
from argparse import RawTextHelpFormatter
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.utils import timezone as tz
|
||||
|
||||
from aircox.models import *
|
||||
|
||||
logger = logging.getLogger('aircox.tools')
|
||||
|
||||
class Actions:
|
||||
@staticmethod
|
||||
def __check_conflicts (item, saved_items):
|
||||
"""
|
||||
Check for conflicts, and update conflictual
|
||||
items if they have been generated during this
|
||||
update.
|
||||
|
||||
It set an attribute 'do_not_save' if the item should not
|
||||
be saved. FIXME: find proper way
|
||||
|
||||
Return the number of conflicts
|
||||
"""
|
||||
conflicts = list(item.get_conflicts())
|
||||
for i, conflict in enumerate(conflicts):
|
||||
if conflict.program == item.program:
|
||||
item.do_not_save = True
|
||||
del conflicts[i]
|
||||
continue
|
||||
|
||||
if conflict.pk in saved_items and \
|
||||
conflict.type != Diffusion.Type.unconfirmed:
|
||||
conflict.type = Diffusion.Type.unconfirmed
|
||||
conflict.save()
|
||||
|
||||
if not conflicts:
|
||||
item.type = Diffusion.Type.normal
|
||||
return 0
|
||||
|
||||
item.type = Diffusion.Type.unconfirmed
|
||||
return len(conflicts)
|
||||
|
||||
@classmethod
|
||||
def update (cl, date, mode):
|
||||
manual = (mode == 'manual')
|
||||
if not manual:
|
||||
saved_items = set()
|
||||
|
||||
count = [0, 0]
|
||||
for schedule in Schedule.objects.filter(program__active = True) \
|
||||
.order_by('initial'):
|
||||
# in order to allow rerun links between diffusions, we save items
|
||||
# by schedule;
|
||||
items = schedule.diffusions_of_month(date, exclude_saved = True)
|
||||
count[0] += len(items)
|
||||
|
||||
if manual:
|
||||
Diffusion.objects.bulk_create(items)
|
||||
else:
|
||||
for item in items:
|
||||
count[1] += cl.__check_conflicts(item, saved_items)
|
||||
if hasattr(item, 'do_not_save'):
|
||||
count[0] -= 1
|
||||
continue
|
||||
|
||||
item.save()
|
||||
saved_items.add(item)
|
||||
|
||||
logger.info('[update] schedule %s: %d new diffusions',
|
||||
str(schedule), len(items),
|
||||
)
|
||||
|
||||
logger.info('[update] %d diffusions have been created, %s', count[0],
|
||||
'do not forget manual approval' if manual else
|
||||
'{} conflicts found'.format(count[1]))
|
||||
|
||||
@staticmethod
|
||||
def clean (date):
|
||||
qs = Diffusion.objects.filter(type = Diffusion.Type.unconfirmed,
|
||||
start__lt = date)
|
||||
logger.info('[clean] %d diffusions will be removed', qs.count())
|
||||
qs.delete()
|
||||
|
||||
@staticmethod
|
||||
def check (date):
|
||||
qs = Diffusion.objects.filter(type = Diffusion.Type.unconfirmed,
|
||||
start__gt = date)
|
||||
items = []
|
||||
for diffusion in qs:
|
||||
schedules = Schedule.objects.filter(program = diffusion.program)
|
||||
for schedule in schedules:
|
||||
if schedule.match(diffusion.start):
|
||||
break
|
||||
else:
|
||||
items.append(diffusion.id)
|
||||
|
||||
logger.info('[check] %d diffusions will be removed', len(items))
|
||||
if len(items):
|
||||
Diffusion.objects.filter(id__in = items).delete()
|
||||
|
||||
|
||||
class Command (BaseCommand):
|
||||
help= __doc__
|
||||
|
||||
def add_arguments (self, parser):
|
||||
parser.formatter_class=RawTextHelpFormatter
|
||||
now = tz.datetime.today()
|
||||
|
||||
group = parser.add_argument_group('action')
|
||||
group.add_argument(
|
||||
'--update', action='store_true',
|
||||
help='generate (unconfirmed) diffusions for the given month. '
|
||||
'These diffusions must be confirmed manually by changing '
|
||||
'their type to "normal"')
|
||||
group.add_argument(
|
||||
'--clean', action='store_true',
|
||||
help='remove unconfirmed diffusions older than the given month')
|
||||
|
||||
group.add_argument(
|
||||
'--check', action='store_true',
|
||||
help='check future unconfirmed diffusions from the given date '
|
||||
'agains\'t schedules and remove it if that do not match any '
|
||||
'schedule')
|
||||
|
||||
group = parser.add_argument_group('date')
|
||||
group.add_argument(
|
||||
'--year', type=int, default=now.year,
|
||||
help='used by update, default is today\'s year')
|
||||
group.add_argument(
|
||||
'--month', type=int, default=now.month,
|
||||
help='used by update, default is today\'s month')
|
||||
group.add_argument(
|
||||
'--next-month', action='store_true',
|
||||
help='set the date to the next month of given date'
|
||||
' (if next month from today'
|
||||
)
|
||||
|
||||
group = parser.add_argument_group('options')
|
||||
group.add_argument(
|
||||
'--mode', type=str, choices=['manual', 'auto'],
|
||||
default='auto',
|
||||
help='manual means that all generated diffusions are unconfirmed, '
|
||||
'thus must be approved manually; auto confirmes all '
|
||||
'diffusions except those that conflicts with others'
|
||||
)
|
||||
|
||||
|
||||
def handle (self, *args, **options):
|
||||
date = tz.datetime(year = options.get('year'),
|
||||
month = options.get('month'),
|
||||
day = 1)
|
||||
date = tz.make_aware(date)
|
||||
if options.get('next_month'):
|
||||
month = options.get('month')
|
||||
date += tz.timedelta(days = 28)
|
||||
if date.month == month:
|
||||
date += tz.timedelta(days = 28)
|
||||
|
||||
date = date.replace(day = 1)
|
||||
|
||||
if options.get('update'):
|
||||
Actions.update(date, mode = options.get('mode'))
|
||||
if options.get('clean'):
|
||||
Actions.clean(date)
|
||||
if options.get('check'):
|
||||
Actions.check(date)
|
||||
|
||||
142
aircox/management/commands/import_playlist.py
Normal file
142
aircox/management/commands/import_playlist.py
Normal file
@ -0,0 +1,142 @@
|
||||
"""
|
||||
Import one or more playlist for the given sound. Attach it to the sound
|
||||
or to the related Diffusion if wanted.
|
||||
|
||||
Playlists are in CSV format, where columns are separated with a
|
||||
'{settings.AIRCOX_IMPORT_PLAYLIST_CSV_DELIMITER}'. Text quote is
|
||||
{settings.AIRCOX_IMPORT_PLAYLIST_CSV_TEXT_QUOTE}.
|
||||
The order of the elements is: {settings.AIRCOX_IMPORT_PLAYLIST_CSV_COLS}
|
||||
|
||||
If 'minutes' or 'seconds' are given, position will be expressed as timed
|
||||
position, instead of position in playlist.
|
||||
"""
|
||||
import os
|
||||
import csv
|
||||
import logging
|
||||
from argparse import RawTextHelpFormatter
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
|
||||
from aircox.models import *
|
||||
import aircox.settings as settings
|
||||
__doc__ = __doc__.format(settings = settings)
|
||||
|
||||
logger = logging.getLogger('aircox.tools')
|
||||
|
||||
|
||||
class Importer:
|
||||
data = None
|
||||
tracks = None
|
||||
|
||||
def __init__(self, related = None, path = None, save = False):
|
||||
if path:
|
||||
self.read(path)
|
||||
if related:
|
||||
self.make_playlist(related, save)
|
||||
|
||||
def reset(self):
|
||||
self.data = None
|
||||
self.tracks = None
|
||||
|
||||
def read(self, path):
|
||||
if not os.path.exists(path):
|
||||
return True
|
||||
with open(path, 'r') as file:
|
||||
self.data = list(csv.reader(
|
||||
file,
|
||||
delimiter = settings.AIRCOX_IMPORT_PLAYLIST_CSV_DELIMITER,
|
||||
quotechar = settings.AIRCOX_IMPORT_PLAYLIST_CSV_TEXT_QUOTE,
|
||||
))
|
||||
|
||||
def __get(self, line, field, default = None):
|
||||
maps = settings.AIRCOX_IMPORT_PLAYLIST_CSV_COLS
|
||||
if field not in maps:
|
||||
return default
|
||||
index = maps.index(field)
|
||||
return line[index] if index < len(line) else default
|
||||
|
||||
def make_playlist(self, related, save = False):
|
||||
"""
|
||||
Make a playlist from the read data, and return it. If save is
|
||||
true, save it into the database
|
||||
"""
|
||||
maps = settings.AIRCOX_IMPORT_PLAYLIST_CSV_COLS
|
||||
tracks = []
|
||||
|
||||
in_seconds = ('minutes' or 'seconds') in maps
|
||||
for index, line in enumerate(self.data):
|
||||
position = \
|
||||
int(self.__get(line, 'minutes', 0)) * 60 + \
|
||||
int(self.__get(line, 'seconds', 0)) \
|
||||
if in_seconds else index
|
||||
|
||||
track, created = Track.objects.get_or_create(
|
||||
related_type = ContentType.objects.get_for_model(related),
|
||||
related_id = related.pk,
|
||||
title = self.__get(line, 'title'),
|
||||
artist = self.__get(line, 'artist'),
|
||||
position = position,
|
||||
)
|
||||
|
||||
track.in_seconds = in_seconds
|
||||
track.info = self.__get(line, 'info')
|
||||
tags = self.__get(line, 'tags')
|
||||
if tags:
|
||||
track.tags.add(*tags.split(','))
|
||||
|
||||
if save:
|
||||
track.save()
|
||||
tracks.append(track)
|
||||
self.tracks = tracks
|
||||
return tracks
|
||||
|
||||
|
||||
class Command (BaseCommand):
|
||||
help= __doc__
|
||||
|
||||
def add_arguments (self, parser):
|
||||
parser.formatter_class=RawTextHelpFormatter
|
||||
now = tz.datetime.today()
|
||||
|
||||
parser.add_argument(
|
||||
'path', metavar='PATH', type=str,
|
||||
help='path of the input playlist to read'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--sound', '-s', type=str,
|
||||
help='generate a playlist for the sound of the given path. '
|
||||
'If not given, try to match a sound with the same path.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--diffusion', '-d', action='store_true',
|
||||
help='try to get the diffusion relative to the sound if it exists'
|
||||
)
|
||||
|
||||
def handle (self, path, *args, **options):
|
||||
# FIXME: absolute/relative path of sounds vs given path
|
||||
if options.get('sound'):
|
||||
related = Sound.objects.filter(
|
||||
path__icontains = options.get('sound')
|
||||
).first()
|
||||
else:
|
||||
path, ext = os.path.splitext(options.get('path'))
|
||||
related = Sound.objects.filter(path__icontains = path).first()
|
||||
|
||||
if not related:
|
||||
logger.error('no sound found in the database for the path ' \
|
||||
'{path}'.format(path=path))
|
||||
return -1
|
||||
|
||||
if options.get('diffusion') and related.diffusion:
|
||||
related = related.diffusion
|
||||
|
||||
importer = Importer(related = related, path = path, save = True)
|
||||
for track in importer.tracks:
|
||||
logger.info('imported track at {pos}: {title}, by '
|
||||
'{artist}'.format(
|
||||
pos = track.position,
|
||||
title = track.title, artist = track.artist
|
||||
)
|
||||
)
|
||||
|
||||
383
aircox/management/commands/sounds_monitor.py
Normal file
383
aircox/management/commands/sounds_monitor.py
Normal file
@ -0,0 +1,383 @@
|
||||
"""
|
||||
Monitor sound files; For each program, check for:
|
||||
- new files;
|
||||
- deleted files;
|
||||
- differences between files and sound;
|
||||
- quality of the files;
|
||||
|
||||
It tries to parse the file name to get the date of the diffusion of an
|
||||
episode and associate the file with it; We use the following format:
|
||||
yyyymmdd[_n][_][name]
|
||||
|
||||
Where:
|
||||
'yyyy' the year of the episode's diffusion;
|
||||
'mm' the month of the episode's diffusion;
|
||||
'dd' the day of the episode's diffusion;
|
||||
'n' the number of the episode (if multiple episodes);
|
||||
'name' the title of the sound;
|
||||
|
||||
|
||||
To check quality of files, call the command sound_quality_check using the
|
||||
parameters given by the setting AIRCOX_SOUND_QUALITY. This script requires
|
||||
Sox (and soxi).
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
import re
|
||||
import logging
|
||||
import subprocess
|
||||
from argparse import RawTextHelpFormatter
|
||||
import atexit
|
||||
|
||||
from watchdog.observers import Observer
|
||||
from watchdog.events import PatternMatchingEventHandler, FileModifiedEvent
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from aircox.models import *
|
||||
import aircox.settings as settings
|
||||
import aircox.utils as utils
|
||||
|
||||
logger = logging.getLogger('aircox.tools')
|
||||
|
||||
class SoundInfo:
|
||||
name = ''
|
||||
sound = None
|
||||
|
||||
year = None
|
||||
month = None
|
||||
day = None
|
||||
n = None
|
||||
duration = None
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
return self._path
|
||||
|
||||
@path.setter
|
||||
def path(self, value):
|
||||
"""
|
||||
Parse file name to get info on the assumption it has the correct
|
||||
format (given in Command.help)
|
||||
"""
|
||||
file_name = os.path.basename(value)
|
||||
file_name = os.path.splitext(file_name)[0]
|
||||
r = re.search('^(?P<year>[0-9]{4})'
|
||||
'(?P<month>[0-9]{2})'
|
||||
'(?P<day>[0-9]{2})'
|
||||
'(_(?P<n>[0-9]+))?'
|
||||
'_?(?P<name>.*)$',
|
||||
file_name)
|
||||
|
||||
if not (r and r.groupdict()):
|
||||
r = { 'name': file_name }
|
||||
logger.info('file name can not be parsed -> %s', value)
|
||||
else:
|
||||
r = r.groupdict()
|
||||
|
||||
self._path = value
|
||||
self.name = r['name'].replace('_', ' ').capitalize()
|
||||
self.year = int(r.get('year')) if 'year' in r else None
|
||||
self.month = int(r.get('month')) if 'month' in r else None
|
||||
self.day = int(r.get('day')) if 'day' in r else None
|
||||
self.n = r.get('n')
|
||||
return r
|
||||
|
||||
def __init__(self, path = ''):
|
||||
self.path = path
|
||||
|
||||
def get_duration(self):
|
||||
p = subprocess.Popen(['soxi', '-D', self.path],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
out, err = p.communicate()
|
||||
if not err:
|
||||
duration = utils.seconds_to_time(int(float(out)))
|
||||
self.duration = duration
|
||||
return duration
|
||||
|
||||
def get_sound(self, kwargs = None, save = True):
|
||||
"""
|
||||
Get or create a sound using self info.
|
||||
|
||||
If the sound is created/modified, get its duration and update it
|
||||
(if save is True, sync to DB), and check for a playlist file.
|
||||
"""
|
||||
sound, created = Sound.objects.get_or_create(
|
||||
path = self.path,
|
||||
defaults = kwargs
|
||||
)
|
||||
if created or sound.check_on_file():
|
||||
logger.info('sound is new or have been modified -> %s', self.path)
|
||||
sound.duration = self.get_duration()
|
||||
sound.name = self.name
|
||||
if save:
|
||||
sound.save()
|
||||
self.sound = sound
|
||||
return sound
|
||||
|
||||
def find_playlist(self, sound):
|
||||
"""
|
||||
Find a playlist file corresponding to the sound path
|
||||
"""
|
||||
import aircox.management.commands.import_playlist \
|
||||
as import_playlist
|
||||
|
||||
path = os.path.splitext(self.sound.path)[0] + '.csv'
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
|
||||
old = Track.objects.get_for(object = sound)
|
||||
if old:
|
||||
return
|
||||
|
||||
import_playlist.Importer(sound, path, save=True)
|
||||
|
||||
def find_diffusion(self, program, save = True):
|
||||
"""
|
||||
For a given program, check if there is an initial diffusion
|
||||
to associate to, using the date info we have. Update self.sound
|
||||
and save it consequently.
|
||||
|
||||
We only allow initial diffusion since there should be no
|
||||
rerun.
|
||||
"""
|
||||
if self.year == None or not self.sound or self.sound.diffusion:
|
||||
return;
|
||||
|
||||
diffusion = Diffusion.objects.filter(
|
||||
program = program,
|
||||
initial__isnull = True,
|
||||
start__year = self.year,
|
||||
start__month = self.month,
|
||||
start__day = self.day,
|
||||
)
|
||||
if not diffusion:
|
||||
return
|
||||
diffusion = diffusion[0]
|
||||
|
||||
logger.info('diffusion %s mathes to sound -> %s', str(diffusion),
|
||||
self.sound.path)
|
||||
self.sound.diffusion = diffusion
|
||||
if save:
|
||||
self.sound.save()
|
||||
return diffusion
|
||||
|
||||
|
||||
class MonitorHandler(PatternMatchingEventHandler):
|
||||
"""
|
||||
Event handler for watchdog, in order to be used in monitoring.
|
||||
"""
|
||||
def __init__(self, subdir):
|
||||
"""
|
||||
subdir: AIRCOX_SOUND_ARCHIVES_SUBDIR or AIRCOX_SOUND_EXCERPTS_SUBDIR
|
||||
"""
|
||||
self.subdir = subdir
|
||||
if self.subdir == settings.AIRCOX_SOUND_ARCHIVES_SUBDIR:
|
||||
self.sound_kwargs = { 'type': Sound.Type.archive }
|
||||
else:
|
||||
self.sound_kwargs = { 'type': Sound.Type.excerpt }
|
||||
|
||||
patterns = ['*/{}/*{}'.format(self.subdir, ext)
|
||||
for ext in settings.AIRCOX_SOUND_FILE_EXT ]
|
||||
super().__init__(patterns=patterns, ignore_directories=True)
|
||||
|
||||
def on_created(self, event):
|
||||
self.on_modified(event)
|
||||
|
||||
def on_modified(self, event):
|
||||
logger.info('sound modified: %s', event.src_path)
|
||||
program = Program.get_from_path(event.src_path)
|
||||
if not program:
|
||||
return
|
||||
|
||||
si = SoundInfo(event.src_path)
|
||||
si.get_sound(self.sound_kwargs, True)
|
||||
if si.year != None:
|
||||
si.find_diffusion(program)
|
||||
|
||||
def on_deleted(self, event):
|
||||
logger.info('sound deleted: %s', event.src_path)
|
||||
sound = Sound.objects.filter(path = event.src_path)
|
||||
if sound:
|
||||
sound = sound[0]
|
||||
sound.type = sound.Type.removed
|
||||
sound.save()
|
||||
|
||||
def on_moved(self, event):
|
||||
logger.info('sound moved: %s -> %s', event.src_path, event.dest_path)
|
||||
sound = Sound.objects.filter(path = event.src_path)
|
||||
if not sound:
|
||||
self.on_modified(
|
||||
FileModifiedEvent(event.dest_path)
|
||||
)
|
||||
return
|
||||
|
||||
sound = sound[0]
|
||||
sound.path = event.dest_path
|
||||
sound.save()
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help= __doc__
|
||||
|
||||
def report(self, program = None, component = None, *content):
|
||||
if not component:
|
||||
logger.info('%s: %s', str(program), ' '.join([str(c) for c in content]))
|
||||
else:
|
||||
logger.info('%s, %s: %s', str(program), str(component),
|
||||
' '.join([str(c) for c in content]))
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.formatter_class=RawTextHelpFormatter
|
||||
parser.add_argument(
|
||||
'-q', '--quality_check', action='store_true',
|
||||
help='Enable quality check using sound_quality_check on all ' \
|
||||
'sounds marqued as not good'
|
||||
)
|
||||
parser.add_argument(
|
||||
'-s', '--scan', action='store_true',
|
||||
help='Scan programs directories for changes, plus check for a '
|
||||
' matching diffusion on sounds that have not been yet assigned'
|
||||
)
|
||||
parser.add_argument(
|
||||
'-m', '--monitor', action='store_true',
|
||||
help='Run in monitor mode, watch for modification in the filesystem '
|
||||
'and react in consequence'
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if options.get('scan'):
|
||||
self.scan()
|
||||
if options.get('quality_check'):
|
||||
self.check_quality(check = (not options.get('scan')) )
|
||||
if options.get('monitor'):
|
||||
self.monitor()
|
||||
|
||||
@staticmethod
|
||||
def check_sounds(qs):
|
||||
"""
|
||||
Only check for the sound existence or update
|
||||
"""
|
||||
# check files
|
||||
for sound in qs:
|
||||
if sound.check_on_file():
|
||||
sound.save(check = False)
|
||||
|
||||
def scan(self):
|
||||
"""
|
||||
For all programs, scan dirs
|
||||
"""
|
||||
logger.info('scan all programs...')
|
||||
programs = Program.objects.filter()
|
||||
|
||||
for program in programs:
|
||||
logger.info('#%d %s', program.id, program.name)
|
||||
self.scan_for_program(
|
||||
program, settings.AIRCOX_SOUND_ARCHIVES_SUBDIR,
|
||||
type = Sound.Type.archive,
|
||||
)
|
||||
self.scan_for_program(
|
||||
program, settings.AIRCOX_SOUND_EXCERPTS_SUBDIR,
|
||||
type = Sound.Type.excerpt,
|
||||
)
|
||||
|
||||
def scan_for_program(self, program, subdir, **sound_kwargs):
|
||||
"""
|
||||
Scan a given directory that is associated to the given program, and
|
||||
update sounds information.
|
||||
"""
|
||||
logger.info('- %s/', subdir)
|
||||
if not program.ensure_dir(subdir):
|
||||
return
|
||||
|
||||
sound_kwargs['program'] = program
|
||||
|
||||
subdir = os.path.join(program.path, subdir)
|
||||
sounds = []
|
||||
|
||||
# sounds in directory
|
||||
for path in os.listdir(subdir):
|
||||
path = os.path.join(subdir, path)
|
||||
if not path.endswith(settings.AIRCOX_SOUND_FILE_EXT):
|
||||
continue
|
||||
|
||||
si = SoundInfo(path)
|
||||
si.get_sound(sound_kwargs, True)
|
||||
si.find_diffusion(program)
|
||||
si.find_playlist(si.sound)
|
||||
sounds.append(si.sound.pk)
|
||||
|
||||
# sounds in db & unchecked
|
||||
sounds = Sound.objects.filter(path__startswith = subdir). \
|
||||
exclude(pk__in = sounds)
|
||||
self.check_sounds(sounds)
|
||||
|
||||
def check_quality(self, check = False):
|
||||
"""
|
||||
Check all files where quality has been set to bad
|
||||
"""
|
||||
import aircox.management.commands.sounds_quality_check \
|
||||
as quality_check
|
||||
|
||||
# get available sound files
|
||||
sounds = Sound.objects.filter(good_quality = False) \
|
||||
.exclude(type = Sound.Type.removed)
|
||||
if check:
|
||||
self.check_sounds(sounds)
|
||||
|
||||
files = [ sound.path for sound in sounds
|
||||
if os.path.exists(sound.path) ]
|
||||
|
||||
# check quality
|
||||
logger.info('quality check...',)
|
||||
cmd = quality_check.Command()
|
||||
cmd.handle( files = files,
|
||||
**settings.AIRCOX_SOUND_QUALITY )
|
||||
|
||||
# update stats
|
||||
logger.info('update stats in database')
|
||||
def update_stats(sound_info, sound):
|
||||
stats = sound_info.get_file_stats()
|
||||
if stats:
|
||||
duration = int(stats.get('length'))
|
||||
sound.duration = utils.seconds_to_time(duration)
|
||||
|
||||
for sound_info in cmd.good:
|
||||
sound = Sound.objects.get(path = sound_info.path)
|
||||
sound.good_quality = True
|
||||
update_stats(sound_info, sound)
|
||||
sound.save(check = False)
|
||||
|
||||
for sound_info in cmd.bad:
|
||||
sound = Sound.objects.get(path = sound_info.path)
|
||||
update_stats(sound_info, sound)
|
||||
sound.save(check = False)
|
||||
|
||||
def monitor(self):
|
||||
"""
|
||||
Run in monitor mode
|
||||
"""
|
||||
archives_handler = MonitorHandler(
|
||||
subdir = settings.AIRCOX_SOUND_ARCHIVES_SUBDIR
|
||||
)
|
||||
excerpts_handler = MonitorHandler(
|
||||
subdir = settings.AIRCOX_SOUND_EXCERPTS_SUBDIR
|
||||
)
|
||||
|
||||
observer = Observer()
|
||||
observer.schedule(archives_handler, settings.AIRCOX_PROGRAMS_DIR,
|
||||
recursive=True)
|
||||
observer.schedule(excerpts_handler, settings.AIRCOX_PROGRAMS_DIR,
|
||||
recursive=True)
|
||||
observer.start()
|
||||
|
||||
def leave():
|
||||
observer.stop()
|
||||
observer.join()
|
||||
atexit.register(leave)
|
||||
|
||||
while True:
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
174
aircox/management/commands/sounds_quality_check.py
Normal file
174
aircox/management/commands/sounds_quality_check.py
Normal file
@ -0,0 +1,174 @@
|
||||
"""
|
||||
Analyse and check files using Sox, prints good and bad files.
|
||||
"""
|
||||
import sys
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
from argparse import RawTextHelpFormatter
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
logger = logging.getLogger('aircox.tools')
|
||||
|
||||
class Stats:
|
||||
attributes = [
|
||||
'DC offset', 'Min level', 'Max level',
|
||||
'Pk lev dB', 'RMS lev dB', 'RMS Pk dB',
|
||||
'RMS Tr dB', 'Flat factor', 'Length s',
|
||||
]
|
||||
|
||||
def __init__ (self, path, **kwargs):
|
||||
"""
|
||||
If path is given, call analyse with path and kwargs
|
||||
"""
|
||||
self.values = {}
|
||||
if path:
|
||||
self.analyse(path, **kwargs)
|
||||
|
||||
def get (self, attr):
|
||||
return self.values.get(attr)
|
||||
|
||||
def parse (self, output):
|
||||
for attr in Stats.attributes:
|
||||
value = re.search(attr + r'\s+(?P<value>\S+)', output)
|
||||
value = value and value.groupdict()
|
||||
if value:
|
||||
try:
|
||||
value = float(value.get('value'))
|
||||
except ValueError:
|
||||
value = None
|
||||
self.values[attr] = value
|
||||
self.values['length'] = self.values['Length s']
|
||||
|
||||
def analyse (self, path, at = None, length = None):
|
||||
"""
|
||||
If at and length are given use them as excerpt to analyse.
|
||||
"""
|
||||
args = ['sox', path, '-n']
|
||||
|
||||
if at is not None and length is not None:
|
||||
args += ['trim', str(at), str(length) ]
|
||||
|
||||
args.append('stats')
|
||||
|
||||
p = subprocess.Popen(args, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
# sox outputs to stderr (my god WHYYYY)
|
||||
out_, out = p.communicate()
|
||||
self.parse(str(out, encoding='utf-8'))
|
||||
|
||||
|
||||
class Sound:
|
||||
path = None # file path
|
||||
sample_length = 120 # default sample length in seconds
|
||||
stats = None # list of samples statistics
|
||||
bad = None # list of bad samples
|
||||
good = None # list of good samples
|
||||
|
||||
def __init__ (self, path, sample_length = None):
|
||||
self.path = path
|
||||
self.sample_length = sample_length if sample_length is not None \
|
||||
else self.sample_length
|
||||
|
||||
def get_file_stats (self):
|
||||
return self.stats and self.stats[0]
|
||||
|
||||
def analyse (self):
|
||||
logger.info('complete file analysis')
|
||||
self.stats = [ Stats(self.path) ]
|
||||
position = 0
|
||||
length = self.stats[0].get('length')
|
||||
|
||||
if not self.sample_length:
|
||||
return
|
||||
|
||||
logger.info('start samples analysis...')
|
||||
while position < length:
|
||||
stats = Stats(self.path, at = position, length = self.sample_length)
|
||||
self.stats.append(stats)
|
||||
position += self.sample_length
|
||||
|
||||
def check (self, name, min_val, max_val):
|
||||
self.good = [ index for index, stats in enumerate(self.stats)
|
||||
if min_val <= stats.get(name) <= max_val ]
|
||||
self.bad = [ index for index, stats in enumerate(self.stats)
|
||||
if index not in self.good ]
|
||||
self.resume()
|
||||
|
||||
def resume (self):
|
||||
view = lambda array: [
|
||||
'file' if index is 0 else
|
||||
'sample {} (at {} seconds)'.format(index, (index-1) * self.sample_length)
|
||||
for index in array
|
||||
]
|
||||
|
||||
if self.good:
|
||||
logger.info(self.path + ' -> good: \033[92m%s\033[0m',
|
||||
', '.join(view(self.good)))
|
||||
if self.bad:
|
||||
logger.info(self.path + ' -> bad: \033[91m%s\033[0m',
|
||||
', '.join(view(self.bad)))
|
||||
|
||||
class Command (BaseCommand):
|
||||
help = __doc__
|
||||
sounds = None
|
||||
|
||||
def add_arguments (self, parser):
|
||||
parser.formatter_class=RawTextHelpFormatter
|
||||
|
||||
parser.add_argument(
|
||||
'files', metavar='FILE', type=str, nargs='+',
|
||||
help='file(s) to analyse'
|
||||
)
|
||||
parser.add_argument(
|
||||
'-s', '--sample_length', type=int, default=120,
|
||||
help='size of sample to analyse in seconds. If not set (or 0), does'
|
||||
' not analyse by sample',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-a', '--attribute', type=str,
|
||||
help='attribute name to use to check, that can be:\n' + \
|
||||
', '.join([ '"{}"'.format(attr) for attr in Stats.attributes ])
|
||||
)
|
||||
parser.add_argument(
|
||||
'-r', '--range', type=float, nargs=2,
|
||||
help='range of minimal and maximal accepted value such as: ' \
|
||||
'--range min max'
|
||||
)
|
||||
parser.add_argument(
|
||||
'-i', '--resume', action='store_true',
|
||||
help='print a resume of good and bad files'
|
||||
)
|
||||
|
||||
def handle (self, *args, **options):
|
||||
# parameters
|
||||
minmax = options.get('range')
|
||||
if not minmax:
|
||||
raise CommandError('no range specified')
|
||||
|
||||
attr = options.get('attribute')
|
||||
if not attr:
|
||||
raise CommandError('no attribute specified')
|
||||
|
||||
# sound analyse and checks
|
||||
self.sounds = [ Sound(path, options.get('sample_length'))
|
||||
for path in options.get('files') ]
|
||||
self.bad = []
|
||||
self.good = []
|
||||
for sound in self.sounds:
|
||||
logger.info('analyse ' + sound.path)
|
||||
sound.analyse()
|
||||
sound.check(attr, minmax[0], minmax[1])
|
||||
if sound.bad:
|
||||
self.bad.append(sound)
|
||||
else:
|
||||
self.good.append(sound)
|
||||
|
||||
# resume
|
||||
if options.get('resume'):
|
||||
for sound in self.good:
|
||||
logger.info('\033[92m+ %s\033[0m', sound.path)
|
||||
for sound in self.bad:
|
||||
logger.info('\033[91m+ %s\033[0m', sound.path)
|
||||
|
||||
346
aircox/management/commands/streamer.py
Normal file
346
aircox/management/commands/streamer.py
Normal file
@ -0,0 +1,346 @@
|
||||
"""
|
||||
Handle the audio streamer and controls it as we want it to be. It is
|
||||
used to:
|
||||
- generate config files and playlists;
|
||||
- monitor Liquidsoap, logs and scheduled programs;
|
||||
- cancels Diffusions that have an archive but could not have been played;
|
||||
- run Liquidsoap
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
import re
|
||||
|
||||
from argparse import RawTextHelpFormatter
|
||||
|
||||
from django.conf import settings as main_settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.utils import timezone as tz
|
||||
|
||||
from aircox.models import Station, Diffusion, Track, Sound, Log
|
||||
|
||||
|
||||
class Monitor:
|
||||
"""
|
||||
Log and launch diffusions for the given station.
|
||||
|
||||
Monitor should be able to be used after a crash a go back
|
||||
where it was playing, so we heavily use logs to be able to
|
||||
do that.
|
||||
|
||||
We keep trace of played items on the generated stream:
|
||||
- sounds played on this stream;
|
||||
- scheduled diffusions
|
||||
- tracks for sounds of streamed programs
|
||||
"""
|
||||
station = None
|
||||
streamer = None
|
||||
cancel_timeout = 60*10
|
||||
"""
|
||||
Time in seconds before a diffusion that have archives is cancelled
|
||||
because it has not been played.
|
||||
"""
|
||||
sync_timeout = 60*10
|
||||
"""
|
||||
Time in minuts before all stream playlists are checked and updated
|
||||
"""
|
||||
sync_next = None
|
||||
"""
|
||||
Datetime of the next sync
|
||||
"""
|
||||
|
||||
def __init__(self, station, **kwargs):
|
||||
self.station = station
|
||||
self.__dict__.update(kwargs)
|
||||
|
||||
def monitor(self):
|
||||
"""
|
||||
Run all monitoring functions.
|
||||
"""
|
||||
if not self.streamer:
|
||||
self.streamer = self.station.streamer
|
||||
|
||||
if not self.streamer.ready():
|
||||
return
|
||||
|
||||
self.trace()
|
||||
self.sync_playlists()
|
||||
self.handle()
|
||||
|
||||
def log(self, **kwargs):
|
||||
"""
|
||||
Create a log using **kwargs, and print info
|
||||
"""
|
||||
log = Log(station = self.station, **kwargs)
|
||||
log.save()
|
||||
log.print()
|
||||
|
||||
def trace(self):
|
||||
"""
|
||||
Check the current_sound of the station and update logs if
|
||||
needed.
|
||||
"""
|
||||
self.streamer.fetch()
|
||||
current_sound = self.streamer.current_sound
|
||||
current_source = self.streamer.current_source
|
||||
if not current_sound or not current_source:
|
||||
return
|
||||
|
||||
log = Log.objects.get_for(model = Sound) \
|
||||
.filter(station = self.station) \
|
||||
.order_by('date').last()
|
||||
|
||||
# only streamed
|
||||
if log and (log.related and not log.related.diffusion):
|
||||
self.trace_sound_tracks(log)
|
||||
|
||||
# TODO: expiration
|
||||
if log and (log.source == current_source.id and \
|
||||
log.related and
|
||||
log.related.path == current_sound):
|
||||
return
|
||||
|
||||
sound = Sound.objects.filter(path = current_sound)
|
||||
self.log(
|
||||
type = Log.Type.play,
|
||||
source = current_source.id,
|
||||
date = tz.now(),
|
||||
related = sound[0] if sound else None,
|
||||
comment = None if sound else current_sound,
|
||||
)
|
||||
|
||||
def trace_sound_tracks(self, log):
|
||||
"""
|
||||
Log tracks for the given sound (for streamed programs); Called by
|
||||
self.trace
|
||||
"""
|
||||
logs = Log.objects.get_for(model = Track) \
|
||||
.filter(pk__gt = log.pk)
|
||||
logs = [ log.related_id for log in logs ]
|
||||
|
||||
tracks = Track.objects.get_for(object = log.related) \
|
||||
.filter(in_seconds = True)
|
||||
if tracks and len(tracks) == len(logs):
|
||||
return
|
||||
|
||||
tracks = tracks.exclude(pk__in = logs).order_by('position')
|
||||
now = tz.now()
|
||||
for track in tracks:
|
||||
pos = log.date + tz.timedelta(seconds = track.position)
|
||||
if pos < now:
|
||||
self.log(
|
||||
type = Log.Type.play,
|
||||
source = log.source,
|
||||
date = pos,
|
||||
related = track
|
||||
)
|
||||
|
||||
def sync_playlists(self):
|
||||
"""
|
||||
Synchronize updated playlists
|
||||
"""
|
||||
now = tz.now()
|
||||
if self.sync_next and self.sync_next < now:
|
||||
return
|
||||
|
||||
self.sync_next = now + tz.timedelta(seconds = self.sync_timeout)
|
||||
|
||||
for source in self.station.sources:
|
||||
if source == self.station.dealer:
|
||||
continue
|
||||
playlist = [ sound.path for sound in
|
||||
source.program.sound_set.all() ]
|
||||
source.playlist = playlist
|
||||
|
||||
def trace_canceled(self):
|
||||
"""
|
||||
Check diffusions that should have been played but did not start,
|
||||
and cancel them
|
||||
"""
|
||||
if not self.cancel_timeout:
|
||||
return
|
||||
|
||||
diffs = Diffusions.objects.get_at().filter(
|
||||
type = Diffusion.Type.normal,
|
||||
sound__type = Sound.Type.archive,
|
||||
)
|
||||
logs = station.get_played(models = Diffusion)
|
||||
|
||||
date = tz.now() - datetime.timedelta(seconds = self.cancel_timeout)
|
||||
for diff in diffs:
|
||||
if logs.filter(related = diff):
|
||||
continue
|
||||
if diff.start < now:
|
||||
diff.type = Diffusion.Type.canceled
|
||||
diff.save()
|
||||
self.log(
|
||||
type = Log.Type.other,
|
||||
related = diff,
|
||||
comment = 'Diffusion canceled after {} seconds' \
|
||||
.format(self.cancel_timeout)
|
||||
)
|
||||
|
||||
def __current_diff(self):
|
||||
"""
|
||||
Return a tuple with the currently running diffusion and the items
|
||||
that still have to be played. If there is not, return None
|
||||
"""
|
||||
station = self.station
|
||||
now = tz.make_aware(tz.datetime.now())
|
||||
|
||||
diff_log = station.get_played(models = Diffusion) \
|
||||
.order_by('date').last()
|
||||
if not diff_log or \
|
||||
not diff_log.related.is_date_in_range(now):
|
||||
return None, []
|
||||
|
||||
# sound has switched? assume it has been (forced to) stopped
|
||||
sounds = station.get_played(models = Sound) \
|
||||
.filter(date__gte = diff_log.date) \
|
||||
.order_by('date')
|
||||
|
||||
if sounds.last() and sounds.last().source != diff_log.source:
|
||||
return diff_log, []
|
||||
|
||||
# last diff is still playing: get the remaining playlist
|
||||
sounds = sounds.filter(
|
||||
source = diff_log.source, pk__gt = diff_log.pk
|
||||
)
|
||||
sounds = [
|
||||
sound.related.path for sound in sounds
|
||||
if sound.related.type != Sound.Type.removed
|
||||
]
|
||||
|
||||
return (
|
||||
diff_log.related,
|
||||
[ path for path in diff_log.related.playlist
|
||||
if path not in sounds ]
|
||||
)
|
||||
|
||||
def __next_diff(self, diff):
|
||||
"""
|
||||
Return the tuple with the next diff that should be played and
|
||||
the playlist
|
||||
|
||||
Note: diff is a log
|
||||
"""
|
||||
station = self.station
|
||||
now = tz.now()
|
||||
|
||||
args = {'start__gt': diff.date } if diff else {}
|
||||
diff = Diffusion.objects.get_at(now).filter(
|
||||
type = Diffusion.Type.normal,
|
||||
sound__type = Sound.Type.archive,
|
||||
**args
|
||||
).distinct().order_by('start').first()
|
||||
return (diff, diff and diff.playlist or [])
|
||||
|
||||
def handle(self):
|
||||
"""
|
||||
Handle scheduled diffusion, trigger if needed, preload playlists
|
||||
and so on.
|
||||
"""
|
||||
station = self.station
|
||||
dealer = station.dealer
|
||||
if not dealer:
|
||||
return
|
||||
now = tz.now()
|
||||
|
||||
# current and next diffs
|
||||
diff, playlist = self.__current_diff()
|
||||
dealer.active = bool(playlist)
|
||||
|
||||
next_diff, next_playlist = self.__next_diff(diff)
|
||||
playlist += next_playlist
|
||||
|
||||
# playlist update
|
||||
if dealer.playlist != playlist:
|
||||
dealer.playlist = playlist
|
||||
if next_diff:
|
||||
self.log(
|
||||
type = Log.Type.load,
|
||||
source = dealer.id,
|
||||
date = now,
|
||||
related = next_diff
|
||||
)
|
||||
|
||||
# dealer.on when next_diff start <= now
|
||||
if next_diff and not dealer.active and \
|
||||
next_diff.start <= now:
|
||||
dealer.active = True
|
||||
self.log(
|
||||
type = Log.Type.play,
|
||||
source = dealer.id,
|
||||
date = now,
|
||||
related = next_diff,
|
||||
)
|
||||
|
||||
|
||||
class Command (BaseCommand):
|
||||
help= __doc__
|
||||
|
||||
def add_arguments (self, parser):
|
||||
parser.formatter_class=RawTextHelpFormatter
|
||||
group = parser.add_argument_group('actions')
|
||||
group.add_argument(
|
||||
'-c', '--config', action='store_true',
|
||||
help='generate configuration files for the stations'
|
||||
)
|
||||
group.add_argument(
|
||||
'-m', '--monitor', action='store_true',
|
||||
help='monitor the scheduled diffusions and log what happens'
|
||||
)
|
||||
group.add_argument(
|
||||
'-r', '--run', action='store_true',
|
||||
help='run the required applications for the stations'
|
||||
)
|
||||
|
||||
group = parser.add_argument_group('options')
|
||||
group.add_argument(
|
||||
'-d', '--delay', type=int,
|
||||
default=1000,
|
||||
help='time to sleep in MILLISECONDS between two updates when we '
|
||||
'monitor'
|
||||
)
|
||||
group.add_argument(
|
||||
'-s', '--station', type=str, action='append',
|
||||
help='name of the station to monitor instead of monitoring '
|
||||
'all stations'
|
||||
)
|
||||
group.add_argument(
|
||||
'-t', '--timeout', type=int,
|
||||
default=600,
|
||||
help='time to wait in SECONDS before canceling a diffusion that '
|
||||
'has not been ran but should have been. If 0, does not '
|
||||
'check'
|
||||
)
|
||||
|
||||
def handle (self, *args,
|
||||
config = None, run = None, monitor = None,
|
||||
station = [], delay = 1000, timeout = 600,
|
||||
**options):
|
||||
|
||||
stations = Station.objects.filter(name__in = station)[:] \
|
||||
if station else Station.objects.all()[:]
|
||||
|
||||
for station in stations:
|
||||
# station.prepare()
|
||||
if config and not run: # no need to write it twice
|
||||
station.streamer.push()
|
||||
if run:
|
||||
station.streamer.process_run()
|
||||
|
||||
if monitor:
|
||||
monitors = [
|
||||
Monitor(station, cancel_timeout = timeout)
|
||||
for station in stations
|
||||
]
|
||||
delay = delay / 1000
|
||||
while True:
|
||||
for monitor in monitors:
|
||||
monitor.monitor()
|
||||
time.sleep(delay)
|
||||
|
||||
if run:
|
||||
for station in stations:
|
||||
station.controller.process_wait()
|
||||
|
||||
Reference in New Issue
Block a user