Merge pull request 'Fix sound monitor issues' (#82) from dev-1.0-sound-monitor-fixes into develop-1.0
Reviewed-on: #82
This commit is contained in:
Thomas Kairos
@ -25,233 +25,23 @@ Sox (and soxi).
|
||||
"""
|
||||
from argparse import RawTextHelpFormatter
|
||||
import concurrent.futures as futures
|
||||
import datetime
|
||||
import atexit
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
import mutagen
|
||||
from watchdog.observers import Observer
|
||||
from watchdog.events import PatternMatchingEventHandler, FileModifiedEvent
|
||||
|
||||
from django.conf import settings as conf
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.utils import timezone as tz
|
||||
from django.utils.translation import gettext as _
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from aircox import settings, utils
|
||||
from aircox.models import Diffusion, Program, Sound, Track
|
||||
from .import_playlist import PlaylistImport
|
||||
from aircox import settings
|
||||
from aircox.models import Program, Sound
|
||||
from aircox.management.sound_file import SoundFile
|
||||
from aircox.management.sound_monitor import MonitorHandler
|
||||
|
||||
logger = logging.getLogger('aircox.commands')
|
||||
|
||||
|
||||
sound_path_re = re.compile(
|
||||
'^(?P<year>[0-9]{4})(?P<month>[0-9]{2})(?P<day>[0-9]{2})'
|
||||
'(_(?P<hour>[0-9]{2})h(?P<minute>[0-9]{2}))?'
|
||||
'(_(?P<n>[0-9]+))?'
|
||||
'_?[ -]*(?P<name>.*)$'
|
||||
)
|
||||
|
||||
|
||||
class SoundFile:
|
||||
path = None
|
||||
info = None
|
||||
path_info = None
|
||||
sound = None
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
|
||||
@property
|
||||
def sound_path(self):
|
||||
""" Relative path name """
|
||||
return self.path.replace(conf.MEDIA_ROOT + '/', '')
|
||||
|
||||
def sync(self, sound=None, program=None, deleted=False, **kwargs):
|
||||
"""
|
||||
Update related sound model and save it.
|
||||
"""
|
||||
if deleted:
|
||||
sound = Sound.objects.path(self.path).first()
|
||||
if sound:
|
||||
sound.type = sound.TYPE_REMOVED
|
||||
sound.check_on_file()
|
||||
sound.save()
|
||||
return sound
|
||||
|
||||
# FIXME: sound.program as not null
|
||||
if not program:
|
||||
program = Program.get_from_path(self.path)
|
||||
logger.info('program from path "%s" -> %s', self.path, program)
|
||||
kwargs['program_id'] = program.pk
|
||||
|
||||
sound, created = Sound.objects.get_or_create(file=self.sound_path, defaults=kwargs) \
|
||||
if not sound else (sound, False)
|
||||
self.sound = sound
|
||||
sound.program = program
|
||||
if created or sound.check_on_file():
|
||||
logger.info('sound is new or have been modified -> %s', self.sound_path)
|
||||
self.read_path()
|
||||
sound.name = self.path_info.get('name')
|
||||
|
||||
self.read_file_info()
|
||||
if self.info is not None:
|
||||
sound.duration = utils.seconds_to_time(self.info.info.length)
|
||||
|
||||
# check for episode
|
||||
if sound.episode is None and self.read_path():
|
||||
self.find_episode(program)
|
||||
|
||||
sound.save()
|
||||
|
||||
# check for playlist
|
||||
self.find_playlist(sound)
|
||||
return sound
|
||||
|
||||
def read_path(self):
|
||||
"""
|
||||
Parse file name to get info on the assumption it has the correct
|
||||
format (given in Command.help). Return True if path contains informations.
|
||||
"""
|
||||
if self.path_info:
|
||||
return 'year' in self.path_info
|
||||
|
||||
name = os.path.splitext(os.path.basename(self.path))[0]
|
||||
match = sound_path_re.search(name)
|
||||
if match:
|
||||
path_info = match.groupdict()
|
||||
for k in ('year', 'month', 'day', 'hour', 'minute'):
|
||||
if path_info.get(k) is not None:
|
||||
path_info[k] = int(path_info[k])
|
||||
self.path_info = path_info
|
||||
return True
|
||||
else:
|
||||
self.path_info = {'name': name}
|
||||
return False
|
||||
|
||||
def read_file_info(self):
|
||||
""" Read file information and metadata. """
|
||||
if os.path.exists(self.path):
|
||||
self.info = mutagen.File(self.path)
|
||||
else:
|
||||
self.info = None
|
||||
|
||||
def find_episode(self, program):
|
||||
"""
|
||||
For a given program, check if there is an initial diffusion
|
||||
to associate to, using the date info we have. Update self.sound
|
||||
and save it consequently.
|
||||
|
||||
We only allow initial diffusion since there should be no
|
||||
rerun.
|
||||
"""
|
||||
pi = self.path_info
|
||||
if 'year' not in pi or not self.sound or self.sound.episode:
|
||||
return None
|
||||
|
||||
if pi.get('hour') is not None:
|
||||
date = tz.datetime(pi.get('year'), pi.get('month'), pi.get('day'),
|
||||
pi.get('hour') or 0, pi.get('minute') or 0)
|
||||
date = tz.get_current_timezone().localize(date)
|
||||
else:
|
||||
date = datetime.date(pi.get('year'), pi.get('month'), pi.get('day'))
|
||||
|
||||
diffusion = program.diffusion_set.at(date).first()
|
||||
if not diffusion:
|
||||
return None
|
||||
|
||||
logger.info('%s <--> %s', self.sound.file.name, str(diffusion.episode))
|
||||
self.sound.episode = diffusion.episode
|
||||
return diffusion
|
||||
|
||||
def find_playlist(self, sound=None, use_meta=True):
|
||||
"""
|
||||
Find a playlist file corresponding to the sound path, such as:
|
||||
my_sound.ogg => my_sound.csv
|
||||
|
||||
Use sound's file metadata if no corresponding playlist has been
|
||||
found and `use_meta` is True.
|
||||
"""
|
||||
if sound is None:
|
||||
sound = self.sound
|
||||
if sound.track_set.count() > 1:
|
||||
return
|
||||
|
||||
# import playlist
|
||||
path_noext, ext = os.path.splitext(self.sound.file.path)
|
||||
path = path_noext + '.csv'
|
||||
if os.path.exists(path):
|
||||
PlaylistImport(path, sound=sound).run()
|
||||
# use metadata
|
||||
elif use_meta:
|
||||
if self.info is None:
|
||||
self.read_file_info()
|
||||
if self.info and self.info.tags:
|
||||
tags = self.info.tags
|
||||
title, artist, album, year = tuple(
|
||||
t and ', '.join(t) for t in (
|
||||
tags.get(k) for k in ('title', 'artist', 'album', 'year'))
|
||||
)
|
||||
title = title or (self.path_info and self.path_info.get('name')) or \
|
||||
os.path.basename(path_noext)
|
||||
info = '{} ({})'.format(album, year) if album and year else \
|
||||
album or year or ''
|
||||
track = Track(sound=sound,
|
||||
position=int(tags.get('tracknumber', 0)),
|
||||
title=title,
|
||||
artist=artist or _('unknown'),
|
||||
info=info)
|
||||
track.save()
|
||||
|
||||
|
||||
class MonitorHandler(PatternMatchingEventHandler):
|
||||
"""
|
||||
Event handler for watchdog, in order to be used in monitoring.
|
||||
"""
|
||||
pool = None
|
||||
|
||||
def __init__(self, subdir, pool):
|
||||
"""
|
||||
subdir: AIRCOX_SOUND_ARCHIVES_SUBDIR or AIRCOX_SOUND_EXCERPTS_SUBDIR
|
||||
"""
|
||||
self.subdir = subdir
|
||||
self.pool = pool
|
||||
if self.subdir == settings.AIRCOX_SOUND_ARCHIVES_SUBDIR:
|
||||
self.sound_kwargs = {'type': Sound.TYPE_ARCHIVE}
|
||||
else:
|
||||
self.sound_kwargs = {'type': Sound.TYPE_EXCERPT}
|
||||
|
||||
patterns = ['*/{}/*{}'.format(self.subdir, ext)
|
||||
for ext in settings.AIRCOX_SOUND_FILE_EXT]
|
||||
super().__init__(patterns=patterns, ignore_directories=True)
|
||||
|
||||
def on_created(self, event):
|
||||
self.on_modified(event)
|
||||
|
||||
def on_modified(self, event):
|
||||
logger.info('sound modified: %s', event.src_path)
|
||||
def updated(event, sound_kwargs):
|
||||
SoundFile(event.src_path).sync(**sound_kwargs)
|
||||
self.pool.submit(updated, event, self.sound_kwargs)
|
||||
|
||||
def on_moved(self, event):
|
||||
logger.info('sound moved: %s -> %s', event.src_path, event.dest_path)
|
||||
def moved(event, sound_kwargs):
|
||||
sound = Sound.objects.filter(file=event.src_path)
|
||||
sound_file = SoundFile(event.dest_path) if not sound else sound
|
||||
sound_file.sync(**sound_kwargs)
|
||||
self.pool.submit(moved, event, self.sound_kwargs)
|
||||
|
||||
def on_deleted(self, event):
|
||||
logger.info('sound deleted: %s', event.src_path)
|
||||
def deleted(event):
|
||||
SoundFile(event.src_path).sync(deleted=True)
|
||||
self.pool.submit(deleted, event)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = __doc__
|
||||
|
||||
|
||||
@ -1,118 +1,16 @@
|
||||
"""
|
||||
Analyse and check files using Sox, prints good and bad files.
|
||||
"""
|
||||
import sys
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
from argparse import RawTextHelpFormatter
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from aircox.management.sound_stats import SoxStats, SoundStats
|
||||
|
||||
logger = logging.getLogger('aircox.commands')
|
||||
|
||||
|
||||
class Stats:
|
||||
attributes = [
|
||||
'DC offset', 'Min level', 'Max level',
|
||||
'Pk lev dB', 'RMS lev dB', 'RMS Pk dB',
|
||||
'RMS Tr dB', 'Flat factor', 'Length s',
|
||||
]
|
||||
|
||||
def __init__(self, path, **kwargs):
|
||||
"""
|
||||
If path is given, call analyse with path and kwargs
|
||||
"""
|
||||
self.values = {}
|
||||
if path:
|
||||
self.analyse(path, **kwargs)
|
||||
|
||||
def get(self, attr):
|
||||
return self.values.get(attr)
|
||||
|
||||
def parse(self, output):
|
||||
for attr in Stats.attributes:
|
||||
value = re.search(attr + r'\s+(?P<value>\S+)', output)
|
||||
value = value and value.groupdict()
|
||||
if value:
|
||||
try:
|
||||
value = float(value.get('value'))
|
||||
except ValueError:
|
||||
value = None
|
||||
self.values[attr] = value
|
||||
self.values['length'] = self.values['Length s']
|
||||
|
||||
def analyse(self, path, at=None, length=None):
|
||||
"""
|
||||
If at and length are given use them as excerpt to analyse.
|
||||
"""
|
||||
args = ['sox', path, '-n']
|
||||
|
||||
if at is not None and length is not None:
|
||||
args += ['trim', str(at), str(length)]
|
||||
|
||||
args.append('stats')
|
||||
|
||||
p = subprocess.Popen(args, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
# sox outputs to stderr (my god WHYYYY)
|
||||
out_, out = p.communicate()
|
||||
self.parse(str(out, encoding='utf-8'))
|
||||
|
||||
|
||||
class SoundStats:
|
||||
path = None # file path
|
||||
sample_length = 120 # default sample length in seconds
|
||||
stats = None # list of samples statistics
|
||||
bad = None # list of bad samples
|
||||
good = None # list of good samples
|
||||
|
||||
def __init__(self, path, sample_length=None):
|
||||
self.path = path
|
||||
self.sample_length = sample_length if sample_length is not None \
|
||||
else self.sample_length
|
||||
|
||||
def get_file_stats(self):
|
||||
return self.stats and self.stats[0]
|
||||
|
||||
def analyse(self):
|
||||
logger.info('complete file analysis')
|
||||
self.stats = [Stats(self.path)]
|
||||
position = 0
|
||||
length = self.stats[0].get('length')
|
||||
|
||||
if not self.sample_length:
|
||||
return
|
||||
|
||||
logger.info('start samples analysis...')
|
||||
while position < length:
|
||||
stats = Stats(self.path, at=position, length=self.sample_length)
|
||||
self.stats.append(stats)
|
||||
position += self.sample_length
|
||||
|
||||
def check(self, name, min_val, max_val):
|
||||
self.good = [index for index, stats in enumerate(self.stats)
|
||||
if min_val <= stats.get(name) <= max_val]
|
||||
self.bad = [index for index, stats in enumerate(self.stats)
|
||||
if index not in self.good]
|
||||
self.resume()
|
||||
|
||||
def resume(self):
|
||||
def view(array): return [
|
||||
'file' if index is 0 else
|
||||
'sample {} (at {} seconds)'.format(
|
||||
index, (index-1) * self.sample_length)
|
||||
for index in array
|
||||
]
|
||||
|
||||
if self.good:
|
||||
logger.info(self.path + ' -> good: \033[92m%s\033[0m',
|
||||
', '.join(view(self.good)))
|
||||
if self.bad:
|
||||
logger.info(self.path + ' -> bad: \033[91m%s\033[0m',
|
||||
', '.join(view(self.bad)))
|
||||
|
||||
|
||||
class Command (BaseCommand):
|
||||
help = __doc__
|
||||
sounds = None
|
||||
@ -132,7 +30,7 @@ class Command (BaseCommand):
|
||||
parser.add_argument(
|
||||
'-a', '--attribute', type=str,
|
||||
help='attribute name to use to check, that can be:\n' +
|
||||
', '.join(['"{}"'.format(attr) for attr in Stats.attributes])
|
||||
', '.join(['"{}"'.format(attr) for attr in SoxStats.attributes])
|
||||
)
|
||||
parser.add_argument(
|
||||
'-r', '--range', type=float, nargs=2,
|
||||
@ -160,7 +58,7 @@ class Command (BaseCommand):
|
||||
self.bad = []
|
||||
self.good = []
|
||||
for sound in self.sounds:
|
||||
logger.info('analyse ' + sound.file.name)
|
||||
logger.info('analyse ' + sound.path)
|
||||
sound.analyse()
|
||||
sound.check(attr, minmax[0], minmax[1])
|
||||
if sound.bad:
|
||||
@ -171,6 +69,6 @@ class Command (BaseCommand):
|
||||
# resume
|
||||
if options.get('resume'):
|
||||
for sound in self.good:
|
||||
logger.info('\033[92m+ %s\033[0m', sound.file.name)
|
||||
logger.info('\033[92m+ %s\033[0m', sound.path)
|
||||
for sound in self.bad:
|
||||
logger.info('\033[91m+ %s\033[0m', sound.file.name)
|
||||
logger.info('\033[91m+ %s\033[0m', sound.path)
|
||||
|
||||
216
aircox/management/sound_file.py
Normal file
216
aircox/management/sound_file.py
Normal file
@ -0,0 +1,216 @@
|
||||
#! /usr/bin/env python3
|
||||
"""
|
||||
Provide SoundFile which is used to link between database and file system.
|
||||
|
||||
|
||||
File name
|
||||
=========
|
||||
It tries to parse the file name to get the date of the diffusion of an
|
||||
episode and associate the file with it; We use the following format:
|
||||
yyyymmdd[_n][_][name]
|
||||
|
||||
Where:
|
||||
'yyyy' the year of the episode's diffusion;
|
||||
'mm' the month of the episode's diffusion;
|
||||
'dd' the day of the episode's diffusion;
|
||||
'n' the number of the episode (if multiple episodes);
|
||||
'name' the title of the sound;
|
||||
|
||||
Sound Quality
|
||||
=============
|
||||
To check quality of files, call the command sound_quality_check using the
|
||||
parameters given by the setting AIRCOX_SOUND_QUALITY. This script requires
|
||||
Sox (and soxi).
|
||||
"""
|
||||
from datetime import date
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
import mutagen
|
||||
|
||||
from django.conf import settings as conf
|
||||
from django.utils import timezone as tz
|
||||
from django.utils.translation import gettext as _
|
||||
|
||||
from aircox import utils
|
||||
from aircox.models import Program, Sound, Track
|
||||
from .commands.import_playlist import PlaylistImport
|
||||
|
||||
logger = logging.getLogger('aircox.commands')
|
||||
|
||||
|
||||
class SoundFile:
|
||||
"""
|
||||
Handle synchronisation between sounds on files and database.
|
||||
"""
|
||||
path = None
|
||||
info = None
|
||||
path_info = None
|
||||
sound = None
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
|
||||
@property
|
||||
def sound_path(self):
|
||||
""" Relative path name """
|
||||
return self.path.replace(conf.MEDIA_ROOT + '/', '')
|
||||
|
||||
@property
|
||||
def episode(self):
|
||||
return self.sound and self.sound.episode
|
||||
|
||||
def sync(self, sound=None, program=None, deleted=False, **kwargs):
|
||||
"""
|
||||
Update related sound model and save it.
|
||||
"""
|
||||
if deleted:
|
||||
return self._on_delete(self.path)
|
||||
|
||||
# FIXME: sound.program as not null
|
||||
if not program:
|
||||
program = Program.get_from_path(self.path)
|
||||
logger.debug('program from path "%s" -> %s', self.path, program)
|
||||
kwargs['program_id'] = program.pk
|
||||
|
||||
if sound:
|
||||
created = False
|
||||
else:
|
||||
sound, created = Sound.objects.get_or_create(
|
||||
file=self.sound_path, defaults=kwargs)
|
||||
|
||||
self.sound = sound
|
||||
self.path_info = self.read_path(self.path)
|
||||
|
||||
sound.program = program
|
||||
if created or sound.check_on_file():
|
||||
sound.name = self.path_info.get('name')
|
||||
self.info = self.read_file_info()
|
||||
if self.info is not None:
|
||||
sound.duration = utils.seconds_to_time(self.info.info.length)
|
||||
|
||||
# check for episode
|
||||
if sound.episode is None and 'year' in self.path_info:
|
||||
sound.episode = self.find_episode(sound, self.path_info)
|
||||
sound.save()
|
||||
|
||||
# check for playlist
|
||||
self.find_playlist(sound)
|
||||
return sound
|
||||
|
||||
def _on_delete(self, path):
|
||||
# TODO: remove from db on delete
|
||||
sound = Sound.objects.path(self.path).first()
|
||||
if sound:
|
||||
sound.type = sound.TYPE_REMOVED
|
||||
sound.check_on_file()
|
||||
sound.save()
|
||||
return sound
|
||||
|
||||
def read_path(self, path):
|
||||
"""
|
||||
Parse path name returning dictionary of extracted info. It can contain:
|
||||
- `year`, `month`, `day`: diffusion date
|
||||
- `hour`, `minute`: diffusion time
|
||||
- `n`: sound arbitrary number (used for sound ordering)
|
||||
- `name`: cleaned name extracted or file name (without extension)
|
||||
"""
|
||||
basename = os.path.basename(path)
|
||||
basename = os.path.splitext(basename)[0]
|
||||
reg_match = self._path_re.search(basename)
|
||||
if reg_match:
|
||||
info = reg_match.groupdict()
|
||||
for k in ('year', 'month', 'day', 'hour', 'minute', 'n'):
|
||||
if info.get(k) is not None:
|
||||
info[k] = int(info[k])
|
||||
|
||||
name = info.get('name')
|
||||
info['name'] = name and self._into_name(name) or basename
|
||||
else:
|
||||
info = {'name': basename}
|
||||
return info
|
||||
|
||||
_path_re = re.compile(
|
||||
'^(?P<year>[0-9]{4})(?P<month>[0-9]{2})(?P<day>[0-9]{2})'
|
||||
'(_(?P<hour>[0-9]{2})h(?P<minute>[0-9]{2}))?'
|
||||
'(_(?P<n>[0-9]+))?'
|
||||
'_?[ -]*(?P<name>.*)$'
|
||||
)
|
||||
|
||||
def _into_name(self, name):
|
||||
name = name.replace('_', ' ')
|
||||
return ' '.join(r.capitalize() for r in name.split(' '))
|
||||
|
||||
def read_file_info(self):
|
||||
""" Read file information and metadata. """
|
||||
return mutagen.File(self.path) if os.path.exists(self.path) else None
|
||||
|
||||
def find_episode(self, sound, path_info):
|
||||
"""
|
||||
For a given program, check if there is an initial diffusion
|
||||
to associate to, using the date info we have. Update self.sound
|
||||
and save it consequently.
|
||||
|
||||
We only allow initial diffusion since there should be no
|
||||
rerun.
|
||||
"""
|
||||
program, pi = sound.program, path_info
|
||||
if 'year' not in pi or not sound or sound.episode:
|
||||
return None
|
||||
|
||||
year, month, day = pi.get('year'), pi.get('month'), pi.get('day')
|
||||
if pi.get('hour') is not None:
|
||||
at = tz.datetime(year, month, day, pi.get('hour', 0),
|
||||
pi.get('minute', 0))
|
||||
at = tz.get_current_timezone().localize(at)
|
||||
else:
|
||||
at = date(year, month, day)
|
||||
|
||||
diffusion = program.diffusion_set.at(at).first()
|
||||
if not diffusion:
|
||||
return None
|
||||
|
||||
logger.debug('%s <--> %s', sound.file.name, str(diffusion.episode))
|
||||
return diffusion.episode
|
||||
|
||||
def find_playlist(self, sound=None, use_meta=True):
|
||||
"""
|
||||
Find a playlist file corresponding to the sound path, such as:
|
||||
my_sound.ogg => my_sound.csv
|
||||
|
||||
Use sound's file metadata if no corresponding playlist has been
|
||||
found and `use_meta` is True.
|
||||
"""
|
||||
if sound is None:
|
||||
sound = self.sound
|
||||
if sound.track_set.count() > 1:
|
||||
return
|
||||
|
||||
# import playlist
|
||||
path_noext, ext = os.path.splitext(self.sound.file.path)
|
||||
path = path_noext + '.csv'
|
||||
if os.path.exists(path):
|
||||
PlaylistImport(path, sound=sound).run()
|
||||
# use metadata
|
||||
elif use_meta:
|
||||
if self.info is None:
|
||||
self.read_file_info()
|
||||
if self.info and self.info.tags:
|
||||
tags = self.info.tags
|
||||
title, artist, album, year = tuple(
|
||||
t and ', '.join(t) for t in (
|
||||
tags.get(k) for k in ('title', 'artist', 'album',
|
||||
'year'))
|
||||
)
|
||||
title = title or (self.path_info and
|
||||
self.path_info.get('name')) or \
|
||||
os.path.basename(path_noext)
|
||||
info = '{} ({})'.format(album, year) if album and year else \
|
||||
album or year or ''
|
||||
track = Track(sound=sound,
|
||||
position=int(tags.get('tracknumber', 0)),
|
||||
title=title,
|
||||
artist=artist or _('unknown'),
|
||||
info=info)
|
||||
track.save()
|
||||
163
aircox/management/sound_monitor.py
Normal file
163
aircox/management/sound_monitor.py
Normal file
@ -0,0 +1,163 @@
|
||||
#! /usr/bin/env python3
|
||||
|
||||
"""
|
||||
Monitor sound files; For each program, check for:
|
||||
- new files;
|
||||
- deleted files;
|
||||
- differences between files and sound;
|
||||
- quality of the files;
|
||||
|
||||
It tries to parse the file name to get the date of the diffusion of an
|
||||
episode and associate the file with it; WNotifye the following format:
|
||||
yyyymmdd[_n][_][name]
|
||||
|
||||
Where:
|
||||
'yyyy' the year Notifyhe episode's diffusion;
|
||||
'mm' the month of the episode's difNotifyon;
|
||||
'dd' the day of the episode's diffusion;
|
||||
'n' the number of the episode (if multiple episodes);
|
||||
'name' the title of the sNotify;
|
||||
|
||||
|
||||
To check quality of files, call the command sound_quality_check using the
|
||||
parameters given by the setting AIRCOX_SOUND_QUALITY. This script requires
|
||||
Sox (and soxi).
|
||||
"""
|
||||
from datetime import datetime, timedelta
|
||||
import logging
|
||||
import time
|
||||
|
||||
from watchdog.events import PatternMatchingEventHandler
|
||||
|
||||
from aircox import settings
|
||||
from aircox.models import Sound
|
||||
|
||||
from .sound_file import SoundFile
|
||||
|
||||
|
||||
logger = logging.getLogger('aircox.commands')
|
||||
|
||||
|
||||
__all__ = ('NotifyHandler', 'CreateHandler', 'DeleteHandler',
|
||||
'MoveHandler', 'ModifiedHandler', 'MonitorHandler',)
|
||||
|
||||
|
||||
class NotifyHandler:
|
||||
future = None
|
||||
log_msg = None
|
||||
|
||||
def __init__(self):
|
||||
self.timestamp = datetime.now()
|
||||
|
||||
def ping(self):
|
||||
self.timestamp = datetime.now()
|
||||
|
||||
def __call__(self, event, path=None, **kw):
|
||||
sound_file = SoundFile(path or event.src_path)
|
||||
if self.log_msg:
|
||||
msg = self.log_msg.format(event=event, sound_file=sound_file)
|
||||
logger.info(msg)
|
||||
|
||||
sound_file.sync(**kw)
|
||||
return sound_file
|
||||
|
||||
|
||||
class CreateHandler(NotifyHandler):
|
||||
log_msg = 'Sound file created: {sound_file.path}'
|
||||
|
||||
|
||||
class DeleteHandler(NotifyHandler):
|
||||
log_msg = 'Sound file deleted: {sound_file.path}'
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
kwargs['deleted'] = True
|
||||
return super().__call__(*args, **kwargs)
|
||||
|
||||
|
||||
class MoveHandler(NotifyHandler):
|
||||
log_msg = 'Sound file moved: {event.src_path} -> {event.dest_path}'
|
||||
|
||||
def __call__(self, event, **kw):
|
||||
sound = Sound.objects.filter(file=event.src_path)
|
||||
# FIXME: this is wrong
|
||||
if sound:
|
||||
kw['sound'] = sound
|
||||
kw['path'] = event.src_path
|
||||
else:
|
||||
kw['path'] = event.dest_path
|
||||
return super().__call__(event, **kw)
|
||||
|
||||
|
||||
class ModifiedHandler(NotifyHandler):
|
||||
timeout_delta = timedelta(seconds=30)
|
||||
log_msg = 'Sound file updated: {sound_file.path}'
|
||||
|
||||
def wait(self):
|
||||
# multiple call of this handler can be done consecutively, we block
|
||||
# its thread using timeout
|
||||
# Note: this method may be subject to some race conflicts, but this
|
||||
# should not be big a real issue.
|
||||
timeout = self.timestamp + self.timeout_delta
|
||||
while datetime.now() < timeout:
|
||||
time.sleep(self.timeout_delta.total_seconds())
|
||||
timeout = self.timestamp + self.timeout_delta
|
||||
|
||||
def __call__(self, event, **kw):
|
||||
self.wait()
|
||||
return super().__call__(event, **kw)
|
||||
|
||||
|
||||
class MonitorHandler(PatternMatchingEventHandler):
|
||||
"""
|
||||
Event handler for watchdog, in order to be used in monitoring.
|
||||
"""
|
||||
pool = None
|
||||
jobs = {}
|
||||
|
||||
def __init__(self, subdir, pool):
|
||||
"""
|
||||
subdir: AIRCOX_SOUND_ARCHIVES_SUBDIR or AIRCOX_SOUND_EXCERPTS_SUBDIR
|
||||
"""
|
||||
self.subdir = subdir
|
||||
self.pool = pool
|
||||
if self.subdir == settings.AIRCOX_SOUND_ARCHIVES_SUBDIR:
|
||||
self.sync_kw = {'type': Sound.TYPE_ARCHIVE}
|
||||
else:
|
||||
self.sync_kw = {'type': Sound.TYPE_EXCERPT}
|
||||
|
||||
patterns = ['*/{}/*{}'.format(self.subdir, ext)
|
||||
for ext in settings.AIRCOX_SOUND_FILE_EXT]
|
||||
super().__init__(patterns=patterns, ignore_directories=True)
|
||||
|
||||
def on_created(self, event):
|
||||
self._submit(CreateHandler(), event, 'new', **self.sync_kw)
|
||||
|
||||
def on_deleted(self, event):
|
||||
self._submit(DeleteHandler(), event, 'del')
|
||||
|
||||
def on_moved(self, event):
|
||||
self._submit(MoveHandler(), event, 'mv', **self.sync_kw)
|
||||
|
||||
def on_modified(self, event):
|
||||
self._submit(ModifiedHandler(), event, 'up', **self.sync_kw)
|
||||
|
||||
def _submit(self, handler, event, job_key_prefix, **kwargs):
|
||||
"""
|
||||
Send handler job to pool if not already running.
|
||||
Return tuple with running job and boolean indicating if its a new one.
|
||||
"""
|
||||
key = job_key_prefix + ':' + event.src_path
|
||||
job = self.jobs.get(key)
|
||||
if job and not job.future.done():
|
||||
job.ping()
|
||||
return job, False
|
||||
|
||||
handler.future = self.pool.submit(handler, event, **kwargs)
|
||||
self.jobs[key] = handler
|
||||
|
||||
def done(r):
|
||||
print(':::: job done', key)
|
||||
if self.jobs.get(key) is handler:
|
||||
del self.jobs[key]
|
||||
handler.future.add_done_callback(done)
|
||||
return handler, True
|
||||
115
aircox/management/sound_stats.py
Normal file
115
aircox/management/sound_stats.py
Normal file
@ -0,0 +1,115 @@
|
||||
"""
|
||||
Provide sound analysis class using Sox.
|
||||
"""
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
logger = logging.getLogger('aircox.commands')
|
||||
|
||||
|
||||
__all__ = ('SoxStats', 'SoundStats')
|
||||
|
||||
|
||||
class SoxStats:
|
||||
"""
|
||||
Run Sox process and parse output
|
||||
"""
|
||||
attributes = [
|
||||
'DC offset', 'Min level', 'Max level',
|
||||
'Pk lev dB', 'RMS lev dB', 'RMS Pk dB',
|
||||
'RMS Tr dB', 'Flat factor', 'Length s',
|
||||
]
|
||||
|
||||
def __init__(self, path, **kwargs):
|
||||
"""
|
||||
If path is given, call analyse with path and kwargs
|
||||
"""
|
||||
self.values = {}
|
||||
if path:
|
||||
self.analyse(path, **kwargs)
|
||||
|
||||
def get(self, attr):
|
||||
return self.values.get(attr)
|
||||
|
||||
def parse(self, output):
|
||||
for attr in self.attributes:
|
||||
value = re.search(attr + r'\s+(?P<value>\S+)', output)
|
||||
value = value and value.groupdict()
|
||||
if value:
|
||||
try:
|
||||
value = float(value.get('value'))
|
||||
except ValueError:
|
||||
value = None
|
||||
self.values[attr] = value
|
||||
self.values['length'] = self.values['Length s']
|
||||
|
||||
def analyse(self, path, at=None, length=None):
|
||||
"""
|
||||
If at and length are given use them as excerpt to analyse.
|
||||
"""
|
||||
args = ['sox', path, '-n']
|
||||
|
||||
if at is not None and length is not None:
|
||||
args += ['trim', str(at), str(length)]
|
||||
|
||||
args.append('stats')
|
||||
|
||||
p = subprocess.Popen(args, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
# sox outputs to stderr (my god WHYYYY)
|
||||
out_, out = p.communicate()
|
||||
self.parse(str(out, encoding='utf-8'))
|
||||
|
||||
|
||||
class SoundStats:
|
||||
path = None # file path
|
||||
sample_length = 120 # default sample length in seconds
|
||||
stats = None # list of samples statistics
|
||||
bad = None # list of bad samples
|
||||
good = None # list of good samples
|
||||
|
||||
def __init__(self, path, sample_length=None):
|
||||
self.path = path
|
||||
self.sample_length = sample_length if sample_length is not None \
|
||||
else self.sample_length
|
||||
|
||||
def get_file_stats(self):
|
||||
return self.stats and self.stats[0]
|
||||
|
||||
def analyse(self):
|
||||
logger.debug('complete file analysis')
|
||||
self.stats = [SoxStats(self.path)]
|
||||
position = 0
|
||||
length = self.stats[0].get('length')
|
||||
|
||||
if not self.sample_length:
|
||||
return
|
||||
|
||||
logger.debug('start samples analysis...')
|
||||
while position < length:
|
||||
stats = SoxStats(self.path, at=position, length=self.sample_length)
|
||||
self.stats.append(stats)
|
||||
position += self.sample_length
|
||||
|
||||
def check(self, name, min_val, max_val):
|
||||
self.good = [index for index, stats in enumerate(self.stats)
|
||||
if min_val <= stats.get(name) <= max_val]
|
||||
self.bad = [index for index, stats in enumerate(self.stats)
|
||||
if index not in self.good]
|
||||
self.resume()
|
||||
|
||||
def resume(self):
|
||||
def view(array): return [
|
||||
'file' if index == 0 else
|
||||
'sample {} (at {} seconds)'.format(
|
||||
index, (index-1) * self.sample_length)
|
||||
for index in array
|
||||
]
|
||||
|
||||
if self.good:
|
||||
logger.debug(self.path + ' -> good: \033[92m%s\033[0m',
|
||||
', '.join(view(self.good)))
|
||||
if self.bad:
|
||||
logger.debug(self.path + ' -> bad: \033[91m%s\033[0m',
|
||||
', '.join(view(self.bad)))
|
||||
@ -9,6 +9,7 @@ from django.utils.translation import gettext_lazy as _
|
||||
|
||||
from taggit.managers import TaggableManager
|
||||
|
||||
from aircox import settings
|
||||
from .program import Program
|
||||
from .episode import Episode
|
||||
|
||||
@ -73,6 +74,8 @@ class SoundQuerySet(models.QuerySet):
|
||||
)
|
||||
|
||||
|
||||
# TODO:
|
||||
# - provide a default name based on program and episode
|
||||
class Sound(models.Model):
|
||||
"""
|
||||
A Sound is the representation of a sound file that can be either an excerpt
|
||||
@ -105,13 +108,14 @@ class Sound(models.Model):
|
||||
)
|
||||
|
||||
def _upload_to(self, filename):
|
||||
subdir = AIRCOX_SOUND_ARCHIVES_SUBDIR if self.type == self.TYPE_ARCHIVE else \
|
||||
AIRCOX_SOUND_EXCERPTS_SUBDIR
|
||||
subdir = settings.AIRCOX_SOUND_ARCHIVES_SUBDIR \
|
||||
if self.type == self.TYPE_ARCHIVE else \
|
||||
settings.AIRCOX_SOUND_EXCERPTS_SUBDIR
|
||||
return os.path.join(self.program.path, subdir, filename)
|
||||
|
||||
file = models.FileField(
|
||||
_('file'), upload_to=_upload_to, max_length=256,
|
||||
db_index=True,
|
||||
db_index=True, unique=True,
|
||||
)
|
||||
duration = models.TimeField(
|
||||
_('duration'),
|
||||
@ -175,6 +179,7 @@ class Sound(models.Model):
|
||||
|
||||
return os.path.exists(self.file.path)
|
||||
|
||||
# TODO: rename to sync_fs()
|
||||
def check_on_file(self):
|
||||
"""
|
||||
Check sound file info again'st self, and update informations if
|
||||
@ -183,7 +188,7 @@ class Sound(models.Model):
|
||||
if not self.file_exists():
|
||||
if self.type == self.TYPE_REMOVED:
|
||||
return
|
||||
logger.info('sound %s: has been removed', self.file.name)
|
||||
logger.debug('sound %s: has been removed', self.file.name)
|
||||
self.type = self.TYPE_REMOVED
|
||||
return True
|
||||
|
||||
@ -202,8 +207,8 @@ class Sound(models.Model):
|
||||
if self.mtime != mtime:
|
||||
self.mtime = mtime
|
||||
self.is_good_quality = None
|
||||
logger.info('sound %s: m_time has changed. Reset quality info',
|
||||
self.file.name)
|
||||
logger.debug('sound %s: m_time has changed. Reset quality info',
|
||||
self.file.name)
|
||||
return True
|
||||
|
||||
return changed
|
||||
|
||||
@ -46,7 +46,7 @@ class Station(models.Model):
|
||||
)
|
||||
default = models.BooleanField(
|
||||
_('default station'),
|
||||
default=True,
|
||||
default=False,
|
||||
help_text=_('use this station as the main one.')
|
||||
)
|
||||
active = models.BooleanField(
|
||||
|
||||
2
aircox/tests/__init__.py
Normal file
2
aircox/tests/__init__.py
Normal file
@ -0,0 +1,2 @@
|
||||
from .management import *
|
||||
|
||||
2
aircox/tests/management/__init__.py
Normal file
2
aircox/tests/management/__init__.py
Normal file
@ -0,0 +1,2 @@
|
||||
from .sound_file import *
|
||||
from .sound_monitor import *
|
||||
73
aircox/tests/management/sound_file.py
Normal file
73
aircox/tests/management/sound_file.py
Normal file
@ -0,0 +1,73 @@
|
||||
from datetime import timedelta
|
||||
|
||||
from django.conf import settings as conf
|
||||
from django.test import TestCase
|
||||
from django.utils import timezone as tz
|
||||
|
||||
from aircox import models
|
||||
from aircox.management.sound_file import SoundFile
|
||||
|
||||
|
||||
__all__ = ('SoundFileTestCase',)
|
||||
|
||||
|
||||
class SoundFileTestCase(TestCase):
|
||||
path_infos = {
|
||||
'test/20220101_10h13_1_sample_1.mp3': {
|
||||
'year': 2022, 'month': 1, 'day': 1, 'hour': 10, 'minute': 13,
|
||||
'n': 1, 'name': 'Sample 1'},
|
||||
'test/20220102_10h13_sample_2.mp3': {
|
||||
'year': 2022, 'month': 1, 'day': 2, 'hour': 10, 'minute': 13,
|
||||
'name': 'Sample 2'},
|
||||
'test/20220103_1_sample_3.mp3': {
|
||||
'year': 2022, 'month': 1, 'day': 3, 'n': 1, 'name': 'Sample 3'},
|
||||
'test/20220104_sample_4.mp3': {
|
||||
'year': 2022, 'month': 1, 'day': 4, 'name': 'Sample 4'},
|
||||
'test/20220105.mp3': {
|
||||
'year': 2022, 'month': 1, 'day': 5, 'name': '20220105'},
|
||||
}
|
||||
subdir_prefix = 'test'
|
||||
sound_files = {k: r for k, r in (
|
||||
(path, SoundFile(conf.MEDIA_ROOT + '/' + path))
|
||||
for path in path_infos.keys()
|
||||
)}
|
||||
|
||||
def test_sound_path(self):
|
||||
for path, sound_file in self.sound_files.items():
|
||||
self.assertEqual(path, sound_file.sound_path)
|
||||
|
||||
def test_read_path(self):
|
||||
for path, sound_file in self.sound_files.items():
|
||||
expected = self.path_infos[path]
|
||||
result = sound_file.read_path(path)
|
||||
# remove None values
|
||||
result = {k: v for k, v in result.items() if v is not None}
|
||||
self.assertEqual(expected, result, "path: {}".format(path))
|
||||
|
||||
def _setup_diff(self, program, info):
|
||||
episode = models.Episode(program=program, title='test-episode')
|
||||
at = tz.datetime(**{
|
||||
k: info[k] for k in ('year', 'month', 'day', 'hour', 'minute')
|
||||
if info.get(k)
|
||||
})
|
||||
at = tz.make_aware(at)
|
||||
diff = models.Diffusion(episode=episode, start=at,
|
||||
end=at+timedelta(hours=1))
|
||||
episode.save()
|
||||
diff.save()
|
||||
return diff
|
||||
|
||||
def test_find_episode(self):
|
||||
station = models.Station(name='test-station')
|
||||
program = models.Program(station=station, title='test')
|
||||
station.save()
|
||||
program.save()
|
||||
|
||||
for path, sound_file in self.sound_files.items():
|
||||
infos = sound_file.read_path(path)
|
||||
diff = self._setup_diff(program, infos)
|
||||
sound = models.Sound(program=diff.program, file=path)
|
||||
result = sound_file.find_episode(sound, infos)
|
||||
self.assertEquals(diff.episode, result)
|
||||
|
||||
# TODO: find_playlist, sync
|
||||
76
aircox/tests/management/sound_monitor.py
Normal file
76
aircox/tests/management/sound_monitor.py
Normal file
@ -0,0 +1,76 @@
|
||||
import concurrent.futures as futures
|
||||
from datetime import datetime, timedelta
|
||||
import time
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
from aircox.management.sound_monitor import \
|
||||
NotifyHandler, MoveHandler, ModifiedHandler, MonitorHandler
|
||||
|
||||
|
||||
__all__ = ('NotifyHandlerTestCase', 'MoveHandlerTestCase',
|
||||
'ModifiedHandlerTestCase', 'MonitorHandlerTestCase',)
|
||||
|
||||
|
||||
class FakeEvent:
|
||||
def __init__(self, **kwargs):
|
||||
self.__dict__.update(**kwargs)
|
||||
|
||||
|
||||
class WaitHandler(NotifyHandler):
|
||||
def __call__(self, timeout=0.5, *args, **kwargs):
|
||||
# using time.sleep make the future done directly, don't know why
|
||||
start = datetime.now()
|
||||
while datetime.now() - start < timedelta(seconds=4):
|
||||
pass
|
||||
|
||||
|
||||
class NotifyHandlerTestCase(TestCase):
|
||||
pass
|
||||
|
||||
|
||||
class MoveHandlerTestCase(TestCase):
|
||||
pass
|
||||
|
||||
|
||||
class ModifiedHandlerTestCase(TestCase):
|
||||
def test_wait(self):
|
||||
handler = ModifiedHandler()
|
||||
handler.timeout_delta = timedelta(seconds=0.1)
|
||||
start = datetime.now()
|
||||
handler.wait()
|
||||
delta = datetime.now() - start
|
||||
self.assertTrue(delta < handler.timeout_delta + timedelta(seconds=0.1))
|
||||
|
||||
def test_wait_ping(self):
|
||||
pool = futures.ThreadPoolExecutor(1)
|
||||
handler = ModifiedHandler()
|
||||
handler.timeout_delta = timedelta(seconds=0.5)
|
||||
|
||||
future = pool.submit(handler.wait)
|
||||
time.sleep(0.3)
|
||||
handler.ping()
|
||||
time.sleep(0.3)
|
||||
self.assertTrue(future.running())
|
||||
|
||||
|
||||
class MonitorHandlerTestCase(TestCase):
|
||||
def setUp(self):
|
||||
pool = futures.ThreadPoolExecutor(2)
|
||||
self.monitor = MonitorHandler('archives', pool)
|
||||
|
||||
def test_submit_new_job(self):
|
||||
event = FakeEvent(src_path='dummy_src')
|
||||
handler = NotifyHandler()
|
||||
result = self.monitor._submit(handler, event, 'up')
|
||||
self.assertIs(handler, result)
|
||||
self.assertIsInstance(handler.future, futures.Future)
|
||||
self.monitor.pool.shutdown()
|
||||
|
||||
def test_submit_job_exists(self):
|
||||
event = FakeEvent(src_path='dummy_src')
|
||||
|
||||
job_1 = self.monitor._submit(WaitHandler(), event, 'up')
|
||||
job_2 = self.monitor._submit(NotifyHandler(), event, 'up')
|
||||
self.assertIs(job_1, job_2)
|
||||
self.monitor.pool.shutdown()
|
||||
@ -12,6 +12,7 @@ bleach~=5.0
|
||||
easy-thumbnails~=2.8
|
||||
tzlocal~=4.2
|
||||
|
||||
dateutils~=0.6
|
||||
mutagen~=1.45
|
||||
Pillow~=9.0
|
||||
psutil~=5.9
|
||||
|
||||
Reference in New Issue
Block a user