ctactic/aircox-radiocampus
1
0
Fork 0
forked from rc/aircox
Code Issues 10 Pull Requests Activity

code quality

Browse Source
This commit is contained in:
bkfox
2023-03-13 17:47:00 +01:00
parent 934817da8a
commit 112770eddf
162 changed files with 4798 additions and 4069 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
aircox/management/commands/archiver.py
View File

@ -1,41 +1,48 @@
"""Handle archiving of logs in order to keep database light and fast.
The logs are archived in gzip files, per day.
"""
Handle archiving of logs in order to keep database light and fast. The
logs are archived in gzip files, per day.
"""
from argparse import RawTextHelpFormatter
import datetime
import logging
from argparse import RawTextHelpFormatter
from django.core.management.base import BaseCommand
from django.utils import timezone as tz
import aircox.settings as settings
from aircox.models import Log, Station
from aircox.models import Log
from aircox.models.log import LogArchiver
logger = logging.getLogger('aircox.commands')
logger = logging.getLogger("aircox.commands")
class Command (BaseCommand):
__all__ = ("Command",)
class Command(BaseCommand):
help = __doc__
def add_arguments(self, parser):
parser.formatter_class = RawTextHelpFormatter
group = parser.add_argument_group('actions')
group = parser.add_argument_group("actions")
group.add_argument(
'-a', '--age', type=int,
"-a",
"--age",
type=int,
default=settings.AIRCOX_LOGS_ARCHIVES_AGE,
help='minimal age in days of logs to archive. Default is '
'settings.AIRCOX_LOGS_ARCHIVES_AGE'
help="minimal age in days of logs to archive. Default is "
"settings.AIRCOX_LOGS_ARCHIVES_AGE",
)
group.add_argument(
'-k', '--keep', action='store_true',
help='keep logs in database instead of deleting them'
"-k",
"--keep",
action="store_true",
help="keep logs in database instead of deleting them",
)
def handle(self, *args, age, keep, **options):
date = datetime.date.today() - tz.timedelta(days=age)
# FIXME: mysql support?
logger.info('archive logs for %s and earlier', date)
logger.info("archive logs for %s and earlier", date)
count = LogArchiver().archive(Log.objects.filter(date__date__lte=date))
logger.info('total log archived %d', count)
logger.info("total log archived %d", count)

90
aircox/management/commands/diffusions.py
View File

@ -1,5 +1,4 @@
"""
Manage diffusions using schedules, to update, clean up or check diffusions.
"""Manage diffusions using schedules, to update, clean up or check diffusions.
A generated diffusion can be unconfirmed, that means that the user must confirm
it by changing its type to "normal". The behaviour is controlled using
@ -13,9 +12,9 @@ from django.core.management.base import BaseCommand
from django.db import transaction
from django.utils import timezone as tz
from aircox.models import Schedule, Diffusion
from aircox.models import Diffusion, Schedule
logger = logging.getLogger('aircox.commands')
logger = logging.getLogger("aircox.commands")
class Actions:
@ -26,20 +25,28 @@ class Actions:
def update(self):
episodes, diffusions = [], []
for schedule in Schedule.objects.filter(program__active=True,
initial__isnull=True):
for schedule in Schedule.objects.filter(
program__active=True, initial__isnull=True
):
eps, diffs = schedule.diffusions_of_month(self.date)
if eps:
episodes += eps
if diffs:
diffusions += diffs
logger.info('[update] %s: %d episodes, %d diffusions and reruns',
str(schedule), len(eps), len(diffs))
logger.info(
"[update] %s: %d episodes, %d diffusions and reruns",
str(schedule),
len(eps),
len(diffs),
)
with transaction.atomic():
logger.info('[update] save %d episodes and %d diffusions',
len(episodes), len(diffusions))
logger.info(
"[update] save %d episodes and %d diffusions",
len(episodes),
len(diffusions),
)
for episode in episodes:
episode.save()
for diffusion in diffusions:
@ -48,9 +55,10 @@ class Actions:
diffusion.save()
def clean(self):
qs = Diffusion.objects.filter(type=Diffusion.TYPE_UNCONFIRMED,
start__lt=self.date)
logger.info('[clean] %d diffusions will be removed', qs.count())
qs = Diffusion.objects.filter(
type=Diffusion.TYPE_UNCONFIRMED, start__lt=self.date
)
logger.info("[clean] %d diffusions will be removed", qs.count())
qs.delete()
@ -61,45 +69,57 @@ class Command(BaseCommand):
parser.formatter_class = RawTextHelpFormatter
today = datetime.date.today()
group = parser.add_argument_group('action')
group = parser.add_argument_group("action")
group.add_argument(
'-u', '--update', action='store_true',
help='generate (unconfirmed) diffusions for the given month. '
'These diffusions must be confirmed manually by changing '
'their type to "normal"'
"-u",
"--update",
action="store_true",
help="generate (unconfirmed) diffusions for the given month. "
"These diffusions must be confirmed manually by changing "
'their type to "normal"',
)
group.add_argument(
'-l', '--clean', action='store_true',
help='remove unconfirmed diffusions older than the given month'
"-l",
"--clean",
action="store_true",
help="remove unconfirmed diffusions older than the given month",
)
group = parser.add_argument_group('date')
group = parser.add_argument_group("date")
group.add_argument(
'--year', type=int, default=today.year,
help='used by update, default is today\'s year')
"--year",
type=int,
default=today.year,
help="used by update, default is today's year",
)
group.add_argument(
'--month', type=int, default=today.month,
help='used by update, default is today\'s month')
"--month",
type=int,
default=today.month,
help="used by update, default is today's month",
)
group.add_argument(
'--next-month', action='store_true',
help='set the date to the next month of given date'
' (if next month from today'
"--next-month",
action="store_true",
help="set the date to the next month of given date"
" (if next month from today",
)
def handle(self, *args, **options):
date = datetime.date(year=options['year'], month=options['month'],
day=1)
if options.get('next_month'):
month = options.get('month')
date = datetime.date(
year=options["year"], month=options["month"], day=1
)
if options.get("next_month"):
month = options.get("month")
date += tz.timedelta(days=28)
if date.month == month:
date += tz.timedelta(days=28)
date = date.replace(day=1)
actions = Actions(date)
if options.get('update'):
if options.get("update"):
actions.update()
if options.get('clean'):
if options.get("clean"):
actions.clean()
if options.get('check'):
if options.get("check"):
actions.check()

124
aircox/management/commands/import_playlist.py
View File

@ -1,5 +1,4 @@
"""
Import one or more playlist for the given sound. Attach it to the provided
"""Import one or more playlist for the given sound. Attach it to the provided
sound.
Playlists are in CSV format, where columns are separated with a
@ -10,23 +9,22 @@ The order of the elements is: {settings.AIRCOX_IMPORT_PLAYLIST_CSV_COLS}
If 'minutes' or 'seconds' are given, position will be expressed as timed
position, instead of position in playlist.
"""
import os
import csv
import logging
import os
from argparse import RawTextHelpFormatter
from django.core.management.base import BaseCommand, CommandError
from django.contrib.contenttypes.models import ContentType
from django.core.management.base import BaseCommand
from aircox import settings
from aircox.models import *
from aircox.models import Sound, Track
__doc__ = __doc__.format(settings=settings)
__all__ = ('PlaylistImport', 'Command')
__all__ = ("PlaylistImport", "Command")
logger = logging.getLogger('aircox.commands')
logger = logging.getLogger("aircox.commands")
class PlaylistImport:
@ -45,62 +43,74 @@ class PlaylistImport:
def run(self):
self.read()
if self.track_kwargs.get('sound') is not None:
if self.track_kwargs.get("sound") is not None:
self.make_playlist()
def read(self):
if not os.path.exists(self.path):
return True
with open(self.path, 'r') as file:
logger.info('start reading csv ' + self.path)
self.data = list(csv.DictReader(
(row for row in file
if not (row.startswith('#') or row.startswith('\ufeff#'))
and row.strip()),
fieldnames=settings.AIRCOX_IMPORT_PLAYLIST_CSV_COLS,
delimiter=settings.AIRCOX_IMPORT_PLAYLIST_CSV_DELIMITER,
quotechar=settings.AIRCOX_IMPORT_PLAYLIST_CSV_TEXT_QUOTE,
))
with open(self.path, "r") as file:
logger.info("start reading csv " + self.path)
self.data = list(
csv.DictReader(
(
row
for row in file
if not (
row.startswith("#") or row.startswith("\ufeff#")
)
and row.strip()
),
fieldnames=settings.AIRCOX_IMPORT_PLAYLIST_CSV_COLS,
delimiter=settings.AIRCOX_IMPORT_PLAYLIST_CSV_DELIMITER,
quotechar=settings.AIRCOX_IMPORT_PLAYLIST_CSV_TEXT_QUOTE,
)
)
def make_playlist(self):
"""Make a playlist from the read data, and return it.
If save is true, save it into the database
"""
Make a playlist from the read data, and return it. If save is
true, save it into the database
"""
if self.track_kwargs.get('sound') is None:
logger.error('related track\'s sound is missing. Skip import of ' +
self.path + '.')
if self.track_kwargs.get("sound") is None:
logger.error(
"related track's sound is missing. Skip import of "
+ self.path
+ "."
)
return
maps = settings.AIRCOX_IMPORT_PLAYLIST_CSV_COLS
tracks = []
logger.info('parse csv file ' + self.path)
has_timestamp = ('minutes' or 'seconds') in maps
logger.info("parse csv file " + self.path)
has_timestamp = ("minutes" or "seconds") in maps
for index, line in enumerate(self.data):
if ('title' or 'artist') not in line:
if ("title" or "artist") not in line:
return
try:
timestamp = int(line.get('minutes') or 0) * 60 + \
int(line.get('seconds') or 0) \
if has_timestamp else None
timestamp = (
int(line.get("minutes") or 0) * 60
+ int(line.get("seconds") or 0)
if has_timestamp
else None
)
track, created = Track.objects.get_or_create(
title=line.get('title'),
artist=line.get('artist'),
title=line.get("title"),
artist=line.get("artist"),
position=index,
**self.track_kwargs
)
track.timestamp = timestamp
track.info = line.get('info')
tags = line.get('tags')
track.info = line.get("info")
tags = line.get("tags")
if tags:
track.tags.add(*tags.lower().split(','))
track.tags.add(*tags.lower().split(","))
except Exception as err:
logger.warning(
'an error occured for track {index}, it may not '
'have been saved: {err}'
.format(index=index, err=err)
"an error occured for track {index}, it may not "
"have been saved: {err}".format(index=index, err=err)
)
continue
@ -116,33 +126,41 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.formatter_class = RawTextHelpFormatter
parser.add_argument(
'path', metavar='PATH', type=str,
help='path of the input playlist to read'
"path",
metavar="PATH",
type=str,
help="path of the input playlist to read",
)
parser.add_argument(
'--sound', '-s', type=str,
help='generate a playlist for the sound of the given path. '
'If not given, try to match a sound with the same path.'
"--sound",
"-s",
type=str,
help="generate a playlist for the sound of the given path. "
"If not given, try to match a sound with the same path.",
)
def handle(self, path, *args, **options):
# FIXME: absolute/relative path of sounds vs given path
if options.get('sound'):
sound = Sound.objects.filter(file__icontains=options.get('sound'))\
.first()
if options.get("sound"):
sound = Sound.objects.filter(
file__icontains=options.get("sound")
).first()
else:
path_, ext = os.path.splitext(path)
sound = Sound.objects.filter(path__icontains=path_).first()
if not sound:
logger.error('no sound found in the database for the path '
'{path}'.format(path=path))
logger.error(
"no sound found in the database for the path "
"{path}".format(path=path)
)
return
# FIXME: auto get sound.episode if any
importer = PlaylistImport(path, sound=sound).run()
for track in importer.tracks:
logger.info('track #{pos} imported: {title}, by {artist}'.format(
pos=track.position, title=track.title, artist=track.artist
))
logger.info(
"track #{pos} imported: {title}, by {artist}".format(
pos=track.position, title=track.title, artist=track.artist
)
)

116
aircox/management/commands/sounds_monitor.py
View File

@ -1,7 +1,7 @@
#! /usr/bin/env python3
"""
Monitor sound files; For each program, check for:
"""Monitor sound files; For each program, check for:
- new files;
- deleted files;
- differences between files and sound;
@ -23,23 +23,22 @@ To check quality of files, call the command sound_quality_check using the
parameters given by the setting AIRCOX_SOUND_QUALITY. This script requires
Sox (and soxi).
"""
from argparse import RawTextHelpFormatter
import concurrent.futures as futures
import atexit
import concurrent.futures as futures
import logging
import os
import time
from watchdog.observers import Observer
from argparse import RawTextHelpFormatter
from django.core.management.base import BaseCommand
from watchdog.observers import Observer
from aircox import settings
from aircox.models import Program, Sound
from aircox.management.sound_file import SoundFile
from aircox.management.sound_monitor import MonitorHandler
from aircox.models import Program, Sound
logger = logging.getLogger('aircox.commands')
logger = logging.getLogger("aircox.commands")
class Command(BaseCommand):
@ -47,39 +46,42 @@ class Command(BaseCommand):
def report(self, program=None, component=None, *content):
if not component:
logger.info('%s: %s', str(program),
' '.join([str(c) for c in content]))
logger.info(
"%s: %s", str(program), " ".join([str(c) for c in content])
)
else:
logger.info('%s, %s: %s', str(program), str(component),
' '.join([str(c) for c in content]))
logger.info(
"%s, %s: %s",
str(program),
str(component),
" ".join([str(c) for c in content]),
)
def scan(self):
"""
For all programs, scan dirs
"""
logger.info('scan all programs...')
"""For all programs, scan dirs."""
logger.info("scan all programs...")
programs = Program.objects.filter()
dirs = []
for program in programs:
logger.info('#%d %s', program.id, program.title)
logger.info("#%d %s", program.id, program.title)
self.scan_for_program(
program, settings.AIRCOX_SOUND_ARCHIVES_SUBDIR,
program,
settings.AIRCOX_SOUND_ARCHIVES_SUBDIR,
type=Sound.TYPE_ARCHIVE,
)
self.scan_for_program(
program, settings.AIRCOX_SOUND_EXCERPTS_SUBDIR,
program,
settings.AIRCOX_SOUND_EXCERPTS_SUBDIR,
type=Sound.TYPE_EXCERPT,
)
dirs.append(os.path.join(program.abspath))
return dirs
def scan_for_program(self, program, subdir, **sound_kwargs):
"""
Scan a given directory that is associated to the given program, and
update sounds information.
"""
logger.info('- %s/', subdir)
"""Scan a given directory that is associated to the given program, and
update sounds information."""
logger.info("- %s/", subdir)
if not program.ensure_dir(subdir):
return
@ -97,37 +99,49 @@ class Command(BaseCommand):
sounds.append(sound_file.sound.pk)
# sounds in db & unchecked
sounds = Sound.objects.filter(file__startswith=subdir). \
exclude(pk__in=sounds)
sounds = Sound.objects.filter(file__startswith=subdir).exclude(
pk__in=sounds
)
self.check_sounds(sounds, program=program)
def check_sounds(self, qs, **sync_kwargs):
""" Only check for the sound existence or update """
"""Only check for the sound existence or update."""
# check files
for sound in qs:
if sound.check_on_file():
SoundFile(sound.file.path).sync(sound=sound, **sync_kwargs)
def monitor(self):
""" Run in monitor mode """
"""Run in monitor mode."""
with futures.ThreadPoolExecutor() as pool:
archives_handler = MonitorHandler(
settings.AIRCOX_SOUND_ARCHIVES_SUBDIR, pool,
type=Sound.TYPE_ARCHIVE)
settings.AIRCOX_SOUND_ARCHIVES_SUBDIR,
pool,
type=Sound.TYPE_ARCHIVE,
)
excerpts_handler = MonitorHandler(
settings.AIRCOX_SOUND_EXCERPTS_SUBDIR, pool,
type=Sound.TYPE_EXCERPT)
settings.AIRCOX_SOUND_EXCERPTS_SUBDIR,
pool,
type=Sound.TYPE_EXCERPT,
)
observer = Observer()
observer.schedule(archives_handler, settings.AIRCOX_PROGRAMS_DIR_ABS,
recursive=True)
observer.schedule(excerpts_handler, settings.AIRCOX_PROGRAMS_DIR_ABS,
recursive=True)
observer.schedule(
archives_handler,
settings.AIRCOX_PROGRAMS_DIR_ABS,
recursive=True,
)
observer.schedule(
excerpts_handler,
settings.AIRCOX_PROGRAMS_DIR_ABS,
recursive=True,
)
observer.start()
def leave():
observer.stop()
observer.join()
atexit.register(leave)
while True:
@ -136,25 +150,31 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.formatter_class = RawTextHelpFormatter
parser.add_argument(
'-q', '--quality_check', action='store_true',
help='Enable quality check using sound_quality_check on all '
'sounds marqued as not good'
"-q",
"--quality_check",
action="store_true",
help="Enable quality check using sound_quality_check on all "
"sounds marqued as not good",
)
parser.add_argument(
'-s', '--scan', action='store_true',
help='Scan programs directories for changes, plus check for a '
' matching diffusion on sounds that have not been yet assigned'
"-s",
"--scan",
action="store_true",
help="Scan programs directories for changes, plus check for a "
" matching diffusion on sounds that have not been yet assigned",
)
parser.add_argument(
'-m', '--monitor', action='store_true',
help='Run in monitor mode, watch for modification in the filesystem '
'and react in consequence'
"-m",
"--monitor",
action="store_true",
help="Run in monitor mode, watch for modification in the "
"filesystem and react in consequence",
)
def handle(self, *args, **options):
if options.get('scan'):
if options.get("scan"):
self.scan()
#if options.get('quality_check'):
# if options.get('quality_check'):
# self.check_quality(check=(not options.get('scan')))
if options.get('monitor'):
if options.get("monitor"):
self.monitor()

71
aircox/management/commands/sounds_quality_check.py
View File

@ -1,17 +1,15 @@
"""
Analyse and check files using Sox, prints good and bad files.
"""
"""Analyse and check files using Sox, prints good and bad files."""
import logging
from argparse import RawTextHelpFormatter
from django.core.management.base import BaseCommand, CommandError
from aircox.management.sound_stats import SoxStats, SoundStats
from aircox.management.sound_stats import SoundStats, SoxStats
logger = logging.getLogger('aircox.commands')
logger = logging.getLogger("aircox.commands")
class Command (BaseCommand):
class Command(BaseCommand):
help = __doc__
sounds = None
@ -19,46 +17,61 @@ class Command (BaseCommand):
parser.formatter_class = RawTextHelpFormatter
parser.add_argument(
'files', metavar='FILE', type=str, nargs='+',
help='file(s) to analyse'
"files",
metavar="FILE",
type=str,
nargs="+",
help="file(s) to analyse",
)
parser.add_argument(
'-s', '--sample_length', type=int, default=120,
help='size of sample to analyse in seconds. If not set (or 0), does'
' not analyse by sample',
"-s",
"--sample_length",
type=int,
default=120,
help="size of sample to analyse in seconds. If not set (or 0), "
"does not analyse by sample",
)
parser.add_argument(
'-a', '--attribute', type=str,
help='attribute name to use to check, that can be:\n' +
', '.join(['"{}"'.format(attr) for attr in SoxStats.attributes])
"-a",
"--attribute",
type=str,
help="attribute name to use to check, that can be:\n"
+ ", ".join(['"{}"'.format(attr) for attr in SoxStats.attributes]),
)
parser.add_argument(
'-r', '--range', type=float, nargs=2,
help='range of minimal and maximal accepted value such as: '
'--range min max'
"-r",
"--range",
type=float,
nargs=2,
help="range of minimal and maximal accepted value such as: "
"--range min max",
)
parser.add_argument(
'-i', '--resume', action='store_true',
help='print a resume of good and bad files'
"-i",
"--resume",
action="store_true",
help="print a resume of good and bad files",
)
def handle(self, *args, **options):
# parameters
minmax = options.get('range')
minmax = options.get("range")
if not minmax:
raise CommandError('no range specified')
raise CommandError("no range specified")
attr = options.get('attribute')
attr = options.get("attribute")
if not attr:
raise CommandError('no attribute specified')
raise CommandError("no attribute specified")
# sound analyse and checks
self.sounds = [SoundStats(path, options.get('sample_length'))
for path in options.get('files')]
self.sounds = [
SoundStats(path, options.get("sample_length"))
for path in options.get("files")
]
self.bad = []
self.good = []
for sound in self.sounds:
logger.info('analyse ' + sound.path)
logger.info("analyse " + sound.path)
sound.analyse()
sound.check(attr, minmax[0], minmax[1])
if sound.bad:
@ -67,8 +80,8 @@ class Command (BaseCommand):
self.good.append(sound)
# resume
if options.get('resume'):
if options.get("resume"):
for sound in self.good:
logger.info('\033[92m+ %s\033[0m', sound.path)
logger.info("\033[92m+ %s\033[0m", sound.path)
for sound in self.bad:
logger.info('\033[91m+ %s\033[0m', sound.path)
logger.info("\033[91m+ %s\033[0m", sound.path)