From 161af3fb1ad465cfa11d00d14fa22ccf75622ef4 Mon Sep 17 00:00:00 2001 From: bkfox Date: Fri, 15 Jul 2016 18:23:08 +0200 Subject: [PATCH] add script to import playlists to sounds or to diffusions --- cms/routes.py | 3 + cms/sections.py | 66 +++++--- cms/views.py | 4 + cms/website.py | 2 + controllers/models.py | 61 ++----- controllers/monitor.py | 65 ++++++-- notes.md | 2 + .../management/commands/import_playlist.py | 154 ++++++++++++++++++ programs/models.py | 119 ++++++++++---- programs/settings.py | 28 +++- programs/utils.py | 1 + website/sections.py | 10 +- 12 files changed, 393 insertions(+), 122 deletions(-) create mode 100644 programs/management/commands/import_playlist.py diff --git a/cms/routes.py b/cms/routes.py index 2a07c69..9f10153 100644 --- a/cms/routes.py +++ b/cms/routes.py @@ -95,6 +95,9 @@ class DetailRoute(Route): class AllRoute(Route): + """ + Retrieve all element of the given model. + """ name = 'all' @classmethod diff --git a/cms/sections.py b/cms/sections.py index 008de08..87f6e3e 100644 --- a/cms/sections.py +++ b/cms/sections.py @@ -87,32 +87,45 @@ class Section(Viewable, View): ! Important Note: values given for rendering are considered as safe HTML in templates. - - Attributes: - * template_name: template to use for rendering - * tag: container's tags - * name: set name/id of the section container - * css_class: css classes of the container - * attr: HTML attributes of the container - * title: title of the section - * header: header of the section - * footer: footer of the section - - * message_empty: if message_empty is not None, print its value as - content of the section instead of hiding it. This works also when - its value is an empty string (prints an empty string). """ template_name = 'aircox/cms/website.html' - + """ + Template used for rendering + """ tag = 'div' + """ + HTML tag used for the container + """ name = '' + """ + Name/ID of the container + """ css_class = '' + """ + CSS classes for the container + """ attrs = None + """ + HTML Attributes of the container + """ title = '' + """ + Safe HTML code for the title + """ header = '' + """ + Safe HTML code for the header + """ footer = '' - + """ + Safe HTML code for the footer + """ message_empty = None + """ + If message_empty is not None, print its value as + content of the section instead of hiding it. This works also when + its value is an empty string (prints an empty string). + """ request = None object = None @@ -284,22 +297,31 @@ class List(Section): Common interface for list configuration. Attributes: - * object_list: force an object list to be used - * url: url to the list in full page - * message_empty: message to print when list is empty (if not hiding) - * fields: fields of the items to render - * image_size: size of the images - * truncate: number of words to keep in content (0 = full content) """ template_name = 'aircox/cms/list.html' object_list = None + """ + Use this object list (default behaviour for lists) + """ url = None + """ + URL to the list in full page; If given, print it + """ paginate_by = 4 fields = [ 'date', 'time', 'image', 'title', 'content', 'info', 'actions' ] + """ + Fields that must be rendered. + """ image_size = '64x64' + """ + Size of the image when rendered in the list + """ truncate = 16 + """ + Number of words to print in content. If 0, print all the content + """ def __init__ (self, items = None, *args, **kwargs): """ diff --git a/cms/views.py b/cms/views.py index 3d1389d..4645977 100644 --- a/cms/views.py +++ b/cms/views.py @@ -249,4 +249,8 @@ class PageView(BaseView, TemplateView): If sections is a list of sections, then render like a detail view; If it is a single section, render it as website.html view; """ + # dirty hack in order to accept a "model" kwargs, to allow "model=None" + # in routes. Cf. website.register (at if model / else) + model = None + diff --git a/cms/website.py b/cms/website.py index 18718f7..e813fcc 100644 --- a/cms/website.py +++ b/cms/website.py @@ -120,6 +120,8 @@ class Website: reg = self.register_model(name, model, as_default) reg.routes.extend(routes) view_kwargs['model'] = model + else: + view_kwargs['model'] = None # init view if not view_kwargs.get('menus'): diff --git a/controllers/models.py b/controllers/models.py index 01e80a7..e88f1aa 100644 --- a/controllers/models.py +++ b/controllers/models.py @@ -39,6 +39,11 @@ class Station(programs.Nameable): max_length = 32, choices = [ (name, name) for name in Plugins.registry.keys() ], ) + active = models.BooleanField( + _('active'), + default = True, + help_text = _('this station is active') + ) plugin = None """ @@ -127,24 +132,22 @@ class Station(programs.Nameable): if self.plugin_name: self.plugin = Plugins.registry.get(self.plugin_name) - def play_logs(self, include_diffusions = True, - include_sounds = True, - exclude_archives = True): + def get_played(self, models, archives = True): """ - Return a queryset with log of played elements on this station. - Ordered by date ascending. - """ - models = [] - if include_diffusions: models.append(programs.Diffusion) - if include_sounds: models.append(programs.Sound) + Return a queryset with log of played elements on this station, + of the given models, ordered by date ascending. + * model: a model or a list of models + * archive: if false, exclude log of diffusion's archives from + the queryset; + """ qs = Log.get_for(model = models) \ .filter(station = station, type = Log.Type.play) - if exclude_archives and self.dealer: + if not archives and self.dealer: qs = qs.exclude( source = self.dealer.id_, related_type = ContentType.objects.get_for_model( - program.Sound + programs.Sound ) ) return qs.order_by('date') @@ -294,7 +297,6 @@ class Source(programs.Nameable): raise ValueError('can not save a dealer source') super().save(*args, **kwargs) - # TODO update controls class Output (models.Model): @@ -326,7 +328,7 @@ class Output (models.Model): ) -class Log(models.Model): +class Log(programs.Related): """ Log sounds and diffusions that are played on the station. @@ -376,16 +378,6 @@ class Log(models.Model): max_length = 512, blank = True, null = True, ) - related_type = models.ForeignKey( - ContentType, - blank = True, null = True, - ) - related_id = models.PositiveIntegerField( - blank = True, null = True, - ) - related = GenericForeignKey( - 'related_type', 'related_id', - ) @property def end(self): @@ -407,29 +399,6 @@ class Log(models.Model): date = programs.date_or_default(date) return self.end < date - @classmethod - def get_for(cl, object = None, model = None): - """ - Return a queryset that filter on the related object. If object is - given, filter using it, otherwise only using model. - - If model is not given, uses object's type. - """ - if not model and object: - model = type(object) - - if type(model) in (list, tuple): - model = [ ContentType.objects.get_for_model(m).id - for m in model ] - qs = cl.objects.filter(related_type__pk__in = model) - else: - model = ContentType.objects.get_for_model(model) - qs = cl.objects.filter(related_type__pk = model.id) - - if object: - qs = qs.filter(related_id = object.pk) - return qs - def print(self): logger.info('log #%s: %s%s', str(self), diff --git a/controllers/monitor.py b/controllers/monitor.py index e66fa30..65f8588 100644 --- a/controllers/monitor.py +++ b/controllers/monitor.py @@ -10,6 +10,11 @@ class Monitor: Monitor should be able to be used after a crash a go back where it was playing, so we heavily use logs to be able to do that. + + We keep trace of played items on the generated stream: + - sounds played on this stream; + - scheduled diffusions + - tracks for sounds of streamed programs """ station = None controller = None @@ -22,11 +27,10 @@ class Monitor: self.station.prepare() self.controller = self.station.controller - self.track() + self.trace() self.handler() - @staticmethod - def log(**kwargs): + def log(self, **kwargs): """ Create a log using **kwargs, and print info """ @@ -34,10 +38,10 @@ class Monitor: log.save() log.print() - def track(self): + def trace(self): """ Check the current_sound of the station and update logs if - needed + needed. """ self.controller.fetch() current_sound = self.controller.current_sound @@ -47,6 +51,11 @@ class Monitor: log = Log.get_for(model = programs.Sound) \ .filter(station = self.station).order_by('date').last() + + # only streamed + if log and not log.related.diffusion: + self.trace_sound_tracks(log) + # TODO: expiration if log and (log.source == current_source.id_ and \ log.related.path == current_sound): @@ -56,12 +65,37 @@ class Monitor: self.log( type = Log.Type.play, source = current_source.id_, - date = tz.make_aware(tz.datetime.now()), - + date = tz.now(), related = sound[0] if sound else None, comment = None if sound else current_sound, ) + def trace_sound_tracks(self, log): + """ + Log tracks for the given sound (for streamed programs); Called by + self.trace + """ + logs = Log.get_for(model = programs.Track) \ + .filter(pk__gt = log.pk) + logs = [ log.pk for log in logs ] + + tracks = programs.Track.get_for(object = log.related) + .filter(pos_in_sec = True) + if len(tracks) == len(logs): + return + + tracks = tracks.exclude(pk__in = logs).order_by('pos') + now = tz.now() + for track in tracks: + pos = log.date + tz.timedelta(seconds = track.pos) + if pos < now: + self.log( + type = Log.Type.play, + source = log.source, + date = pos, + related = track + ) + def __current_diff(self): """ Return a tuple with the currently running diffusion and the items @@ -70,24 +104,23 @@ class Monitor: station = self.station now = tz.make_aware(tz.datetime.now()) - diff_log = Log.get_for(model = programs.Diffusion) \ - .filter(station = station, type = Log.Type.play) \ - .order_by('date').last() + diff_log = station.get_played(models = programs.Diffusion) \ + .order_by('date').last() if not diff_log or \ not diff_log.related.is_date_in_my_range(now): return None, [] # sound has switched? assume it has been (forced to) stopped - sound_log = Log.get_for(model = programs.Sound) \ - .filter(station = station).order_by('date').last() - if sound_log and sound_log.source != diff_log.source: + sounds = station.get_played(models = programs.Sound) + last_sound = sounds.order_by('date').last() + if last_sound and last_sound.source != diff_log.source: return None, [] # last diff is still playing: get the remaining playlist - sounds = Log.get_for(model = programs.Sound) \ - .filter(station = station, source = diff_log.source) \ - .filter(pk__gt = diff.log.pk) + sounds = sounds.filter( + source = diff_log.source, pk__gt = diff_log.pk + ) sounds = [ sound.path for sound in sounds if not sound.removed ] return ( diff --git a/notes.md b/notes.md index 5e802f7..61ca17f 100644 --- a/notes.md +++ b/notes.md @@ -10,6 +10,7 @@ - users - tests: - sound_monitor + - import_playlist - liquidsoap: - models to template -> note @@ -52,5 +53,6 @@ - comments -> remove/edit by the author - integrate logs for tracks + in on air +- get_for "model" -> "models" diff --git a/programs/management/commands/import_playlist.py b/programs/management/commands/import_playlist.py new file mode 100644 index 0000000..9b95be5 --- /dev/null +++ b/programs/management/commands/import_playlist.py @@ -0,0 +1,154 @@ +""" +Import one or more playlist for the given sound. Attach it to the sound +or to the related Diffusion if wanted. + +We support different formats: +- plain text: a track per line, where columns are separated with a + '{settings.AIRCOX_IMPORT_PLAYLIST_PLAIN_DELIMITER}'. + The order of the elements is: {settings.AIRCOX_IMPORT_PLAYLIST_PLAIN_COLS} +- csv: CSV file where columns are separated with a + '{settings.AIRCOX_IMPORT_PLAYLIST_CSV_DELIMITER)}'. Text quote is + {settings.AIRCOX_IMPORT_PLAYLIST_CSV_TEXT_QUOTE}. + The order of the elements is: {settings.AIRCOX_IMPORT_PLAYLIST_CSV_COLS} + +If 'minutes' or 'seconds' are given, position will be expressed as timed +position, instead of position in playlist. + +Base the format detection using file extension. If '.csv', uses CSV importer, +otherwise plain text one. +""" +import os +import csv +import logging +from argparse import RawTextHelpFormatter + +from django.core.management.base import BaseCommand, CommandError + +from aircox.programs.models import * +import aircox.programs.settings as settings +__doc__ = __doc__.format(settings) + +logger = logging.getLogger('aircox.tools') + + +class Importer: + type = None + data = None + tracks = None + + def __init__(self, related = None, path = None): + if path: + self.read(path) + if related: + self.make_playlist(related, True) + + def reset(self): + self.type = None + self.data = None + self.tracks = None + + def read(self, path): + if not os.path.exists(path): + return True + + with open(path, 'r') as file: + sp, *ext = os.path.splitext(path)[1] + if ext[0] and ext[0] == 'csv': + self.type = 'csv' + self.data = csv.reader( + file, + delimiter = settings.AIRCOX_IMPORT_PLAYLIST_CSV_DELIMITER, + quotechar = settings.AIRCOX_IMPORT_PLAYLIST_CSV_TEXT_QUOTE, + ) + else: + self.type = 'plain' + self.data = [ + line.split(settings.AIRCOX_IMPORT_PLAYLIST_PLAIN_DELIMITER) + for line in file.readlines() + ] + + def __get(self, line, field, default = None): + maps = settings.AIRCOX_IMPORT_CSV_COLS + if field not in maps: + return default + index = maps.index(field) + return line[index] if index < len(line) else default + + def make_playlist(self, related, save = False): + """ + Make a playlist from the read data, and return it. If save is + true, save it into the database + """ + maps = settings.AIRCOX_IMPORT_CSV_COLS if self.type == 'csv' else \ + settings.AIRCOX_IMPORT_PLAIN_COLS + tracks = [] + + for index, line in enumerate(self.data): + if ('minutes' or 'seconds') in maps: + kwargs['pos_in_secs'] = True + kwargs['pos'] = int(self.__get(line, 'minutes', 0)) * 60 + \ + int(self.__get(line, 'seconds', 0)) + else: + kwargs['pos'] = index + + kwargs['related'] = related + kwargs.update({ + k: self.__get(line, k) for k in maps + if k not in ('minutes', 'seconds') + }) + + track = Track(**kwargs) + # FIXME: bulk_create? + if save: + track.save() + tracks.append(track) + self.tracks = tracks + return tracks + + +class Command (BaseCommand): + help= __doc__ + + def add_arguments (self, parser): + parser.formatter_class=RawTextHelpFormatter + now = tz.datetime.today() + + parser.add_argument( + 'path', type=str, + help='path of the input playlist to read' + ) + parser.add_argument( + '--sound', '-s', type=str, + help='generate a playlist for the sound of the given path. ' + 'If not given, try to match a sound with the same path.' + ) + parser.add_argument( + '--diffusion', '-d', action='store_true', + help='try to get the diffusion relative to the sound if it exists' + ) + + def handle (self, path, *args, **options): + # FIXME: absolute/relative path of sounds vs given path + if options.get('sound'): + related = Sound.objects.filter(path__icontains = path).first() + else: + path, ext = os.path.splitext(options.get('path')) + related = Sound.objects.filter(path__icontains = path).first() + + if not related: + logger.error('no sound found in the database for the path ' \ + '{path}'.format(path=path)) + return -1 + + if options.get('diffusion') and related.diffusion: + related = related.diffusion + + importer = Importer(related = related, path = path) + for track in importer.tracks: + logger.log('imported track at {pos}: {name}, by ' + '{artist}'.format( + pos = track.pos, + name = track.name, artist = track.artist + ) + ) + diff --git a/programs/models.py b/programs/models.py index 7b2b0bf..f859987 100755 --- a/programs/models.py +++ b/programs/models.py @@ -44,6 +44,48 @@ def date_or_default(date, no_time = False): return date +class Related(models.Model): + """ + Add a field "related" of type GenericForeignKey, plus utilities. + """ + related_type = models.ForeignKey( + ContentType, + blank = True, null = True, + ) + related_id = models.PositiveIntegerField( + blank = True, null = True, + ) + related = GenericForeignKey( + 'related_type', 'related_id', + ) + + @classmethod + def get_for(cl, object = None, model = None): + """ + Return a queryset that filter on the given object or model(s) + + * object: if given, use its type and pk; match on models only. + * model: one model or list of models + """ + if not model and object: + model = type(object) + + if type(model) in (list, tuple): + model = [ ContentType.objects.get_for_model(m).id + for m in model ] + qs = cl.objects.filter(related_type__pk__in = model) + else: + model = ContentType.objects.get_for_model(model) + qs = cl.objects.filter(related_type__pk = model.id) + + if object: + qs = qs.filter(related_id = object.pk) + return qs + + class Meta: + abstract = True + + class Nameable(models.Model): name = models.CharField ( _('name'), @@ -66,40 +108,6 @@ class Nameable(models.Model): abstract = True -class Track(Nameable): - """ - Track of a playlist of a diffusion. The position can either be expressed - as the position in the playlist or as the moment in seconds it started. - """ - # There are no nice solution for M2M relations ship (even without - # through) in django-admin. So we unfortunately need to make one- - # to-one relations and add a position argument - diffusion = models.ForeignKey( - 'Diffusion', - ) - artist = models.CharField( - _('artist'), - max_length = 128, - ) - # position can be used to specify a position in seconds for stream - # programs or a position in the playlist - position = models.SmallIntegerField( - default = 0, - help_text=_('position in the playlist'), - ) - tags = TaggableManager( - verbose_name=_('tags'), - blank=True, - ) - - def __str__(self): - return ' '.join([self.artist, ':', self.name ]) - - class Meta: - verbose_name = _('Track') - verbose_name_plural = _('Tracks') - - class Sound(Nameable): """ A Sound is the representation of a sound file that can be either an excerpt @@ -114,6 +122,7 @@ class Sound(Nameable): 'Diffusion', verbose_name = _('diffusion'), blank = True, null = True, + help_text = _('this is set for scheduled programs') ) type = models.SmallIntegerField( verbose_name = _('type'), @@ -713,3 +722,45 @@ class Diffusion(models.Model): ('programming', _('edit the diffusion\'s planification')), ) + +class Track(Nameable,Related): + """ + Track of a playlist of an object. The position can either be expressed + as the position in the playlist or as the moment in seconds it started. + """ + # There are no nice solution for M2M relations ship (even without + # through) in django-admin. So we unfortunately need to make one- + # to-one relations and add a position argument + artist = models.CharField( + _('artist'), + max_length = 128, + ) + position = models.SmallIntegerField( + default = 0, + help_text=_('position in the playlist'), + ) + info = models.CharField( + _('information'), + max_length = 128, + blank = True, null = True, + help_text=_('additional informations about this track, such as ' + 'the version, if is it a remix, features, etc.'), + ) + tags = TaggableManager( + verbose_name=_('tags'), + blank=True, + ) + pos_in_secs = models.BooleanField( + _('use seconds'), + default = False, + help_text=_('position in the playlist is expressed in seconds') + ) + + def __str__(self): + return ' '.join([self.artist, ':', self.name ]) + + class Meta: + verbose_name = _('Track') + verbose_name_plural = _('Tracks') + + diff --git a/programs/settings.py b/programs/settings.py index e892609..dc1e984 100755 --- a/programs/settings.py +++ b/programs/settings.py @@ -37,9 +37,33 @@ ensure('AIRCOX_SOUND_QUALITY', { ) # Extension of sound files -ensure('AIRCOX_SOUND_FILE_EXT', - ('.ogg','.flac','.wav','.mp3','.opus')) +ensure( + 'AIRCOX_SOUND_FILE_EXT', + ('.ogg','.flac','.wav','.mp3','.opus') +) # Stream for the scheduled diffusions ensure('AIRCOX_SCHEDULED_STREAM', 0) + +# Import playlist: columns for plain text files +ensure( + 'AIRCOX_IMPORT_PLAYLIST_PLAIN_COLS', + ('artist', 'title', 'tags', 'version') +) +# Import playlist: delimiter for plain text files +ensure('AIRCOX_IMPORT_PLAYLIST_PLAIN_DELIMITER', '--') + +# Import playlist: columns for CSV file +ensure( + 'AIRCOX_IMPORT_PLAYLIST_CSV_COLS', + ('artist', 'title', 'minutes', 'seconds', 'tags', 'version') +) +# Import playlist: column delimiter of csv text files +ensure('AIRCOX_IMPORT_PLAYLIST_CSV_DELIMITER', ';') +# Import playlist: text delimiter of csv text files +ensure('AIRCOX_IMPORT_PLAYLIST_CSV_TEXT_QUOTE', '"') + + + + diff --git a/programs/utils.py b/programs/utils.py index 5f3f1aa..5d4482c 100644 --- a/programs/utils.py +++ b/programs/utils.py @@ -1,5 +1,6 @@ import datetime + def to_timedelta (time): """ Transform a datetime or a time instance to a timedelta, diff --git a/website/sections.py b/website/sections.py index f677387..f04a2dd 100644 --- a/website/sections.py +++ b/website/sections.py @@ -229,8 +229,7 @@ class Playlist(sections.List): message_empty = '' def get_object_list(self): - tracks = programs.Track.objects \ - .filter(diffusion = self.object.related) \ + tracks = programs.Track.get_for(object = self.object.related) \ .order_by('position') return [ sections.ListItem(title=track.name, content=track.artist) for track in tracks ] @@ -336,3 +335,10 @@ class Schedule(Diffusions): return None +class Logs(Schedule): + """ + Return a list of played stream sounds and diffusions. + """ + template_name = 'aircox/website/schedule.html' + # HERE -- + rename aircox/website/schedule to dated_list +