diff --git a/README.md b/README.md index 1d4e461..0fdb350 100755 --- a/README.md +++ b/README.md @@ -51,13 +51,14 @@ Python modules: * `Pillow`: `aircox.cms` (needed by `wagtail`) * Django's required database modules (remember to install `mysqlclient` if you plan to use a MySql database server) -External applications: +External applications & modules: * `liquidsoap`: `aircox` (generation of the audio streams) * `sox`: `aircox` (check sounds quality and metadatas) * `gunicorn`: WSGI server to be used in production (installed along as dependency) * `supervisord`: supervisor * note there might be external dependencies for python's Pillow too * sqlite, mysql or any database library that you need to run a database, that is supported by Django (+ eventual python deps) +* gzip: archive logs ### Setup environment All scripts and files assumes that: diff --git a/aircox/management/commands/streamer.py b/aircox/management/commands/streamer.py index afbcdaa..55006a8 100755 --- a/aircox/management/commands/streamer.py +++ b/aircox/management/commands/streamer.py @@ -131,10 +131,6 @@ class Monitor: if not current_sound or not current_source: return - # last log can be anything, so we need to keep track of the last - # sound log too - # sound on air can be of a diffusion or a stream. - log = self.last_log # sound on air changed @@ -142,7 +138,7 @@ class Monitor: (log.sound and log.sound.path != current_sound): sound = Sound.objects.filter(path = current_sound).first() - # find diff + # find an eventual diff last_diff = self.last_diff_start diff = None if not last_diff.is_expired(): @@ -157,11 +153,11 @@ class Monitor: date = tz.now(), sound = sound, diffusion = diff, - # keep sound path (if sound is removed, we keep that info) + # if sound is removed, we keep sound path info comment = current_sound, ) - # tracks -- only for sound's + # tracks -- only for streams if not log.diffusion: self.trace_sound_tracks(log) @@ -219,7 +215,7 @@ class Monitor: type = Diffusion.Type.normal, sound__type = Sound.Type.archive, ) - logs = station.played(diffusion__isnull = False) + logs = station.raw_on_air(diffusion__isnull = False) date = tz.now() - datetime.timedelta(seconds = self.cancel_timeout) for diff in diffs: @@ -244,7 +240,7 @@ class Monitor: station = self.station now = tz.now() - log = station.played(diffusion__isnull = False) \ + log = station.raw_on_air(diffusion__isnull = False) \ .select_related('diffusion') \ .order_by('date').last() if not log or not log.diffusion.is_date_in_range(now): @@ -252,7 +248,7 @@ class Monitor: return None, [] # last sound source change: end of file reached or forced to stop - sounds = station.played(sound__isnull = False) \ + sounds = station.raw_on_air(sound__isnull = False) \ .filter(date__gte = log.date) \ .order_by('date') diff --git a/aircox/models.py b/aircox/models.py index ae04f08..215bf95 100755 --- a/aircox/models.py +++ b/aircox/models.py @@ -229,11 +229,11 @@ class Station(Nameable): self.__prepare_controls() return self.__streamer - def played(self, *args, **kwargs): + def raw_on_air(self, *args, **kwargs): """ - Call Log.objects.played for this station + Forward call to Log.objects.on_air for this station """ - return Log.objects.played(self, *args, **kwargs) + return Log.objects.on_air(self, *args, **kwargs) def on_air(self, date = None, count = 0): """ @@ -244,11 +244,10 @@ class Station(Nameable): * count: number of items to retrieve if not zero; If date is not specified, count MUST be set to a non-zero value. - Be careful with what you which for: the result is a plain list. - It is different from Station.played method since it filters out - elements that should have not been on air, such as a stream that - has been played when there was a live diffusion. + It is different from Station.raw_on_air method since it filters + out elements that should have not been on air, such as a stream + that has been played when there was a live diffusion. """ # FIXME: as an iterator? # TODO argument to get sound instead of tracks @@ -264,7 +263,9 @@ class Station(Nameable): if date: logs = Log.objects.at(self, date) - diffs = Diffusion.objects.at(self, date, type = Diffusion.Type.normal) \ + diffs = Diffusion.objects \ + .at(self, date, + type = Diffusion.Type.normal) \ .order_by('-start') else: logs = Log.objects @@ -274,7 +275,7 @@ class Station(Nameable): q = models.Q(diffusion__isnull = False) | \ models.Q(track__isnull = False) - logs = logs.filter(q).order_by('-date') + logs = logs.filter(q, type = Log.Type.on_air).order_by('-date') # filter out tracks played when there was a diffusion n = 0 @@ -508,6 +509,7 @@ class Schedule(models.Model): 'self', verbose_name = _('initial schedule'), blank = True, null = True, + on_delete=models.SET_NULL, help_text = 'this schedule is a rerun of this one', ) @@ -793,12 +795,14 @@ class Diffusion(models.Model): 'self', verbose_name = _('initial diffusion'), blank = True, null = True, + on_delete=models.SET_NULL, help_text = _('the diffusion is a rerun of this one') ) # port = models.ForeignKey( # 'self', # verbose_name = _('port'), # blank = True, null = True, + # on_delete=models.SET_NULL, # help_text = _('use this input port'), # ) conflicts = models.ManyToManyField( @@ -934,6 +938,7 @@ class Sound(Nameable): 'Diffusion', verbose_name = _('diffusion'), blank = True, null = True, + on_delete=models.SET_NULL, help_text = _('initial diffusion related it') ) type = models.SmallIntegerField( @@ -1214,16 +1219,13 @@ class LogManager(models.Manager): return self.station(station, qs) if station else qs # TODO: rename on_air + rename Station.on_air into sth like regular_on_air - def played(self, station, archives = True, date = None, **kwargs): + def on_air(self, station, date = None, **kwargs): """ Return a queryset of the played elements' log for the given station and model. This queryset is ordered by date ascending - * station: related station - * archives: if false, exclude log of diffusion's archives from - the queryset; - * date: get played logs at the given date only - * include_live: include diffusion that have no archive + * station: return logs occuring on this station + * date: only return logs that occured at this date * kwargs: extra filter kwargs """ if date: @@ -1231,15 +1233,7 @@ class LogManager(models.Manager): else: qs = self - qs = qs.filter( - type__in = (Log.Type.start, Log.Type.on_air), **kwargs - ) - - if not archives and station.dealer: - qs = qs.exclude( - source = station.dealer.id, - sound__isnull = False - ) + qs = qs.filter(type = Log.Type.on_air, **kwargs) return qs.order_by('date') @staticmethod @@ -1300,10 +1294,11 @@ class LogManager(models.Manager): for log in qs ] + # Note: since we use Yaml, we can just append new logs when file + # exists yet <3 with gzip.open(path, 'ab') as archive: data = yaml.dump(logs).encode('utf8') archive.write(data) - # TODO: delete logs if not keep: qs.delete() @@ -1385,18 +1380,21 @@ class Log(models.Model): verbose_name = _('Diffusion'), blank = True, null = True, db_index = True, + on_delete=models.SET_NULL, ) sound = models.ForeignKey( Sound, verbose_name = _('Sound'), blank = True, null = True, db_index = True, + on_delete=models.SET_NULL, ) track = models.ForeignKey( Track, verbose_name = _('Track'), blank = True, null = True, db_index = True, + on_delete=models.SET_NULL, ) objects = LogManager() diff --git a/aircox/templates/aircox/controllers/stats.html b/aircox/templates/aircox/controllers/stats.html index aff464f..cacdc8e 100755 --- a/aircox/templates/aircox/controllers/stats.html +++ b/aircox/templates/aircox/controllers/stats.html @@ -26,7 +26,6 @@ - {% for stats in statistics %}
@@ -54,7 +53,7 @@ {% for track in item.tracks %} - + {{ track.date|time:"H:i" }} {% trans "Track" %} {{ track.artist }} -- {{ track.title }} {{ track.version }} @@ -75,6 +74,23 @@ {% endwith %} {% endwith %} + + + {% for tag, count, average in stats.tags %} {{ tag }}: {{ average|floatformat }}% ({{ count }})
{% endfor %} diff --git a/aircox/views.py b/aircox/views.py index 7f13bbe..cb6584c 100755 --- a/aircox/views.py +++ b/aircox/views.py @@ -1,3 +1,4 @@ +import os import json import datetime @@ -10,6 +11,7 @@ from django.utils.translation import ugettext as _, ugettext_lazy from django.utils import timezone as tz import aircox.models as models +import aircox.settings as settings class Stations: @@ -136,40 +138,25 @@ class Monitor(View,TemplateResponseMixin,LoginRequiredMixin): class StatisticsView(View,TemplateResponseMixin,LoginRequiredMixin): + """ + View for statistics. + """ + # we cannot manipulate queryset, since we have to be able to read from archives template_name = 'aircox/controllers/stats.html' - class Taggable: - tags = None - - def add_tags(self, qs): - if not self.tags: - self.tags = {} - - qs = qs.values('tags__name').annotate(count = Count('tags__name')) \ - .order_by('tags__name') - for q in qs: - key = q['tags__name'] - if not key: - continue - - value = q['count'] - if key not in self.tags: - self.tags[key] = value - else: - self.tags[key] += value - - class Item (Taggable): - type = '' + class Item: date = None + end = None name = None related = None tracks = None tags = None + col = None def __init__(self, **kwargs): self.__dict__.update(kwargs) - class Stats (Taggable): + class Stats: station = None date = None items = None @@ -180,11 +167,30 @@ class StatisticsView(View,TemplateResponseMixin,LoginRequiredMixin): - tracks_count: total count of tracks """ count = 0 + #rows = None def __init__(self, **kwargs): self.items = [] + # self.rows = [] self.__dict__.update(kwargs) + # Note: one row contains a column for diffusions and one for streams + #def append(self, log): + # if log.col == 0: + # self.rows.append((log, [])) + # return + # + # if self.rows: + # row = self.rows[len(self.rows)-1] + # last = row[0] or row[1][len(row[1])-1] + # if last.date < log.date < last.end: + # row[1].append(log) + # return + # + # # all other cases: new row + # self.rows.append((None, [log])) + + def get_stats(self, station, date): """ Return statistics for the given station and date. @@ -192,53 +198,50 @@ class StatisticsView(View,TemplateResponseMixin,LoginRequiredMixin): stats = self.Stats(station = station, date = date, items = [], tags = {}) - last_item = None - for elm in reversed(station.on_air(date)): - qs = None + qs = station.raw_on_air(date = date) \ + .prefetch_related('diffusion', 'sound', 'track', + 'track__tags') + sound_log = None + for log in qs: + rel = None item = None - if type(elm) == models.Diffusion: - qs = models.Track.objects.get_for(object = elm) + + if log.diffusion: + rel = log.diffusion item = self.Item( + name = rel.program.name, type = _('Diffusion'), - date = elm.date, - name = elm.program.name, - related = elm, - tracks = qs[:] + col = 0, + tracks = models.Track.objects.get_for(object = rel) + .prefetch_related('tags'), ) - item.add_tags(qs) - stats.items.append(item) - stats.count += len(item.tracks) - else: - # type is Track (related object of a track is a sound) - stream = elm.related.related - qs = models.Track.objects.filter(pk = elm.related.pk) + sound_log = None + elif log.sound: + rel = log.sound + item = self.Item( + name = rel.program.name + ': ' + os.path.basename(rel.path), + type = _('Stream'), + col = 1, + tracks = [], + ) + sound_log = item + elif log.track: + # append to last sound log + if not sound_log: + # TODO: create item ? should never happen + continue - if last_item and last_item.related == stream: - item = last_item - else: - item = self.Item( - type = _('Stream'), - date = elm.date, - name = stream.path, - related = stream, - tracks = [] - ) - stats.items.append(item) + sound_log.tracks.append(log.track) + sound_log.end = log.end + sound_log + continue - elm.related.date = elm.date - item.tracks.append(elm.related) - item.date = min(elm.date, item.date) - item.add_tags(qs) - stats.count += 1 + item.date = log.date + item.end = log.end + item.related = rel + # stats.append(item) + stats.items.append(item) - last_item = item - stats.add_tags(qs) - - stats.tags = [ - (name, count, count / stats.count * 100) - for name, count in stats.tags.items() - ] - stats.tags.sort(key=lambda s: s[0]) return stats def get_context_data(self, **kwargs): diff --git a/requirements.txt b/requirements.txt index 88464bc..d59d998 100755 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ Django>=1.10.3 django-taggit>=0.18.3 watchdog>=0.8.3 psutil>=5.0.1 +pyyaml>=3.12 dateutils>=0.6.6 bleach>=1.4.3 django-htmlmin>=0.10.0