rewrite a bit stats for later updates; rename played into (raw_)on_air

README.md

@@ -51,13 +51,14 @@ Python modules:
 * `Pillow`: `aircox.cms` (needed by `wagtail`)
 * Django's required database modules (remember to install `mysqlclient` if you plan to use a MySql database server)
 
-External applications:
+External applications & modules:
 * `liquidsoap`: `aircox` (generation of the audio streams)
 * `sox`: `aircox` (check sounds quality and metadatas)
 * `gunicorn`: WSGI server to be used in production (installed along as dependency)
 * `supervisord`: supervisor
 * note there might be external dependencies for python's Pillow too
 * sqlite, mysql or any database library that you need to run a database, that is supported by Django (+ eventual python deps)
+* gzip: archive logs
 
 ### Setup environment
 All scripts and files assumes that:

aircox/management/commands/streamer.py

@@ -131,10 +131,6 @@ class Monitor:
         if not current_sound or not current_source:
             return
 
-        # last log can be anything, so we need to keep track of the last
-        # sound log too
-        # sound on air can be of a diffusion or a stream.
-
         log = self.last_log
 
         # sound on air changed
@@ -142,7 +138,7 @@ class Monitor:
                 (log.sound and log.sound.path != current_sound):
             sound = Sound.objects.filter(path = current_sound).first()
 
-            # find diff
+            # find an eventual diff
             last_diff = self.last_diff_start
             diff = None
             if not last_diff.is_expired():
@@ -157,11 +153,11 @@ class Monitor:
                 date = tz.now(),
                 sound = sound,
                 diffusion = diff,
-                # keep sound path (if sound is removed, we keep that info)
+                # if sound is removed, we keep sound path info
                 comment = current_sound,
             )
 
-        # tracks -- only for sound's
+        # tracks -- only for streams
         if not log.diffusion:
             self.trace_sound_tracks(log)
 
@@ -219,7 +215,7 @@ class Monitor:
             type = Diffusion.Type.normal,
             sound__type = Sound.Type.archive,
         )
-        logs = station.played(diffusion__isnull = False)
+        logs = station.raw_on_air(diffusion__isnull = False)
 
         date = tz.now() - datetime.timedelta(seconds = self.cancel_timeout)
         for diff in diffs:
@@ -244,7 +240,7 @@ class Monitor:
         station = self.station
         now = tz.now()
 
-        log = station.played(diffusion__isnull = False) \
+        log = station.raw_on_air(diffusion__isnull = False) \
                      .select_related('diffusion') \
                      .order_by('date').last()
         if not log or not log.diffusion.is_date_in_range(now):
@@ -252,7 +248,7 @@ class Monitor:
             return None, []
 
         # last sound source change: end of file reached or forced to stop
-        sounds = station.played(sound__isnull = False) \
+        sounds = station.raw_on_air(sound__isnull = False) \
                         .filter(date__gte = log.date) \
                         .order_by('date')
 

aircox/models.py

@@ -229,11 +229,11 @@ class Station(Nameable):
         self.__prepare_controls()
         return self.__streamer
 
-    def played(self, *args, **kwargs):
+    def raw_on_air(self, *args, **kwargs):
         """
-        Call Log.objects.played for this station
+        Forward call to Log.objects.on_air for this station
         """
-        return Log.objects.played(self, *args, **kwargs)
+        return Log.objects.on_air(self, *args, **kwargs)
 
     def on_air(self, date = None, count = 0):
         """
@@ -244,11 +244,10 @@ class Station(Nameable):
         * count: number of items to retrieve if not zero;
 
         If date is not specified, count MUST be set to a non-zero value.
-        Be careful with what you which for: the result is a plain list.
 
-        It is different from Station.played method since it filters out
+        It is different from Station.raw_on_air method since it filters
-        elements that should have not been on air, such as a stream that
+        out elements that should have not been on air, such as a stream
-        has been played when there was a live diffusion.
+        that has been played when there was a live diffusion.
         """
         # FIXME: as an iterator?
         # TODO argument to get sound instead of tracks
@@ -264,7 +263,9 @@ class Station(Nameable):
 
         if date:
             logs = Log.objects.at(self, date)
-            diffs = Diffusion.objects.at(self, date, type = Diffusion.Type.normal) \
+            diffs = Diffusion.objects \
+                             .at(self, date,
+                                 type = Diffusion.Type.normal) \
                              .order_by('-start')
         else:
             logs = Log.objects
@@ -274,7 +275,7 @@ class Station(Nameable):
 
         q = models.Q(diffusion__isnull = False) | \
             models.Q(track__isnull = False)
-        logs = logs.filter(q).order_by('-date')
+        logs = logs.filter(q, type = Log.Type.on_air).order_by('-date')
 
         # filter out tracks played when there was a diffusion
         n = 0
@@ -508,6 +509,7 @@ class Schedule(models.Model):
         'self',
         verbose_name = _('initial schedule'),
         blank = True, null = True,
+        on_delete=models.SET_NULL,
         help_text = 'this schedule is a rerun of this one',
     )
 
@@ -793,12 +795,14 @@ class Diffusion(models.Model):
         'self',
         verbose_name = _('initial diffusion'),
         blank = True, null = True,
+        on_delete=models.SET_NULL,
         help_text = _('the diffusion is a rerun of this one')
     )
     # port = models.ForeignKey(
     #    'self',
     #    verbose_name = _('port'),
     #    blank = True, null = True,
+    #    on_delete=models.SET_NULL,
     #    help_text = _('use this input port'),
     # )
     conflicts = models.ManyToManyField(
@@ -934,6 +938,7 @@ class Sound(Nameable):
         'Diffusion',
         verbose_name = _('diffusion'),
         blank = True, null = True,
+        on_delete=models.SET_NULL,
         help_text = _('initial diffusion related it')
     )
     type = models.SmallIntegerField(
@@ -1214,16 +1219,13 @@ class LogManager(models.Manager):
         return self.station(station, qs) if station else qs
 
     # TODO: rename on_air + rename Station.on_air into sth like regular_on_air
-    def played(self, station, archives = True, date = None, **kwargs):
+    def on_air(self, station, date = None, **kwargs):
         """
         Return a queryset of the played elements' log for the given
         station and model. This queryset is ordered by date ascending
 
-        * station: related station
+        * station: return logs occuring on this station
-        * archives: if false, exclude log of diffusion's archives from
+        * date: only return logs that occured at this date
-            the queryset;
-        * date: get played logs at the given date only
-        * include_live: include diffusion that have no archive
         * kwargs: extra filter kwargs
         """
         if date:
@@ -1231,15 +1233,7 @@ class LogManager(models.Manager):
         else:
             qs = self
 
-        qs = qs.filter(
+        qs = qs.filter(type = Log.Type.on_air, **kwargs)
-            type__in = (Log.Type.start, Log.Type.on_air), **kwargs
-        )
-
-        if not archives and station.dealer:
-            qs = qs.exclude(
-                source = station.dealer.id,
-                sound__isnull = False
-            )
         return qs.order_by('date')
 
     @staticmethod
@@ -1300,10 +1294,11 @@ class LogManager(models.Manager):
             for log in qs
         ]
 
+        # Note: since we use Yaml, we can just append new logs when file
+        # exists yet <3
         with gzip.open(path, 'ab') as archive:
             data = yaml.dump(logs).encode('utf8')
             archive.write(data)
-            # TODO: delete logs
 
         if not keep:
             qs.delete()
@@ -1385,18 +1380,21 @@ class Log(models.Model):
         verbose_name = _('Diffusion'),
         blank = True, null = True,
         db_index = True,
+        on_delete=models.SET_NULL,
     )
     sound = models.ForeignKey(
         Sound,
         verbose_name = _('Sound'),
         blank = True, null = True,
         db_index = True,
+        on_delete=models.SET_NULL,
     )
     track = models.ForeignKey(
         Track,
         verbose_name = _('Track'),
         blank = True, null = True,
         db_index = True,
+        on_delete=models.SET_NULL,
     )
 
     objects = LogManager()

aircox/templates/aircox/controllers/stats.html

@@ -26,7 +26,6 @@
 </form>
 </header>
 
-
 {% for stats in statistics %}
 <section class="station">
     <header>
@@ -54,7 +53,7 @@
             </tr>
 
             {% for track in item.tracks %}
-            <tr class="subdata">
+            <tr class="subdata" tags="{{ track.tags.all|join:', '}}">
                 <td>{{ track.date|time:"H:i" }}</td>
                 <td>{% trans "Track" %}</td>
                 <td>{{ track.artist }} -- <emph>{{ track.title }}</emph> {{ track.version }}</td>
@@ -75,6 +74,23 @@
                 {% endwith %}
                 {% endwith %}
             </th>
+            <th>
+            <script>
+            var tracks = document.querySelectorAll('.subdata[tags]');
+            var tags = {}
+            for(var i = 0; i < tracks.length; i++) {
+                var tags_ = tracks[i].getAttribute('tags').split(', ');
+                for(var j = 0; j < tags_.length; j++) {
+                    var tag = tags_[j];
+                    tags[tag] = (tags[tag] || 0) + 1;
+                }
+            }
+
+            for(var tag in tags) {
+                document.write('<span>' + tag + ': <b>' + tags[tag] + '</b><br>');
+            }
+            </script>
+            </th>
             <th>{% for tag, count, average in stats.tags %}
                 <span>{{ tag }}: <b>{{ average|floatformat }}%</b> ({{ count }})<br>
             {% endfor %}

aircox/views.py

@@ -1,3 +1,4 @@
+import os
 import json
 import datetime
 
@@ -10,6 +11,7 @@ from django.utils.translation import ugettext as _, ugettext_lazy
 from django.utils import timezone as tz
 
 import aircox.models as models
+import aircox.settings as settings
 
 
 class Stations:
@@ -136,40 +138,25 @@ class Monitor(View,TemplateResponseMixin,LoginRequiredMixin):
 
 
 class StatisticsView(View,TemplateResponseMixin,LoginRequiredMixin):
+    """
+    View for statistics.
+    """
+    # we cannot manipulate queryset, since we have to be able to read from archives
     template_name = 'aircox/controllers/stats.html'
 
-    class Taggable:
+    class Item:
-        tags = None
-
-        def add_tags(self, qs):
-            if not self.tags:
-                self.tags = {}
-
-            qs = qs.values('tags__name').annotate(count = Count('tags__name')) \
-                    .order_by('tags__name')
-            for q in qs:
-                key = q['tags__name']
-                if not key:
-                    continue
-
-                value = q['count']
-                if key not in self.tags:
-                    self.tags[key] = value
-                else:
-                    self.tags[key] += value
-
-    class Item (Taggable):
-        type = ''
         date = None
+        end = None
         name = None
         related = None
         tracks = None
         tags = None
+        col = None
 
         def __init__(self, **kwargs):
             self.__dict__.update(kwargs)
 
-    class Stats (Taggable):
+    class Stats:
         station = None
         date = None
         items = None
@@ -180,11 +167,30 @@ class StatisticsView(View,TemplateResponseMixin,LoginRequiredMixin):
             - tracks_count: total count of tracks
         """
         count = 0
+        #rows = None
 
         def __init__(self, **kwargs):
             self.items = []
+        #    self.rows = []
             self.__dict__.update(kwargs)
 
+        # Note: one row contains a column for diffusions and one for streams
+        #def append(self, log):
+        #    if log.col == 0:
+        #        self.rows.append((log, []))
+        #        return
+        #
+        #    if self.rows:
+        #        row = self.rows[len(self.rows)-1]
+        #        last = row[0] or row[1][len(row[1])-1]
+        #        if last.date < log.date < last.end:
+        #            row[1].append(log)
+        #            return
+        #
+        #    # all other cases: new row
+        #    self.rows.append((None, [log]))
+
+
     def get_stats(self, station, date):
         """
         Return statistics for the given station and date.
@@ -192,53 +198,50 @@ class StatisticsView(View,TemplateResponseMixin,LoginRequiredMixin):
         stats = self.Stats(station = station, date = date,
                            items = [], tags = {})
 
-        last_item = None
+        qs = station.raw_on_air(date = date) \
-        for elm in reversed(station.on_air(date)):
+                    .prefetch_related('diffusion', 'sound', 'track',
-            qs = None
+                                      'track__tags')
+        sound_log = None
+        for log in qs:
+            rel = None
             item = None
-            if type(elm) == models.Diffusion:
+
-                qs = models.Track.objects.get_for(object = elm)
+            if log.diffusion:
+                rel = log.diffusion
                 item = self.Item(
+                    name = rel.program.name,
                     type = _('Diffusion'),
-                    date = elm.date,
+                    col = 0,
-                    name = elm.program.name,
+                    tracks = models.Track.objects.get_for(object = rel)
-                    related = elm,
+                                         .prefetch_related('tags'),
-                    tracks = qs[:]
                 )
-                item.add_tags(qs)
+                sound_log = None
-                stats.items.append(item)
+            elif log.sound:
-                stats.count += len(item.tracks)
+                rel = log.sound
-            else:
+                item = self.Item(
-                # type is Track (related object of a track is a sound)
+                    name = rel.program.name + ': ' + os.path.basename(rel.path),
-                stream = elm.related.related
+                    type = _('Stream'),
-                qs = models.Track.objects.filter(pk = elm.related.pk)
+                    col = 1,
+                    tracks = [],
+                )
+                sound_log = item
+            elif log.track:
+                # append to last sound log
+                if not sound_log:
+                    # TODO: create item ? should never happen
+                    continue
 
-                if last_item and last_item.related == stream:
+                sound_log.tracks.append(log.track)
-                    item = last_item
+                sound_log.end = log.end
-                else:
+                sound_log
-                    item = self.Item(
+                continue
-                        type = _('Stream'),
-                        date = elm.date,
-                        name = stream.path,
-                        related = stream,
-                        tracks = []
-                    )
-                    stats.items.append(item)
 
-                elm.related.date = elm.date
+            item.date = log.date
-                item.tracks.append(elm.related)
+            item.end = log.end
-                item.date = min(elm.date, item.date)
+            item.related = rel
-                item.add_tags(qs)
+            # stats.append(item)
-                stats.count += 1
+            stats.items.append(item)
 
-            last_item = item
-            stats.add_tags(qs)
-
-        stats.tags = [
-            (name, count, count / stats.count * 100)
-            for name, count in stats.tags.items()
-        ]
-        stats.tags.sort(key=lambda s: s[0])
         return stats
 
     def get_context_data(self, **kwargs):

requirements.txt

@@ -3,6 +3,7 @@ Django>=1.10.3
 django-taggit>=0.18.3
 watchdog>=0.8.3
 psutil>=5.0.1
+pyyaml>=3.12
 dateutils>=0.6.6
 bleach>=1.4.3
 django-htmlmin>=0.10.0