Feature: Write-up stats

README.md

@@ -58,6 +58,18 @@ Note: Using the localhost IP address as above will route to the default app (cur
 
 Note: if the specification for the VM changes, e.g. what's in the `Vagrantfile`, those changes will need to be retrospectively applied to any existing machines. Either `vagrant reload --provision` the existing machine, or `vagrant destroy; vagrant up` to create a fresh machine to this spec.
 
+#### Backup, and restoring from it
+
+A production install should automatically backup the database and key files to a cloud service.
+
+A django management command is included to apply this backup to a local machine, i.e. follow the above procedure with this.
+
+```
+vagrant up
+vagrant ssh
+python3.6 /folk_rnn_webapp/folk_rnn_site/manage.py applybackup
+```
+
 #### Tests
 ```
 vagrant up
@@ -99,3 +111,13 @@ After some time, the server should be deployed and the webapp running (see note
 As before, `vagrant ssh` to log in to the server, and `vagrant provision` to update. 
 
 Note: there is a bug in `vagrant up` for the first time, the provisioning scripts that should run unprivileged instead run as root. However, rather than `vagrant destroy; vagrant up` to rebuild a machine, do this: `vagrant halt`, pause, `vagrant rebuild`, pause, `vagrant provision`. This will keep the IP address and provision correctly. So for a new server, do the rebuild dance and all should be good (alternatively, `chown vagrant:vagrant` the directory `/var/opt/folk_rnn_task/tune` and the contents of `/folk_rnn_static`).
+
+## Dataset
+
+Browse to `<composer url>/dataset` to download a CSV of all the tunes generated by the composer app.
+
+Browse to `<archiver url>/dataset` to download a CSV of all the tunes and settings contributed to the archiver app.
+
+On a host machine, run the following django management command `python3.6 /folk_rnn_webapp/folk_rnn_site/manage.py stats` to view i. a tune-centric view of activity, ii. a composer-session-centric view of activity, and iii. descriptive statistics suitable for academic write-up.
+
+All datasets are anonymous.

folk_rnn_site/backup/management/commands/applybackup.py

@@ -0,0 +1,45 @@
+import os
+import tarfile
+import logging
+from tempfile import TemporaryDirectory
+
+from django.core.management.base import BaseCommand
+from django.core.management import call_command
+
+from .backup import Command as BackupCommand
+
+logger = logging.getLogger(__name__)
+
+class Command(BaseCommand):
+    """"
+    Download the latest production backup and apply to local machine
+    """
+    help = 'Download the latest production backup and apply to local machine'
+
+    def handle(self, *args, **kwargs):
+        '''
+        Process the command (i.e. the django manage.py entrypoint)
+        '''
+        logger.info('Apply Backup starting.')
+
+        backup = BackupCommand()
+
+        with TemporaryDirectory() as tmp:
+            logger.info('Downloading latest production backup...')
+            db_path, log_path, tunes_path = backup.download_latest_production_backup(to_dir=tmp)
+
+            logger.info('Applying database...')
+            with tarfile.open(name=db_path, mode='r:bz2') as tar:
+                tar.extractall(path=tmp)
+                call_command('flush', interactive=False) # Ideally drop, create and migrate
+                call_command('loaddata', os.path.join(tmp, 'db_data.json'))
+
+            logger.info('Applying logs...')
+            with tarfile.open(name=log_path, mode='r:bz2') as tar:
+                tar.extractall(path='/')
+
+            logger.info('Applying tune...')
+            with tarfile.open(name=tunes_path, mode='r:bz2') as tar:
+                tar.extractall(path='/')
+
+        logger.info('Apply Backup finished.')

folk_rnn_site/backup/management/commands/backup.py

@@ -145,11 +145,7 @@ def archive_store_folder(self, folder_path):
                 logger.error(f'archive_store_folder: {e}') # It should... # Do not retry. Log the error and fail.
         return body['name']
 
-    def list_stored_files(self):
-        """
-        Utility function, typically for interactive use
-        """
-        print('Listing all stored files')
+    def _list_stored_files(self):
         next_page_token = 'no token for first page'
         while next_page_token is not None:
             kwargs = {'orderBy':'createdTime desc'}
@@ -161,7 +157,15 @@ def list_stored_files(self):
             else:
                 next_page_token = None
             for meta in drive_list['files']:
-                print(self.drive.files().get(fileId=meta['id'], fields='name,size,createdTime').execute())
+                yield self.drive.files().get(fileId=meta['id'], fields='id, name,size,createdTime').execute()
+
+    def list_stored_files(self):
+        """
+        Utility function, typically for interactive use
+        """
+        print('Listing all stored files')
+        for file_info in self._list_stored_files():
+            print(f"{file_info['name']:60}{file_info['size']:10}{file_info['createdTime']}")
 
     def download_file(self, file_id, file_path):
         """
@@ -196,7 +200,27 @@ def download_file_named(self, filename):
             raise ValueError
         self.download_file(file_id, filename)
 
-
+    def download_latest_production_backup(self, to_dir=''):
+        names = [
+            ['db_data_backup_production_', None, None],
+            ['folk_rnn_webapp_backup_production_', None, None],
+            ['tunes_backup_production_', None, None],
+            ]
+
+        # relies on _list_stored_files's newest-first ordering request
+        for file_info in self._list_stored_files():
+            for name in names:
+                if file_info.get('name').startswith(name[0]):
+                    name[1] = file_info.get('id')
+                    name[2] = file_info.get('name')
+            if all(x[1] for x in names):
+                for name in names:
+                    download_path = os.path.join(to_dir, name[2])
+                    self.download_file(name[1], download_path)
+                    name.append(download_path)
+                break
+        return ([x[3] for x in names])
+
     def delete_file(self, file_id):
         """
         Utility function, typically for interactive use

folk_rnn_site/composer/consumers.py

@@ -235,10 +235,10 @@ def receive_json(self, content):
                 # untrusted input
                 to_log = f"URL: {content['url']}. State: {content['state']}"
                 # poor mans check
-                if len(to_log) < 400:
+                if len(to_log) < 1024:
                     self.log_use(to_log)
                 else:
-                    self.log_use(to_log[:400])
+                    self.log_use(to_log[:1024])
                     logger.warning('(worryingly)long state_notification')
             elif content['type'] in ['midi_play', 'midi_download']:
                 self.log_use(f"{content['type']} of tune {content['tune_id']}")

folk_rnn_site/folk_rnn_site/settings.py

@@ -55,6 +55,7 @@
     'composer',
     'archiver',
     'backup',
+    'stats',
 ]
 
 MIDDLEWARE = [