Retry failed soundcloud uploads

config/config.py

@@ -33,6 +33,7 @@
 SOUNDCLOUD_PASSWORD = env.get("SOUNDCLOUD_PASSWORD", '')
 
 MAX_SOUNDCLOUD_BATCH = int(env.get("MAX_SOUNDCLOUD_BATCH", '1'))
+MAX_SOUNDCLOUD_RETRIES = int(env.get("MAX_SOUNDCLOUD_RETRIE", '3'))
 SOUNDCLOUD_PERIOD_MINUTES = env.get("SOUNDCLOUD_PERIOD_MINUTES", '5')
 
 S3_BUCKET = "pmg-assets"

migrations/versions/46085f5c8e36_add_soundcloud_upload_try_logging_table.py

@@ -0,0 +1,33 @@
+"""Add soundcloud upload try logging table
+
+Revision ID: 46085f5c8e36
+Revises: 1e3be3ff186d
+Create Date: 2017-02-10 11:20:12.308503
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '46085f5c8e36'
+down_revision = '1e3be3ff186d'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('soundcloud_track_retry',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
+    sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
+    sa.Column('soundcloud_track_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['soundcloud_track_id'], ['soundcloud_track.id'], name=op.f('fk_soundcloud_track_retry_soundcloud_track_id_soundcloud_track')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_soundcloud_track_retry'))
+    )
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('soundcloud_track_retry')
+    ### end Alembic commands ###

pmg/models/soundcloud_track.py

@@ -1,8 +1,9 @@
 from datetime import datetime
 from pmg import db, app
 from pmg.models import File, EventFile
+from requests.exceptions import HTTPError
 from soundcloud import Client
-from sqlalchemy import desc, func
+from sqlalchemy import asc, desc, func
 from sqlalchemy.orm import backref
 import logging
 
@@ -68,12 +69,15 @@ def new_from_file(cls, client, file):
         soundcloud_track = cls(file=file)
         db.session.add(soundcloud_track)
         db.session.commit()
+        cls._upload(client, soundcloud_track, file)
 
+    @staticmethod
+    def _upload(client, soundcloud_track, file):
         with file.open() as file_handle:
             logging.info("Uploading to SoundCloud: %s" % file)
             track = client.post('/tracks', track={
                 'title': file.title,
-                'description': cls._html_description(file),
+                'description': SoundcloudTrack._html_description(file),
                 'sharing': 'public',
                 'asset_data': file_handle,
                 'license': 'cc-by',
@@ -131,6 +135,7 @@ def sync(cls):
                         password=app.config['SOUNDCLOUD_PASSWORD'])
         cls.upload_files(client)
         cls.sync_upload_state(client)
+        cls.handle_failed(client)
 
     @classmethod
     def get_unstarted_query(cls):
@@ -168,3 +173,70 @@ def sync_upload_state(cls, client):
                            .order_by(cls.created_at).all()
         for track in tracks:
             track.sync_state(client)
+
+    @classmethod
+    def handle_failed(cls, client):
+        for track_id, retries in db.session.query(cls.id,
+                                                  func.count(SoundcloudTrackRetry.id).label('retries'))\
+                                           .filter(cls.state == 'failed')\
+                                           .outerjoin(SoundcloudTrackRetry)\
+                                           .group_by(cls.id)\
+                                           .order_by('retries')\
+                                           .limit(app.config['MAX_SOUNDCLOUD_BATCH']):
+            if retries <= app.config['MAX_SOUNDCLOUD_RETRIES']:
+                soundcloud_track = db.session.query(cls).get(track_id)
+                # Sometimes tracks apparently go from failed to finished. Yeah.
+                soundcloud_track.sync_state(client)
+                if soundcloud_track.state == 'failed':
+                    logging.info("Retrying failed soundcloud track %r" % soundcloud_track)
+                    retry = SoundcloudTrackRetry(soundcloud_track=soundcloud_track)
+                    db.session.add(retry)
+                    db.session.commit()
+                    try:
+                        client.delete(soundcloud_track.uri)
+                    except HTTPError as delete_result:
+                        # Handle brokenness at SoundCloud where deleting a track
+                        # results with an HTTP 500 response yet the track
+                        # is gone (HTTP 404) when you try to GET it.
+                        if delete_result.response.status_code == 500:
+                            try:
+                                client.get(soundcloud_track.uri)
+                                # If the delete gves a 500 and the GET doesn't
+                                # give a 404, something bad happened.
+                                raise Exception("Tried to delete but track %s " +\
+                                                "still there" % soundcloud_track.uri)
+                            except HTTPError as get_result:
+                                if get_result.response.status_code != 404:
+                                    raise Exception("Can't tell if track %s " +\
+                                                    "that we attempted to delete "+\
+                                                    "is still there." % soundcloud_track.uri)
+                    # If we get here we expect that we've successfully deleted
+                    # the failed track from SoundCloud.
+                    # Indicate that we've started uploading
+                    soundcloud_track.state = None
+                    db.session.commit()
+                    cls._upload(client, soundcloud_track, soundcloud_track.file)
+
+
+class SoundcloudTrackRetry(db.Model):
+    __tablename__ = "soundcloud_track_retry"
+
+    id = db.Column(db.Integer, primary_key=True)
+    created_at = db.Column(
+        db.DateTime(timezone=True),
+        nullable=False,
+        default=datetime.utcnow
+    )
+    updated_at = db.Column(
+        db.DateTime(timezone=True),
+        nullable=False,
+        default=datetime.utcnow,
+        onupdate=func.current_timestamp()
+    )
+    soundcloud_track_id = db.Column(
+        db.Integer,
+        db.ForeignKey('soundcloud_track.id'),
+        nullable=False,
+        unique=False
+    )
+    soundcloud_track = db.relationship('SoundcloudTrack', backref=backref('retries', lazy='joined'), lazy=True)