Integrated LikedSavedDatabase

* LikedSavedDatabase now records submissions and their associated
files after downloading. This will pave the way for more cool features
in the future
* Split up redditUserImageScraper main function to be a bit more readable

LikedSavedDatabase.py

@@ -2,6 +2,9 @@
 import sqlite3
 import submission as Submissions
 
+# Global database
+db = None
+
 class LikedSavedDatabase:
     def __init__(self, databaseFilePath):
         isNewDatabase = not os.path.exists(databaseFilePath)
@@ -46,6 +49,13 @@ def addSubmission(self, submission):
                        submission.getAsList())
         self.save()
 
+    def addSubmissions(self, submissions):
+        cursor = self.dbConnection.cursor()
+
+        cursor.executemany("insert or ignore into Submissions values (NULL,?,?,?,?,?,?,?,?)",
+                       Submissions.getAsList_generator(submissions))
+        self.save()
+
     def printSubmissions(self):
         cursor = self.dbConnection.cursor()
 
@@ -73,11 +83,23 @@ def addSubmissionToCollection(self, submissionId, collectionId):
                        (submissionId, collectionId))
         self.save()
 
-    def associateFileToSubmission(self, filePath, submissionId):
+    def associateFileToSubmissionId(self, filePath, submissionId):
         cursor = self.dbConnection.cursor()
         cursor.execute("insert or ignore into FilesToSubmissions values (?,?)",
                        (filePath, submissionId))
         self.save()
+
+    def associateFileToSubmission(self, filePath, submission):
+        cursor = self.dbConnection.cursor()
+        cursor.execute("select * from Submissions where postUrl=?", (submission.postUrl,))
+        submissionInDb = cursor.fetchone()
+        if submissionInDb:
+            submissionId = submissionInDb[0]
+            cursor.execute("insert or ignore into FilesToSubmissions values (?,?)",
+                           (filePath, submissionId))
+            self.save()
+        else:
+            print("DB error: couldn't find submission from post URL {}".format(submission.postUrl))
 
     def getAllSubmissionsInCollection(self, collectionId):
         cursor = self.dbConnection.cursor()
@@ -126,10 +148,14 @@ def testOnRealSubmissions():
             print("Couldn't find {}".format(title))
         else:
             db.addSubmissionToCollection(dbSubmission[0], dbCollection[0])
-            db.associateFileToSubmission("{}.png".format(title), dbSubmission[0])
+            db.associateFileToSubmissionId("{}.png".format(title), dbSubmission[0])
 
     print(db.getAllSubmissionsInCollection(dbCollection[0]))
     print(db.getAllFiles())
+
+def initializeFromSettings(userSettings):
+    global db
+    db = LikedSavedDatabase(userSettings['Database'])
 
 if __name__ == '__main__':
     #testDatabase()

LikedSavedDownloaderServer.py

@@ -12,7 +12,7 @@
 import json
 import multiprocessing
 
-from utilities import sort_naturally
+import utilities
 import settings
 import redditUserImageScraper
 
@@ -44,10 +44,6 @@ def generateSavedImagesCache(outputDir):
 
     print('Finished creating Liked Saved cache ({} images/videos)'.format(len(savedImagesCache)))
 
-def outputPathToServerPath(path):
-    # This is a little weird
-    return 'output' + path.split(settings.settings['Output_dir'])[1]
-
 def getRandomImage(filteredImagesCache=None, randomImageFilter=''):
     if not savedImagesCache:
         generateSavedImagesCache(settings.settings['Output_dir'])
@@ -59,7 +55,7 @@ def getRandomImage(filteredImagesCache=None, randomImageFilter=''):
 
     print('\tgetRandomImage(): Chose random image {} (filter {})'.format(randomImage, randomImageFilter))
 
-    serverPath = outputPathToServerPath(randomImage)
+    serverPath = utilities.outputPathToServerPath(randomImage)
 
     return randomImage, serverPath
 
@@ -377,7 +373,7 @@ def on_message(self, message):
                 for file in files:
                     if file.endswith(supportedExtensions):
                         imagesInFolder.append(os.path.join(root, file))
-            sort_naturally(imagesInFolder)
+            utilities.sort_naturally(imagesInFolder)
             currentImageIndex = imagesInFolder.index(fullImagePath)
             if currentImageIndex >= 0:
                 action = 'setImage'
@@ -389,7 +385,7 @@ def on_message(self, message):
                     nextImageIndex = len(imagesInFolder) - 1
 
                 fullImagePath = imagesInFolder[nextImageIndex]
-                serverImagePath = outputPathToServerPath(fullImagePath)
+                serverImagePath = utilities.outputPathToServerPath(fullImagePath)
 
         if command == 'setFilter':
             newFilter = parsedMessage['filter']

imageSaver.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+import LikedSavedDatabase
 import imgurpython as imgur
 import logger
 import os
@@ -304,8 +305,9 @@ def saveAllImgurAlbums(outputDir, imgurAuth, subredditAlbums, soft_retrieve_imgs
 
         numAlbums = len(albums)
         for albumIndex, album in enumerate(albums):
-            albumTitle = album[0]
-            albumUrl = cleanImgurAlbumUrl(album[1])
+            albumSubmission = album[0]
+            albumTitle = album[1]
+            albumUrl = cleanImgurAlbumUrl(album[2])
             logger.log('\t[' + percentageComplete(albumIndex, numAlbums) + '] ' 
                 + '\t' + albumTitle + ' (' + albumUrl + ')')
 
@@ -358,6 +360,8 @@ def saveAllImgurAlbums(outputDir, imgurAuth, subredditAlbums, soft_retrieve_imgs
                 if not soft_retrieve_imgs:
                     # Retrieve the image and save it
                     urlretrieve(imageUrl, saveFilePath)
+                    LikedSavedDatabase.db.associateFileToSubmission(
+                            utilities.outputPathToDatabasePath(saveFilePath), albumSubmission)
 
                 logger.log('\t\t[' + percentageComplete(imageIndex, numImages) + '] ' 
                     + ' [save] ' + imageUrl + ' saved to "' + saveAlbumPath + '"')
@@ -425,9 +429,9 @@ def saveAllImages(outputDir, submissions, imgur_auth = None, only_download_album
                 else:
                     # We're going to save Imgur Albums at a separate stage
                     if subredditDir in imgurAlbumsToSave:
-                        imgurAlbumsToSave[subredditDir].append((submissionTitle, url))
+                        imgurAlbumsToSave[subredditDir].append((submission, submissionTitle, url))
                     else:
-                        imgurAlbumsToSave[subredditDir] = [(submissionTitle, url)]
+                        imgurAlbumsToSave[subredditDir] = [(submission, submissionTitle, url)]
                     continue
             elif only_download_albums:
                 continue
@@ -497,6 +501,9 @@ def saveAllImages(outputDir, submissions, imgur_auth = None, only_download_album
                 # Retrieve the image and save it
                 try:
                     urlretrieve(url, saveFilePath)
+
+                    LikedSavedDatabase.db.associateFileToSubmission(
+                            utilities.outputPathToDatabasePath(saveFilePath), submission)
                 except IOError as e:
                     logger.log('[ERROR] IOError: Url {0} raised exception:\n\t{1} {2}'
                         .format(url, e.errno, e.strerror))