adding a batch importer to download everything from flickr.

ranjan-rk · Dec 30, 2012 · 88e3baa · 88e3baa
1 parent e9db69c
commit 88e3baa
Show file tree

Hide file tree

Showing 19 changed files with 1,640 additions and 243 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,5 @@
+*.idea
+target
+*iml
+*ipr
+*iws
diff --git a/batch-importer/pom.xml b/batch-importer/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <artifactId>spring-social-flickr</artifactId>
+        <groupId>org.springframework.social</groupId>
+        <version>0.0.1-SNAPSHOT</version>
+    </parent>
+    <artifactId>batch-importer</artifactId>
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.social</groupId>
+            <artifactId>spring-social-flickr-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-orm</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-jdbc</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-tx</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.batch</groupId>
+            <artifactId>spring-batch-integration</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.batch</groupId>
+            <artifactId>spring-batch-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>aopalliance</groupId>
+            <artifactId>aopalliance</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-aop</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>cglib</groupId>
+            <artifactId>cglib</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context-support</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-beans</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>javax.inject</groupId>
+            <artifactId>javax.inject</artifactId>
+        </dependency>
+    </dependencies>
+</project>
diff --git a/batch-importer/spring-social-batch-importer.iml b/batch-importer/spring-social-batch-importer.iml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
+    <output url="file://$MODULE_DIR$/target/classes" />
+    <output-test url="file://$MODULE_DIR$/target/test-classes" />
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/main/resources" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
+      <sourceFolder url="file://$MODULE_DIR$/src/test/resources" isTestSource="true" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="module" module-name="spring-social-flickr-core" />
+    <orderEntry type="library" name="Maven: org.springframework.social:spring-social-core:1.0.2.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework.social:spring-social-test:1.0.2.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-web:3.2.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-context:3.2.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-core:3.2.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.1" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-aop:3.2.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: aopalliance:aopalliance:1.0" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-beans:3.2.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-expression:3.2.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework.social:spring-social-web:1.0.2.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-webmvc:3.2.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: javax.inject:javax.inject:1" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.9.4" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.9.4" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.9.4" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework.security:spring-security-core:3.1.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework.security:spring-security-crypto:3.1.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: xom:xom:1.2.5" level="project" />
+    <orderEntry type="library" name="Maven: xml-apis:xml-apis:1.3.03" level="project" />
+    <orderEntry type="library" name="Maven: xerces:xercesImpl:2.8.0" level="project" />
+    <orderEntry type="library" name="Maven: xalan:xalan:2.7.0" level="project" />
+    <orderEntry type="library" name="Maven: jaxen:jaxen:1.1.1" level="project" />
+    <orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" />
+    <orderEntry type="library" name="Maven: jdom:jdom:1.0" level="project" />
+    <orderEntry type="library" name="Maven: com.h2database:h2:1.3.159" level="project" />
+    <orderEntry type="library" name="Maven: postgresql:postgresql:8.4-702.jdbc4" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-orm:3.2.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-jdbc:3.2.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-tx:3.2.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework.batch:spring-batch-integration:1.2.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework.batch:spring-batch-core:2.1.7.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework.batch:spring-batch-infrastructure:2.1.7.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: com.thoughtworks.xstream:xstream:1.3" level="project" />
+    <orderEntry type="library" name="Maven: xpp3:xpp3_min:1.1.4c" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jettison:jettison:1.1" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework.integration:spring-integration-core:2.0.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
+    <orderEntry type="library" name="Maven: cglib:cglib:2.2.2" level="project" />
+    <orderEntry type="library" name="Maven: asm:asm:3.3.1" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-context-support:3.2.0.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.5.6" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.5.6" level="project" />
+    <orderEntry type="library" name="Maven: log4j:log4j:1.2.14" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.8.2" level="project" />
+  </component>
+</module>
+
diff --git a/...n/java/org/springframework/social/importer/DelegatingFlickrPhotoAlbumPhotoItemReader.java b/...n/java/org/springframework/social/importer/DelegatingFlickrPhotoAlbumPhotoItemReader.java
@@ -0,0 +1,73 @@
+package org.springframework.social.importer;
+
+import org.springframework.batch.item.*;
+import org.springframework.batch.item.database.JdbcCursorItemReader;
+import org.springframework.social.flickr.api.MediaEnum;
+import org.springframework.social.flickr.api.PhotoSizeEnum;
+import org.springframework.social.flickr.api.Photoset;
+import org.springframework.social.flickr.api.impl.FlickrTemplate;
+
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+/**
+ * Simple Item reader that reads all the photos for a given {@link PhotoSet}
+ * and then passes that information onto an {@link org.springframework.batch.item.ItemWriter}.
+ *
+ * @author Josh Long
+ */
+public class DelegatingFlickrPhotoAlbumPhotoItemReader implements ItemReader<Photo>, ItemStream {
+
+    private JdbcCursorItemReader<PhotoSet> delegatingPhotoSetItemReader;
+    private FlickrTemplate flickrTemplate;
+    private PhotoSet photoSet;
+    private Queue<org.springframework.social.flickr.api.Photo> photoCollection = new ConcurrentLinkedQueue<org.springframework.social.flickr.api.Photo>();
+
+    public DelegatingFlickrPhotoAlbumPhotoItemReader(FlickrTemplate flickrTemplate, JdbcCursorItemReader<PhotoSet> delegatingPhotoSetItemReader) {
+        this.flickrTemplate = flickrTemplate;
+        this.delegatingPhotoSetItemReader = delegatingPhotoSetItemReader;
+    }
+
+    @Override
+    public Photo read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
+
+        // if theres nothing in the photo collection...
+        if (photoCollection.size() == 0) {
+
+            // then load a PhotoSet
+            photoSet = this.delegatingPhotoSetItemReader.read();
+
+            // if theres no PhotoSet, then we're done, no more photos to read
+            if (null == photoSet)
+                return null;
+
+            // if there is a PhotoSet, then load its PhotoDetails
+            Photoset photosSet = flickrTemplate.photosetOperations().getPhotos(photoSet.getId(), null, null, null, null, MediaEnum.PHOTOS);
+            for (org.springframework.social.flickr.api.Photo p : photosSet.getPhoto()) {
+                photoCollection.add(p);
+            }
+        }
+
+        org.springframework.social.flickr.api.Photo photo = photoCollection.isEmpty() ? null : photoCollection.remove();
+        if (null == photo)
+            return null;
+
+        // downloads the 'large' image
+        return new Photo(photo.getId(), photo.getUrl(PhotoSizeEnum.b), photo.getTitle(), null, photoSet.getId());
+    }
+
+    @Override
+    public void open(ExecutionContext executionContext) throws ItemStreamException {
+        delegatingPhotoSetItemReader.open(executionContext);
+    }
+
+    @Override
+    public void update(ExecutionContext executionContext) throws ItemStreamException {
+        delegatingPhotoSetItemReader.update(executionContext);
+    }
+
+    @Override
+    public void close() throws ItemStreamException {
+        delegatingPhotoSetItemReader.close();
+    }
+}
diff --git a/batch-importer/src/main/java/org/springframework/social/importer/FlickrImporter.java b/batch-importer/src/main/java/org/springframework/social/importer/FlickrImporter.java
@@ -0,0 +1,114 @@
+package org.springframework.social.importer;
+
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.batch.core.JobParametersBuilder;
+import org.springframework.batch.core.launch.JobLauncher;
+import org.springframework.context.Lifecycle;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.scheduling.concurrent.ConcurrentTaskScheduler;
+import org.springframework.util.Assert;
+
+import java.io.File;
+import java.util.Date;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * component that manages Spring Batch jobs to import photos from Flickr and
+ * downloads them to a local cache where they can be used.
+ *
+ * @author Josh Long
+ */
+public class FlickrImporter implements Lifecycle {
+
+    private volatile JobLauncher jobLauncher;
+
+    private volatile Job importFlickrPhotosJob;
+
+    private volatile Map<File, JobExecution> mapOfFilesToRunningJobs = new ConcurrentHashMap<File, JobExecution>();
+
+    private volatile TaskScheduler scheduler;
+
+    public FlickrImporter(Job importFlickrPhotosJob, JobLauncher jobLauncher, TaskScheduler s) {
+        this.importFlickrPhotosJob = importFlickrPhotosJob;
+        this.jobLauncher = jobLauncher;
+        this.scheduler = s;
+    }
+
+    /**
+     * call this to kick off the import job.
+     *
+     * @param file the directory to which the imported photos should be written
+     */
+    public void importPhotosToDirectory(
+            String at,
+            String atSecret,
+            String consumerKey,
+            String consumerSecret,
+            File file) throws Throwable {
+
+        Assert.notNull(file, "you must provide a non-null File object.");
+        Assert.isTrue(file.exists(), "the " + file.getAbsolutePath() + " must exist.");
+        Assert.isTrue(file.canWrite(), "we must be able to write to " + file.getAbsolutePath() + ".");
+
+        JobParameters jp = new JobParametersBuilder()
+                .addDate("when", new Date())
+                .addString("accessToken", at)
+                .addString("accessTokenSecret", atSecret)
+                .addString("consumerKey", consumerKey)
+                .addString("consumerSecret", consumerSecret)
+                .addString("output", file.getAbsolutePath())
+                .toJobParameters();
+
+        JobExecution jobExecution = jobLauncher.run(this.importFlickrPhotosJob, jp);
+
+        this.mapOfFilesToRunningJobs.put(file, jobExecution);
+    }
+
+    /**
+     * tests to see if any jobs can be removed and, if so, does.
+     * <p/>
+     * todo we should re-work this in terms of {@link java.lang.ref.WeakReference weak references} and {@link java.util.WeakHashMap weak hash map}.
+     */
+    public static class JobCleanupRunnable implements Runnable {
+
+        private volatile Map<File, JobExecution> executionMap;
+
+        public JobCleanupRunnable(Map<File, JobExecution> ex) {
+            this.executionMap = ex;
+        }
+
+        @Override
+        public void run() {
+            for (Map.Entry<File, JobExecution> e : executionMap.entrySet())
+                if (!e.getValue().isRunning())
+                    executionMap.remove(e.getKey());
+
+        }
+    }
+
+
+    @Override
+    public void start() {
+        // we don't have a particular obligation to do anything here..
+        if (null == this.scheduler) {
+            this.scheduler = new ConcurrentTaskScheduler();
+        }
+        this.scheduler.scheduleAtFixedRate(new JobCleanupRunnable(this.mapOfFilesToRunningJobs), 1000);
+    }
+
+    @Override
+    public void stop() {
+        for (JobExecution jobExecution : this.mapOfFilesToRunningJobs.values()) {
+            jobExecution.stop();
+        }
+
+    }
+
+    @Override
+    public boolean isRunning() {
+        return this.mapOfFilesToRunningJobs.size() > 0;
+    }
+}