Permalink
Browse files

adding a batch importer to download everything from flickr.

  • Loading branch information...
1 parent e9db69c commit 88e3baa69842db29a8bc2eb2270bcc33613ecdb5 @joshlong joshlong committed Dec 30, 2012
View
@@ -0,0 +1,5 @@
+*.idea
+target
+*iml
+*ipr
+*iws
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>spring-social-flickr</artifactId>
+ <groupId>org.springframework.social</groupId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+ <artifactId>batch-importer</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.social</groupId>
+ <artifactId>spring-social-flickr-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-orm</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-web</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-jdbc</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-tx</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.batch</groupId>
+ <artifactId>spring-batch-integration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.batch</groupId>
+ <artifactId>spring-batch-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>aopalliance</groupId>
+ <artifactId>aopalliance</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-aop</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>cglib</groupId>
+ <artifactId>cglib</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context-support</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>javax.inject</groupId>
+ <artifactId>javax.inject</artifactId>
+ </dependency>
+ </dependencies>
+</project>
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+ <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
+ <output url="file://$MODULE_DIR$/target/classes" />
+ <output-test url="file://$MODULE_DIR$/target/test-classes" />
+ <exclude-output />
+ <content url="file://$MODULE_DIR$">
+ <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+ <sourceFolder url="file://$MODULE_DIR$/src/main/resources" isTestSource="false" />
+ <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
+ <sourceFolder url="file://$MODULE_DIR$/src/test/resources" isTestSource="true" />
+ <excludeFolder url="file://$MODULE_DIR$/target" />
+ </content>
+ <orderEntry type="inheritedJdk" />
+ <orderEntry type="sourceFolder" forTests="false" />
+ <orderEntry type="module" module-name="spring-social-flickr-core" />
+ <orderEntry type="library" name="Maven: org.springframework.social:spring-social-core:1.0.2.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework.social:spring-social-test:1.0.2.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework:spring-web:3.2.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework:spring-context:3.2.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework:spring-core:3.2.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.1" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework:spring-aop:3.2.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: aopalliance:aopalliance:1.0" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework:spring-beans:3.2.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework:spring-expression:3.2.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework.social:spring-social-web:1.0.2.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework:spring-webmvc:3.2.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: javax.inject:javax.inject:1" level="project" />
+ <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.9.4" level="project" />
+ <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.9.4" level="project" />
+ <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.9.4" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework.security:spring-security-core:3.1.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework.security:spring-security-crypto:3.1.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: xom:xom:1.2.5" level="project" />
+ <orderEntry type="library" name="Maven: xml-apis:xml-apis:1.3.03" level="project" />
+ <orderEntry type="library" name="Maven: xerces:xercesImpl:2.8.0" level="project" />
+ <orderEntry type="library" name="Maven: xalan:xalan:2.7.0" level="project" />
+ <orderEntry type="library" name="Maven: jaxen:jaxen:1.1.1" level="project" />
+ <orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" />
+ <orderEntry type="library" name="Maven: jdom:jdom:1.0" level="project" />
+ <orderEntry type="library" name="Maven: com.h2database:h2:1.3.159" level="project" />
+ <orderEntry type="library" name="Maven: postgresql:postgresql:8.4-702.jdbc4" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework:spring-orm:3.2.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework:spring-jdbc:3.2.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework:spring-tx:3.2.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework.batch:spring-batch-integration:1.2.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework.batch:spring-batch-core:2.1.7.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework.batch:spring-batch-infrastructure:2.1.7.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: com.thoughtworks.xstream:xstream:1.3" level="project" />
+ <orderEntry type="library" name="Maven: xpp3:xpp3_min:1.1.4c" level="project" />
+ <orderEntry type="library" name="Maven: org.codehaus.jettison:jettison:1.1" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework.integration:spring-integration-core:2.0.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
+ <orderEntry type="library" name="Maven: cglib:cglib:2.2.2" level="project" />
+ <orderEntry type="library" name="Maven: asm:asm:3.3.1" level="project" />
+ <orderEntry type="library" name="Maven: org.springframework:spring-context-support:3.2.0.RELEASE" level="project" />
+ <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.5.6" level="project" />
+ <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.5.6" level="project" />
+ <orderEntry type="library" name="Maven: log4j:log4j:1.2.14" level="project" />
+ <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.8.2" level="project" />
+ </component>
+</module>
+
@@ -0,0 +1,73 @@
+package org.springframework.social.importer;
+
+import org.springframework.batch.item.*;
+import org.springframework.batch.item.database.JdbcCursorItemReader;
+import org.springframework.social.flickr.api.MediaEnum;
+import org.springframework.social.flickr.api.PhotoSizeEnum;
+import org.springframework.social.flickr.api.Photoset;
+import org.springframework.social.flickr.api.impl.FlickrTemplate;
+
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+/**
+ * Simple Item reader that reads all the photos for a given {@link PhotoSet}
+ * and then passes that information onto an {@link org.springframework.batch.item.ItemWriter}.
+ *
+ * @author Josh Long
+ */
+public class DelegatingFlickrPhotoAlbumPhotoItemReader implements ItemReader<Photo>, ItemStream {
+
+ private JdbcCursorItemReader<PhotoSet> delegatingPhotoSetItemReader;
+ private FlickrTemplate flickrTemplate;
+ private PhotoSet photoSet;
+ private Queue<org.springframework.social.flickr.api.Photo> photoCollection = new ConcurrentLinkedQueue<org.springframework.social.flickr.api.Photo>();
+
+ public DelegatingFlickrPhotoAlbumPhotoItemReader(FlickrTemplate flickrTemplate, JdbcCursorItemReader<PhotoSet> delegatingPhotoSetItemReader) {
+ this.flickrTemplate = flickrTemplate;
+ this.delegatingPhotoSetItemReader = delegatingPhotoSetItemReader;
+ }
+
+ @Override
+ public Photo read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
+
+ // if theres nothing in the photo collection...
+ if (photoCollection.size() == 0) {
+
+ // then load a PhotoSet
+ photoSet = this.delegatingPhotoSetItemReader.read();
+
+ // if theres no PhotoSet, then we're done, no more photos to read
+ if (null == photoSet)
+ return null;
+
+ // if there is a PhotoSet, then load its PhotoDetails
+ Photoset photosSet = flickrTemplate.photosetOperations().getPhotos(photoSet.getId(), null, null, null, null, MediaEnum.PHOTOS);
+ for (org.springframework.social.flickr.api.Photo p : photosSet.getPhoto()) {
+ photoCollection.add(p);
+ }
+ }
+
+ org.springframework.social.flickr.api.Photo photo = photoCollection.isEmpty() ? null : photoCollection.remove();
+ if (null == photo)
+ return null;
+
+ // downloads the 'large' image
+ return new Photo(photo.getId(), photo.getUrl(PhotoSizeEnum.b), photo.getTitle(), null, photoSet.getId());
+ }
+
+ @Override
+ public void open(ExecutionContext executionContext) throws ItemStreamException {
+ delegatingPhotoSetItemReader.open(executionContext);
+ }
+
+ @Override
+ public void update(ExecutionContext executionContext) throws ItemStreamException {
+ delegatingPhotoSetItemReader.update(executionContext);
+ }
+
+ @Override
+ public void close() throws ItemStreamException {
+ delegatingPhotoSetItemReader.close();
+ }
+}
@@ -0,0 +1,114 @@
+package org.springframework.social.importer;
+
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.batch.core.JobParametersBuilder;
+import org.springframework.batch.core.launch.JobLauncher;
+import org.springframework.context.Lifecycle;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.scheduling.concurrent.ConcurrentTaskScheduler;
+import org.springframework.util.Assert;
+
+import java.io.File;
+import java.util.Date;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * component that manages Spring Batch jobs to import photos from Flickr and
+ * downloads them to a local cache where they can be used.
+ *
+ * @author Josh Long
+ */
+public class FlickrImporter implements Lifecycle {
+
+ private volatile JobLauncher jobLauncher;
+
+ private volatile Job importFlickrPhotosJob;
+
+ private volatile Map<File, JobExecution> mapOfFilesToRunningJobs = new ConcurrentHashMap<File, JobExecution>();
+
+ private volatile TaskScheduler scheduler;
+
+ public FlickrImporter(Job importFlickrPhotosJob, JobLauncher jobLauncher, TaskScheduler s) {
+ this.importFlickrPhotosJob = importFlickrPhotosJob;
+ this.jobLauncher = jobLauncher;
+ this.scheduler = s;
+ }
+
+ /**
+ * call this to kick off the import job.
+ *
+ * @param file the directory to which the imported photos should be written
+ */
+ public void importPhotosToDirectory(
+ String at,
+ String atSecret,
+ String consumerKey,
+ String consumerSecret,
+ File file) throws Throwable {
+
+ Assert.notNull(file, "you must provide a non-null File object.");
+ Assert.isTrue(file.exists(), "the " + file.getAbsolutePath() + " must exist.");
+ Assert.isTrue(file.canWrite(), "we must be able to write to " + file.getAbsolutePath() + ".");
+
+ JobParameters jp = new JobParametersBuilder()
+ .addDate("when", new Date())
+ .addString("accessToken", at)
+ .addString("accessTokenSecret", atSecret)
+ .addString("consumerKey", consumerKey)
+ .addString("consumerSecret", consumerSecret)
+ .addString("output", file.getAbsolutePath())
+ .toJobParameters();
+
+ JobExecution jobExecution = jobLauncher.run(this.importFlickrPhotosJob, jp);
+
+ this.mapOfFilesToRunningJobs.put(file, jobExecution);
+ }
+
+ /**
+ * tests to see if any jobs can be removed and, if so, does.
+ * <p/>
+ * todo we should re-work this in terms of {@link java.lang.ref.WeakReference weak references} and {@link java.util.WeakHashMap weak hash map}.
+ */
+ public static class JobCleanupRunnable implements Runnable {
+
+ private volatile Map<File, JobExecution> executionMap;
+
+ public JobCleanupRunnable(Map<File, JobExecution> ex) {
+ this.executionMap = ex;
+ }
+
+ @Override
+ public void run() {
+ for (Map.Entry<File, JobExecution> e : executionMap.entrySet())
+ if (!e.getValue().isRunning())
+ executionMap.remove(e.getKey());
+
+ }
+ }
+
+
+ @Override
+ public void start() {
+ // we don't have a particular obligation to do anything here..
+ if (null == this.scheduler) {
+ this.scheduler = new ConcurrentTaskScheduler();
+ }
+ this.scheduler.scheduleAtFixedRate(new JobCleanupRunnable(this.mapOfFilesToRunningJobs), 1000);
+ }
+
+ @Override
+ public void stop() {
+ for (JobExecution jobExecution : this.mapOfFilesToRunningJobs.values()) {
+ jobExecution.stop();
+ }
+
+ }
+
+ @Override
+ public boolean isRunning() {
+ return this.mapOfFilesToRunningJobs.size() > 0;
+ }
+}
Oops, something went wrong. Retry.

0 comments on commit 88e3baa

Please sign in to comment.