Permalink
Browse files

Improved The Space ingestion to support both stage and production env…

…ironments.
  • Loading branch information...
1 parent 6f2748f commit bf1bb55ea1a8620cffe6684922895790ba5854ec @sbtourist sbtourist committed May 3, 2012
@@ -40,12 +40,14 @@
private final String BASE_CATEGORY_URI = "http://thespace.org/by/genre/";
private final String EPISODE_TYPE = "episode";
//
+ private final String url;
private final SimpleHttpClient client;
private final AdapterLog log;
private final ContentResolver contentResolver;
private final ContentWriter contentWriter;
- public TheSpaceItemProcessor(SimpleHttpClient client, AdapterLog log, ContentResolver contentResolver, ContentWriter contentWriter) {
+ public TheSpaceItemProcessor(String url, SimpleHttpClient client, AdapterLog log, ContentResolver contentResolver, ContentWriter contentWriter) {
+ this.url = url;
this.client = client;
this.log = log;
this.contentResolver = contentResolver;
@@ -134,14 +136,14 @@ private void fillItem(Item episode, JsonNode node, ObjectMapper mapper) throws E
Iterator<JsonNode> clips = node.get("available_clips").getElements();
while (clips.hasNext()) {
String cPid = clips.next().get("pid").asText();
- JsonNode clip = client.get(new SimpleHttpRequest<JsonNode>(TheSpaceUpdater.BASE_API_URL + "/items/" + cPid + ".json", new JSonNodeHttpResponseTransformer(mapper)));
+ JsonNode clip = client.get(new SimpleHttpRequest<JsonNode>(url + "/items/" + cPid + ".json", new JSonNodeHttpResponseTransformer(mapper)));
episode.addClip(getClip(mapper, clip.get("programme"), episode));
}
Iterator<JsonNode> versions = node.get("versions").getElements();
while (versions.hasNext()) {
String vPid = versions.next().get("pid").asText();
- JsonNode version = client.get(new SimpleHttpRequest<JsonNode>(TheSpaceUpdater.BASE_API_URL + "/items/" + vPid + ".json", new JSonNodeHttpResponseTransformer(mapper)));
+ JsonNode version = client.get(new SimpleHttpRequest<JsonNode>(url + "/items/" + vPid + ".json", new JSonNodeHttpResponseTransformer(mapper)));
episode.addVersion(getVersion(mapper, version.get("version"), episode));
}
} catch (Exception ex) {
@@ -164,7 +166,7 @@ private void fillEpisode(Episode episode, JsonNode node, ObjectMapper mapper) th
if (series == null) {
series = new Series();
}
- fillSeries(series, mapper, client.get(new SimpleHttpRequest<JsonNode>(TheSpaceUpdater.BASE_API_URL + "/items/" + pPid + ".json", new JSonNodeHttpResponseTransformer(mapper))).get("programme"));
+ fillSeries(series, mapper, client.get(new SimpleHttpRequest<JsonNode>(url + "/items/" + pPid + ".json", new JSonNodeHttpResponseTransformer(mapper))).get("programme"));
series.setChildRefs(Iterables.concat(series.getChildRefs(), ImmutableList.of(episode.childRef())));
episode.setParentRef(ParentRef.parentRefFrom(series));
contentWriter.createOrUpdate(series);
@@ -224,7 +226,7 @@ private Series fillSeries(Series series, ObjectMapper mapper, JsonNode node) thr
Iterator<JsonNode> clips = node.get("available_clips").getElements();
while (clips.hasNext()) {
String cPid = clips.next().get("pid").asText();
- JsonNode clip = client.get(new SimpleHttpRequest<JsonNode>(TheSpaceUpdater.BASE_API_URL + "/items/" + cPid + ".json", new JSonNodeHttpResponseTransformer(mapper)));
+ JsonNode clip = client.get(new SimpleHttpRequest<JsonNode>(url + "/items/" + cPid + ".json", new JSonNodeHttpResponseTransformer(mapper)));
series.addClip(getClip(mapper, clip.get("programme"), series));
}
@@ -269,7 +271,7 @@ private Clip getClip(ObjectMapper mapper, JsonNode node, Content parent) throws
Iterator<JsonNode> versions = node.get("versions").getElements();
while (versions.hasNext()) {
String vPid = versions.next().get("pid").asText();
- JsonNode version = client.get(new SimpleHttpRequest<JsonNode>(TheSpaceUpdater.BASE_API_URL + "/items/" + vPid + ".json", new JSonNodeHttpResponseTransformer(mapper)));
+ JsonNode version = client.get(new SimpleHttpRequest<JsonNode>(url + "/items/" + vPid + ".json", new JSonNodeHttpResponseTransformer(mapper)));
clip.addVersion(getVersion(mapper, version.get("version"), clip));
}
@@ -14,12 +14,14 @@
*/
public class TheSpaceItemsProcessor {
+ private final String url;
private final SimpleHttpClient client;
private final AdapterLog log;
private final ContentResolver contentResolver;
private final ContentWriter contentWriter;
- public TheSpaceItemsProcessor(SimpleHttpClient client, AdapterLog log, ContentResolver contentResolver, ContentWriter contentWriter) {
+ public TheSpaceItemsProcessor(String url, SimpleHttpClient client, AdapterLog log, ContentResolver contentResolver, ContentWriter contentWriter) {
+ this.url = url;
this.client = client;
this.log = log;
this.contentResolver = contentResolver;
@@ -29,14 +31,14 @@ public TheSpaceItemsProcessor(SimpleHttpClient client, AdapterLog log, ContentRe
public void process(JsonNode items) throws Exception {
ObjectMapper mapper = new ObjectMapper();
//
- TheSpaceItemProcessor processor = new TheSpaceItemProcessor(client, log, contentResolver, contentWriter);
+ TheSpaceItemProcessor processor = new TheSpaceItemProcessor(url, client, log, contentResolver, contentWriter);
//
Iterator<JsonNode> results = items.get("results").getElements();
while (results.hasNext()) {
JsonNode item = results.next();
String pid = item.get("pid").asText();
try {
- JsonNode node = client.get(new SimpleHttpRequest<JsonNode>(TheSpaceUpdater.BASE_API_URL + "/items/" + pid + ".json", new JSonNodeHttpResponseTransformer(mapper)));
+ JsonNode node = client.get(new SimpleHttpRequest<JsonNode>(url + "/items/" + pid + ".json", new JSonNodeHttpResponseTransformer(mapper)));
processor.process(node.get("programme"));
} catch (Exception ex) {
log.record(new AdapterLogEntry(AdapterLogEntry.Severity.WARN).withCause(ex));
@@ -1,28 +1,28 @@
package org.atlasapi.remotesite.space;
import com.metabroadcast.common.properties.Configurer;
+import com.metabroadcast.common.scheduling.RepetitionRule;
import javax.annotation.PostConstruct;
import org.atlasapi.persistence.content.ContentWriter;
import org.atlasapi.persistence.logging.AdapterLog;
import org.atlasapi.persistence.logging.AdapterLogEntry;
import org.atlasapi.persistence.logging.AdapterLogEntry.Severity;
-import org.joda.time.LocalTime;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.metabroadcast.common.scheduling.RepetitionRules;
-import com.metabroadcast.common.scheduling.RepetitionRules.Daily;
import com.metabroadcast.common.scheduling.SimpleScheduler;
import org.atlasapi.persistence.content.ContentGroupResolver;
import org.atlasapi.persistence.content.ContentGroupWriter;
import org.atlasapi.persistence.content.ContentResolver;
+import org.joda.time.Duration;
@Configuration
public class TheSpaceModule {
- private final static Daily DAILY = RepetitionRules.daily(new LocalTime(4, 30, 0));
+ private final static RepetitionRule HOURLY = RepetitionRules.every(Duration.standardHours(1));
private @Autowired
SimpleScheduler scheduler;
private @Autowired
@@ -38,12 +38,12 @@
@PostConstruct
public void startBackgroundTasks() throws Exception {
- scheduler.schedule(theSpaceUpdater().withName("TheSpace Updater"), DAILY);
+ scheduler.schedule(theSpaceUpdater().withName("TheSpace Updater"), HOURLY);
log.record(new AdapterLogEntry(Severity.INFO).withSource(getClass()).withDescription("Installed TheSpace updater"));
}
@Bean
public TheSpaceUpdater theSpaceUpdater() throws Exception {
- return new TheSpaceUpdater(contentResolver, contentWriter, groupResolver, groupWriter, log, Configurer.get("thespace.keystore.path").get(), Configurer.get("thespace.keystore.password").get());
+ return new TheSpaceUpdater(contentResolver, contentWriter, groupResolver, groupWriter, log, Configurer.get("thespace.keystore.path").get(), Configurer.get("thespace.keystore.password").get(), Configurer.get("thespace.url").get());
}
}
@@ -16,14 +16,16 @@
*/
public class TheSpacePlaylistsProcessor {
+ private final String url;
private final SimpleHttpClient client;
private final AdapterLog log;
private final ContentResolver contentResolver;
private final ContentWriter contentWriter;
private final ContentGroupResolver groupResolver;
private final ContentGroupWriter groupWriter;
- public TheSpacePlaylistsProcessor(SimpleHttpClient client, AdapterLog log, ContentResolver contentResolver, ContentWriter contentWriter, ContentGroupResolver groupResolver, ContentGroupWriter groupWriter) {
+ public TheSpacePlaylistsProcessor(String url, SimpleHttpClient client, AdapterLog log, ContentResolver contentResolver, ContentWriter contentWriter, ContentGroupResolver groupResolver, ContentGroupWriter groupWriter) {
+ this.url = url;
this.client = client;
this.log = log;
this.contentResolver = contentResolver;
@@ -42,7 +44,7 @@ public void process(JsonNode items) throws Exception {
JsonNode item = results.next();
String pid = item.get("pid").asText();
try {
- JsonNode node = client.get(new SimpleHttpRequest<JsonNode>(TheSpaceUpdater.BASE_API_URL + "/items/" + pid + ".json", new JSonNodeHttpResponseTransformer(mapper)));
+ JsonNode node = client.get(new SimpleHttpRequest<JsonNode>(url + "/items/" + pid + ".json", new JSonNodeHttpResponseTransformer(mapper)));
processor.process(node.get("programme"));
} catch (Exception ex) {
log.record(new AdapterLogEntry(AdapterLogEntry.Severity.WARN).withCause(ex));
@@ -22,35 +22,39 @@
*/
class TheSpaceUpdater extends ScheduledTask {
- public static final String BASE_API_URL = "https://web.stage.thespace.org/";
- //
private final Timestamper timestamper = new SystemClock();
private final ContentResolver contentResolver;
private final ContentWriter contentWriter;
private final ContentGroupResolver groupResolver;
private final ContentGroupWriter groupWriter;
private final AdapterLog log;
+ private final String url;
private SimpleHttpClient client;
- public TheSpaceUpdater(ContentResolver contentResolver, ContentWriter contentWriter, ContentGroupResolver groupResolver, ContentGroupWriter groupWriter, AdapterLog log, String keystore, String password) throws Exception {
+ public TheSpaceUpdater(ContentResolver contentResolver, ContentWriter contentWriter, ContentGroupResolver groupResolver, ContentGroupWriter groupWriter, AdapterLog log, String keystore, String password, String url) throws Exception {
this.contentResolver = contentResolver;
this.contentWriter = contentWriter;
this.groupResolver = groupResolver;
this.groupWriter = groupWriter;
this.log = log;
- this.client = new RequestLimitingSimpleHttpClient(HttpClients.httpsClient(this.getClass().getClassLoader().getResource(keystore), password), 10);
+ this.url = url;
+ if (keystore == null || keystore.isEmpty() || password == null) {
+ this.client = new RequestLimitingSimpleHttpClient(HttpClients.webserviceClient(), 10);
+ } else {
+ this.client = new RequestLimitingSimpleHttpClient(HttpClients.httpsClient(this.getClass().getClassLoader().getResource(keystore), password), 10);
+ }
}
@Override
public void runTask() {
try {
Timestamp start = timestamper.timestamp();
- log.record(new AdapterLogEntry(AdapterLogEntry.Severity.INFO).withDescription("TheSpace update started from " + BASE_API_URL).withSource(getClass()));
+ log.record(new AdapterLogEntry(AdapterLogEntry.Severity.INFO).withDescription("TheSpace update started from " + url).withSource(getClass()));
- TheSpaceItemsProcessor itemsProcessor = new TheSpaceItemsProcessor(client, log, contentResolver, contentWriter);
- TheSpacePlaylistsProcessor playlistsProcessor = new TheSpacePlaylistsProcessor(client, log, contentResolver, contentWriter, groupResolver, groupWriter);
- JsonNode items = client.get(new SimpleHttpRequest<JsonNode>(BASE_API_URL + "/items.json", new JSonNodeHttpResponseTransformer(new ObjectMapper())));
- JsonNode playlists = client.get(new SimpleHttpRequest<JsonNode>(BASE_API_URL + "/items/playlists.json", new JSonNodeHttpResponseTransformer(new ObjectMapper())));
+ TheSpaceItemsProcessor itemsProcessor = new TheSpaceItemsProcessor(url, client, log, contentResolver, contentWriter);
+ TheSpacePlaylistsProcessor playlistsProcessor = new TheSpacePlaylistsProcessor(url, client, log, contentResolver, contentWriter, groupResolver, groupWriter);
+ JsonNode items = client.get(new SimpleHttpRequest<JsonNode>(url + "/items.json", new JSonNodeHttpResponseTransformer(new ObjectMapper())));
+ JsonNode playlists = client.get(new SimpleHttpRequest<JsonNode>(url + "/items/playlists.json", new JSonNodeHttpResponseTransformer(new ObjectMapper())));
itemsProcessor.process(items);
playlistsProcessor.process(playlists);
@@ -110,4 +110,14 @@ lovefilm.oauth.api.key=
lovefilm.oauth.api.secret=
thespace.keystore.path=
-thespace.keystore.password=
+thespace.keystore.password=
+thespace.url=
+
+notifications.email.from=atlas-admin@metabroadcast.com
+notifications.email.fromFriendlyName=Atlas
+
+notifications.email.username=atlas-admin@metabroadcast.com
+notifications.email.password=
+notifications.email.host=smtp.gmail.com
+
+notifications.email.to=atlas@metabroadcast.com
@@ -30,4 +30,5 @@ lovefilm.oauth.api.key=
lovefilm.oauth.api.secret=
thespace.keystore.path=
-thespace.keystore.password=
+thespace.keystore.password=
+thespace.url=
@@ -60,7 +60,7 @@ public Object answer(InvocationOnMock invocation) throws Throwable {
}
}).when(groupWriter).createOrUpdate(any(ContentGroup.class));
- TheSpaceUpdater updater = new TheSpaceUpdater(contentResolver, contentWriter, groupResolver, groupWriter, log, "", "");
+ TheSpaceUpdater updater = new TheSpaceUpdater(contentResolver, contentWriter, groupResolver, groupWriter, log, "", "", "https://web.stage.thespace.org/");
updater.runTask();
}
}

0 comments on commit bf1bb55

Please sign in to comment.