Skip to content

Commit

Permalink
diffing catalog and website scan works
Browse files Browse the repository at this point in the history
  • Loading branch information
@cpesch
cpesch committed Feb 8, 2015
1 parent fb7058b commit f538504
Show file tree
Hide file tree
Showing 3 changed files with 119 additions and 35 deletions.
49 changes: 44 additions & 5 deletions download-tools/src/main/java/slash/navigation/download/tools/ScanWebsite.java
View file
Expand Up @@ -27,10 +27,17 @@
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import slash.navigation.datasources.DataSource;
import slash.navigation.datasources.DataSourceService;
import slash.navigation.datasources.File;
import slash.navigation.datasources.Map;
import slash.navigation.datasources.Theme;
import slash.navigation.download.tools.base.BaseDownloadTool;
import slash.navigation.download.tools.helpers.AnchorFilter;
import slash.navigation.download.tools.helpers.AnchorParser;
import slash.navigation.rest.Get;

import javax.xml.bind.JAXBException;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
Expand All @@ -46,13 +53,13 @@
* @author Christian Pesch
*/

public class ScanWebsite {
public class ScanWebsite extends BaseDownloadTool {
private static final Logger log = Logger.getLogger(ScanWebsite.class.getName());
private static final String URL_ARGUMENT = "url";
private static final String BASE_URL_ARGUMENT = "baseUrl";
private static final String EXTENSION_ARGUMENT = "extension";
private static final String ID_ARGUMENT = "id";

private String url, baseUrl;
private String url, baseUrl, id;
private Set<String> extensions;

private String appendURIs(String uri, String anchor) {
Expand Down Expand Up @@ -93,9 +100,38 @@ private List<String> collectUris() throws IOException {
return asList(sortedUris);
}

private void scan() throws IOException {
private Set<String> collectURIs(DataSource source) {
Set<String> result = new HashSet<>();
for(File file : source.getFiles())
result.add(file.getUri());
for(Map map : source.getMaps())
result.add(map.getUri());
for(Theme theme : source.getThemes())
result.add(theme.getUri());
return result;
}

private void scan() throws IOException, JAXBException {
List<String> collectedUris = collectUris();
log.info("Collected URIs: " + collectedUris + " (" + collectedUris.size() + " elements)");

DataSourceService service = loadDataSources(getDataSourcesTarget());
DataSource source = service.getDataSourceById(id);
if (source == null)
throw new IllegalArgumentException("Unknown data source: " + id);
if (!url.equals(source.getBaseUrl()))
throw new IllegalArgumentException("Data source URL: " + source.getBaseUrl() + " doesn't match URL: " + url);

Set<String> files = collectURIs(source);

Set<String> addedUris = new HashSet<>(collectedUris);
addedUris.removeAll(files);

Set<String> removedUris = new HashSet<>(files);
removedUris.removeAll(collectedUris);

log.info("Added URIs: " + addedUris + " (" + addedUris.size() + " elements)");
log.info("Removed URIs: " + removedUris + " (" + removedUris.size() + " elements)");
}

private void run(String[] args) throws Exception {
Expand All @@ -104,8 +140,9 @@ private void run(String[] args) throws Exception {
extensions = extensionArguments != null ? new HashSet<>(asList(extensionArguments)) : null;
url = line.getOptionValue(URL_ARGUMENT);
baseUrl = line.getOptionValue(BASE_URL_ARGUMENT);
if(baseUrl == null)
if (baseUrl == null)
baseUrl = url;
id = line.getOptionValue(ID_ARGUMENT);
scan();
System.exit(0);
}
Expand All @@ -120,6 +157,8 @@ private CommandLine parseCommandLine(String[] args) throws ParseException {
withDescription("URL to use as a base for resources").create());
options.addOption(OptionBuilder.withArgName(EXTENSION_ARGUMENT).hasArgs().withLongOpt("extension").
withDescription("Extensions to scan for").create());
options.addOption(OptionBuilder.withArgName(ID_ARGUMENT).hasArgs().isRequired().withLongOpt("id").
withDescription("ID of the data source").create());
try {
return parser.parse(options, args);
} catch (ParseException e) {
Expand Down
43 changes: 13 additions & 30 deletions download-tools/src/main/java/slash/navigation/download/tools/SnapshotCatalog.java
View file
Expand Up @@ -31,19 +31,17 @@
import slash.navigation.download.Download;
import slash.navigation.download.DownloadManager;
import slash.navigation.download.FileAndChecksum;
import slash.navigation.download.tools.base.BaseDownloadTool;

import javax.xml.bind.JAXBException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;

import static java.lang.String.format;
import static slash.common.io.Directories.ensureDirectory;
import static slash.common.io.Directories.getApplicationDirectory;
import static slash.common.io.Directories.getTemporaryDirectory;
import static slash.navigation.download.Action.Copy;

Expand All @@ -53,19 +51,12 @@
* @author Christian Pesch
*/

public class SnapshotCatalog {
public class SnapshotCatalog extends BaseDownloadTool {
private static final Logger log = Logger.getLogger(SnapshotCatalog.class.getName());
private static final String URL_ARGUMENT = "url";
private static final String[] EDITIONS = {"Online", "Offline"};
private DownloadManager downloadManager = new DownloadManager(new File(getTemporaryDirectory(), "snapshot-queue.xml"));

private File getDataSourcesTarget() {
return ensureDirectory(getApplicationDirectory("snapshot").getAbsolutePath());
}

private File getEditionTarget() {
return ensureDirectory(new File(getDataSourcesTarget(), "editions"));
}
private String url;
private DownloadManager downloadManager = new DownloadManager(new File(getTemporaryDirectory(), "snapshot-queue.xml"));

private void deleteAll(File directory) throws IOException {
File[] files = directory.listFiles();
Expand All @@ -79,34 +70,25 @@ private void deleteAll(File directory) throws IOException {
}
}

private DataSourceService loadDataSources(File directory) throws JAXBException, FileNotFoundException {
DataSourceService dataSourceService = new DataSourceService();
File[] files = directory.listFiles();
if (files != null)
for (File file : files)
dataSourceService.load(new FileInputStream(file));
return dataSourceService;
}

private void snapshot(String baseUrl) throws IOException, JAXBException {
private void snapshot() throws IOException, JAXBException {
downloadManager.clearQueue();
deleteAll(getDataSourcesTarget());

snapshotEditions(baseUrl, getEditionTarget());
snapshotEditions(getEditionTarget());
DataSourceService dataSourceService = loadDataSources(getEditionTarget());
snapshotDataSources(dataSourceService, getDataSourcesTarget());
}

private void snapshotEditions(String baseUrl, File target) {
private void snapshotEditions(File target) {
List<Download> downloads = new ArrayList<>();
// TODO remove hardcoded EDITIONS by scanning http://www.routeconverter.com/datasources/edition/
for (String edition : EDITIONS) {
String uri = edition.toLowerCase() + ".xml";
String url = baseUrl + "edition/" + uri;
String editionUri = edition.toLowerCase() + ".xml";
String editionUrl = url + "edition/" + editionUri;
Download download = downloadManager.queueForDownload("RouteConverter " + edition + " Edition: Catalog of Data Sources",
url, Copy, null, new FileAndChecksum(new File(target, uri), null), null);
editionUrl, Copy, null, new FileAndChecksum(new File(target, editionUri), null), null);
downloads.add(download);
log.info(format("Downloading '%s'", url));
log.info(format("Downloading '%s'", editionUrl));
}
downloadManager.waitForCompletion(downloads);
}
Expand All @@ -128,7 +110,8 @@ url, Copy, null, new FileAndChecksum(new File(directory, file.getUri().toLowerCa

private void run(String[] args) throws Exception {
CommandLine line = parseCommandLine(args);
snapshot(line.getOptionValue(URL_ARGUMENT));
url = line.getOptionValue(URL_ARGUMENT);
snapshot();
System.exit(0);
}

Expand Down
62 changes: 62 additions & 0 deletions download-tools/src/main/java/slash/navigation/download/tools/base/BaseDownloadTool.java
View file
@@ -0,0 +1,62 @@
/*
This file is part of RouteConverter.
RouteConverter is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
RouteConverter is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with RouteConverter; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Copyright (C) 2007 Christian Pesch. All Rights Reserved.
*/
package slash.navigation.download.tools.base;

import slash.navigation.datasources.DataSourceService;

import javax.xml.bind.JAXBException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FilenameFilter;

import static slash.common.io.Directories.ensureDirectory;
import static slash.common.io.Directories.getApplicationDirectory;

/**
* Base for the download tools.
*
* @author Christian Pesch
*/

public class BaseDownloadTool {
protected static final String URL_ARGUMENT = "url";

protected File getDataSourcesTarget() {
return ensureDirectory(getApplicationDirectory("snapshot").getAbsolutePath());
}

protected File getEditionTarget() {
return ensureDirectory(new File(getDataSourcesTarget(), "editions"));
}

protected DataSourceService loadDataSources(File directory) throws JAXBException, FileNotFoundException {
DataSourceService dataSourceService = new DataSourceService();
File[] files = directory.listFiles(new FilenameFilter() {
public boolean accept(File dir, String name) {
return name.endsWith(".xml");
}
});
if (files != null)
for (File file : files)
dataSourceService.load(new FileInputStream(file));
return dataSourceService;
}
}

0 comments on commit f538504

Please sign in to comment.