Multi-threading (#71)

src/org/spdx/crossref/CrossRef.java

@@ -0,0 +1,71 @@
+/**
+ * 
+ */
+package org.spdx.crossref;
+
+import org.spdx.rdfparser.SpdxRdfConstants;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+/**
+ * @author Smith Tanjong
+ *
+ */
+public class CrossRef {
+	String match;
+	String url;
+	Boolean isValid;
+	Boolean isLive;
+	String timestamp;
+	Boolean isWayBackLink;
+
+	public CrossRef(String url, Boolean isValid, Boolean isLive, Boolean isWayBackLink, String match, String timestamp) {
+		this.url = url;
+		this.isValid = isValid;
+		this.isLive = isLive;
+		this.isWayBackLink = isWayBackLink;
+		this.match = match;
+		this.timestamp = timestamp;
+	}
+
+	public CrossRef() {
+		this.url = null;
+		this.isValid = null;
+		this.isLive = null;
+		this.isWayBackLink = null;
+		this.match = null;
+		this.timestamp = null;
+	}
+
+	public void setUrl(String url) {
+		this.url = url;
+	}
+
+	public void setDetails(Boolean isValid, Boolean isLive, Boolean isWayBackLink, String match, String timestamp) {
+		this.isValid = isValid;
+		this.isLive = isLive;
+		this.isWayBackLink = isWayBackLink;
+		this.match = match;
+		this.timestamp = timestamp;
+	}
+
+	public String toString(){
+		String crossRefDetails = String.format("{%s: %s,%s: %b,%s: %b,%s: %b,%s: %s,%s: %s}",
+				SpdxRdfConstants.PROP_CROSS_REF_URL, url,
+				SpdxRdfConstants.PROP_CROSS_REF_IS_VALID, isValid,
+				SpdxRdfConstants.PROP_CROSS_REF_IS_LIVE, isLive,
+				SpdxRdfConstants.PROP_CROSS_REF_WAYBACK_LINK, isWayBackLink,
+				SpdxRdfConstants.PROP_CROSS_REF_MATCH, match,
+				SpdxRdfConstants.PROP_CROSS_REF_TIMESTAMP, timestamp);
+		return crossRefDetails;
+	}
+
+	public JsonObject toJsonObject() {
+		Gson g = new Gson();		
+		JsonElement el = g.toJsonTree(this);
+		return el.getAsJsonObject();
+	}
+
+}

src/org/spdx/crossref/CrossRefHelper.java

@@ -0,0 +1,98 @@
+/**
+ * Copyright (c) 2014 Source Auditor Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *
+*/
+package org.spdx.crossref;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.spdx.licensexml.LicenseXmlDocument;
+import org.spdx.rdfparser.SpdxRdfConstants;
+
+/**
+ * Helper class that provides details for each url in the array it receives
+ * @author Smith Tanjong
+ *
+ */
+public class CrossRefHelper implements Callable<String[]> {
+	static final Logger logger = LoggerFactory.getLogger(CrossRefHelper.class.getName());
+
+	String[] crossRefUrls;
+
+	/**
+	 * @param crossRefUrls an array of url in string format
+	 */
+    public CrossRefHelper(String[] crossRefUrls) {
+        this.crossRefUrls = crossRefUrls;
+    }
+
+    /**
+	 * @param crossRefUrls the array of urls
+	 * @return urlDetails the Array of string containing the url details
+	 */
+	public static String[] buildUrlDetails(String[] crossRefUrls) {
+		String[] urlDetails = new String[crossRefUrls.length];
+		for (int i = 0; i < crossRefUrls.length; i++) {
+			String url = crossRefUrls[i];
+			CrossRef crossRefDetails = new CrossRef();
+			crossRefDetails.setUrl(url);
+			ExecutorService executorService = Executors.newFixedThreadPool(10);
+
+			Future<Boolean> isValid = executorService.submit(new Valid(url));
+			Future<Boolean> isLive = executorService.submit(new Live(url));
+			Future<Boolean> isWayback = executorService.submit(new Wayback(url));
+			Future<String> timestamp = executorService.submit(new Timestamp());
+
+			try {
+				Boolean isValidUrl = isValid.get(10, TimeUnit.SECONDS);
+		    	Boolean isLiveUrl = isValidUrl ? isLive.get(10, TimeUnit.SECONDS) : false;
+		    	Boolean isWaybackUrl = isValidUrl ? isWayback.get(5, TimeUnit.SECONDS) : false;
+		    	String currentDate = timestamp.get(5, TimeUnit.SECONDS);
+		    	String match = isValidUrl ? "--" : "--";
+		    	crossRefDetails.setDetails(isValidUrl, isLiveUrl, isWaybackUrl, match, currentDate);
+		    	urlDetails[i] = crossRefDetails.toString();
+		    } catch (Exception e) {
+		        // interrupts if there is any possible error
+		    	isValid.cancel(true);
+		    	isLive.cancel(true);
+		    	isWayback.cancel(true);
+		    	timestamp.cancel(true);
+		    	logger.error("Interrupted.",e.getMessage());
+		    	crossRefDetails.setUrl(url);
+		    	crossRefDetails.setDetails(Valid.urlValidator(url), false, Wayback.isWayBackUrl(url), "--", Timestamp.getTimestamp());
+		    	urlDetails[i] = crossRefDetails.toString();
+		    } finally {
+		    	executorService.shutdown();
+		    }
+		    try {
+				executorService.awaitTermination(1000, TimeUnit.SECONDS);
+			} catch (InterruptedException e) {
+				logger.error("Interrupted while waiting for termination",e.getMessage());
+			}
+		}
+		return urlDetails;
+	}
+
+	@Override
+	public String[] call() throws Exception {
+		return buildUrlDetails(crossRefUrls);
+	}
+}

src/org/spdx/crossref/Live.java

@@ -0,0 +1,65 @@
+/**
+ * Copyright (c) 2014 Source Auditor Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *
+*/
+package org.spdx.crossref;
+
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.concurrent.Callable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Determines whether the url is live or not
+ * @author Smith Tanjong
+ *
+ */
+public class Live implements Callable<Boolean>  {
+	static final Logger logger = LoggerFactory.getLogger(Live.class.getName());
+	String url;
+
+	/**
+	 * @param url the url in string form
+	 */
+    public Live(String url) {
+        this.url = url;
+    }
+
+    /**
+	 * @param url the url in string form
+	 * @return true/false if the url is live or not
+	 */
+	public static boolean urlLinkExists(String URLName){
+	    try {
+	      HttpURLConnection.setFollowRedirects(false);
+	      HttpURLConnection con = (HttpURLConnection) new URL(URLName).openConnection();
+	      con.setRequestMethod("HEAD");
+	      con.setConnectTimeout(8500);
+	      return (con.getResponseCode() == HttpURLConnection.HTTP_OK || con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED);
+	    }
+	    catch (Exception e) {
+	    	logger.warn("Failed checking live status.",e.getMessage());
+	        return false;
+	    }
+	  }
+
+	@Override
+	public Boolean call() throws Exception {
+		return urlLinkExists(url);
+	}
+
+}

src/org/spdx/crossref/Match.java

@@ -0,0 +1,22 @@
+/**
+ * Copyright (c) 2014 Source Auditor Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *
+*/
+
+package org.spdx.crossref;
+
+public class Match {
+
+}

src/org/spdx/crossref/Timestamp.java

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) 2014 Source Auditor Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *
+*/
+
+package org.spdx.crossref;
+
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.FormatStyle;
+import java.util.concurrent.Callable;
+
+/**
+ * Gets the current timestamp, to be added to the url details
+ * @author Smith Tanjong
+ *
+ */
+public class Timestamp implements Callable<String> {
+
+	/**
+	 * @return timestamp the current timestamp in UTC
+	 */
+    public static String getTimestamp(){
+		// Get current timestamp
+    	DateTimeFormatter isoDateTime = DateTimeFormatter.ISO_DATE_TIME;
+		DateTimeFormatter formatter = isoDateTime.ofLocalizedDateTime( FormatStyle.SHORT ).ofPattern("YYYY-MM-dd - HH:mm:ss");
+		String timeStamp = ZonedDateTime.now( ZoneOffset.UTC ).format( formatter );
+		return timeStamp.toString();
+	}
+
+	@Override
+	public String call() throws Exception {
+		return getTimestamp();
+	}
+
+}

src/org/spdx/crossref/UrlConstants.java

@@ -0,0 +1,35 @@
+/**
+ * Copyright (c) 2014 Source Auditor Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *
+*/
+
+package org.spdx.crossref;
+
+/**
+ * Constants which are useful for the check on the validity and url kind of the url
+ * @author Smith Tanjong
+ *
+ */
+public class UrlConstants {
+	public static final String [] INVALID_URL_DOMAINS = {"localhost", "127.0.0.1"};
+	public static final String [] WAYBACK_URLS = {"web.archive.org", "wayback.archive.org"};
+
+	public static final Integer CROSS_REF_INDEX_URL = 0;
+	public static final Integer CROSS_REF_INDEX_ISVALID = 1;
+	public static final Integer CROSS_REF_INDEX_ISLIVE = 2;
+	public static final Integer CROSS_REF_INDEX_ISWAYBACKLINK = 3;
+	public static final Integer CROSS_REF_INDEX_MATCH = 4;
+	public static final Integer CROSS_REF_INDEX_TIMESTAMP = 5;
+}