Field add clean 1 (#70)

* Add helper methods to get field values, write new fields to json fiel via json writer and setter methods in the tools repo

* Add helper methods to get field values, write new fields to json fiel via json writer and setter methods in the tools repo

* Undo change: Add back print statement

* Change JSONArray to Array of strings

* Correct merge error

* Update rdf and html documents

* Remove whitespace from all files

* Comment license text code

* rename seealsoDetails to crossrefs

Test/org/spdx/licenselistpublisher/UnitTestHelper.java

@@ -95,7 +95,7 @@ public static boolean isArraysEquivalent(RdfModelObject[] a1,
 	/**
 	 * @param filePath Path for file
 	 * @return Text from the file
-	 * @throws IOException 
+	 * @throws IOException
 	 */
 	public static String fileToText(String filePath) throws IOException {
 		return Files.toString(new File(filePath), Charset.forName("UTF-8"));

Test/org/spdx/licensexml/LicenseXmlDocumentTest.java

@@ -45,7 +45,7 @@
  *
  */
 public class LicenseXmlDocumentTest {
-	
+
 	static final String TEST_FILE_PATH = "TestFiles" + File.separator + "test-license.xml";
 	static final String TEST_OPTIONAL_FILE_PATH = "TestFiles" + File.separator + "test-optional-annotations.xml";
 	private static final String TEST_LICENSE_COMMENT = "Test note";
@@ -86,8 +86,8 @@ public class LicenseXmlDocumentTest {
 			"Last Paragraph exc <<var;name=\"altteste\";original=\"Alternate Text exc\";match=\".+\">> Non matching line. e<<beginOptional>> Optional text exc<<endOptional>>";
 	private static final String TEST_DEP_LICENSE_VERSION = "2.2";
 	private static final String AGPL3ONLY_FILE_PATH = "TestFiles" + File.separator + "AGPL-3.0-only.xml";
-	
-	
+
+
 
 	/**
 	 * @throws java.lang.Exception
@@ -112,7 +112,7 @@ public void testLicenseXmlDocumentFile() throws Exception {
 		new LicenseXmlDocument(licenseFile);
 		// I guess if we don't get any exceptions, it passed
 	}
-	
+
 	@Test
 	public void testOptionalAnnotations() throws Exception {
 		File licenseFile = new File(TEST_OPTIONAL_FILE_PATH);
@@ -129,8 +129,8 @@ public void testOptionalAnnotations() throws Exception {
 
 	/**
 	 * Test method for {@link org.spdx.licensexml.LicenseXmlDocument#getListedLicense()}.
-	 * @throws LicenseXmlException 
-	 * @throws InvalidSPDXAnalysisException 
+	 * @throws LicenseXmlException
+	 * @throws InvalidSPDXAnalysisException
 	 */
 	@Test
 	public void testGetListedLicense() throws LicenseXmlException, InvalidSPDXAnalysisException {
@@ -165,8 +165,8 @@ public void testGetListedLicense() throws LicenseXmlException, InvalidSPDXAnalys
 
 	/**
 	 * Test method for {@link org.spdx.licensexml.LicenseXmlDocument#getLicenseException()}.
-	 * @throws LicenseXmlException 
-	 * @throws InvalidSPDXAnalysisException 
+	 * @throws LicenseXmlException
+	 * @throws InvalidSPDXAnalysisException
 	 */
 	@Test
 	public void testGetLicenseException() throws LicenseXmlException, InvalidSPDXAnalysisException {
@@ -191,7 +191,7 @@ public void testParserBehavior() throws ParserConfigurationException, SAXExcepti
 		String result = LicenseXmlHelper.dumpLicenseDom((Element) doc.getDocumentElement().getElementsByTagName("license").item(0));
 		assertTrue(result.length() > 0);
 	}
-	
+
 	@Test
 	public void testRegressionAgpl3Only() throws LicenseXmlException, InvalidSPDXAnalysisException, InvalidLicenseTemplateException {
 		File licenseFile = new File(AGPL3ONLY_FILE_PATH);

pom.xml

@@ -71,6 +71,11 @@
 		    <artifactId>commons-validator</artifactId>
 		    <version>1.6</version>
 		</dependency>
+		<dependency>
+		    <groupId>org.jsoup</groupId>
+		    <artifactId>jsoup</artifactId>
+		    <version>1.13.1</version>
+		</dependency>
 	</dependencies>
 	<profiles>
 		<profile>

resources/htmlTemplate/LicenseHTMLTemplate.html

@@ -43,7 +43,7 @@
       .page h2, h3, h4, h5 {
         color: #4597cb;
       }
-      
+
       .page h1 {
       font-size: 2em;
       }
@@ -59,19 +59,19 @@
       #footer-copyright {
         margin-top: 25px;
       }
-      
+
       .replacable-license-text {
       color: #CC0000;
       }
-      
+
       .replacable-license-text p var {
       color: #CC0000;
       }
-      
+
       .optional-license-text {
       color: #0000cc;
       }
-      
+
       .optional-license-text p var {
       color: #0000cc;
       }
@@ -111,17 +111,17 @@
       <a href="/" title="Home" rel="home" id="logo">
         <img src="https://spdx.org/sites/cpstandard/files/logo_spdx_250.png" alt="Home" />
       </a>
-      
+
       <div id="name-and-slogan">
         <div id="site-name">
           <h1><a href="/" title="Home" rel="home">Software Package Data Exchange (SPDX)</a></h1>
-        </div>    
+        </div>
       </div>
-      
+
     </div>
   </div> <!-- /header -->
 
-  <div id="highlighted">  
+  <div id="highlighted">
       <div class="region region-highlighted">
       </div>
   </div>
@@ -149,14 +149,17 @@ <h2>Other web pages for this license</h2>
           <div style="margin-left: 20px;">
             <ul>
              {{#otherWebPages}}
-             <li><a href="{{&url}}" rel="rdfs:seeAlso">{{site}}</a></li>
+             <li>
+               <a href="{{&url}}" rel="rdfs:seeAlso" isValid={{isValid}} isLive={{isLive}} isWayBackLink={{isWayBackLink}} match={{match}} timestamp={{timestamp}}>{{site}}</a>
+               <!--Tentative way of displaying information <p>isValid: {{isValid}}, isLive: {{isLive}}, isWayBackLink: {{isWayBackLink}}, match: {{match}}, timestamp: {{timestamp}}</p> -->
+             </li>
              {{/otherWebPages}}
              {{^otherWebPages}}
              <li>None</li>
              {{/otherWebPages}}
            </ul>
           </div>
-          
+
       <div property="spdx:isOsiApproved" style="display: none;">{{osiApproved}}</div>
 
       <h2 id="notes">Notes</h2>

src/org/spdx/htmltemplates/ExceptionHtml.java

@@ -39,16 +39,16 @@
  *
  */
 public class ExceptionHtml {
-	
+
 	static final String TEMPLATE_CLASS_PATH = "resources" + "/" + "htmlTemplate";
 	static final String TEMPLATE_ROOT_PATH = "resources" + File.separator + "htmlTemplate";
 	static final String HTML_TEMPLATE = "ExceptionHTMLTemplate.html";
-	
+
 	Map<String, Object> mustacheMap = Maps.newHashMap();
-	
+
 	/**
 	 * @param exception
-	 * @throws InvalidLicenseTemplateException 
+	 * @throws InvalidLicenseTemplateException
 	 */
 	public ExceptionHtml(ListedLicenseException exception) throws InvalidLicenseTemplateException {
 		List<String> alSourceUrls = Lists.newArrayList();
@@ -70,8 +70,8 @@ public ExceptionHtml(ListedLicenseException exception) throws InvalidLicenseTemp
 	/**
 	 * @param exceptionHtmlFile
 	 * @param exceptionHtmlTocReference
-	 * @throws IOException 
-	 * @throws MustacheException 
+	 * @throws IOException
+	 * @throws MustacheException
 	 */
 	public void writeToFile(File exceptionHtmlFile,
 			String exceptionHtmlTocReference) throws IOException, MustacheException {

src/org/spdx/htmltemplates/ExceptionHtmlToc.java

@@ -42,14 +42,14 @@ public class ExceptionHtmlToc {
 	static final String TEMPLATE_CLASS_PATH = "resources" + "/" + "htmlTemplate";
 	static final String TEMPLATE_ROOT_PATH = "resources" + File.separator + "htmlTemplate";
 	static final String HTML_TEMPLATE = "ExceptionsTocHTMLTemplate.html";
-	
+
 	public static class DeprecatedExceptionRow {
 		private int refNumber;
 		String licenseExceptionId;
 		private String reference;
 		private String exceptionName;
 		private String deprecatedVersion;
-		
+
 		public DeprecatedExceptionRow(String licenseExceptionId, String exceptionName,
 				int refNumber, String reference, String deprecatedVersion) {
 			this.licenseExceptionId = licenseExceptionId;
@@ -198,7 +198,7 @@ public void setLicenseExceptionId(String licenseExceptionId) {
 		}
 
 		private String licenseExceptionId;
-		
+
 		public ExceptionRow(String licenseExceptionId, String exceptionName,
 				int refNumber, String reference) {
 			this.licenseExceptionId = licenseExceptionId;
@@ -207,10 +207,10 @@ public ExceptionRow(String licenseExceptionId, String exceptionName,
 			this.reference = reference;
 		}
 	}
-	
+
 	List<ExceptionRow> exceptions = Lists.newArrayList();
 	List<DeprecatedExceptionRow> deprecatedExceptions = Lists.newArrayList();
-	
+
 	int currentRefNum = 1;
 
 	/**
@@ -221,25 +221,25 @@ public ExceptionRow(String licenseExceptionId, String exceptionName,
 	public void addException(LicenseException exception,
 			String exceptionHTMLReference) {
 		exceptions.add(new ExceptionRow(
-				StringEscapeUtils.escapeHtml4(exception.getLicenseExceptionId()), 
-				StringEscapeUtils.escapeHtml4(exception.getName()),			
+				StringEscapeUtils.escapeHtml4(exception.getLicenseExceptionId()),
+				StringEscapeUtils.escapeHtml4(exception.getName()),
 				currentRefNum++, exceptionHTMLReference));
 	}
-	
+
 	public void addDeprecatedException(LicenseException exception,
 			String exceptionHTMLReference, String deprecatedVersion) {
 		deprecatedExceptions.add(new DeprecatedExceptionRow(
-				StringEscapeUtils.escapeHtml4(exception.getLicenseExceptionId()), 
-				StringEscapeUtils.escapeHtml4(exception.getName()),			
+				StringEscapeUtils.escapeHtml4(exception.getLicenseExceptionId()),
+				StringEscapeUtils.escapeHtml4(exception.getName()),
 				currentRefNum++, exceptionHTMLReference, deprecatedVersion));
 	}
 
 	/**
 	 * Creates and writes an Exception Table of Contents file
 	 * @param exceptionTocFile file to write to
 	 * @param version Version of the License List
-	 * @throws IOException 
-	 * @throws MustacheException 
+	 * @throws IOException
+	 * @throws MustacheException
 	 */
 	public void writeToFile(File exceptionTocFile, String version) throws MustacheException, IOException {
 
@@ -251,7 +251,7 @@ public void writeToFile(File exceptionTocFile, String version) throws MustacheEx
 			public int compare(ExceptionRow arg0, ExceptionRow arg1) {
 				return arg0.getLicenseExceptionId().compareTo(arg1.getLicenseExceptionId());
 			}
-			
+
 		});
 		mustacheMap.put("listedExceptions", exceptions);
 		mustacheMap.put("deprecatedExceptions", deprecatedExceptions);

src/org/spdx/htmltemplates/InvalidLicenseHtmlTemplateException.java

@@ -1,23 +1,23 @@
 /**
- * 
+ *
  */
 package org.spdx.htmltemplates;
 
 /**
  * @author Gary O'Neall
- * 
+ *
  * Exceptions related to HTML license templates
  *
  */
 public class InvalidLicenseHtmlTemplateException extends Exception {
 
 	/**
-	 * 
+	 *
 	 */
 	private static final long serialVersionUID = 1L;
 
 	/**
-	 * 
+	 *
 	 */
 	public InvalidLicenseHtmlTemplateException() {
 		super();

src/org/spdx/htmltemplates/LicenseHTMLFile.java

@@ -27,6 +27,7 @@
 
 import org.apache.commons.lang3.StringEscapeUtils;
 import org.spdx.html.InvalidLicenseTemplateException;
+import org.spdx.licensexml.UrlHelper;
 import org.spdx.rdfparser.license.SpdxListedLicense;
 
 import com.github.mustachejava.DefaultMustacheFactory;
@@ -42,14 +43,14 @@
  *
  */
 public class LicenseHTMLFile {
-	
+
 	static final String TEMPLATE_CLASS_PATH = "resources" + "/" + "htmlTemplate";
 	static final String TEMPLATE_ROOT_PATH = "resources" + File.separator + "htmlTemplate";
 	static final String TEMPLATE_FILE_NAME = "LicenseHTMLTemplate.html";
 	static final boolean USE_SITE = false;	// set to true to use the site name for the link of external web pages
 
 	static final Pattern SITE_PATTERN = Pattern.compile("http://(.*)\\.(.*)(\\.org|\\.com|\\.net|\\.info)");
-	
+
 	/**
 	 * Parses a URL and stores the site name and the original URL
 	 * @author Gary O'Neall
@@ -69,7 +70,27 @@ public void setUrl(String url) {
 		public String getSite() {
 			return getSiteFromUrl(url);
 		}
-
+
+		public boolean getIsValid() {
+			return UrlHelper.urlValidator(url);
+		}
+
+		public boolean getIsLive() {
+			return UrlHelper.urlLinkExists(url);
+		}
+
+		public String getMatch() {
+			return "--";
+		}
+
+		public boolean getIsWayBackLink() {
+			return UrlHelper.isWayBackUrl(url);
+		}
+
+		public String getTimestamp() {
+			return UrlHelper.getTimestamp();
+		}
+
 		@SuppressWarnings("unused")
 		private String getSiteFromUrl(String url) {
 			Matcher matcher = SITE_PATTERN.matcher(url);
@@ -91,11 +112,11 @@ private String getSiteFromUrl(String url) {
 	public LicenseHTMLFile(SpdxListedLicense license) {
 		this.license = license;
 	}
-	
+
 	public LicenseHTMLFile() {
 		this(null);
 	}
-	
+
 	/**
 	 * @return the license
 	 */
@@ -109,7 +130,7 @@ public SpdxListedLicense getLicense() {
 	public void setLicense(SpdxListedLicense license) {
 		this.license = license;
 	}
-	
+
 	public void writeToFile(File htmlFile, String tableOfContentsReference) throws IOException, MustacheException, InvalidLicenseTemplateException {
 		FileOutputStream stream = null;
 		OutputStreamWriter writer = null;
@@ -137,12 +158,12 @@ public void writeToFile(File htmlFile, String tableOfContentsReference) throws I
 			if (stream != null) {
 				stream.close();
 			}
-		}		
+		}
 	}
 	/**
 	 * @return
-	 * @throws  
-	 * @throws LicenseTemplateRuleException 
+	 * @throws
+	 * @throws LicenseTemplateRuleException
 	 */
 	private Map<String, Object> buildMustachMap() throws InvalidLicenseTemplateException {
 			Map<String, Object> retval = Maps.newHashMap();