Implement configurable comments

The lexer may now optionally treat ';', '#', or '%' as the start of a line comment. Fix #1
io7m-com · Nov 24, 2017 · dc45253 · dc45253
1 parent db4f4ba
commit dc45253
Show file tree

Hide file tree

Showing 19 changed files with 485 additions and 28 deletions.
diff --git a/README-CHANGES.xml b/README-CHANGES.xml
@@ -48,17 +48,26 @@
         <c:change date="2016-10-04T00:00:00+00:00" summary="Add command line tool for formatting s-expressions."/>
       </c:changes>
     </c:release>
-    <c:release date="2017-11-16T16:32:57+00:00" ticket-system="com.github.io7m.jsx" version="0.8.0">
+    <c:release date="2017-11-24T00:00:00+00:00" ticket-system="com.github.io7m.jsx" version="0.8.0">
       <c:changes>
         <c:change date="2016-11-10T00:00:00+00:00" summary="Add combinators for validating S-expressions."/>
         <c:change date="2017-04-06T00:00:00+00:00" summary="Rename project. Use the new primogenitor POM and 2017 project conventions."/>
         <c:change compatible="false" date="2017-04-06T00:00:00+00:00" summary="Use jlexing 0.2.1, numerous API changes for consistency (see japicmp report for details)."/>
         <c:change compatible="false" date="2017-11-16T00:00:00+00:00" summary="All modules are now Java 9 modules. JDK 9 is now required."/>
-        <c:change compatible="false" date="2017-11-16T16:32:57+00:00" summary="Removed dependency on com.io7m.jnull."/>
+        <c:change compatible="false" date="2017-11-16T00:00:00+00:00" summary="Removed dependency on com.io7m.jnull."/>
+      </c:changes>
+    </c:release>
+    <c:release date="2017-11-24T13:17:34+00:00" ticket-system="com.github.io7m.jsx" version="0.9.0">
+      <c:changes>
+        <c:change date="2017-11-24T00:00:00+00:00" summary="Add configurable comment tokens">
+          <c:tickets>
+            <c:ticket id="1"/>
+          </c:tickets>
+        </c:change>
       </c:changes>
     </c:release>
   </c:releases>
   <c:ticket-systems>
-    <c:ticket-system default="false" id="com.github.io7m.jsx" url="http://github.com/io7m/jsx/issues/"/>
+    <c:ticket-system default="true" id="com.github.io7m.jsx" url="http://github.com/io7m/jsx/issues/"/>
   </c:ticket-systems>
 </c:changelog>
diff --git a/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/lexer/JSXLexerComment.java b/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/lexer/JSXLexerComment.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright © 2016 <code@io7m.com> http://io7m.com
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
+ * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package com.io7m.jsx.api.lexer;
+
+/**
+ * The available types of comments.
+ */
+
+public enum JSXLexerComment
+{
+  /**
+   * Comments start with {@code #}
+   */
+
+  COMMENT_HASH('#'),
+
+  /**
+   * Comments start with {@code %}
+   */
+
+  COMMENT_PERCENT('%'),
+
+  /**
+   * Comments start with {@code ;}
+   */
+
+  COMMENT_SEMICOLON(';');
+
+  private final int token;
+
+  /**
+   * @return The comment character codepoint
+   */
+
+  public int token()
+  {
+    return this.token;
+  }
+
+  JSXLexerComment(
+    final int in_token)
+  {
+    this.token = in_token;
+  }
+}
diff --git a/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/lexer/JSXLexerConfigurationType.java b/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/lexer/JSXLexerConfigurationType.java
@@ -20,6 +20,7 @@
 import org.immutables.value.Value;
 
 import java.nio.file.Path;
+import java.util.EnumSet;
 import java.util.Optional;
 
 /**
@@ -58,4 +59,15 @@ default boolean newlinesInQuotedStrings()
 
   @Value.Parameter(order = 2)
   Optional<Path> file();
+
+  /**
+   * @return The string(s) used to start line comments
+   */
+
+  @Value.Parameter(order = 3)
+  @Value.Default
+  default EnumSet<JSXLexerComment> comments()
+  {
+    return EnumSet.noneOf(JSXLexerComment.class);
+  }
 }
diff --git a/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenComment.java b/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenComment.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright © 2016 <code@io7m.com> http://io7m.com
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
+ * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package com.io7m.jsx.api.tokens;
+
+import com.io7m.jlexing.core.LexicalPosition;
+import com.io7m.jsx.api.lexer.JSXLexerComment;
+
+import java.nio.file.Path;
+import java.util.Objects;
+
+/**
+ * A comment.
+ */
+
+public final class TokenComment implements TokenType
+{
+  private final LexicalPosition<Path> lex;
+  private final String text;
+  private final JSXLexerComment comment;
+
+  /**
+   * @param in_lex     The lexical information
+   * @param in_comment The comment type
+   * @param in_text    The token text
+   */
+
+  public TokenComment(
+    final LexicalPosition<Path> in_lex,
+    final JSXLexerComment in_comment,
+    final String in_text)
+  {
+    this.lex = Objects.requireNonNull(in_lex, "Lexical information");
+    this.comment = Objects.requireNonNull(in_comment, "Comment");
+    this.text = Objects.requireNonNull(in_text, "Text");
+  }
+
+  /**
+   * @return The comment type
+   */
+
+  public JSXLexerComment comment()
+  {
+    return this.comment;
+  }
+
+  @Override
+  public LexicalPosition<Path> lexical()
+  {
+    return this.lex;
+  }
+
+  /**
+   * @return The text content of the token
+   */
+
+  public String text()
+  {
+    return this.text;
+  }
+
+  @Override
+  public <A, E extends Exception> A matchToken(
+    final TokenMatcherType<A, E> m)
+    throws E
+  {
+    return m.comment(this);
+  }
+
+  @Override
+  public String toString()
+  {
+    final StringBuilder builder = new StringBuilder(64);
+    builder.append("[TokenComment ");
+    builder.append(this.lex);
+    builder.append(": ");
+    builder.append(this.text);
+    builder.append("]");
+    return Objects.requireNonNull(builder.toString(), "Result string");
+  }
+}
diff --git a/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenEOF.java b/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenEOF.java
@@ -17,9 +17,9 @@
 package com.io7m.jsx.api.tokens;
 
 import com.io7m.jlexing.core.LexicalPosition;
-import java.util.Objects;
 
 import java.nio.file.Path;
+import java.util.Objects;
 
 /**
  * End of file.
@@ -36,7 +36,7 @@ public final class TokenEOF implements TokenType
   public TokenEOF(
     final LexicalPosition<Path> in_lex)
   {
-    this.lex = in_lex;
+    this.lex = Objects.requireNonNull(in_lex, "Lexical");
   }
 
   @Override

diff --git a/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenLeftParenthesis.java b/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenLeftParenthesis.java
@@ -17,9 +17,9 @@
 package com.io7m.jsx.api.tokens;
 
 import com.io7m.jlexing.core.LexicalPosition;
-import java.util.Objects;
 
 import java.nio.file.Path;
+import java.util.Objects;
 
 /**
  * Left parenthesis '('.
@@ -36,7 +36,7 @@ public final class TokenLeftParenthesis implements TokenType
   public TokenLeftParenthesis(
     final LexicalPosition<Path> in_lex)
   {
-    this.lex = in_lex;
+    this.lex = Objects.requireNonNull(in_lex, "Lexical");
   }
 
   @Override

diff --git a/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenLeftSquare.java b/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenLeftSquare.java
@@ -17,9 +17,9 @@
 package com.io7m.jsx.api.tokens;
 
 import com.io7m.jlexing.core.LexicalPosition;
-import java.util.Objects;
 
 import java.nio.file.Path;
+import java.util.Objects;
 
 /**
  * Left square bracket '['.
@@ -36,7 +36,7 @@ public final class TokenLeftSquare implements TokenType
   public TokenLeftSquare(
     final LexicalPosition<Path> in_lex)
   {
-    this.lex = in_lex;
+    this.lex = Objects.requireNonNull(in_lex, "Lexical");
   }
 
   @Override

diff --git a/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenMatcherType.java b/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenMatcherType.java
@@ -122,4 +122,18 @@ A rightSquare(
   A symbol(
     TokenSymbol t)
     throws E;
+
+  /**
+   * Match a token.
+   *
+   * @param t The token
+   *
+   * @return A value of {@code A}
+   *
+   * @throws E If required
+   */
+
+  A comment(
+    TokenComment t)
+    throws E;
 }
diff --git a/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenQuotedString.java b/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenQuotedString.java
@@ -17,9 +17,9 @@
 package com.io7m.jsx.api.tokens;
 
 import com.io7m.jlexing.core.LexicalPosition;
-import java.util.Objects;
 
 import java.nio.file.Path;
+import java.util.Objects;
 
 /**
  * A quoted string.

diff --git a/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenRightParenthesis.java b/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenRightParenthesis.java
@@ -17,9 +17,9 @@
 package com.io7m.jsx.api.tokens;
 
 import com.io7m.jlexing.core.LexicalPosition;
-import java.util.Objects;
 
 import java.nio.file.Path;
+import java.util.Objects;
 
 /**
  * Right parenthesis ')'.
@@ -36,7 +36,7 @@ public final class TokenRightParenthesis implements TokenType
   public TokenRightParenthesis(
     final LexicalPosition<Path> in_lex)
   {
-    this.lex = in_lex;
+    this.lex = Objects.requireNonNull(in_lex, "Lexical");
   }
 
   @Override

diff --git a/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenRightSquare.java b/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenRightSquare.java
@@ -17,9 +17,9 @@
 package com.io7m.jsx.api.tokens;
 
 import com.io7m.jlexing.core.LexicalPosition;
-import java.util.Objects;
 
 import java.nio.file.Path;
+import java.util.Objects;
 
 /**
  * Right square bracket ']'.
@@ -36,7 +36,7 @@ public final class TokenRightSquare implements TokenType
   public TokenRightSquare(
     final LexicalPosition<Path> in_lex)
   {
-    this.lex = in_lex;
+    this.lex = Objects.requireNonNull(in_lex, "Lexical");
   }
 
   @Override

diff --git a/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenSymbol.java b/com.io7m.jsx.parser.api/src/main/java/com/io7m/jsx/api/tokens/TokenSymbol.java
@@ -17,9 +17,9 @@
 package com.io7m.jsx.api.tokens;
 
 import com.io7m.jlexing.core.LexicalPosition;
-import java.util.Objects;
 
 import java.nio.file.Path;
+import java.util.Objects;
 
 /**
  * A symbol.