inject comments

java/codebuff.iml

@@ -18,5 +18,6 @@
     <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.11" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
     <orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.4" level="project" />
+    <orderEntry type="library" name="Maven: com.google.guava:guava:19.0" level="project" />
   </component>
 </module>

java/codebuff.ipr

@@ -220,6 +220,17 @@
     <mapping directory="$PROJECT_DIR$/.." vcs="Git" />
   </component>
   <component name="libraryTable">
+    <library name="Maven: com.google.guava:guava:19.0">
+      <CLASSES>
+        <root url="jar://$MAVEN_REPOSITORY$/com/google/guava/guava/19.0/guava-19.0.jar!/" />
+      </CLASSES>
+      <JAVADOC>
+        <root url="jar://$MAVEN_REPOSITORY$/com/google/guava/guava/19.0/guava-19.0-javadoc.jar!/" />
+      </JAVADOC>
+      <SOURCES>
+        <root url="jar://$MAVEN_REPOSITORY$/com/google/guava/guava/19.0/guava-19.0-sources.jar!/" />
+      </SOURCES>
+    </library>
     <library name="Maven: junit:junit:4.11">
       <CLASSES>
         <root url="jar://$MAVEN_REPOSITORY$/junit/junit/4.11/junit-4.11.jar!/" />

java/pom.xml

@@ -47,6 +47,11 @@
 			<artifactId>commons-lang3</artifactId>
 			<version>3.4</version>
 		</dependency>
+		<dependency>
+			<groupId>com.google.guava</groupId>
+			<artifactId>guava</artifactId>
+			<version>19.0</version>
+		</dependency>
 	</dependencies>
 
 	<build>

java/src/org/antlr/codebuff/Formatter.java

@@ -1,5 +1,6 @@
 package org.antlr.codebuff;
 
+import com.google.common.base.CharMatcher;
 import org.antlr.v4.runtime.CommonToken;
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.ParserRuleContext;
@@ -112,6 +113,8 @@ public void processToken(int indexIntoRealTokens, int tokenIndexInStream) {
 		CommonToken curToken = (CommonToken)tokens.get(tokenIndexInStream);
 		String tokText = curToken.getText();
 
+		emitCommentsToTheLeft(tokenIndexInStream);
+
 		int[] features = getNodeFeatures(tokenToNodeMap, doc, tokenIndexInStream, line, tabSize);
 		// must set "prev end column" value as token stream doesn't have it;
 		// we're tracking it as we emit tokens
@@ -215,6 +218,50 @@ else if ( (align&0xFF)==CAT_INDENT_FROM_ANCESTOR_FIRST_TOKEN ) {
 		charPosInLine += n;
 	}
 
+	/** Look into the token stream to get the comments to the left of current
+	 *  token. Emit all whitespace and comments except for whitespace at the
+	 *  end as we'll inject that per newline prediction.
+	 */
+	public void emitCommentsToTheLeft(int tokenIndexInStream) {
+		List<Token> hiddenTokensToLeft = tokens.getHiddenTokensToLeft(tokenIndexInStream);
+		if ( hiddenTokensToLeft!=null ) {
+			// if at least one is not whitespace, assume it's a comment and print all hidden stuff including whitespace
+			boolean hasComment = false;
+			for (Token hidden : hiddenTokensToLeft) {
+				String hiddenText = hidden.getText();
+				if ( !hiddenText.matches("\\s+") ) {
+					hasComment = true;
+					break;
+				}
+			}
+			if ( hasComment ) {
+				// avoid whitespace at end of sequence as we'll inject that
+				int last = -1;
+				for (int i=hiddenTokensToLeft.size()-1; i>=0; i--) {
+					Token hidden = hiddenTokensToLeft.get(i);
+					String hiddenText = hidden.getText();
+					if ( !hiddenText.matches("\\s+") ) {
+						last = i;
+						break;
+					}
+				}
+				List<Token> stripped = hiddenTokensToLeft.subList(0, last+1);
+				for (Token hidden : stripped) {
+					String hiddenText = hidden.getText();
+					output.append(hiddenText);
+					if ( hiddenText.matches("\\n+") ) {
+						line += CharMatcher.is('\n').countIn(hiddenText);
+						charPosInLine = 0;
+					}
+					else {
+						// if a comment or plain ' ', must count char position
+						charPosInLine += hiddenText.length();
+					}
+				}
+			}
+		}
+	}
+
 	public TokenPositionAnalysis getTokenAnalysis(int[] features, int indexIntoRealTokens, int tokenIndexInStream,
 	                                              int injectNewline,
 	                                              int alignWithPrevious,