Makes HiddenTokenSequencer configurable (save/restore). Adds test.

The HiddenTokenSequencer now asks an advisor if hidden state should be
saved/restored to work around a sequencing problem.

Test added to show that sequencing gets start/end wrong.

org.cloudsmith.geppetto.pp.dsl.tests/src/org/cloudsmith/geppetto/pp/dsl/tests/TestPPFormattingFailing.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2011 Cloudsmith Inc. and other contributors, as listed below.
+ * Copyright (c) 2012 Cloudsmith Inc. and other contributors, as listed below.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -16,6 +16,7 @@
 import org.cloudsmith.geppetto.pp.dsl.formatting.PPSemanticLayout;
 import org.cloudsmith.geppetto.pp.dsl.formatting.PPStylesheetProvider;
 import org.cloudsmith.geppetto.pp.dsl.ppformatting.PPIndentationInformation;
+import org.cloudsmith.geppetto.pp.dsl.tests.utils.DebugHiddenTokenSequencer;
 import org.cloudsmith.xtext.dommodel.IDomNode;
 import org.cloudsmith.xtext.dommodel.formatter.CSSDomFormatter;
 import org.cloudsmith.xtext.dommodel.formatter.DomNodeLayoutFeeder;
@@ -24,8 +25,10 @@
 import org.cloudsmith.xtext.dommodel.formatter.ILayoutManager;
 import org.cloudsmith.xtext.dommodel.formatter.css.DomCSS;
 import org.cloudsmith.xtext.serializer.DomBasedSerializer;
+import org.cloudsmith.xtext.serializer.acceptor.IHiddenTokenSequencerAdvisor;
 import org.eclipse.emf.ecore.EObject;
 import org.eclipse.xtext.formatting.IIndentationInformation;
+import org.eclipse.xtext.junit.serializer.DebugSequenceAcceptor;
 import org.eclipse.xtext.resource.XtextResource;
 import org.eclipse.xtext.serializer.ISerializer;
 import org.eclipse.xtext.serializer.diagnostic.ISerializationDiagnostic.Acceptor;
@@ -37,12 +40,13 @@
 import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
+import com.google.inject.Module;
 import com.google.inject.Provider;
 import com.google.inject.name.Names;
 import com.google.inject.util.Modules;
 
 /**
- * @author henrik
+ * Triggers problems with computation of the current hidden state during sequencing.
  * 
  */
 public class TestPPFormattingFailing extends AbstractPuppetTests {
@@ -63,7 +67,48 @@ public ReplaceRegion format(IDomNode dom, ITextRegion regionToFormat, IFormattin
 
 	}
 
-	public static class TestSetup extends PPTestSetup {
+	public static class DebugHiddenProvider implements Provider<IHiddenTokenSequencer> {
+
+		@Override
+		public IHiddenTokenSequencer get() {
+			DebugHiddenTokenSequencer result = new DebugHiddenTokenSequencer();
+			TestPPFormattingFailing.theDebugAcceptor = result;
+			return result;
+		}
+
+	}
+
+	/**
+	 * Advices a hidden token sequencer to *not* save and restore the hidden state when processing
+	 * rulecalls.
+	 * 
+	 */
+	public static class NonSaveRestorHiddenStateAdvise implements IHiddenTokenSequencerAdvisor {
+
+		@Override
+		public boolean shouldSaveRestoreState() {
+			return false;
+		}
+
+	}
+
+	public static class TestSetupDebugOutput extends TestSpecificSetup {
+		public static class TestDebugModule extends TestSpecificSetup.TestModule {
+
+			@Override
+			public void configureIHiddenTokenSequencer(Binder binder) {
+				binder.bind(IHiddenTokenSequencer.class).toProvider(DebugHiddenProvider.class);
+				binder.bind(IHiddenTokenSequencerAdvisor.class).to(NonSaveRestorHiddenStateAdvise.class);
+			}
+		}
+
+		@Override
+		public Module getModule() {
+			return new TestDebugModule();
+		}
+	}
+
+	public static class TestSpecificSetup extends PPTestSetup {
 		public static class TestModule extends PPTestModule {
 
 			@Override
@@ -72,8 +117,8 @@ public void configure(Binder binder) {
 				binder.bind(ISerializer.class).to(DomBasedSerializer.class);
 				binder.bind(IDomModelFormatter.class).to(DebugFormatter.class);
 				// Want serializer to insert empty WS even if there is no node model
-				binder.bind(IHiddenTokenSequencer.class).to(
-					org.cloudsmith.xtext.serializer.acceptor.HiddenTokenSequencer.class);
+				// binder.bind(IHiddenTokenSequencer.class).to(
+				// org.cloudsmith.xtext.serializer.acceptor.HiddenTokenSequencer.class);
 
 				// Bind the default style sheet (TODO: should use a test specific sheet)
 				binder.bind(DomCSS.class).toProvider(PPStylesheetProvider.class);
@@ -82,20 +127,47 @@ public void configure(Binder binder) {
 				// binder.bind(IIndentationInformation.class).to(IIndentationInformation.Default.class);
 
 				binder.bind(ILayoutManager.class).annotatedWith(Names.named("Default")).to(PPSemanticLayout.class);
+
+				// configureIHiddenTokenSequencer(binder);
+			}
+
+			public void configureIHiddenTokenSequencer(Binder binder) {
+				// bind the real HiddenTokenSequencer
+				binder.bind(IHiddenTokenSequencer.class).to(
+					org.cloudsmith.xtext.serializer.acceptor.HiddenTokenSequencer.class);
+				// advise it to not save / restore hidden state to trigger error
+				binder.bind(IHiddenTokenSequencerAdvisor.class).to(NonSaveRestorHiddenStateAdvise.class);
+
 			}
 		}
 
 		@Override
 		public Injector createInjector() {
 			return Guice.createInjector(Modules.override(new org.cloudsmith.geppetto.pp.dsl.PPRuntimeModule()).with(
-				new TestModule()));
+				getModule()));
+		}
+
+		public Module getModule() {
+			return new TestModule();
 		}
 	}
 
+	public static DebugSequenceAcceptor theDebugAcceptor;
+
 	private String doubleQuote(String s) {
 		return '"' + s + '"';
 	}
 
+	@Override
+	public void setUp() throws Exception {
+		super.setUp();
+		// Produces debug output for the hidden sequencer
+		with(TestSetupDebugOutput.class);
+
+		// // Runs with configuration that shows problem
+		// with(TestSpecificSetup.class);
+	}
+
 	/**
 	 * Don't want to test serializer as this test barfs on some PP constructs (with and without serialization
 	 * is not the same result).
@@ -114,17 +186,20 @@ protected boolean shouldTestSerializer(XtextResource resource) {
 	 * @throws Exception
 	 */
 	public void test_Serialize_DoubleQuotedString_1() throws Exception {
-		String original = "before${var}/after${1+2}$$${$var}";
-		String formatted = doubleQuote("before${var}/after${1 + 2}$$${$var}");
+		// String original = "before${var}/after${1+2}$$${$var}";
+		// String formatted = doubleQuote("before${var}/after${1 + 2}$$${$var}");
+		String code = doubleQuote("${1+2}");
+		String formatted = doubleQuote("${1 + 2}");
 		formatted += "\n";
-		String code = doubleQuote(original);
 		XtextResource r = getResourceFromString(code);
 		EObject result = r.getContents().get(0);
 		assertTrue("Should be a PuppetManifest", result instanceof PuppetManifest);
 		result = ((PuppetManifest) result).getStatements().get(0);
 		assertTrue("Should be a DoubleQuotedString", result instanceof DoubleQuotedString);
 
 		String s = serializeFormatted(r.getContents().get(0));
+		if(theDebugAcceptor != null)
+			System.err.println(theDebugAcceptor.toString());
 		assertEquals("Serialization of interpolated string should produce same result", formatted, s);
 	}
 }

org.cloudsmith.geppetto.pp.dsl.tests/src/org/cloudsmith/geppetto/pp/dsl/tests/utils/DebugHiddenTokenSequencer.java

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) 2012 Cloudsmith Inc. and other contributors, as listed below.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ * 
+ * Contributors:
+ *   Cloudsmith
+ * 
+ */
+package org.cloudsmith.geppetto.pp.dsl.tests.utils;
+
+import org.eclipse.emf.ecore.EObject;
+import org.eclipse.xtext.junit.serializer.DebugSequenceAcceptor;
+import org.eclipse.xtext.serializer.acceptor.ISequenceAcceptor;
+import org.eclipse.xtext.serializer.diagnostic.ISerializationDiagnostic.Acceptor;
+import org.eclipse.xtext.serializer.sequencer.IHiddenTokenSequencer;
+
+/**
+ * A DebugSequenceAcceptor that implements IHiddenTokenSequencer.
+ * 
+ */
+public class DebugHiddenTokenSequencer extends DebugSequenceAcceptor implements IHiddenTokenSequencer {
+
+	@Override
+	public void init(EObject context, EObject semanticObject, ISequenceAcceptor sequenceAcceptor, Acceptor errorAcceptor) {
+		add("HIDDEN INIT", "", "", -1, NO_NODE);
+
+	}
+
+}

org.cloudsmith.geppetto.pp.dsl/src/org/cloudsmith/xtext/serializer/acceptor/HiddenTokenSequencer.java

@@ -52,6 +52,9 @@
  */
 public class HiddenTokenSequencer implements IHiddenTokenSequencer, ISyntacticSequenceAcceptor {
 
+	@Inject
+	protected IHiddenTokenSequencerAdvisor advisor;
+
 	@Inject
 	protected IHiddenTokenHelper hiddenTokenHelper;
 
@@ -448,7 +451,8 @@ protected void push(RuleCall rc) {
 	}
 
 	private void restoreHidden() {
-		currentHidden = hiddenStack.remove(hiddenStack.size() - 1);
+		if(advisor.shouldSaveRestoreState())
+			currentHidden = hiddenStack.remove(hiddenStack.size() - 1);
 	}
 
 	/**
@@ -457,6 +461,9 @@ private void restoreHidden() {
 	 * @param eobj
 	 */
 	private void saveHidden(EObject eobj) {
+		if(!advisor.shouldSaveRestoreState())
+			return;
+
 		hiddenStack.add(currentHidden);
 		ParserRule r = closestContainingParserRule(eobj);
 		if(r != null && r.isDefinesHiddenTokens())

org.cloudsmith.geppetto.pp.dsl/src/org/cloudsmith/xtext/serializer/acceptor/IHiddenTokenSequencerAdvisor.java

@@ -0,0 +1,33 @@
+/**
+ * Copyright (c) 2011 Cloudsmith Inc. and other contributors, as listed below.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ * 
+ * Contributors:
+ *   Cloudsmith
+ * 
+ */
+package org.cloudsmith.xtext.serializer.acceptor;
+
+import com.google.inject.ImplementedBy;
+
+/**
+ * Helps with advice to the HiddenTokenSequencer
+ * 
+ */
+@ImplementedBy(org.cloudsmith.xtext.serializer.acceptor.IHiddenTokenSequencerAdvisor.Default.class)
+public interface IHiddenTokenSequencerAdvisor {
+
+	public static class Default implements IHiddenTokenSequencerAdvisor {
+
+		@Override
+		public boolean shouldSaveRestoreState() {
+			return true;
+		}
+
+	}
+
+	public boolean shouldSaveRestoreState();
+}