Made PreprocessorMacro and TokenList public (#676)

### Changes

With this PR, `PreprocessorMacro` and its subclasses are no longer package-private but public instead. Same goes for `TokenList`.

### Reasons for the changes

When parsing C/C++ code with CDT as a standalone library I want to track certain macro expansions using the [MacroExpander](https://github.com/eclipse-cdt/cdt/blob/main/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/MacroExpander.java).

It is not actually part of the public API but declared as `public`. It takes an argument `macroDictionary` of type `CharArrayMap<PreprocessorMacro>` in its constructor. Currently, `PreprocessorMacro` is package-privat and thus, not available without shadowing (i.e., implementing my custom macro tracker in the package `org.eclipse.cdt.internal.core.parser.scanner`).

It questionable whether it is a good design decision to have public and private API mixed together here (rendering the usage of the constructor of `MacroExpander` impossible).

Also, another problem occurs once you take the route of shadowing to have access to `PreprocessorMacro`: the CDT jars are all signed during release (makes sense) but, sadly speaking, that will result in a `SecurityException` when trying to call their API from your own (unsigned) code which lives in the shadowed package (in my case in `org.eclipse.cdt.internal.core.parser.scanner`).

### Problem solved

MacroExpander can now be used from the outside because `PreprocessorMacro` and `TokenList` are no longer package-private. Shadowing is no longer needed. Signed jars are no longer a problem.

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CounterMacro.java

@@ -0,0 +1,37 @@
+/*******************************************************************************
+ * Copyright (c) 2007, 2020 Wind River Systems, Inc. and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Markus Schorn - initial API and implementation
+ *     Alexander Fedorov (ArSysOp) - Bug 561992
+ *******************************************************************************/
+package org.eclipse.cdt.internal.core.parser.scanner;
+
+import org.eclipse.cdt.core.parser.IToken;
+
+public final class CounterMacro extends DynamicMacro {
+	private static final char[] ZERO = { '0' };
+
+	private long fValue;
+
+	public CounterMacro(char[] name) {
+		super(name);
+	}
+
+	@Override
+	public Token execute(MacroExpander expander) {
+		return new TokenWithImage(IToken.tINTEGER, null, 0, 0, String.valueOf(fValue++).toCharArray());
+	}
+
+	@Override
+	public char[] getExpansionImage() {
+		return ZERO;
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/DateMacro.java

@@ -0,0 +1,51 @@
+/*******************************************************************************
+ * Copyright (c) 2007, 2020 Wind River Systems, Inc. and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Markus Schorn - initial API and implementation
+ *     Alexander Fedorov (ArSysOp) - Bug 561992
+ *******************************************************************************/
+package org.eclipse.cdt.internal.core.parser.scanner;
+
+import java.text.DateFormatSymbols;
+import java.util.Calendar;
+
+import org.eclipse.cdt.core.parser.IToken;
+
+public final class DateMacro extends DynamicMacro {
+	public DateMacro(char[] name) {
+		super(name);
+	}
+
+	@Override
+	public Token execute(MacroExpander expander) {
+		return new TokenWithImage(IToken.tSTRING, null, 0, 0, createDate());
+	}
+
+	private char[] createDate() {
+		char[] charArray;
+		StringBuilder buffer = new StringBuilder("\""); //$NON-NLS-1$
+		Calendar cal = Calendar.getInstance();
+		DateFormatSymbols dfs = new DateFormatSymbols();
+		buffer.append(dfs.getShortMonths()[cal.get(Calendar.MONTH)]);
+		buffer.append(" "); //$NON-NLS-1$
+		append(buffer, cal.get(Calendar.DAY_OF_MONTH));
+		buffer.append(" "); //$NON-NLS-1$
+		buffer.append(cal.get(Calendar.YEAR));
+		buffer.append("\""); //$NON-NLS-1$
+		charArray = buffer.toString().toCharArray();
+		return charArray;
+	}
+
+	@Override
+	public char[] getExpansionImage() {
+		return createDate();
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/DynamicMacro.java

@@ -0,0 +1,49 @@
+/*******************************************************************************
+ * Copyright (c) 2007, 2020 Wind River Systems, Inc. and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Markus Schorn - initial API and implementation
+ *     Alexander Fedorov (ArSysOp) - Bug 561992
+ *******************************************************************************/
+package org.eclipse.cdt.internal.core.parser.scanner;
+
+import org.eclipse.cdt.internal.core.parser.scanner.Lexer.LexerOptions;
+
+public abstract class DynamicMacro extends PreprocessorMacro {
+
+	public DynamicMacro(char[] name) {
+		super(name);
+	}
+
+	@Override
+	public final char[] getExpansion() {
+		return getExpansionImage();
+	}
+
+	public abstract Token execute(MacroExpander expander);
+
+	@Override
+	public TokenList getTokens(MacroDefinitionParser mdp, LexerOptions lexOptions, MacroExpander expander) {
+		TokenList result = new TokenList();
+		result.append(execute(expander));
+		return result;
+	}
+
+	final protected void append(StringBuilder buffer, int value) {
+		if (value < 10)
+			buffer.append("0"); //$NON-NLS-1$
+		buffer.append(value);
+	}
+
+	@Override
+	public final boolean isDynamic() {
+		return true;
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/FileMacro.java

@@ -0,0 +1,36 @@
+/*******************************************************************************
+ * Copyright (c) 2007, 2020 Wind River Systems, Inc. and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Markus Schorn - initial API and implementation
+ *     Alexander Fedorov (ArSysOp) - Bug 561992
+ *******************************************************************************/
+package org.eclipse.cdt.internal.core.parser.scanner;
+
+import org.eclipse.cdt.core.parser.IToken;
+
+public final class FileMacro extends DynamicMacro {
+	public FileMacro(char[] name) {
+		super(name);
+	}
+
+	@Override
+	public Token execute(MacroExpander expander) {
+		StringBuffer buffer = new StringBuffer("\""); //$NON-NLS-1$
+		buffer.append(expander.getCurrentFilename().replace("\\", "\\\\")); //$NON-NLS-1$//$NON-NLS-2$
+		buffer.append('\"');
+		return new TokenWithImage(IToken.tSTRING, null, 0, 0, buffer.toString().toCharArray());
+	}
+
+	@Override
+	public char[] getExpansionImage() {
+		return "\"file\"".toCharArray(); //$NON-NLS-1$
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/FunctionStyleMacro.java

@@ -0,0 +1,119 @@
+/*******************************************************************************
+ * Copyright (c) 2007, 2020 Wind River Systems, Inc. and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Markus Schorn - initial API and implementation
+ *     Alexander Fedorov (ArSysOp) - Bug 561992
+ *******************************************************************************/
+package org.eclipse.cdt.internal.core.parser.scanner;
+
+import org.eclipse.cdt.core.parser.Keywords;
+
+public class FunctionStyleMacro extends ObjectStyleMacro {
+	public static final int NO_VAARGS = 0; // M(a)
+	public static final int VAARGS = 1; // M(...)
+	public static final int NAMED_VAARGS = 2; // M(a...)
+
+	final private char[][] fParamList;
+	final private int fHasVarArgs;
+	private char[] fSignature;
+
+	public FunctionStyleMacro(char[] name, char[][] paramList, int hasVarArgs, char[] expansion) {
+		this(name, paramList, hasVarArgs, 0, expansion.length, null, new CharArray(expansion));
+	}
+
+	public FunctionStyleMacro(char[] name, char[][] paramList, int hasVarArgs, AbstractCharArray expansion,
+			int expansionOffset, int expansionEndOffset) {
+		this(name, paramList, hasVarArgs, expansionOffset, expansionEndOffset, null, expansion);
+	}
+
+	public FunctionStyleMacro(char[] name, char[][] paramList, int hasVarArgs, int expansionFileOffset,
+			int endFileOffset, TokenList expansion, AbstractCharArray source) {
+		super(name, expansionFileOffset, endFileOffset, expansion, source);
+		fParamList = paramList;
+		fHasVarArgs = hasVarArgs;
+	}
+
+	@Override
+	public char[][] getParameterList() {
+		final int length = fParamList.length;
+		if (fHasVarArgs == NO_VAARGS || length == 0) {
+			return fParamList;
+		}
+		char[][] result = new char[length][];
+		System.arraycopy(fParamList, 0, result, 0, length - 1);
+		if (fHasVarArgs == VAARGS) {
+			result[length - 1] = Keywords.cpELLIPSIS;
+		} else {
+			final char[] param = fParamList[length - 1];
+			final int plen = param.length;
+			final int elen = Keywords.cpELLIPSIS.length;
+			final char[] rp = new char[plen + elen];
+			System.arraycopy(param, 0, rp, 0, plen);
+			System.arraycopy(Keywords.cpELLIPSIS, 0, rp, plen, elen);
+			result[length - 1] = rp;
+		}
+		return result;
+	}
+
+	@Override
+	public char[][] getParameterPlaceholderList() {
+		return fParamList;
+	}
+
+	public char[] getSignature() {
+		if (fSignature != null) {
+			return fSignature;
+		}
+
+		StringBuffer result = new StringBuffer();
+		result.append(getName());
+		result.append('(');
+
+		final int lastIdx = fParamList.length - 1;
+		if (lastIdx >= 0) {
+			for (int i = 0; i < lastIdx; i++) {
+				result.append(fParamList[i]);
+				result.append(',');
+			}
+			switch (fHasVarArgs) {
+			case VAARGS:
+				result.append(Keywords.cpELLIPSIS);
+				break;
+			case NAMED_VAARGS:
+				result.append(fParamList[lastIdx]);
+				result.append(Keywords.cpELLIPSIS);
+				break;
+			default:
+				result.append(fParamList[lastIdx]);
+				break;
+			}
+		}
+		result.append(')');
+		final int len = result.length();
+		final char[] sig = new char[len];
+		result.getChars(0, len, sig, 0);
+		fSignature = sig;
+		return sig;
+	}
+
+	/**
+	 * Returns one of {@link FunctionStyleMacro#NO_VAARGS}, {@link #VAARGS} or {@link #NAMED_VAARGS}.
+	 */
+	@Override
+	public int hasVarArgs() {
+		return fHasVarArgs;
+	}
+
+	@Override
+	public boolean isFunctionStyle() {
+		return true;
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LineMacro.java

@@ -0,0 +1,34 @@
+/*******************************************************************************
+ * Copyright (c) 2007, 2020 Wind River Systems, Inc. and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Markus Schorn - initial API and implementation
+ *     Alexander Fedorov (ArSysOp) - Bug 561992
+ *******************************************************************************/
+package org.eclipse.cdt.internal.core.parser.scanner;
+
+import org.eclipse.cdt.core.parser.IToken;
+
+public final class LineMacro extends DynamicMacro {
+	public LineMacro(char[] name) {
+		super(name);
+	}
+
+	@Override
+	public Token execute(MacroExpander expander) {
+		int lineNumber = expander.getCurrentLineNumber();
+		return new TokenWithImage(IToken.tINTEGER, null, 0, 0, Long.toString(lineNumber).toCharArray());
+	}
+
+	@Override
+	public char[] getExpansionImage() {
+		return new char[] { '1' };
+	}
+}