From 9d372d3d8846e7a2db55fe8642f1c06d824dfb91 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Wed, 29 Jul 2026 13:19:34 +0200 Subject: [PATCH 01/11] Align array and scalar semantics with gawk --- README.md | 2 +- .../java/io/jawk/gawk/AbstractGawkSuite.java | 18 ++ src/it/java/io/jawk/gawk/GawkExtensionIT.java | 4 +- src/it/java/io/jawk/gawk/GawkIT.java | 32 +-- src/main/java/io/jawk/backend/AVM.java | 196 +++++++++++------- .../java/io/jawk/backend/RuntimeStack.java | 4 + src/main/java/io/jawk/frontend/AwkParser.java | 121 ++--------- src/main/java/io/jawk/jrt/HashAssocArray.java | 2 +- .../java/io/jawk/jrt/SortedAssocArray.java | 2 +- src/site/markdown/compatibility.md.vm | 2 + src/test/java/io/jawk/AwkTest.java | 36 ++++ 11 files changed, 226 insertions(+), 193 deletions(-) diff --git a/README.md b/README.md index e7e10f41..d79117de 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Jawk is a pure Java implementation of [AWK](https://en.wikipedia.org/wiki/AWK). Jawk fully implements POSIX AWK, and adds support for the most commonly used gawk-specific features: - Builtins, available by default through the built-in GNU Awk compatibility extension: `asort()`, `asorti()`, `typeof()`, `isarray()`, `mkbool()`, `gensub()`, `patsplit()`, `strtonum()`, `systime()`, `mktime()`, `strftime()`, `bindtextdomain()`, `dcgettext()`, `dcngettext()`, and `PROCINFO["sorted_in"]`-controlled array traversal -- Arrays of arrays (`a[i][j]`) and typed regexp literals (`@/re/`) +- Arrays of arrays (`a[i][j]`), including gawk-compatible runtime typing when arrays, scalars, and subarrays are passed to functions, and typed regexp literals (`@/re/`) - Source inclusion with `@include`, namespaces with `@namespace` and `ns::name`, and indirect function calls such as `@functionName(args)` - `BEGINFILE` / `ENDFILE` special patterns, with the `ERRNO` and `ARGIND` special variables, so a script can hook into the command-line file processing loop and skip unreadable files without a fatal error - The `nextfile` statement diff --git a/src/it/java/io/jawk/gawk/AbstractGawkSuite.java b/src/it/java/io/jawk/gawk/AbstractGawkSuite.java index a6a1e961..466592f1 100644 --- a/src/it/java/io/jawk/gawk/AbstractGawkSuite.java +++ b/src/it/java/io/jawk/gawk/AbstractGawkSuite.java @@ -23,12 +23,15 @@ */ import static org.junit.Assume.assumeTrue; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import io.jawk.AwkTestSupport; import io.jawk.CompatibilityTestResources; +import io.jawk.jrt.AwkRuntimeException; /** * Shared helpers for the explicit gawk compatibility integration suites @@ -55,6 +58,21 @@ protected static String gawkText(String fileName) throws IOException { return new String(Files.readAllBytes(gawkPath(fileName)), StandardCharsets.UTF_8); } + protected static void assertGawkRuntimeFailure(String testName) throws Exception { + AwkTestSupport.TestResult result = AwkTestSupport + .awkTest("GAWK " + testName) + .script(gawkText(testName + ".awk")) + .expectThrow(AwkRuntimeException.class) + .run(); + result.assertExpected(); + + String expectedTranscript = gawkText(testName + ".ok"); + String expectedMessage = expectedTranscript.contains(" as an array") ? + "scalar as an array" : "array in a scalar context"; + String actualMessage = result.thrownException().getMessage(); + assertTrue(actualMessage, actualMessage.contains(expectedMessage)); + } + protected static void skip(String reason) { assumeTrue(reason, false); } diff --git a/src/it/java/io/jawk/gawk/GawkExtensionIT.java b/src/it/java/io/jawk/gawk/GawkExtensionIT.java index 61836a34..da07a8be 100644 --- a/src/it/java/io/jawk/gawk/GawkExtensionIT.java +++ b/src/it/java/io/jawk/gawk/GawkExtensionIT.java @@ -50,7 +50,7 @@ public void test_aadelete1() throws Exception { @Test public void test_aadelete2() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("aadelete2"); } @Test @@ -848,7 +848,7 @@ public void test_mdim5() throws Exception { @Test public void test_mdim6() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("mdim6"); } @Test diff --git a/src/it/java/io/jawk/gawk/GawkIT.java b/src/it/java/io/jawk/gawk/GawkIT.java index a395a7ff..618e7c0a 100644 --- a/src/it/java/io/jawk/gawk/GawkIT.java +++ b/src/it/java/io/jawk/gawk/GawkIT.java @@ -98,7 +98,7 @@ public void test_arrayind3() throws Exception { @Test public void test_arrayparm() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("arrayparm"); } @Test @@ -153,17 +153,17 @@ public void test_arryref2() throws Exception { @Test public void test_arryref3() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("arryref3"); } @Test public void test_arryref4() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("arryref4"); } @Test public void test_arryref5() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("arryref5"); } @Test @@ -178,37 +178,37 @@ public void test_arynasty() throws Exception { @Test public void test_aryprm1() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("aryprm1"); } @Test public void test_aryprm2() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("aryprm2"); } @Test public void test_aryprm3() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("aryprm3"); } @Test public void test_aryprm4() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("aryprm4"); } @Test public void test_aryprm5() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("aryprm5"); } @Test public void test_aryprm6() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("aryprm6"); } @Test public void test_aryprm7() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("aryprm7"); } @Test @@ -651,7 +651,7 @@ public void test_fnarray2() throws Exception { @Test public void test_fnaryscl() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("fnaryscl"); } @Test @@ -1530,7 +1530,7 @@ public void test_printfchar() throws Exception { @Test public void test_prmarscl() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("prmarscl"); } @Test @@ -1845,17 +1845,17 @@ public void test_rswhite() throws Exception { @Test public void test_scalar() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("scalar"); } @Test public void test_sclforin() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("sclforin"); } @Test public void test_sclifin() throws Exception { - skip(NON_ZERO_TRANSCRIPT_REASON); + assertGawkRuntimeFailure("sclifin"); } @Test diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 9d88610f..62296015 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -1468,10 +1468,7 @@ private void executeTuples(PositionTracker position) VariableTuple variableTuple = (VariableTuple) tuple; long offset = variableTuple.getVariableOffset(); boolean isGlobal = variableTuple.isGlobal(); - Object o1 = runtimeStack.getVariable(offset, isGlobal); - if (o1 == null) { - o1 = BLANK; - } + Object o1 = resolveVariable(offset, isGlobal, false); Object o2 = pop(); double d1 = JRT.toDouble(o1); double d2 = JRT.toDouble(o2); @@ -1672,23 +1669,17 @@ private void executeTuples(PositionTracker position) DereferenceTuple dereferenceTuple = (DereferenceTuple) tuple; boolean isGlobal = dereferenceTuple.isGlobal(); long offset = dereferenceTuple.getVariableOffset(); - Object o = runtimeStack.getVariable(offset, isGlobal); - if (o == null) { - if (dereferenceTuple.isArray()) { - // is_array - push(runtimeStack.setVariable(offset, newAwkArray(), isGlobal)); - } else { - push(runtimeStack.setVariable(offset, BLANK, isGlobal)); - } - } else { - push(o); - } + push(resolveVariable(offset, isGlobal, dereferenceTuple.isArray())); position.next(); break; } case PEEK_DEREFERENCE: { VariableTuple variableTuple = (VariableTuple) tuple; - push(runtimeStack.getVariable(variableTuple.getVariableOffset(), variableTuple.isGlobal())); + Object value = runtimeStack + .getVariable(variableTuple.getVariableOffset(), variableTuple.isGlobal()); + push( + value instanceof ArgumentReference ? + ((ArgumentReference) value).currentValue() : value); position.next(); break; } @@ -1698,8 +1689,8 @@ private void executeTuples(PositionTracker position) .getVariable(variableTuple.getVariableOffset(), variableTuple.isGlobal()); push( new IndirectArgumentReference( + runtimeStack.getVariableFrame(variableTuple.isGlobal()), variableTuple.getVariableOffset(), - variableTuple.isGlobal(), scalarValue)); position.next(); break; @@ -2016,9 +2007,7 @@ private void executeTuples(PositionTracker position) break; } if (!(o instanceof Map)) { - throw new AwkRuntimeException( - position.lineNumber(), - "Cannot get a key list (via 'in') of a non associative array. arg = " + o.getClass() + ", " + o); + throw new AwkRuntimeException("Attempting to use a scalar as an array."); } @SuppressWarnings("unchecked") Map map = (Map) o; @@ -2211,10 +2200,11 @@ private void executeTuples(PositionTracker position) Address funcAddr = callTuple.getAddress(); long numFormalParams = callTuple.getNumFormalParams(); long numActualParams = callTuple.getNumActualParams(); + Object[] actualArguments = popArguments(numActualParams); + resolveUserFunctionArguments(actualArguments); runtimeStack.pushFrame(numFormalParams, position.currentIndex()); - // Arguments are stacked, so first in the stack is the last for the function - for (long i = numActualParams - 1; i >= 0; i--) { - runtimeStack.setVariable(i, pop(), false); // false = local + for (int i = 0; i < actualArguments.length; i++) { + runtimeStack.setVariable(i, actualArguments[i], false); } position.jump(funcAddr); // position.next(); @@ -2227,7 +2217,7 @@ private void executeTuples(PositionTracker position) String qualifiedName = normalizeIndirectFunctionName(requestedName); IndirectFunctionTarget target = callTuple.getUserFunctions().get(qualifiedName); if (target != null) { - resolveIndirectArguments(actualArguments, target); + resolveUserFunctionArguments(actualArguments); long formalCount = target.getNumFormalParams(); if (actualArguments.length > formalCount) { jrt @@ -2453,7 +2443,7 @@ private void executeTuples(PositionTracker position) break; } if (!(arr instanceof Map)) { - throw new AwkRuntimeException("Attempting to test membership on a non-associative-array."); + throw new AwkRuntimeException("Attempting to use a scalar as an array."); } @SuppressWarnings("unchecked") Map aa = (Map) arr; @@ -2832,17 +2822,24 @@ private String normalizeIndirectFunctionName(String functionName) { return functionName.startsWith("awk::") ? functionName.substring("awk::".length()) : functionName; } - private void resolveIndirectArguments( - Object[] actualArgumentsParam, - IndirectFunctionTarget target) { + private void resolveUserFunctionArguments(Object[] actualArgumentsParam) { for (int index = 0; index < actualArgumentsParam.length; index++) { - actualArgumentsParam[index] = resolveIndirectArgument( - actualArgumentsParam[index], - target.isArrayParameter(index), - false); + actualArgumentsParam[index] = resolveUserFunctionArgument(actualArgumentsParam[index]); } } + private Object resolveUserFunctionArgument(Object argument) { + if (!(argument instanceof ArgumentReference)) { + return argument; + } + ArgumentReference reference = (ArgumentReference) argument; + Object snapshot = reference.snapshot(); + if (snapshot instanceof ArgumentReference) { + return resolveUserFunctionArgument(snapshot); + } + return isUntyped(snapshot) ? reference : snapshot; + } + private void resolveIndirectArguments( Object[] actualArgumentsParam, BuiltinFunction builtin) { @@ -2878,24 +2875,14 @@ private Object resolveIndirectArgument( Object argument, boolean arrayArgument, boolean rawValueArgument) { - if (!(argument instanceof IndirectArgumentReference)) { - if (!(argument instanceof IndirectArrayArgumentReference)) { - return argument; - } - IndirectArrayArgumentReference reference = (IndirectArrayArgumentReference) argument; - return arrayArgument ? - ensureArrayInArray(reference.map, reference.key) : reference.scalarValue; + if (!(argument instanceof ArgumentReference)) { + return argument; } - IndirectArgumentReference reference = (IndirectArgumentReference) argument; - if (!arrayArgument) { - return reference.scalarValue == null && !rawValueArgument ? BLANK : reference.scalarValue; + ArgumentReference reference = (ArgumentReference) argument; + if (rawValueArgument) { + return reference.currentValue(); } - Object value = runtimeStack.getVariable(reference.offset, reference.global); - if (value == null) { - value = newAwkArray(); - runtimeStack.setVariable(reference.offset, value, reference.global); - } - return value; + return resolveArgumentReference(reference, arrayArgument); } private Object invokeIndirectBuiltin( @@ -3736,9 +3723,10 @@ private String execSubOrGSub(boolean isGsub) { */ private void assign(long l, Object value, boolean isGlobal, PositionTracker position, boolean push) { value = JRT.untypedToBlank(value); + checkScalar(value); // check if curr value already refers to an array - if (runtimeStack.getVariable(l, isGlobal) instanceof Map) { - throw new AwkRuntimeException(position.lineNumber(), "cannot assign anything to an unindexed associative array"); + if (resolveVariable(l, isGlobal, false) instanceof Map) { + throw new AwkRuntimeException(position.lineNumber(), "Attempting to use an array in a scalar context."); } if (push) { push(value); @@ -3757,6 +3745,11 @@ private void assignArray(long offset, Object arrIdx, Object rhs, boolean isGloba private void assignMapElement(Map array, Object arrIdx, Object rhs) { checkScalar(arrIdx); rhs = JRT.untypedToBlank(rhs); + checkScalar(rhs); + if (JRT.containsAwkKey(array, arrIdx) + && JRT.getAssocArrayValue(array, arrIdx) instanceof Map) { + throw new AwkRuntimeException("Attempting to use an array in a scalar context."); + } array.put(arrIdx, rhs); push(rhs); } @@ -3766,8 +3759,8 @@ private void assignMapElement(Map array, Object arrIdx, Object r * is placed back into that variable. */ private Object inc(long l, boolean isGlobal) { - Object o = runtimeStack.getVariable(l, isGlobal); - if (o == null || o instanceof UninitializedObject) { + Object o = resolveVariable(l, isGlobal, false); + if (o instanceof UninitializedObject) { o = ZERO; runtimeStack.setVariable(l, o, isGlobal); } @@ -3781,8 +3774,8 @@ private Object inc(long l, boolean isGlobal) { * is placed back into that variable. */ private Object dec(long l, boolean isGlobal) { - Object o = runtimeStack.getVariable(l, isGlobal); - if (o == null || o instanceof UninitializedObject) { + Object o = resolveVariable(l, isGlobal, false); + if (o instanceof UninitializedObject) { o = ZERO; runtimeStack.setVariable(l, o, isGlobal); } @@ -4172,21 +4165,45 @@ private Map newAwkArray() { } private Map ensureMapVariable(long offset, boolean isGlobal) { - Object value = runtimeStack.getVariable(offset, isGlobal); - if (value == null || value.equals(BLANK) || value instanceof UninitializedObject) { - Map map = newAwkArray(); - runtimeStack.setVariable(offset, map, isGlobal); - return map; - } - return toMap(value); + return toMap(resolveVariable(offset, isGlobal, true)); } private Map getMapVariable(long offset, boolean isGlobal) { + return toMap(resolveVariable(offset, isGlobal, true)); + } + + private Object resolveVariable(long offset, boolean isGlobal, boolean arrayContext) { Object value = runtimeStack.getVariable(offset, isGlobal); - if (value == null || value.equals(BLANK) || value instanceof UninitializedObject) { - return null; + if (value instanceof ArgumentReference) { + value = resolveArgumentReference((ArgumentReference) value, arrayContext); + runtimeStack.setVariable(offset, value, isGlobal); + return value; + } + if (!isUntyped(value)) { + return value; + } + value = arrayContext ? newAwkArray() : BLANK; + runtimeStack.setVariable(offset, value, isGlobal); + return value; + } + + private Object resolveArgumentReference(ArgumentReference reference, boolean arrayContext) { + Object value = reference.currentValue(); + if (value instanceof ArgumentReference) { + value = resolveArgumentReference((ArgumentReference) value, arrayContext); + reference.setValue(value); + return value; } - return toMap(value); + if (!isUntyped(value)) { + return value; + } + value = arrayContext ? newAwkArray() : BLANK; + reference.setValue(value); + return value; + } + + private boolean isUntyped(Object value) { + return value == null || value instanceof UntypedObject; } /** @@ -4198,7 +4215,7 @@ private Map getMapVariable(long offset, boolean isGlobal) { */ private Map toMap(Object value) { if (!(value instanceof Map)) { - throw new AwkRuntimeException("Attempting to treat a scalar as an array."); + throw new AwkRuntimeException("Attempting to use a scalar as an array."); } @SuppressWarnings("unchecked") Map map = (Map) value; @@ -4257,19 +4274,43 @@ private Object normalizeExternalVariableValue(Object value) { private static final UninitializedObject BLANK = new UninitializedObject(); - private static final class IndirectArgumentReference { + private interface ArgumentReference { + + Object snapshot(); + + Object currentValue(); + + void setValue(Object value); + } + + private static final class IndirectArgumentReference implements ArgumentReference { + private final Object[] frame; private final long offset; - private final boolean global; private final Object scalarValue; - private IndirectArgumentReference(long offsetParam, boolean globalParam, Object scalarValueParam) { + private IndirectArgumentReference(Object[] frameParam, long offsetParam, Object scalarValueParam) { + frame = frameParam; offset = offsetParam; - global = globalParam; scalarValue = scalarValueParam; } + + @Override + public Object snapshot() { + return scalarValue; + } + + @Override + public Object currentValue() { + return frame[(int) offset]; + } + + @Override + public void setValue(Object value) { + frame[(int) offset] = value; + } } - private static final class IndirectArrayArgumentReference { + private static final class IndirectArrayArgumentReference implements ArgumentReference { private final Map map; private final Object key; private final Object scalarValue; @@ -4282,6 +4323,21 @@ private IndirectArrayArgumentReference( key = keyParam; scalarValue = scalarValueParam; } + + @Override + public Object snapshot() { + return scalarValue; + } + + @Override + public Object currentValue() { + return JRT.getAssocArrayValue(map, key); + } + + @Override + public void setValue(Object value) { + map.put(key, value); + } } /** diff --git a/src/main/java/io/jawk/backend/RuntimeStack.java b/src/main/java/io/jawk/backend/RuntimeStack.java index a881c816..4586b7aa 100644 --- a/src/main/java/io/jawk/backend/RuntimeStack.java +++ b/src/main/java/io/jawk/backend/RuntimeStack.java @@ -153,6 +153,10 @@ Object getVariable(long offset, boolean isGlobal) { } } + Object[] getVariableFrame(boolean isGlobal) { + return isGlobal ? globals : locals; + } + Object setVariable(long offset, Object val, boolean isGlobal) { if (isGlobal) { globals[(int) offset] = val; diff --git a/src/main/java/io/jawk/frontend/AwkParser.java b/src/main/java/io/jawk/frontend/AwkParser.java index 10bb90ba..e490b4da 100644 --- a/src/main/java/io/jawk/frontend/AwkParser.java +++ b/src/main/java/io/jawk/frontend/AwkParser.java @@ -2098,7 +2098,7 @@ AST STATEMENT() throws IOException { } else if (token == Token.KW_BREAK) { stmt = BREAK_STATEMENT(); } else { - stmt = EXPRESSION_STATEMENT(true, false); // allow in keyword, do Token.NOT allow non-statement ASTs + stmt = EXPRESSION_STATEMENT(true); } terminator(); return stmt; @@ -2108,15 +2108,12 @@ AST STATEMENT() throws IOException { return stmt; } - AST EXPRESSION_STATEMENT(boolean allowInKeyword, boolean allowNonStatementAsts) throws IOException { + AST EXPRESSION_STATEMENT(boolean allowInKeyword) throws IOException { // true = allow comparators // false = do Token.NOT allow multi-dimensional array indices // return new ExpressionStatementAst(ASSIGNMENT_EXPRESSION(true, allowInKeyword, false)); AST exprAst = ASSIGNMENT_EXPRESSION(null, true, allowInKeyword, false); - if (!allowNonStatementAsts && exprAst.hasFlag(AstFlag.NON_STATEMENT)) { - throw parserException("Not a valid statement."); - } return new ExpressionStatementAst(exprAst); } @@ -2277,7 +2274,7 @@ AST OPT_SIMPLE_STATEMENT(boolean allowInKeyword) throws IOException { return PRINTF_STATEMENT(); } else { // allow non-statement ASTs - return EXPRESSION_STATEMENT(allowInKeyword, true); + return EXPRESSION_STATEMENT(allowInKeyword); } } @@ -2541,11 +2538,8 @@ private void populateArrayOperandTuples( String errorMessage) { if (arrayAst instanceof IDAst) { IDAst idAst = (IDAst) arrayAst; - if (idAst.isScalar()) { - arrayAst.throwSemanticException(errorMessage); - } idAst.setArray(true); - idAst.populateTuples(tuples); + tuples.dereference(idAst.offset, true, idAst.isGlobal); return; } if (arrayAst instanceof ArrayReferenceAst) { @@ -2613,7 +2607,6 @@ private int populateActualParameters( private int populateActualParametersUpTo( AwkTuples tuples, FunctionCallParamListAst params, - Set arrayParameterIndexes, int parameterIndex, int maxParameterCount) { /* @@ -2634,19 +2627,20 @@ private int populateActualParametersUpTo( populateActualParametersUpTo( tuples, (FunctionCallParamListAst) params.getAst2(), - arrayParameterIndexes, parameterIndex + 1, maxParameterCount); return 0; } - if (arrayParameterIndexes.contains(Integer.valueOf(parameterIndex))) { - populateArrayOperandTuples( - params.getAst1(), - tuples, - true, - "Parameter position " + (parameterIndex + 1) + " must be an array or subarray."); + AST argument = params.getAst1(); + if (argument instanceof IDAst + && !isJrtManagedSpecialName(((IDAst) argument).id)) { + IDAst idAst = (IDAst) argument; + tuples.pushIndirectArgument(idAst.offset, idAst.isGlobal); + } else if (argument instanceof ArrayReferenceAst) { + ((ArrayReferenceAst) argument).populateTargetReferenceTuples(tuples); + tuples.pushIndirectArrayArgument(); } else { - params.getAst1().populateTuples(tuples); + argument.populateTuples(tuples); } if (params.getAst2() == null) { return 1; @@ -2654,7 +2648,6 @@ private int populateActualParametersUpTo( return 1 + populateActualParametersUpTo( tuples, (FunctionCallParamListAst) params.getAst2(), - arrayParameterIndexes, parameterIndex + 1, maxParameterCount); } @@ -4012,9 +4005,6 @@ public int populateTuples(AwkTuples tuples) { getAst2().populateTuples(tuples); // here, stack contains one value if (getAst1() instanceof IDAst) { IDAst idAst = (IDAst) getAst1(); - if (idAst.isArray()) { - throw new SemanticException("Cannot use " + idAst + " as a scalar. It is an array."); - } idAst.setScalar(true); boolean isSpecial = isJrtManagedSpecialName(idAst.id); if (isSpecial) { @@ -4087,9 +4077,6 @@ public int populateTuples(AwkTuples tuples) { ArrayReferenceAst arr = (ArrayReferenceAst) getAst1(); if (arr.getAst1() instanceof IDAst) { IDAst idAst = (IDAst) arr.getAst1(); - if (idAst.isScalar()) { - throw new SemanticException("Cannot use " + idAst + " as an array. It is a scalar."); - } idAst.setArray(true); } arr.populateTargetReferenceTuples(tuples); @@ -4526,47 +4513,6 @@ int paramCount() { return count; } - void checkActualToFormalParameters(AST actualParamList) { - AST aPtr = actualParamList; - FunctionDefParamListAst fPtr = (FunctionDefParamListAst) getAst1(); - // Extra actual parameters (accepted with a gawk-style runtime - // warning) have no formal counterpart to validate against. - while (aPtr != null && fPtr != null) { - // actual parameter - AST aparam = aPtr.getAst1(); - // formal function parameter - AST fparam = symbolTable.getFunctionParameterIDAST(id, fPtr.id); - - if (fparam.isArray()) { - if (aparam instanceof IDAst) { - IDAst aparamIdAst = (IDAst) aparam; - if (aparamIdAst.isScalar()) { - aparam - .throwSemanticException( - id + ": Actual parameter (" + aparam - + ") is a scalar, but formal parameter is used like an array."); - } - aparamIdAst.setArray(true); - } else if (!(aparam instanceof ArrayReferenceAst)) { - aparam - .throwSemanticException( - id + ": Actual parameter (" + aparam + ") is not an array or subarray reference."); - } - } else if (fparam.isScalar() && aparam instanceof IDAst) { - IDAst aparamIdAst = (IDAst) aparam; - if (aparamIdAst.isArray()) { - aparam - .throwSemanticException( - id + ": Actual parameter (" + aparam - + ") is an array, but formal parameter is used like a scalar."); - } - aparamIdAst.setScalar(true); - } - // next - aPtr = aPtr.getAst2(); - fPtr = (FunctionDefParamListAst) fPtr.getAst1(); - } - } } private final class FunctionCallAst extends ScalarExpressionAst { @@ -4585,14 +4531,6 @@ private FunctionCallAst(FunctionProxy functionProxy, AST params) { * The checks performed are: *
    *
  • Make sure the function is defined. - *
  • The number of actual parameters does not - * exceed the number of formal parameters. - *
  • Matches actual parameters to formal parameter - * usage with respect to whether they are - * scalars, arrays, or either. - * (This determination is based on how - * the formal parameters are used within - * the function block.) *
* A failure of any one of these checks * results in a SemanticException. @@ -4605,10 +4543,6 @@ public void semanticAnalysis() throws SemanticException { if (!functionProxy.isDefined()) { throw new SemanticException("function " + functionProxy + " not defined"); } - int formalParamCount = functionProxy.getFunctionParamCount(); - if (getAst1() != null && formalParamCount > 0) { - functionProxy.checkActualToFormalParameters(getAst1()); - } } @Override @@ -4625,15 +4559,11 @@ public int populateTuples(AwkTuples tuples) { actualParamCountLocal = populateActualParametersUpTo( tuples, (FunctionCallParamListAst) getAst1(), - collectArrayParameterIndexes(functionProxy.functionDefAst), 0, functionProxy.getFunctionParamCount()); } int formalParamCount = functionProxy.getFunctionParamCount(); - if (getAst1() != null && formalParamCount > 0) { - functionProxy.checkActualToFormalParameters(getAst1()); - } if (actualParamCount() > formalParamCount) { // gawk accepts the call but reports it each time it runs tuples.warning(extraArgumentWarning()); @@ -4944,18 +4874,12 @@ private void populateSubTuples(AwkTuples tuples, boolean isGsub) { AST ptr = getAst1().getAst2().getAst2().getAst1(); if (ptr instanceof IDAst) { IDAst idAst = (IDAst) ptr; - if (idAst.isArray()) { - throw new SemanticException("sub cannot accept an unindexed array as its 3rd argument"); - } idAst.setScalar(true); tuples.subForVariable(idAst.offset, idAst.isGlobal, isGsub); } else if (ptr instanceof ArrayReferenceAst) { ArrayReferenceAst arrAst = (ArrayReferenceAst) ptr; if (arrAst.getAst1() instanceof IDAst) { IDAst idAst = (IDAst) arrAst.getAst1(); - if (idAst.isScalar()) { - throw new SemanticException("Cannot use " + idAst + " as an array."); - } idAst.setArray(true); } arrAst.populateTargetReferenceTuples(tuples); @@ -5133,7 +5057,9 @@ public int populateTuples(AwkTuples tuples) { // Use JRT-managed reads for specials pushSpecialVariable(tuples, id); } else { - tuples.dereference(offset, isArray(), isGlobal); + // Bare identifiers are scalar uses. Array-only contexts emit a + // typed dereference through populateArrayOperandTuples(). + tuples.dereference(offset, false, isGlobal); } popSourceLineNumber(tuples); return 1; @@ -5223,6 +5149,9 @@ private void populateTargetValueTuples(AwkTuples tuples) { private void populateContainerTuples(AwkTuples tuples) { if (getAst1() instanceof ArrayReferenceAst) { ((ArrayReferenceAst) getAst1()).populateArrayValueTuples(tuples, true); + } else if (getAst1() instanceof IDAst) { + IDAst idAst = (IDAst) getAst1(); + tuples.dereference(idAst.offset, true, idAst.isGlobal); } else { getAst1().populateTuples(tuples); } @@ -6146,18 +6075,12 @@ public int populateTuples(AwkTuples tuples) { ArrayReferenceAst arrAst = (ArrayReferenceAst) getAst1(); if (arrAst.getAst1() instanceof IDAst) { IDAst idAst = (IDAst) arrAst.getAst1(); - if (idAst.isScalar()) { - throw new SemanticException("delete: Cannot use a scalar as an array."); - } idAst.setArray(true); } arrAst.populateTargetReferenceTuples(tuples); tuples.deleteMapElement(); } else if (getAst1() instanceof IDAst) { IDAst idAst = (IDAst) getAst1(); - if (idAst.isScalar()) { - throw new SemanticException("delete: Cannot delete a scalar."); - } idAst.setArray(true); tuples.deleteArray(idAst.offset, idAst.isGlobal); } else { @@ -6282,9 +6205,6 @@ public String toString() { return super.toString() + " (" + id + ")"; } - private void checkActualToFormalParameters(AST actualParams) { - functionDefAst.checkActualToFormalParameters(actualParams); - } } /** @@ -6485,9 +6405,6 @@ IDAst getFunctionParameterIDAST(String functionName, String fIdString) { AST addArrayID(String id) throws ParserException { IDAst retVal = getID(id); retVal.markReferenced(); - if (retVal.isScalar()) { - throw parserException("Cannot use " + retVal + " as an array."); - } retVal.setArray(true); return retVal; } diff --git a/src/main/java/io/jawk/jrt/HashAssocArray.java b/src/main/java/io/jawk/jrt/HashAssocArray.java index ef8f81a9..bba861d8 100644 --- a/src/main/java/io/jawk/jrt/HashAssocArray.java +++ b/src/main/java/io/jawk/jrt/HashAssocArray.java @@ -119,6 +119,6 @@ public String getMapVersion() { */ @Override public String toString() { - throw new AwkRuntimeException("Cannot evaluate an unindexed array."); + throw new AwkRuntimeException("Attempting to use an array in a scalar context."); } } diff --git a/src/main/java/io/jawk/jrt/SortedAssocArray.java b/src/main/java/io/jawk/jrt/SortedAssocArray.java index 9e942e6b..00e7fe05 100644 --- a/src/main/java/io/jawk/jrt/SortedAssocArray.java +++ b/src/main/java/io/jawk/jrt/SortedAssocArray.java @@ -142,6 +142,6 @@ public String getMapVersion() { */ @Override public String toString() { - throw new AwkRuntimeException("Cannot evaluate an unindexed array."); + throw new AwkRuntimeException("Attempting to use an array in a scalar context."); } } diff --git a/src/site/markdown/compatibility.md.vm b/src/site/markdown/compatibility.md.vm index a0fb10aa..7b7db3b1 100644 --- a/src/site/markdown/compatibility.md.vm +++ b/src/site/markdown/compatibility.md.vm @@ -135,6 +135,8 @@ GNU Awk (gawk) [GNU Awk](https://www.gnu.org/software/gawk/) compatibility gathered from explicit Java integration suites that mirror the main gawk test families: core behavior, extensions, locale-sensitive behavior, and optional features. +Array and scalar types follow gawk's runtime rules across function parameters and nested array references. An untyped argument remains linked to its caller until its first scalar or array use fixes the type; conflicting assignment, membership, iteration, deletion, or subarray use then fails at runtime. +
diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index fdc5c52b..ce5f934c 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -829,6 +829,42 @@ public void testArraysOfArraysRejectScalarAsArray() throws Exception { .runAndAssert(); } + @Test + public void testArrayParameterDefersUntypedActualUntilScalarUse() throws Exception { + AwkTestSupport + .awkTest("untyped array parameter follows caller scalarization") + .script("function f(x) { a = 1; x[1] = 2 } BEGIN { f(a) }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void testArrayParameterDefersUntypedActualUntilArrayUse() throws Exception { + AwkTestSupport + .awkTest("untyped array parameter follows caller array materialization") + .script("function f(x) { x[1] = 2; a = 1 } BEGIN { f(a) }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void testArrayParameterRejectsAlreadyScalarActualAtRuntime() throws Exception { + AwkTestSupport + .awkTest("scalar actual remains scalar in array parameter") + .script("function f(x) { x[1] = 2 } BEGIN { a = 1; f(a) }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void testFunctionResultCannotOverwriteSubarray() throws Exception { + AwkTestSupport + .awkTest("function result cannot overwrite a subarray") + .script("function f(x) { x[1] = 42 } BEGIN { a[0] = f(a[0]) }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + @Test public void testArraysOfArraysReportLineNumberWhenScalarUsedAsArray() throws Exception { assertRuntimeExceptionLineNumber( From 6cf94ebf2e17557cc1f3ff189e37c4a1a3f985d8 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Wed, 29 Jul 2026 14:32:12 +0200 Subject: [PATCH 02/11] Unwrap forwarded argument references for raw value extensions --- src/main/java/io/jawk/backend/AVM.java | 10 ++++++++-- src/test/java/io/jawk/GawkExtensionTest.java | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 62296015..5bdeb9b0 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -1679,7 +1679,7 @@ private void executeTuples(PositionTracker position) .getVariable(variableTuple.getVariableOffset(), variableTuple.isGlobal()); push( value instanceof ArgumentReference ? - ((ArgumentReference) value).currentValue() : value); + resolveRawArgumentReference((ArgumentReference) value) : value); position.next(); break; } @@ -2880,7 +2880,7 @@ private Object resolveIndirectArgument( } ArgumentReference reference = (ArgumentReference) argument; if (rawValueArgument) { - return reference.currentValue(); + return resolveRawArgumentReference(reference); } return resolveArgumentReference(reference, arrayArgument); } @@ -4187,6 +4187,12 @@ private Object resolveVariable(long offset, boolean isGlobal, boolean arrayConte return value; } + private Object resolveRawArgumentReference(ArgumentReference reference) { + Object value = reference.currentValue(); + return value instanceof ArgumentReference ? + resolveRawArgumentReference((ArgumentReference) value) : value; + } + private Object resolveArgumentReference(ArgumentReference reference, boolean arrayContext) { Object value = reference.currentValue(); if (value instanceof ArgumentReference) { diff --git a/src/test/java/io/jawk/GawkExtensionTest.java b/src/test/java/io/jawk/GawkExtensionTest.java index 9d449d6e..01d523fe 100644 --- a/src/test/java/io/jawk/GawkExtensionTest.java +++ b/src/test/java/io/jawk/GawkExtensionTest.java @@ -849,6 +849,20 @@ public void indirectTypeofPreservesUntypedVariables() throws Exception { .runAndAssert(); } + @Test + public void rawValueExtensionsUnwrapForwardedUntypedArguments() throws Exception { + AwkTestSupport + .awkTest("raw value extensions receive forwarded untyped arguments") + .script( + "function forward(value) { inspect(value) } " + + "function inspect(value) { " + + "callback = \"typeof\"; print typeof(value), @callback(value) " + + "} " + + "BEGIN { forward(unset) }") + .expectLines("untyped untyped") + .runAndAssert(); + } + @Test public void typeofReportsStrnumForNumericInputFields() throws Exception { AwkTestSupport From b840329db8522b9c862941559dd8150a1343d1c2 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Wed, 29 Jul 2026 15:43:50 +0200 Subject: [PATCH 03/11] Preserve call-time values for untyped scalar arguments --- src/main/java/io/jawk/backend/AVM.java | 43 ++++++++++++++++++++------ src/test/java/io/jawk/AwkTest.java | 20 ++++++++++++ 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 5bdeb9b0..0d59378e 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -4188,23 +4188,31 @@ private Object resolveVariable(long offset, boolean isGlobal, boolean arrayConte } private Object resolveRawArgumentReference(ArgumentReference reference) { - Object value = reference.currentValue(); + Object value = reference.snapshot(); return value instanceof ArgumentReference ? resolveRawArgumentReference((ArgumentReference) value) : value; } private Object resolveArgumentReference(ArgumentReference reference, boolean arrayContext) { - Object value = reference.currentValue(); + Object value = arrayContext ? reference.currentValue() : reference.snapshot(); if (value instanceof ArgumentReference) { value = resolveArgumentReference((ArgumentReference) value, arrayContext); - reference.setValue(value); + if (arrayContext) { + reference.setValue(value); + } else { + reference.setScalarValue(value); + } return value; } if (!isUntyped(value)) { return value; } value = arrayContext ? newAwkArray() : BLANK; - reference.setValue(value); + if (arrayContext) { + reference.setValue(value); + } else { + reference.setScalarValue(value); + } return value; } @@ -4287,6 +4295,8 @@ private interface ArgumentReference { Object currentValue(); void setValue(Object value); + + void setScalarValue(Object value); } private static final class IndirectArgumentReference implements ArgumentReference { @@ -4314,12 +4324,17 @@ public Object currentValue() { public void setValue(Object value) { frame[(int) offset] = value; } + + @Override + public void setScalarValue(Object value) { + // Scalar arguments are copied at call time. + } } private static final class IndirectArrayArgumentReference implements ArgumentReference { private final Map map; private final Object key; - private final Object scalarValue; + private Object detachedValue; private IndirectArrayArgumentReference( Map mapParam, @@ -4327,22 +4342,32 @@ private IndirectArrayArgumentReference( Object scalarValueParam) { map = mapParam; key = keyParam; - scalarValue = scalarValueParam; + detachedValue = scalarValueParam; } @Override public Object snapshot() { - return scalarValue; + return detachedValue; } @Override public Object currentValue() { - return JRT.getAssocArrayValue(map, key); + return JRT.containsAwkKey(map, key) ? + JRT.getAssocArrayValue(map, key) : detachedValue; } @Override public void setValue(Object value) { - map.put(key, value); + if (JRT.containsAwkKey(map, key)) { + map.put(key, value); + } else { + detachedValue = value; + } + } + + @Override + public void setScalarValue(Object value) { + setValue(value); } } diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index ce5f934c..3a75786c 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -847,6 +847,26 @@ public void testArrayParameterDefersUntypedActualUntilArrayUse() throws Exceptio .runAndAssert(); } + @Test + public void testUntypedScalarParameterKeepsCallTimeValue() throws Exception { + AwkTestSupport + .awkTest("untyped scalar parameter keeps its call-time value") + .script("function f(x) { a = 5; print \"[\" x \"]\"; print a } BEGIN { f(a) }") + .expectLines("[]", "5") + .runAndAssert(); + } + + @Test + public void testDeletedUntypedSubarrayParameterRemainsDetached() throws Exception { + AwkTestSupport + .awkTest("deleted untyped subarray parameter remains detached") + .script( + "function f(x) { delete a[1]; x[2] = 3; print x[2] } " + + "BEGIN { f(a[1]); print (1 in a) }") + .expectLines("3", "0") + .runAndAssert(); + } + @Test public void testArrayParameterRejectsAlreadyScalarActualAtRuntime() throws Exception { AwkTestSupport From 319ce8e9e35e65cb78fd6db497b0a2078a462bd3 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Wed, 29 Jul 2026 16:59:21 +0200 Subject: [PATCH 04/11] Materialize unchanged scalar arguments only --- src/main/java/io/jawk/backend/AVM.java | 12 +++++++--- src/test/java/io/jawk/AwkTest.java | 31 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 0d59378e..6f33ab29 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -4216,7 +4216,7 @@ private Object resolveArgumentReference(ArgumentReference reference, boolean arr return value; } - private boolean isUntyped(Object value) { + private static boolean isUntyped(Object value) { return value == null || value instanceof UntypedObject; } @@ -4327,7 +4327,9 @@ public void setValue(Object value) { @Override public void setScalarValue(Object value) { - // Scalar arguments are copied at call time. + if (isUntyped(currentValue())) { + setValue(value); + } } } @@ -4367,7 +4369,11 @@ public void setValue(Object value) { @Override public void setScalarValue(Object value) { - setValue(value); + if (!JRT.containsAwkKey(map, key)) { + detachedValue = value; + } else if (isUntyped(JRT.getAssocArrayValue(map, key))) { + map.put(key, value); + } } } diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index 3a75786c..0ea248d5 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -856,6 +856,37 @@ public void testUntypedScalarParameterKeepsCallTimeValue() throws Exception { .runAndAssert(); } + @Test + public void testUntypedScalarParameterScalarizesUnchangedCaller() throws Exception { + AwkTestSupport + .awkTest("untyped scalar parameter scalarizes its unchanged caller") + .script("function f(x) { print x } BEGIN { f(a); a[1] = 1 }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void testUntypedScalarParameterPreservesChangedArrayElement() throws Exception { + AwkTestSupport + .awkTest("untyped scalar parameter preserves a changed array element") + .script( + "function f(x) { a[1] = 7; print \"[\" x \"]\" } " + + "BEGIN { f(a[1]); print a[1] }") + .expectLines("[]", "7") + .runAndAssert(); + } + + @Test + public void testUntypedScalarParameterPreservesChangedSubarrayElement() throws Exception { + AwkTestSupport + .awkTest("untyped scalar parameter preserves a changed subarray element") + .script( + "function f(x) { a[1][2] = 7; print \"[\" x \"]\" } " + + "BEGIN { f(a[1]); print a[1][2] }") + .expectLines("[]", "7") + .runAndAssert(); + } + @Test public void testDeletedUntypedSubarrayParameterRemainsDetached() throws Exception { AwkTestSupport From 2a0b6403a4e5b1d23cb758279700a7f82631da62 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Wed, 29 Jul 2026 21:13:40 +0200 Subject: [PATCH 05/11] Observe live array transitions in arguments --- src/main/java/io/jawk/backend/AVM.java | 19 ++++++++++++++- src/main/java/io/jawk/frontend/AwkParser.java | 2 +- .../java/io/jawk/intermediate/AwkTuples.java | 7 ++++++ src/main/java/io/jawk/intermediate/Tuple.java | 12 ++++++++++ src/test/java/io/jawk/AwkTest.java | 24 ++++++++++++++++--- src/test/java/io/jawk/GawkExtensionTest.java | 11 +++++++++ 6 files changed, 70 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 6f33ab29..4139480a 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -73,6 +73,7 @@ import io.jawk.intermediate.Tuple.PushLongTuple; import io.jawk.intermediate.Tuple.PushStringTuple; import io.jawk.intermediate.Tuple.RegexTuple; +import io.jawk.intermediate.Tuple.ScalarPopTuple; import io.jawk.intermediate.Tuple.SubstitutionVariableTuple; import io.jawk.intermediate.Tuple.VariableTuple; import io.jawk.intermediate.UninitializedObject; @@ -1161,7 +1162,10 @@ private void executeTuples(PositionTracker position) } case POP: { // stack[0] = item to pop from the stack - pop(); + Object discarded = pop(); + if (tuple instanceof ScalarPopTuple) { + checkScalar(discarded); + } position.next(); break; } @@ -4188,12 +4192,25 @@ private Object resolveVariable(long offset, boolean isGlobal, boolean arrayConte } private Object resolveRawArgumentReference(ArgumentReference reference) { + Object currentValue = readCurrentArgumentValue(reference); + if (currentValue instanceof Map) { + return currentValue; + } Object value = reference.snapshot(); return value instanceof ArgumentReference ? resolveRawArgumentReference((ArgumentReference) value) : value; } + private Object readCurrentArgumentValue(ArgumentReference reference) { + Object value = reference.currentValue(); + return value instanceof ArgumentReference ? + readCurrentArgumentValue((ArgumentReference) value) : value; + } + private Object resolveArgumentReference(ArgumentReference reference, boolean arrayContext) { + if (!arrayContext) { + checkScalar(readCurrentArgumentValue(reference)); + } Object value = arrayContext ? reference.currentValue() : reference.snapshot(); if (value instanceof ArgumentReference) { value = resolveArgumentReference((ArgumentReference) value, arrayContext); diff --git a/src/main/java/io/jawk/frontend/AwkParser.java b/src/main/java/io/jawk/frontend/AwkParser.java index e490b4da..c7ab4edd 100644 --- a/src/main/java/io/jawk/frontend/AwkParser.java +++ b/src/main/java/io/jawk/frontend/AwkParser.java @@ -3975,7 +3975,7 @@ public int populateTuples(AwkTuples tuples) { pushSourceLineNumber(tuples); int exprCount = getAst1().populateTuples(tuples); if (exprCount == 1) { - tuples.pop(); + tuples.popScalar(); } popSourceLineNumber(tuples); return 0; diff --git a/src/main/java/io/jawk/intermediate/AwkTuples.java b/src/main/java/io/jawk/intermediate/AwkTuples.java index 10d0fda4..d3481ae1 100644 --- a/src/main/java/io/jawk/intermediate/AwkTuples.java +++ b/src/main/java/io/jawk/intermediate/AwkTuples.java @@ -152,6 +152,13 @@ public void pop() { queue.add(new Tuple.NoOperandTuple(Opcode.POP)); } + /** + * Discards a value that was evaluated in scalar context. + */ + public void popScalar() { + queue.add(new Tuple.ScalarPopTuple()); + } + /** *

* push. diff --git a/src/main/java/io/jawk/intermediate/Tuple.java b/src/main/java/io/jawk/intermediate/Tuple.java index b35f46a6..2f7fa9fc 100644 --- a/src/main/java/io/jawk/intermediate/Tuple.java +++ b/src/main/java/io/jawk/intermediate/Tuple.java @@ -154,6 +154,18 @@ public String toString() { } } + /** + * Tuple for discarding an expression-statement value after validating that it + * is scalar. + */ + public static final class ScalarPopTuple extends NoOperandTuple { + private static final long serialVersionUID = 1L; + + ScalarPopTuple() { + super(Opcode.POP); + } + } + /** * Tuple for JRT-managed built-in variable operations. */ diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index 0ea248d5..ea3ce79f 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -829,6 +829,15 @@ public void testArraysOfArraysRejectScalarAsArray() throws Exception { .runAndAssert(); } + @Test + public void testDiscardedArrayExpressionRejectsScalarContext() throws Exception { + AwkTestSupport + .awkTest("discarded array expression rejects scalar context") + .script("BEGIN { a[1] = 1; a; print \"continued\" }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + @Test public void testArrayParameterDefersUntypedActualUntilScalarUse() throws Exception { AwkTestSupport @@ -847,6 +856,15 @@ public void testArrayParameterDefersUntypedActualUntilArrayUse() throws Exceptio .runAndAssert(); } + @Test + public void testScalarParameterRejectsCallerArrayTransition() throws Exception { + AwkTestSupport + .awkTest("scalar parameter rejects caller array transition") + .script("function f(x, y) { x[1] = 1; print y } BEGIN { f(a, a) }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + @Test public void testUntypedScalarParameterKeepsCallTimeValue() throws Exception { AwkTestSupport @@ -877,13 +895,13 @@ public void testUntypedScalarParameterPreservesChangedArrayElement() throws Exce } @Test - public void testUntypedScalarParameterPreservesChangedSubarrayElement() throws Exception { + public void testUntypedScalarParameterRejectsChangedSubarrayElement() throws Exception { AwkTestSupport - .awkTest("untyped scalar parameter preserves a changed subarray element") + .awkTest("untyped scalar parameter rejects a changed subarray element") .script( "function f(x) { a[1][2] = 7; print \"[\" x \"]\" } " + "BEGIN { f(a[1]); print a[1][2] }") - .expectLines("[]", "7") + .expectThrow(AwkRuntimeException.class) .runAndAssert(); } diff --git a/src/test/java/io/jawk/GawkExtensionTest.java b/src/test/java/io/jawk/GawkExtensionTest.java index 01d523fe..2e883f43 100644 --- a/src/test/java/io/jawk/GawkExtensionTest.java +++ b/src/test/java/io/jawk/GawkExtensionTest.java @@ -863,6 +863,17 @@ public void rawValueExtensionsUnwrapForwardedUntypedArguments() throws Exception .runAndAssert(); } + @Test + public void rawValueExtensionsObserveCallerArrayTransitions() throws Exception { + AwkTestSupport + .awkTest("raw value extensions observe caller array transitions") + .script( + "function f(x) { a[1] = 1; print typeof(x), isarray(x) } " + + "BEGIN { f(a) }") + .expectLines("array 1") + .runAndAssert(); + } + @Test public void typeofReportsStrnumForNumericInputFields() throws Exception { AwkTestSupport From 1c5e87b11a647e116e770f0d809dee4ddd79bf10 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 30 Jul 2026 12:27:01 +0200 Subject: [PATCH 06/11] Keep reinserted element arguments detached --- src/main/java/io/jawk/backend/AVM.java | 32 ++++++++++++++++++++++---- src/test/java/io/jawk/AwkTest.java | 11 +++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 4139480a..9d72a0a6 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -42,6 +42,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.WeakHashMap; import java.util.function.BiConsumer; import java.util.regex.Pattern; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -128,6 +129,8 @@ public class AVM implements VariableManager, Closeable { // operand stack private Deque operandStack = new ArrayDeque(); + private final Set indirectArrayArgumentReferences = Collections + .newSetFromMap(new WeakHashMap()); private List arguments; private boolean sortedArrayKeys; private final Map baseInitialVariables; @@ -704,6 +707,7 @@ private void resetRuntimeState(List runtimeArguments, Map runtimeArguments, Map variableOverrides) { // Reset the AVM-owned state that must not leak across executions. operandStack.clear(); + indirectArrayArgumentReferences.clear(); environOffset = NULL_OFFSET; argcOffset = NULL_OFFSET; argvOffset = NULL_OFFSET; @@ -1704,7 +1708,9 @@ private void executeTuples(PositionTracker position) checkScalar(idx); Map map = toMap(pop()); Object scalarValue = JRT.getAssocArrayValue(map, idx); - push(new IndirectArrayArgumentReference(map, idx, scalarValue)); + IndirectArrayArgumentReference reference = new IndirectArrayArgumentReference(map, idx, scalarValue); + indirectArrayArgumentReferences.add(reference); + push(reference); position.next(); break; } @@ -2349,6 +2355,7 @@ private void executeTuples(PositionTracker position) checkScalar(key); if (aa != null) { aa.remove(key); + detachMissingArrayArgumentReferences(aa); } position.next(); break; @@ -2360,6 +2367,7 @@ private void executeTuples(PositionTracker position) checkScalar(key); Map aa = toMap(pop()); aa.remove(key); + detachMissingArrayArgumentReferences(aa); position.next(); break; } @@ -2373,6 +2381,7 @@ private void executeTuples(PositionTracker position) Map array = getMapVariable(offset, isGlobal); if (array != null) { array.clear(); + detachMissingArrayArgumentReferences(array); } position.next(); break; @@ -3113,6 +3122,7 @@ private Object splitIntoArray(Object source, Object target, Object separator, in @SuppressWarnings("unchecked") Map assocArray = (Map) target; assocArray.clear(); + detachMissingArrayArgumentReferences(assocArray); long cnt = 0; while (tokenizer.hasMoreElements()) { Object value = tokenizer.nextElement(); @@ -4233,6 +4243,12 @@ private Object resolveArgumentReference(ArgumentReference reference, boolean arr return value; } + private void detachMissingArrayArgumentReferences(Map map) { + for (IndirectArrayArgumentReference reference : indirectArrayArgumentReferences) { + reference.detachIfMissing(map); + } + } + private static boolean isUntyped(Object value) { return value == null || value instanceof UntypedObject; } @@ -4354,6 +4370,7 @@ private static final class IndirectArrayArgumentReference implements ArgumentRef private final Map map; private final Object key; private Object detachedValue; + private boolean detached; private IndirectArrayArgumentReference( Map mapParam, @@ -4362,6 +4379,7 @@ private IndirectArrayArgumentReference( map = mapParam; key = keyParam; detachedValue = scalarValueParam; + detached = !JRT.containsAwkKey(map, key); } @Override @@ -4371,13 +4389,13 @@ public Object snapshot() { @Override public Object currentValue() { - return JRT.containsAwkKey(map, key) ? + return !detached && JRT.containsAwkKey(map, key) ? JRT.getAssocArrayValue(map, key) : detachedValue; } @Override public void setValue(Object value) { - if (JRT.containsAwkKey(map, key)) { + if (!detached && JRT.containsAwkKey(map, key)) { map.put(key, value); } else { detachedValue = value; @@ -4386,12 +4404,18 @@ public void setValue(Object value) { @Override public void setScalarValue(Object value) { - if (!JRT.containsAwkKey(map, key)) { + if (detached || !JRT.containsAwkKey(map, key)) { detachedValue = value; } else if (isUntyped(JRT.getAssocArrayValue(map, key))) { map.put(key, value); } } + + private void detachIfMissing(Map candidateMap) { + if (!detached && map == candidateMap && !JRT.containsAwkKey(map, key)) { + detached = true; + } + } } /** diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index ea3ce79f..7b35a19f 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -916,6 +916,17 @@ public void testDeletedUntypedSubarrayParameterRemainsDetached() throws Exceptio .runAndAssert(); } + @Test + public void testReinsertedUntypedSubarrayParameterRemainsDetached() throws Exception { + AwkTestSupport + .awkTest("reinserted untyped subarray parameter remains detached") + .script( + "function f(x) { delete a[1]; a[1][9] = 9; x[2] = 2; print x[2] } " + + "BEGIN { f(a[1]); print (2 in a[1]); print a[1][9] }") + .expectLines("2", "0", "9") + .runAndAssert(); + } + @Test public void testArrayParameterRejectsAlreadyScalarActualAtRuntime() throws Exception { AwkTestSupport From 65cb012628a23f8008245b24d5235f2a0be81dbb Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 30 Jul 2026 15:50:47 +0200 Subject: [PATCH 07/11] Handle polymorphic and replaced array arguments --- src/main/java/io/jawk/backend/AVM.java | 49 ++++++++++++++++++- src/main/java/io/jawk/frontend/AwkParser.java | 14 +++++- src/test/java/io/jawk/AwkTest.java | 9 ++++ src/test/java/io/jawk/GawkExtensionTest.java | 11 +++++ 4 files changed, 80 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 9d72a0a6..6ca08ab2 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -38,6 +38,7 @@ import java.util.Deque; import java.util.Enumeration; import java.util.HashMap; +import java.util.IdentityHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -2823,7 +2824,11 @@ private void execLength(CountTuple tuple) { push(jrt.jrtGetInputField(0).toString().length()); return; } - push(lengthOf(pop())); + Object value = pop(); + if (value instanceof ArgumentReference) { + value = resolveLengthArgumentReference((ArgumentReference) value); + } + push(lengthOf(value)); } private Object lengthOf(Object value) { @@ -3023,7 +3028,13 @@ private Object invokeExtension( + "' does not extend " + AbstractExtension.class.getName()); } - Object result = function.invoke((AbstractExtension) extension, args); + Map attachedValues = captureAttachedArrayArgumentValues(); + Object result; + try { + result = function.invoke((AbstractExtension) extension, args); + } finally { + detachReplacedArrayArgumentReferences(attachedValues); + } if (blockResult && result instanceof BlockObject) { result = new BlockManager().block((BlockObject) result); } @@ -4243,6 +4254,28 @@ private Object resolveArgumentReference(ArgumentReference reference, boolean arr return value; } + private Object resolveLengthArgumentReference(ArgumentReference reference) { + Object currentValue = readCurrentArgumentValue(reference); + return currentValue instanceof Map ? currentValue : resolveArgumentReference(reference, false); + } + + private Map captureAttachedArrayArgumentValues() { + Map values = new IdentityHashMap(); + for (IndirectArrayArgumentReference reference : indirectArrayArgumentReferences) { + if (reference.isAttached()) { + values.put(reference, reference.currentValue()); + } + } + return values; + } + + private void detachReplacedArrayArgumentReferences( + Map attachedValues) { + for (Map.Entry entry : attachedValues.entrySet()) { + entry.getKey().detachIfReplaced(entry.getValue()); + } + } + private void detachMissingArrayArgumentReferences(Map map) { for (IndirectArrayArgumentReference reference : indirectArrayArgumentReferences) { reference.detachIfMissing(map); @@ -4416,6 +4449,18 @@ private void detachIfMissing(Map candidateMap) { detached = true; } } + + private boolean isAttached() { + return !detached && JRT.containsAwkKey(map, key); + } + + private void detachIfReplaced(Object previousValue) { + if (!detached + && (!JRT.containsAwkKey(map, key) + || JRT.getAssocArrayValue(map, key) != previousValue)) { + detached = true; + } + } } /** diff --git a/src/main/java/io/jawk/frontend/AwkParser.java b/src/main/java/io/jawk/frontend/AwkParser.java index c7ab4edd..3a079e25 100644 --- a/src/main/java/io/jawk/frontend/AwkParser.java +++ b/src/main/java/io/jawk/frontend/AwkParser.java @@ -4807,7 +4807,19 @@ private void populateLengthTuples(AwkTuples tuples) { if (getAst1() == null) { tuples.length(0); } else { - int ast1Result = getAst1().populateTuples(tuples); + AST params = getAst1(); + AST argument = params instanceof FunctionCallParamListAst + && params.getAst2() == null ? + params.getAst1() : params; + int ast1Result; + if (argument instanceof IDAst + && !isJrtManagedSpecialName(((IDAst) argument).id)) { + IDAst idAst = (IDAst) argument; + tuples.pushIndirectArgument(idAst.offset, idAst.isGlobal); + ast1Result = 1; + } else { + ast1Result = argument.populateTuples(tuples); + } if (ast1Result != 1) { throw new SemanticException("length requires at least one argument"); } diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index 7b35a19f..b049d62d 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -865,6 +865,15 @@ public void testScalarParameterRejectsCallerArrayTransition() throws Exception { .runAndAssert(); } + @Test + public void testLengthParameterObservesCallerArrayTransition() throws Exception { + AwkTestSupport + .awkTest("length parameter observes caller array transition") + .script("function f(x, y) { y[1] = 1; print length(x) } BEGIN { f(a, a) }") + .expectLines("1") + .runAndAssert(); + } + @Test public void testUntypedScalarParameterKeepsCallTimeValue() throws Exception { AwkTestSupport diff --git a/src/test/java/io/jawk/GawkExtensionTest.java b/src/test/java/io/jawk/GawkExtensionTest.java index 2e883f43..6f46d714 100644 --- a/src/test/java/io/jawk/GawkExtensionTest.java +++ b/src/test/java/io/jawk/GawkExtensionTest.java @@ -683,6 +683,17 @@ public void asortWithDestinationLeavesSourceUntouched() throws Exception { .runAndAssert(); } + @Test + public void asortReplacementKeepsElementArgumentsDetached() throws Exception { + AwkTestSupport + .awkTest("asort replacement keeps element arguments detached") + .script( + "function f(x) { b[3][4] = 9; asort(b, a); x[2] = 2; print (2 in a[1]) } " + + "BEGIN { f(a[1]) }") + .expectLines("0") + .runAndAssert(); + } + @Test public void asortiSortsByIndex() throws Exception { AwkTestSupport From 46a071d612ca9b0ff7f1634fbfc98a41ff8b2528 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 30 Jul 2026 16:35:24 +0200 Subject: [PATCH 08/11] Distinguish missing arrays from assigned blanks --- src/main/java/io/jawk/backend/AVM.java | 12 +++++++----- src/test/java/io/jawk/AwkTest.java | 18 ++++++++++++++++++ src/test/java/io/jawk/GawkExtensionTest.java | 9 +++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 6ca08ab2..a5a7d3d6 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -1740,10 +1740,10 @@ private void executeTuples(PositionTracker position) checkScalar(idx); Map map = toMap(pop()); if (map instanceof AssocArray && !JRT.containsAwkKey(map, idx)) { - push(BLANK); + push(AssocArray.UNTYPED); } else { Object value = map.get(idx); - push(value != null ? value : BLANK); + push(value != null ? value : AssocArray.UNTYPED); } position.next(); break; @@ -2012,7 +2012,7 @@ private void executeTuples(PositionTracker position) } case KEYLIST: { Object o = pop(); - if (o == null || o instanceof UninitializedObject) { + if (isUntyped(o)) { push(new ArrayDeque<>()); position.next(); break; @@ -2451,7 +2451,7 @@ private void executeTuples(PositionTracker position) Object arr = pop(); Object arg = pop(); checkScalar(arg); - if (arr == null || arr instanceof UninitializedObject) { + if (isUntyped(arr)) { push(ZERO); position.next(); break; @@ -4214,7 +4214,9 @@ private Object resolveVariable(long offset, boolean isGlobal, boolean arrayConte private Object resolveRawArgumentReference(ArgumentReference reference) { Object currentValue = readCurrentArgumentValue(reference); - if (currentValue instanceof Map) { + if (currentValue instanceof Map + || currentValue instanceof UninitializedObject + && !(currentValue instanceof UntypedObject)) { return currentValue; } Object value = reference.snapshot(); diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index b049d62d..a73296ac 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -838,6 +838,24 @@ public void testDiscardedArrayExpressionRejectsScalarContext() throws Exception .runAndAssert(); } + @Test + public void testAssignedBlankRejectsForInArrayContext() throws Exception { + AwkTestSupport + .awkTest("assigned blank rejects for-in array context") + .script("BEGIN { print \"[\" a \"]\"; for (i in a); print \"continued\" }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void testAssignedBlankRejectsMembershipArrayContext() throws Exception { + AwkTestSupport + .awkTest("assigned blank rejects membership array context") + .script("BEGIN { print \"[\" a \"]\"; print (1 in a); print \"continued\" }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + @Test public void testArrayParameterDefersUntypedActualUntilScalarUse() throws Exception { AwkTestSupport diff --git a/src/test/java/io/jawk/GawkExtensionTest.java b/src/test/java/io/jawk/GawkExtensionTest.java index 6f46d714..dcc0253f 100644 --- a/src/test/java/io/jawk/GawkExtensionTest.java +++ b/src/test/java/io/jawk/GawkExtensionTest.java @@ -885,6 +885,15 @@ public void rawValueExtensionsObserveCallerArrayTransitions() throws Exception { .runAndAssert(); } + @Test + public void rawValueExtensionsObserveCallerScalarization() throws Exception { + AwkTestSupport + .awkTest("raw value extensions observe caller scalarization") + .script("function f(x, y) { print x; print typeof(y) } BEGIN { f(a, a) }") + .expect("\nunassigned\n") + .runAndAssert(); + } + @Test public void typeofReportsStrnumForNumericInputFields() throws Exception { AwkTestSupport From 9da30d430476bb2d0e948f47b83907542020f564 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 30 Jul 2026 19:35:59 +0200 Subject: [PATCH 09/11] Route special variables through managed reads in array contexts Special scalars such as FS or NR used where an array is required now go through their JRT-managed read instead of a plain global dereference, so runtime array validation rejects them like gawk does (FS[1]=2, 1 in FS, for (k in NR), split(s, FS)). Co-Authored-By: Claude Fable 5 --- src/main/java/io/jawk/frontend/AwkParser.java | 12 ++++++-- src/test/java/io/jawk/AwkTest.java | 30 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/jawk/frontend/AwkParser.java b/src/main/java/io/jawk/frontend/AwkParser.java index 3a079e25..f7fce386 100644 --- a/src/main/java/io/jawk/frontend/AwkParser.java +++ b/src/main/java/io/jawk/frontend/AwkParser.java @@ -2539,7 +2539,11 @@ private void populateArrayOperandTuples( if (arrayAst instanceof IDAst) { IDAst idAst = (IDAst) arrayAst; idAst.setArray(true); - tuples.dereference(idAst.offset, true, idAst.isGlobal); + if (isJrtManagedSpecialName(idAst.id)) { + idAst.populateTuples(tuples); + } else { + tuples.dereference(idAst.offset, true, idAst.isGlobal); + } return; } if (arrayAst instanceof ArrayReferenceAst) { @@ -5163,7 +5167,11 @@ private void populateContainerTuples(AwkTuples tuples) { ((ArrayReferenceAst) getAst1()).populateArrayValueTuples(tuples, true); } else if (getAst1() instanceof IDAst) { IDAst idAst = (IDAst) getAst1(); - tuples.dereference(idAst.offset, true, idAst.isGlobal); + if (isJrtManagedSpecialName(idAst.id)) { + idAst.populateTuples(tuples); + } else { + tuples.dereference(idAst.offset, true, idAst.isGlobal); + } } else { getAst1().populateTuples(tuples); } diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index a73296ac..21624eaf 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -856,6 +856,36 @@ public void testAssignedBlankRejectsMembershipArrayContext() throws Exception { .runAndAssert(); } + @Test + public void testSpecialScalarRejectsArrayElementAssignment() throws Exception { + AwkTestSupport + .awkTest("special scalar rejects array element assignment") + .script("BEGIN { FS[1] = 2; print FS }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void testSpecialScalarRejectsArrayOnlyContexts() throws Exception { + AwkTestSupport + .awkTest("special scalar rejects array membership") + .script("BEGIN { print (1 in FS) }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + + AwkTestSupport + .awkTest("special scalar rejects for-in") + .script("BEGIN { for (key in NR); print \"continued\" }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + + AwkTestSupport + .awkTest("special scalar rejects split destination") + .script("BEGIN { split(\"a\", FS) }") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + @Test public void testArrayParameterDefersUntypedActualUntilScalarUse() throws Exception { AwkTestSupport From e78f9dc332438646e5f6e0c9e19327b5f398662d Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 30 Jul 2026 21:21:03 +0200 Subject: [PATCH 10/11] Track element argument references per call frame Passing array elements or variables to user functions allocated a live reference per argument and registered element references in a weakly held set that every delete, split, and extension call swept, making such loops quadratic between garbage collections (32k iterations took 62s; 100k did not finish). Typed arguments are now materialized directly when pushed, so a reference is only created when the argument is genuinely untyped, and the remaining element references are adopted by the receiving call frame and released when it pops. The tracking stack is empty unless an untyped element argument is in flight, making the sweeps no-ops for normal scripts: 1M-iteration loops now run in a few seconds. Co-Authored-By: Claude Fable 5 --- src/main/java/io/jawk/backend/AVM.java | 115 ++++++++++++------ .../java/io/jawk/backend/RuntimeStack.java | 4 + src/test/java/io/jawk/AwkTest.java | 22 ++++ 3 files changed, 106 insertions(+), 35 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index a5a7d3d6..7e798b7a 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -43,7 +43,6 @@ import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.WeakHashMap; import java.util.function.BiConsumer; import java.util.regex.Pattern; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -130,8 +129,11 @@ public class AVM implements VariableManager, Closeable { // operand stack private Deque operandStack = new ArrayDeque(); - private final Set indirectArrayArgumentReferences = Collections - .newSetFromMap(new WeakHashMap()); + // Untyped array elements currently passed as user-function arguments, in + // call order. Each reference lives exactly as long as the call frame that + // received it, so this stack stays empty unless an untyped element + // argument is in flight. + private final Deque elementArgumentReferences = new ArrayDeque(); private List arguments; private boolean sortedArrayKeys; private final Map baseInitialVariables; @@ -708,7 +710,7 @@ private void resetRuntimeState(List runtimeArguments, Map runtimeArguments, Map variableOverrides) { // Reset the AVM-owned state that must not leak across executions. operandStack.clear(); - indirectArrayArgumentReferences.clear(); + elementArgumentReferences.clear(); environOffset = NULL_OFFSET; argcOffset = NULL_OFFSET; argvOffset = NULL_OFFSET; @@ -1696,11 +1698,19 @@ private void executeTuples(PositionTracker position) VariableTuple variableTuple = (VariableTuple) tuple; Object scalarValue = runtimeStack .getVariable(variableTuple.getVariableOffset(), variableTuple.isGlobal()); - push( - new IndirectArgumentReference( - runtimeStack.getVariableFrame(variableTuple.isGlobal()), - variableTuple.getVariableOffset(), - scalarValue)); + // Typed values are passed as plain values; only untyped + // variables need a live link back to the caller's slot. + if (scalarValue instanceof ArgumentReference) { + push(resolveUserFunctionArgument(scalarValue)); + } else if (isUntyped(scalarValue)) { + push( + new IndirectArgumentReference( + runtimeStack.getVariableFrame(variableTuple.isGlobal()), + variableTuple.getVariableOffset(), + scalarValue)); + } else { + push(scalarValue); + } position.next(); break; } @@ -1709,9 +1719,11 @@ private void executeTuples(PositionTracker position) checkScalar(idx); Map map = toMap(pop()); Object scalarValue = JRT.getAssocArrayValue(map, idx); - IndirectArrayArgumentReference reference = new IndirectArrayArgumentReference(map, idx, scalarValue); - indirectArrayArgumentReferences.add(reference); - push(reference); + // Typed elements are passed as plain values; only untyped + // elements need a live link back to their containing array. + push( + isUntyped(scalarValue) ? + new IndirectArrayArgumentReference(map, idx, scalarValue) : scalarValue); position.next(); break; } @@ -2212,8 +2224,8 @@ private void executeTuples(PositionTracker position) long numFormalParams = callTuple.getNumFormalParams(); long numActualParams = callTuple.getNumActualParams(); Object[] actualArguments = popArguments(numActualParams); - resolveUserFunctionArguments(actualArguments); runtimeStack.pushFrame(numFormalParams, position.currentIndex()); + adoptElementArgumentReferences(actualArguments); for (int i = 0; i < actualArguments.length; i++) { runtimeStack.setVariable(i, actualArguments[i], false); } @@ -2228,7 +2240,6 @@ private void executeTuples(PositionTracker position) String qualifiedName = normalizeIndirectFunctionName(requestedName); IndirectFunctionTarget target = callTuple.getUserFunctions().get(qualifiedName); if (target != null) { - resolveUserFunctionArguments(actualArguments); long formalCount = target.getNumFormalParams(); if (actualArguments.length > formalCount) { jrt @@ -2245,6 +2256,7 @@ private void executeTuples(PositionTracker position) activeProfilingFunctions.push(new ActiveFunction(qualifiedName, tupleStartNanos)); } runtimeStack.pushFrame(formalCount, position.currentIndex()); + adoptElementArgumentReferences(actualArguments); int copiedArgumentCount = Math.min(actualArguments.length, (int) formalCount); for (int i = 0; i < copiedArgumentCount; i++) { runtimeStack.setVariable(i, actualArguments[i], false); @@ -2307,6 +2319,7 @@ private void executeTuples(PositionTracker position) break; } case RETURN_FROM_FUNCTION: { + releaseElementArgumentReferences(); position.jump(runtimeStack.popFrame()); push(runtimeStack.getReturnValue()); position.next(); @@ -2406,10 +2419,7 @@ private void executeTuples(PositionTracker position) // If in BEGIN or in a rule, jump to the END section if (!withinEndBlocks && exitAddress != null) { - // clear runtime stack - runtimeStack.popAllFrames(); - // clear operand stack - operandStack.clear(); + resetCallState(); position.jump(exitAddress); } else { // Exit immediately with ExitException @@ -2721,25 +2731,16 @@ private void executeTuples(PositionTracker position) } throw ee; } catch (IOException ioe) { - // clear runtime stack - runtimeStack.popAllFrames(); - // clear operand stack - operandStack.clear(); + resetCallState(); throw ioe; } catch (RuntimeException re) { - // clear runtime stack - runtimeStack.popAllFrames(); - // clear operand stack - operandStack.clear(); + resetCallState(); if (re instanceof AwkSandboxException) { throw re; } throw new AwkRuntimeException(position.lineNumber(), re.getMessage(), re); } catch (AssertionError ae) { - // clear runtime stack - runtimeStack.popAllFrames(); - // clear operand stack - operandStack.clear(); + resetCallState(); throw ae; } @@ -2840,9 +2841,44 @@ private String normalizeIndirectFunctionName(String functionName) { return functionName.startsWith("awk::") ? functionName.substring("awk::".length()) : functionName; } - private void resolveUserFunctionArguments(Object[] actualArgumentsParam) { - for (int index = 0; index < actualArgumentsParam.length; index++) { - actualArgumentsParam[index] = resolveUserFunctionArgument(actualArgumentsParam[index]); + /** + * Unwinds every call frame and clears the per-call runtime state, after an + * {@code exit} statement or an abandoned execution. + */ + private void resetCallState() { + runtimeStack.popAllFrames(); + elementArgumentReferences.clear(); + operandStack.clear(); + } + + /** + * Registers the untyped element references received by the call frame that + * was just pushed, so array mutations occurring during the call can detach + * them. References forwarded from an enclosing call keep their original + * owner. + */ + private void adoptElementArgumentReferences(Object[] actualArguments) { + for (Object argument : actualArguments) { + if (argument instanceof IndirectArrayArgumentReference) { + IndirectArrayArgumentReference reference = (IndirectArrayArgumentReference) argument; + if (reference.ownerFrame < 0) { + reference.ownerFrame = runtimeStack.frameCount(); + elementArgumentReferences.push(reference); + } + } + } + } + + /** + * Drops the element references owned by the call frame that is about to be + * popped. Frames are strictly nested, so the owned references always sit on + * top of the tracking stack. + */ + private void releaseElementArgumentReferences() { + int depth = runtimeStack.frameCount(); + while (!elementArgumentReferences.isEmpty() + && elementArgumentReferences.peek().ownerFrame == depth) { + elementArgumentReferences.pop(); } } @@ -4262,8 +4298,11 @@ private Object resolveLengthArgumentReference(ArgumentReference reference) { } private Map captureAttachedArrayArgumentValues() { + if (elementArgumentReferences.isEmpty()) { + return Collections.emptyMap(); + } Map values = new IdentityHashMap(); - for (IndirectArrayArgumentReference reference : indirectArrayArgumentReferences) { + for (IndirectArrayArgumentReference reference : elementArgumentReferences) { if (reference.isAttached()) { values.put(reference, reference.currentValue()); } @@ -4279,7 +4318,10 @@ private void detachReplacedArrayArgumentReferences( } private void detachMissingArrayArgumentReferences(Map map) { - for (IndirectArrayArgumentReference reference : indirectArrayArgumentReferences) { + if (elementArgumentReferences.isEmpty()) { + return; + } + for (IndirectArrayArgumentReference reference : elementArgumentReferences) { reference.detachIfMissing(map); } } @@ -4406,6 +4448,9 @@ private static final class IndirectArrayArgumentReference implements ArgumentRef private final Object key; private Object detachedValue; private boolean detached; + // Depth of the call frame that received this reference as an + // argument; negative until the call is entered. + private int ownerFrame = -1; private IndirectArrayArgumentReference( Map mapParam, diff --git a/src/main/java/io/jawk/backend/RuntimeStack.java b/src/main/java/io/jawk/backend/RuntimeStack.java index 4586b7aa..49dc351b 100644 --- a/src/main/java/io/jawk/backend/RuntimeStack.java +++ b/src/main/java/io/jawk/backend/RuntimeStack.java @@ -157,6 +157,10 @@ Object[] getVariableFrame(boolean isGlobal) { return isGlobal ? globals : locals; } + int frameCount() { + return localsStack.size(); + } + Object setVariable(long offset, Object val, boolean isGlobal) { if (isGlobal) { globals[(int) offset] = val; diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index 21624eaf..6a9035ce 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -1002,6 +1002,28 @@ public void testFunctionResultCannotOverwriteSubarray() throws Exception { .runAndAssert(); } + @Test(timeout = 30000) + public void testElementArgumentTrackingStaysLinear() throws Exception { + // Element arguments passed in a hot loop, combined with delete, must not + // accumulate tracking state across calls (quadratic before the + // frame-scoped reference lifecycle). + AwkTestSupport + .awkTest("untyped element argument tracking stays linear") + .script( + "function f(x) { delete c[1]; c[1] = 1 } " + + "BEGIN { c[1] = 1; for (i = 0; i < 50000; i++) f(a[i]); print \"done\" }") + .expectLines("done") + .runAndAssert(); + + AwkTestSupport + .awkTest("typed element argument tracking stays linear") + .script( + "function f(x) { delete c[1]; c[1] = 1 } " + + "BEGIN { c[1] = 1; for (i = 0; i < 50000; i++) { a[i] = i; f(a[i]) } print \"done\" }") + .expectLines("done") + .runAndAssert(); + } + @Test public void testArraysOfArraysReportLineNumberWhenScalarUsedAsArray() throws Exception { assertRuntimeExceptionLineNumber( From 4bc266fb6491c8547c958fae8927f64fef05226b Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 30 Jul 2026 21:55:20 +0200 Subject: [PATCH 11/11] Trim per-call overhead from argument handling Pop user-function arguments directly into the callee frame instead of materializing an Object[] per call, and skip the argument-reference check when reading global variables, which can never hold one. Brings call-heavy and variable-heavy scripts back to pre-runtime-typing throughput (within benchmark noise). Co-Authored-By: Claude Fable 5 --- src/main/java/io/jawk/backend/AVM.java | 29 ++++++++++++++++---------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 7e798b7a..64b780e5 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -2223,11 +2223,12 @@ private void executeTuples(PositionTracker position) Address funcAddr = callTuple.getAddress(); long numFormalParams = callTuple.getNumFormalParams(); long numActualParams = callTuple.getNumActualParams(); - Object[] actualArguments = popArguments(numActualParams); runtimeStack.pushFrame(numFormalParams, position.currentIndex()); - adoptElementArgumentReferences(actualArguments); - for (int i = 0; i < actualArguments.length; i++) { - runtimeStack.setVariable(i, actualArguments[i], false); + // Arguments are stacked, so first in the stack is the last for the function + for (long i = numActualParams - 1; i >= 0; i--) { + Object argument = pop(); + adoptElementArgumentReference(argument); + runtimeStack.setVariable(i, argument, false); // false = local } position.jump(funcAddr); // position.next(); @@ -2859,12 +2860,16 @@ private void resetCallState() { */ private void adoptElementArgumentReferences(Object[] actualArguments) { for (Object argument : actualArguments) { - if (argument instanceof IndirectArrayArgumentReference) { - IndirectArrayArgumentReference reference = (IndirectArrayArgumentReference) argument; - if (reference.ownerFrame < 0) { - reference.ownerFrame = runtimeStack.frameCount(); - elementArgumentReferences.push(reference); - } + adoptElementArgumentReference(argument); + } + } + + private void adoptElementArgumentReference(Object argument) { + if (argument instanceof IndirectArrayArgumentReference) { + IndirectArrayArgumentReference reference = (IndirectArrayArgumentReference) argument; + if (reference.ownerFrame < 0) { + reference.ownerFrame = runtimeStack.frameCount(); + elementArgumentReferences.push(reference); } } } @@ -4235,7 +4240,9 @@ private Map getMapVariable(long offset, boolean isGlobal) { private Object resolveVariable(long offset, boolean isGlobal, boolean arrayContext) { Object value = runtimeStack.getVariable(offset, isGlobal); - if (value instanceof ArgumentReference) { + // Argument references are only ever stored in local parameter slots, + // so global reads skip the reference check entirely. + if (!isGlobal && value instanceof ArgumentReference) { value = resolveArgumentReference((ArgumentReference) value, arrayContext); runtimeStack.setVariable(offset, value, isGlobal); return value;