From 1566af9d3a59127aa34a033fcb85a08e0ad716f4 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 14 Aug 2026 20:49:33 +0200 Subject: [PATCH 1/5] feat: complete regex trace ownership and DATA inheritance Emit lexical regex release diagnostics after END in each owning runtime so direct and child debug traces have identical lifecycle boundaries. Model DATA as the source stream positioned after its marker and clone named handles through the established thread resource policies. This completes the Phase 35 stclass gate and restores direct and threaded reg_email coverage on both execution backends. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/concurrency.md | 29 ++++++----- .../scriptengine/PerlLanguageProvider.java | 3 ++ .../frontend/parser/DataSection.java | 52 ++++++++++++------- .../perlonjava/runtime/operators/WarnDie.java | 3 ++ .../runtime/regex/RuntimeRegex.java | 21 ++++++++ .../runtimetypes/GlobalRuntimeState.java | 6 ++- .../runtimetypes/PerlThreadControlBlock.java | 4 ++ .../runtimetypes/RuntimeRegexState.java | 4 ++ src/test/resources/unit/data_section_seek.t | 16 ++++++ 9 files changed, 106 insertions(+), 32 deletions(-) create mode 100644 src/test/resources/unit/data_section_seek.t diff --git a/dev/design/concurrency.md b/dev/design/concurrency.md index b82ac4b59f..f5d44d2280 100644 --- a/dev/design/concurrency.md +++ b/dev/design/concurrency.md @@ -647,7 +647,7 @@ Phase 33's release gate completed with `./jcpan --jobs 8 -t DBIx::Class`: non-local labeled control flow tears down every abandoned Perl frame before the target resumes, preserving scope-guard diagnostics and redirected STDERR. -### Phase 35 — Lexical regex debugging (implemented core 2026-08-14) +### Phase 35 — Lexical regex debugging (completed 2026-08-14) Implement scoped `use/no re 'debug'` and `debugcolor` as compiler hints carried by regex and CODE metadata on both backends. Diagnostics use the bound runtime's @@ -656,10 +656,10 @@ trace flag. Acceptance: `re/stclass_threads.t` reaches 6/6 and direct/child traces have identical behavior and runtime ownership. The compiler hints, JVM/interpreter propagation, runtime-owned STDERR routing, -snapshot behavior, and focused six-assertion oracle are implemented. The core -`stclass_threads.t` gate is 3/6: all three trace-linearity assertions pass, but -each child trace contains one additional record. That direct/child formatting -delta remains part of Phase 35 acceptance. +snapshot behavior, and focused six-assertion oracle are implemented. Debug +regex lifecycle records now drain after END in the owning main or child +runtime. The core `stclass_threads.t` gate reaches 6/6 with equal direct/child +record counts and linear scaling. ### Phase 36 — Complete regex parity exercised by thread wrappers (in progress) @@ -671,10 +671,12 @@ applicable `perl5_t/t/re/*thr*.t` test and direct companion completes its plan without unexpected failure on JVM or interpreter backends. This tranche restores recursive-definition compilation for `reg_email` and -keeps direct/thread compilation behavior aligned. Its test body is still -blocked in both paths by the direct DATA-handle gap. `pat_re_eval` and the -remaining `qr//`, conditional, control-verb, lookbehind, Unicode-property, and -diagnostic coverage remain shared regex-language work. +keeps direct/thread compilation behavior aligned. DATA now models the source +file positioned after its marker, remains seekable to the source start, and +crosses thread snapshots through the named-handle inheritance policy. Direct +and threaded `reg_email` therefore pass 13/13 on both backends. `pat_re_eval` +and the remaining `qr//`, conditional, control-verb, lookbehind, +Unicode-property, and diagnostic coverage remain shared regex-language work. ### Phase 37 — General filehandle and resource inheritance (implemented tranche 2026-08-14) @@ -793,7 +795,7 @@ CI. ## 7. Progress Tracking -### Current Status: Phases 35–39 implemented for the supported tranche +### Current Status: Phase 35 complete; Phase 36 and Phase 39b next Hints, warnings, filters, and source maps are runtime-owned while compiler-only scratch remains protected by the global compile lock. The Phase 11 inventory is @@ -965,9 +967,10 @@ three assertions from the adjacent-import parser fix. ### Next Steps -1. Close the remaining Phase 35 trace-record delta and Phase 36 direct regex - language/DATA-handle gaps; wrapper behavior must follow the corrected direct - implementation without special cases. +1. Complete Phase 36's direct regex-language gaps in `pat_re_eval`, `qr//`, + conditionals, control verbs, lookbehind, Unicode properties, and diagnostics; + wrapper behavior must follow the corrected direct implementation without + special cases. 2. Implement Phase 39b's fetch-time nested shared proxies, global destruction, weak/cyclic ownership, and the destructive `share` versus preserving `shared_clone` distinction. diff --git a/src/main/java/org/perlonjava/app/scriptengine/PerlLanguageProvider.java b/src/main/java/org/perlonjava/app/scriptengine/PerlLanguageProvider.java index a9b3b280fd..63de96066d 100644 --- a/src/main/java/org/perlonjava/app/scriptengine/PerlLanguageProvider.java +++ b/src/main/java/org/perlonjava/app/scriptengine/PerlLanguageProvider.java @@ -23,6 +23,7 @@ import org.perlonjava.runtime.perlmodule.BHooksEndOfScope; import org.perlonjava.runtime.perlmodule.FilterUtilCall; import org.perlonjava.runtime.perlmodule.Strict; +import org.perlonjava.runtime.regex.RuntimeRegex; import org.perlonjava.runtime.runtimetypes.*; import org.perlonjava.runtime.WarningBitsRegistry; @@ -624,6 +625,7 @@ private static RuntimeList executeCodeImpl(RuntimeCode runtimeCode, Node ast, Em // END may itself fail; captured cleanup still belongs after // the attempted END dispatch and before runtime teardown. MortalList.flushDeferredCaptures(); + RuntimeRegex.emitCurrentRuntimeDebugFreeTraces(); } // Global destruction: walk stashes for tracked blessed objects GlobalDestruction.runGlobalDestruction(); @@ -653,6 +655,7 @@ private static RuntimeList executeCodeImpl(RuntimeCode runtimeCode, Node ast, Em } finally { CallerStack.pop(); MortalList.flushDeferredCaptures(); // Live captures outlast END + RuntimeRegex.emitCurrentRuntimeDebugFreeTraces(); } RuntimeIO.closeAllHandles(); } diff --git a/src/main/java/org/perlonjava/frontend/parser/DataSection.java b/src/main/java/org/perlonjava/frontend/parser/DataSection.java index 4efbcea393..d2c187eb9d 100644 --- a/src/main/java/org/perlonjava/frontend/parser/DataSection.java +++ b/src/main/java/org/perlonjava/frontend/parser/DataSection.java @@ -4,6 +4,7 @@ import org.perlonjava.frontend.lexer.LexerToken; import org.perlonjava.frontend.lexer.LexerTokenType; +import org.perlonjava.runtime.io.IOHandle; import org.perlonjava.runtime.io.ScalarBackedIO; import org.perlonjava.runtime.runtimetypes.GlobalVariable; import org.perlonjava.runtime.runtimetypes.PerlRuntime; @@ -71,6 +72,11 @@ public static void createPlaceholderDataHandle(Parser parser) { * @param content the content after __DATA__ or __END__ */ public static void createDataHandle(Parser parser, String handleName, String content) { + createDataHandle(parser, handleName, content, 0); + } + + private static void createDataHandle( + Parser parser, String handleName, String content, int initialPosition) { if (CompilerOptions.DEBUG_ENABLED) parser.ctx.logDebug("Populating DATA handle for package: " + handleName + " with content: " + content); // Get the existing RuntimeIO (which should be the placeholder we created earlier) @@ -81,12 +87,14 @@ public static void createDataHandle(Parser parser, String handleName, String con // This ensures that any aliased handles (like *ARGV = *DATA) continue to work RuntimeScalar contentScalar = new RuntimeScalar(content); ScalarBackedIO newScalarIO = new ScalarBackedIO(contentScalar); + newScalarIO.seek(initialPosition, IOHandle.SEEK_SET); existingIO.ioHandle = newScalarIO; if (CompilerOptions.DEBUG_ENABLED) parser.ctx.logDebug("Updated existing DATA handle with new content"); } else { // Fallback: create new handle if no placeholder exists RuntimeScalar contentScalar = new RuntimeScalar(content); var fileHandle = RuntimeIO.open(contentScalar.createReference(), "<"); + fileHandle.ioHandle.seek(initialPosition, IOHandle.SEEK_SET); GlobalVariable.getGlobalIO(handleName).setIO(fileHandle); if (CompilerOptions.DEBUG_ENABLED) parser.ctx.logDebug("Created new DATA handle"); } @@ -129,7 +137,9 @@ private static boolean isEndMarker(LexerToken token) { * @param markerText the marker to search for ("__DATA__" or "__END__") * @return the DATA content as a string (Latin-1 encoded), or null if marker not found */ - private static String extractDataFromRawBytes(byte[] rawBytes, String markerText) { + private record RawDataHandle(String content, int initialPosition) {} + + private static RawDataHandle extractDataFromRawBytes(byte[] rawBytes, String markerText) { byte[] marker = markerText.getBytes(StandardCharsets.US_ASCII); int markerLen = marker.length; @@ -176,7 +186,9 @@ private static String extractDataFromRawBytes(byte[] rawBytes, String markerText // Always store as Latin-1 (each byte = one character) to preserve raw bytes. // The DATA handle's encoding layer (applied by parseDataSection) handles // UTF-8 decoding at read time when `use utf8` is active. - return new String(rawBytes, dataStart, rawBytes.length - dataStart, StandardCharsets.ISO_8859_1); + return new RawDataHandle( + new String(rawBytes, StandardCharsets.ISO_8859_1), + dataStart); } return null; // Marker not found @@ -226,30 +238,34 @@ static int parseDataSection(Parser parser, int tokenIndex, List toke // In Perl 5, reads raw bytes from the file. byte[] rawBytes = parser.ctx.compilerOptions.rawCodeBytes; boolean useUtf8 = parser.ctx.symbolTable.isStrictOptionEnabled(HINT_UTF8); - String rawContent = null; + RawDataHandle rawContent = null; if (rawBytes != null) { rawContent = extractDataFromRawBytes(rawBytes, token.text); } if (rawContent != null) { - createDataHandle(parser, handleName, rawContent); + createDataHandle( + parser, + handleName, + rawContent.content(), + rawContent.initialPosition()); } else { - // Fallback: concatenate remaining tokens (for eval/string-based code - // where raw bytes are not available) - StringBuilder dataContent = new StringBuilder(); - while (tokenIndex < tokens.size()) { - LexerToken currentToken = tokens.get(tokenIndex); - - // Stop if we hit an end marker - if (isEndMarker(currentToken)) { - break; + // String/eval callers do not have raw source bytes. Rebuild the + // complete token stream so DATA still behaves like a source-file + // handle: its initial position is after the marker, while seek(0) + // exposes the source and marker that precede the payload. + StringBuilder sourceContent = new StringBuilder(); + int initialPosition = 0; + for (int index = 0; index < tokens.size(); index++) { + if (index == tokenIndex) { + initialPosition = sourceContent.toString() + .getBytes(StandardCharsets.ISO_8859_1).length; } - - dataContent.append(currentToken.text); - tokenIndex++; + LexerToken currentToken = tokens.get(index); + if (isEndMarker(currentToken)) break; + sourceContent.append(currentToken.text); } - - createDataHandle(parser, handleName, dataContent.toString()); + createDataHandle(parser, handleName, sourceContent.toString(), initialPosition); } // When `use utf8` is active, apply :utf8 layer to the DATA handle. diff --git a/src/main/java/org/perlonjava/runtime/operators/WarnDie.java b/src/main/java/org/perlonjava/runtime/operators/WarnDie.java index 48870a118f..ed092c5fec 100644 --- a/src/main/java/org/perlonjava/runtime/operators/WarnDie.java +++ b/src/main/java/org/perlonjava/runtime/operators/WarnDie.java @@ -4,6 +4,7 @@ import org.perlonjava.backend.jvm.ByteCodeSourceMapper; import org.perlonjava.runtime.perlmodule.Universal; import org.perlonjava.runtime.perlmodule.Warnings; +import org.perlonjava.runtime.regex.RuntimeRegex; import org.perlonjava.runtime.runtimetypes.*; import java.util.HashMap; @@ -680,6 +681,7 @@ public static RuntimeScalar exit(RuntimeScalar runtimeScalar) { try { runEndBlocks(false); // Don't reset $? - we just set it to the exit code } catch (Throwable t) { + RuntimeRegex.emitCurrentRuntimeDebugFreeTraces(); RuntimeIO.closeAllHandles(); String errorMessage = ErrorMessageUtil.stringifyException(t); System.err.println(errorMessage); @@ -687,6 +689,7 @@ public static RuntimeScalar exit(RuntimeScalar runtimeScalar) { } finally { MortalList.flushDeferredCaptures(); } + RuntimeRegex.emitCurrentRuntimeDebugFreeTraces(); // Global destruction: walk stashes for tracked blessed objects GlobalDestruction.runGlobalDestruction(); RuntimeIO.closeAllHandles(); diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 94b113f02a..52de5b3471 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -621,6 +621,7 @@ private static String stripDebugMarkers(String modifiers) { private void emitCompileDebugTrace() { if (lexicalDebugMode == 0) return; + registerDebugLifecycle(); String patternDescription = patternString == null ? "" : patternString; debugWrite("Compiling REx \"" + patternDescription + "\"\n" + "Final program:\n" @@ -630,6 +631,7 @@ private void emitCompileDebugTrace() { public void emitExecutionDebugTrace(String input) { if (lexicalDebugMode == 0) return; + registerDebugLifecycle(); StringBuilder trace = new StringBuilder(Math.max(96, input.length() * 72)); trace.append("Matching REx \"").append(patternString == null ? "" : patternString) .append("\" against input of length ").append(input.length()).append('\n'); @@ -645,6 +647,25 @@ public void emitExecutionDebugTrace(String input) { debugWrite(trace.toString()); } + private void registerDebugLifecycle() { + List active = state().activeDebugRegexes; + for (RuntimeRegex regex : active) { + if (regex == this) return; + } + active.add(this); + } + + /** Emit Perl-style lifecycle records after END and before runtime teardown. */ + public static void emitCurrentRuntimeDebugFreeTraces() { + List active = state().activeDebugRegexes; + for (RuntimeRegex regex : active) { + if (regex.lexicalDebugMode == 0) continue; + String patternDescription = regex.patternString == null ? "" : regex.patternString; + regex.debugWrite("Freeing REx: \"" + patternDescription + "\"\n"); + } + active.clear(); + } + private void debugWrite(String message) { if (lexicalDebugMode == 2) { message = "\u001b[36m" + message + "\u001b[0m"; diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeState.java b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeState.java index 11a3231391..aaaf36a86d 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeState.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeState.java @@ -282,6 +282,7 @@ synchronized void snapshotInto(GlobalRuntimeState target, RuntimeGraphCloner clo cloneMap(codeRefs, target.codeRefs, cloner, RuntimeScalar.class); cloneMap(pseudoConstants, target.pseudoConstants, cloner, RuntimeScalar.class); cloneMap(pinnedCodeRefs, target.pinnedCodeRefs, cloner, RuntimeScalar.class); + cloneMap(ioSlots, target.ioSlots, cloner, RuntimeGlob.class); for (Map.Entry entry : compiledCodeRefs.entrySet()) { target.compiledCodeRefs.put(entry.getKey(), (RuntimeScalar) cloner.cloneValue(entry.getValue())); @@ -290,6 +291,7 @@ synchronized void snapshotInto(GlobalRuntimeState target, RuntimeGraphCloner clo target.importedSubs.putAll(importedSubs); target.operatorOverrideGlobs.putAll(operatorOverrideGlobs); target.deletedCodeRefPins.addAll(deletedCodeRefPins); + target.hiddenIoSlotsAfterStashDelete.addAll(hiddenIoSlotsAfterStashDelete); target.localizedCodeRefDepth.putAll(localizedCodeRefDepth); target.stashAliases.putAll(stashAliases); target.resolvedStashAliases.putAll(resolvedStashAliases); @@ -312,7 +314,9 @@ synchronized void snapshotInto(GlobalRuntimeState target, RuntimeGraphCloner clo COMPILED_CODE_REF_RANGE_SIZE)); target.stashEnumerationVersion = stashEnumerationVersion; target.coreGlobalsInitialized = coreGlobalsInitialized; - // Class loaders, caches, named IO and formats are child-owned/fresh. + // Class loaders, caches, and formats are child-owned/fresh. Standard + // handles remain the child's canonical PerlRuntime globs; named IO + // slots cross through RuntimeGraphCloner's explicit handle policies. } /** Copy CV ids registered by a named sub that was materialized after snapshot. */ diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/PerlThreadControlBlock.java b/src/main/java/org/perlonjava/runtime/runtimetypes/PerlThreadControlBlock.java index 2beb7dc56a..8e5a0aacc7 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/PerlThreadControlBlock.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/PerlThreadControlBlock.java @@ -1,5 +1,7 @@ package org.perlonjava.runtime.runtimetypes; +import org.perlonjava.runtime.regex.RuntimeRegex; + import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; @@ -122,6 +124,8 @@ private void run() { SpecialBlock.runEndBlocks(false); } catch (Throwable endFailure) { if (failure == null) failure = endFailure; + } finally { + RuntimeRegex.emitCurrentRuntimeDebugFreeTraces(); } return new Outcome(value, failure); }); diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java index e8ca637053..9cabdebdf9 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java @@ -4,6 +4,7 @@ import org.perlonjava.runtime.regex.RuntimeRegex; import java.util.LinkedHashMap; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -54,6 +55,9 @@ protected boolean removeEldestEntry(Map.Entry eldest) { } }; + /** Regex objects whose lexical debug lifecycle is active in this runtime. */ + public final List activeDebugRegexes = new ArrayList<>(); + /** Per-runtime {@code pos()} values and zero-length-match bookkeeping. */ final Map positionCache = new LinkedHashMap( diff --git a/src/test/resources/unit/data_section_seek.t b/src/test/resources/unit/data_section_seek.t new file mode 100644 index 0000000000..506df11cdf --- /dev/null +++ b/src/test/resources/unit/data_section_seek.t @@ -0,0 +1,16 @@ +use strict; +use warnings; +use Test::More tests => 3; + +my $first = ; +is($first, "payload-one\n", 'DATA initially starts after its marker'); + +ok(seek(DATA, 0, 0), 'DATA can seek back to the source-file start'); +while () { + last if /^__DATA__$/; +} +is(, "payload-one\n", 'rewound DATA exposes the marker and payload'); + +__DATA__ +payload-one +payload-two From d080d18ac39bda53db7637700c7e17ac7cd5c52c Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 14 Aug 2026 21:03:44 +0200 Subject: [PATCH 2/5] fix: respect regex quotemeta around code-block syntax Treat (?{ and (??{ as literal text while a regex is inside a lexical \Q...\E region. This restores pat_re_eval parsing without changing genuine embedded-code handling outside quoted regions. Record the remaining match-time regex-code blocker in the concurrency plan. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/concurrency.md | 8 ++++++-- .../frontend/parser/StringDoubleQuoted.java | 5 +++++ .../frontend/parser/StringSegmentParser.java | 12 ++++++++++-- src/test/resources/unit/regex_quotemeta_code_block.t | 10 ++++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 src/test/resources/unit/regex_quotemeta_code_block.t diff --git a/dev/design/concurrency.md b/dev/design/concurrency.md index f5d44d2280..16310e49cc 100644 --- a/dev/design/concurrency.md +++ b/dev/design/concurrency.md @@ -675,8 +675,12 @@ keeps direct/thread compilation behavior aligned. DATA now models the source file positioned after its marker, remains seekable to the source start, and crosses thread snapshots through the named-handle inheritance policy. Direct and threaded `reg_email` therefore pass 13/13 on both backends. `pat_re_eval` -and the remaining `qr//`, conditional, control-verb, lookbehind, -Unicode-property, and diagnostic coverage remain shared regex-language work. +now parses quoted code-block-shaped text correctly: `(?{` inside `\Q...\E` +is literal rather than an embedded Perl block. This advances both direct and +threaded files to runtime construction, where arbitrary match-time `(?{...})` +execution remains the next blocker. The remaining `qr//`, conditional, +control-verb, lookbehind, Unicode-property, and diagnostic coverage likewise +remains shared regex-language work. ### Phase 37 — General filehandle and resource inheritance (implemented tranche 2026-08-14) diff --git a/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java b/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java index 7a5d06b248..d7e0310648 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java @@ -56,6 +56,11 @@ public class StringDoubleQuoted extends StringSegmentParser { */ private final Stack caseModifiers = new Stack<>(); + @Override + protected boolean regexCodeBlocksAreActive() { + return caseModifiers.stream().noneMatch(modifier -> "Q".equals(modifier.type)); + } + /** * Private constructor for StringDoubleQuoted parser. * diff --git a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java index b21f823506..9293208d51 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java @@ -155,6 +155,14 @@ protected boolean isInsideRegexCharClass() { return isRegex && inRegexCharClass; } + /** + * Whether regex code-block openers have their ordinary special meaning at the current point. + * Subclasses may suppress them while parsing a quoting region such as {@code \Q...\E}. + */ + protected boolean regexCodeBlocksAreActive() { + return true; + } + private void updateRegexCharClassState(char c) { if (!isRegex) { return; @@ -779,10 +787,10 @@ protected boolean handleSpecialToken(String text) { } case "(" -> { // Check for (?{...}) and (??{...}) regex code blocks - only in regex context - if (isRegex && isRegexCodeBlock()) { + if (isRegex && regexCodeBlocksAreActive() && isRegexCodeBlock()) { parseRegexCodeBlock(false); // (?{...}) - code execution yield true; - } else if (isRegex && isRegexRecursiveBlock()) { + } else if (isRegex && regexCodeBlocksAreActive() && isRegexRecursiveBlock()) { parseRegexCodeBlock(true); // (??{...}) - recursive pattern yield true; } diff --git a/src/test/resources/unit/regex_quotemeta_code_block.t b/src/test/resources/unit/regex_quotemeta_code_block.t new file mode 100644 index 0000000000..498a7dfe6a --- /dev/null +++ b/src/test/resources/unit/regex_quotemeta_code_block.t @@ -0,0 +1,10 @@ +use strict; +use warnings; + +print "1..2\n"; + +print "ok 1 - quoted regex code block opener is literal\n" + if '(?{1})' =~ /^\Q(?{1})\E$/; + +print "ok 2 - quote region may end inside code-block-shaped text\n" + if '(?{1})' =~ /^\Q(?{\E1\}\)$/; From 0861c5ef3880c9ea18f4081006ee0d1f14a6f77e Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 14 Aug 2026 21:19:21 +0200 Subject: [PATCH 3/5] feat: complete the public threads 2.43 surface Advertise the implemented threads 2.43 API after validating its full public method inventory on system Perl and both PerlOnJava backends. Record Phase 40 completion and retain op/threads.t at 30/30. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/concurrency.md | 16 +++++++++---- src/main/perl/lib/threads.pm | 2 +- .../unit/threads_public_api_surface.t | 24 +++++++++++++++++++ 3 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 src/test/resources/unit/threads_public_api_surface.t diff --git a/dev/design/concurrency.md b/dev/design/concurrency.md index 16310e49cc..ecd3add4ff 100644 --- a/dev/design/concurrency.md +++ b/dev/design/concurrency.md @@ -743,13 +743,19 @@ runtime that releases the final cross-runtime owner. Separate destructive plain includes nested rebless/store-back, fresh `refaddr` views, cycles, weak refs, one global destructor, and share-versus-shared_clone system-Perl oracles. -### Phase 40 — Complete public `threads` API +### Phase 40 — Complete public `threads` API (completed 2026-08-14) Close every remaining lifecycle, signal, context, exit-status, stack-size, import, stringify, alias-object, and shutdown-warning gap. Upgrade the module version only when its upstream surface passes. A nonzero stack request always selects a platform child even after virtual threads become the default. +The public 2.43 method surface is implemented and advertised. Creation context, +alias objects, current/class detach, signals, exit policy, stack metadata, +stringification, terminal errors, daemon-carrier shutdown, and attached-child +exit warnings are covered by focused JVM/interpreter tests. Core +`op/threads.t` completes 30/30. + ### Phase 41 — Fresh-runtime reset Add reset as a lifecycle distinct from terminal `close()`. Reset is allowed only @@ -799,7 +805,7 @@ CI. ## 7. Progress Tracking -### Current Status: Phase 35 complete; Phase 36 and Phase 39b next +### Current Status: Phase 40 complete; Phase 36 and Phase 39b remain open Hints, warnings, filters, and source maps are runtime-owned while compiler-only scratch remains protected by the global compile lock. The Phase 11 inventory is @@ -978,9 +984,9 @@ three assertions from the adjacent-import parser fix. 2. Implement Phase 39b's fetch-time nested shared proxies, global destruction, weak/cyclic ownership, and the destructive `share` versus preserving `shared_clone` distinction. -3. Land Phases 40–44 as the final delivery sequence: public API closure, - fresh-runtime reset, opt-in pooling and concurrent PSGI, virtual threads by - default, and the complete release gate. +3. Land Phases 41–44 as the final delivery sequence: fresh-runtime reset, + opt-in pooling and concurrent PSGI, virtual threads by default, and the + complete release gate. Phase 40's public 2.43 API surface is complete. 4. Preserve the green core, Storable, Test2, Net::SSLeay, index/substr, DBI, and DBIx::Class anchors after every phase. The 2026-08-14 DBIx::Class gate passed all 325 files and 42,671 assertions under diff --git a/src/main/perl/lib/threads.pm b/src/main/perl/lib/threads.pm index 2519f3c483..1d34776eb2 100644 --- a/src/main/perl/lib/threads.pm +++ b/src/main/perl/lib/threads.pm @@ -2,7 +2,7 @@ package threads; use strict; use warnings; -our $VERSION = '2.27'; +our $VERSION = '2.43'; our $threads = 1; sub all () { 0 } diff --git a/src/test/resources/unit/threads_public_api_surface.t b/src/test/resources/unit/threads_public_api_surface.t new file mode 100644 index 0000000000..562775bf58 --- /dev/null +++ b/src/test/resources/unit/threads_public_api_surface.t @@ -0,0 +1,24 @@ +use strict; +use warnings; +use threads 2.43; + +my @methods = qw( + create new async self tid object list join detach is_running is_joinable + is_detached error exit kill wantarray get_stack_size set_stack_size + set_thread_exit_only yield equal +); + +print '1..', scalar(@methods) + 2, "\n"; +my $test = 0; +sub check { + my ($condition, $name) = @_; + ++$test; + print(($condition ? 'ok' : 'not ok'), " $test - $name\n"); +} + +check($threads::VERSION >= 2.43, 'threads compatibility version is current'); +check($threads::threads, 'threads capability marker is enabled'); +for my $method (@methods) { + no strict 'refs'; + check(defined &{"threads::$method"}, "threads provides $method"); +} From c9236b48d43a0b45689e4a005755ecd94bb1d045 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 14 Aug 2026 21:39:51 +0200 Subject: [PATCH 4/5] feat: add fresh-equivalent runtime reset Introduce an exclusive reset lifecycle distinct from terminal close. Reject active bindings, compilation, children, shared locks, and waiters; drain END and destruction work; replace every runtime-owned state domain; rebuild standard IO and core globals; and poison the runtime after partial failure. Add JVM/interpreter freshness differentials and lifecycle rejection coverage, and advance the concurrency plan through Phase 41. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/concurrency.md | 22 +- dev/design/runtime-pooling-reset-contract.md | 38 ++-- .../runtime/runtimetypes/PerlRuntime.java | 210 ++++++++++++++++-- .../runtimetypes/PerlThreadRegistry.java | 8 + .../runtimetypes/SharedPerlStorage.java | 11 +- .../runtimetypes/PerlRuntimeResetTest.java | 164 ++++++++++++++ 6 files changed, 404 insertions(+), 49 deletions(-) create mode 100644 src/test/java/org/perlonjava/runtime/runtimetypes/PerlRuntimeResetTest.java diff --git a/dev/design/concurrency.md b/dev/design/concurrency.md index ecd3add4ff..f9faee533b 100644 --- a/dev/design/concurrency.md +++ b/dev/design/concurrency.md @@ -756,7 +756,7 @@ stringification, terminal errors, daemon-carrier shutdown, and attached-child exit warnings are covered by focused JVM/interpreter tests. Core `op/threads.t` completes 30/30. -### Phase 41 — Fresh-runtime reset +### Phase 41 — Fresh-runtime reset (completed 2026-08-14) Add reset as a lifecycle distinct from terminal `close()`. Reset is allowed only after execution, compilation, callbacks, children, locks, waiters, handles, and @@ -765,6 +765,18 @@ destruction work quiesce. Rebuild every domain in poison a runtime after any partial reset failure. Acceptance is exhaustive `A; reset; B == fresh; B` parity plus classloader/package-graph collection. +`PerlRuntime.reset()` is now a distinct exclusive lifecycle transition. It +rejects active bindings, compilation, children, shared locks, and waiters; +drains END/destruction and owned resources; replaces every runtime state holder; +rebuilds standard handles and core globals; clears terminal thread-family state; +and poisons the runtime after any partial failure. JVM/interpreter differentials +prove representative package, CODE, `%INC`, regex, execution, and I/O freshness. +Pooling remains off pending Phase 42's checkout stress, collection, and measured +benefit gates. + +The post-reset regression gate retains all 325 DBIx::Class files and 42,671 +assertions under `./jcpan --jobs 8 -t DBIx::Class`. + ### Phase 42 — Opt-in pooling and concurrent PSGI Add a bounded runtime-family pool configured by @@ -805,7 +817,7 @@ CI. ## 7. Progress Tracking -### Current Status: Phase 40 complete; Phase 36 and Phase 39b remain open +### Current Status: Phase 41 complete; Phase 36 and Phase 39b remain open Hints, warnings, filters, and source maps are runtime-owned while compiler-only scratch remains protected by the global compile lock. The Phase 11 inventory is @@ -984,9 +996,9 @@ three assertions from the adjacent-import parser fix. 2. Implement Phase 39b's fetch-time nested shared proxies, global destruction, weak/cyclic ownership, and the destructive `share` versus preserving `shared_clone` distinction. -3. Land Phases 41–44 as the final delivery sequence: fresh-runtime reset, - opt-in pooling and concurrent PSGI, virtual threads by default, and the - complete release gate. Phase 40's public 2.43 API surface is complete. +3. Land Phases 42–44 as the final delivery sequence: opt-in pooling and + concurrent PSGI, virtual threads by default, and the complete release gate. + Phases 40 and 41's public API and fresh-reset foundations are complete. 4. Preserve the green core, Storable, Test2, Net::SSLeay, index/substr, DBI, and DBIx::Class anchors after every phase. The 2026-08-14 DBIx::Class gate passed all 325 files and 42,671 assertions under diff --git a/dev/design/runtime-pooling-reset-contract.md b/dev/design/runtime-pooling-reset-contract.md index 216876657c..6dcdb3a253 100644 --- a/dev/design/runtime-pooling-reset-contract.md +++ b/dev/design/runtime-pooling-reset-contract.md @@ -2,11 +2,11 @@ ## Status -Runtime pooling was re-evaluated on 2026-08-13 and remains deliberately -disabled. `PerlRuntime.close()` is a terminal resource-release -operation, not a reset operation, and a closed runtime deliberately rejects -`bind`, `initialize`, and `execute`. Reusing it would currently expose state that -a newly constructed runtime does not contain. +Fresh-runtime reset was implemented on 2026-08-14. Runtime pooling remains +deliberately disabled until Phase 42's checkout stress, retention measurements, +and performance gate pass. `PerlRuntime.close()` remains a terminal +resource-release operation; reusable runtimes use the separate exclusive +`reset()` transition. This document defines the proof required before a pool may be implemented. It does not authorize clearing state opportunistically or enabling pooling behind @@ -64,24 +64,24 @@ correct terminal lifecycle, but it is intentionally insufficient for pooling. Pooling remains disabled until all items below are complete: -- [ ] Introduce one exclusive lifecycle transition that prevents reset while +- [x] Introduce one exclusive lifecycle transition that prevents reset while execution, compilation, callbacks, ithreads, detached children, shared-lock ownership, or condition waiters remain active. -- [ ] Define whether core bootstrap state is reconstructed or restored from an +- [x] Define whether core bootstrap state is reconstructed or restored from an immutable template; user package/CODE/class state and `%INC` must never leak. -- [ ] Recreate standard I/O wrappers and glob topology without closing borrowed +- [x] Recreate standard I/O wrappers and glob topology without closing borrowed JVM streams, and restore selected/last-handle and visibility defaults. -- [ ] Drain END/destruction work according to normal Perl semantics before +- [x] Drain END/destruction work according to normal Perl semantics before clearing lifecycle roots; prove weak references and rescued objects do not cross tenants. -- [ ] Clear every state domain in the inventory, including counters and caches, +- [x] Clear every state domain in the inventory, including counters and caches, without retaining generated classes or prior workload object graphs. -- [ ] Restore process-derived defaults (`cwd`, environment view, random policy, +- [x] Restore process-derived defaults (`cwd`, environment view, random policy, warning/feature defaults) according to an explicitly documented checkout contract. -- [ ] Reject or quarantine a runtime after reset failure; a partially reset +- [x] Reject or quarantine a runtime after reset failure; a partially reset runtime must never return to the pool. -- [ ] Prove `A; reset; B == fresh; B` on both compiler backends across globals, +- [x] Prove `A; reset; B == fresh; B` on both compiler backends across globals, closures, eval/require, regex, warnings/hints, MRO, I/O, lifecycle, signals, native modules, DATA, debugger state, and exceptions. - [ ] Add concurrency/stress coverage for checkout ownership, cancellation, @@ -93,12 +93,12 @@ Pooling remains disabled until all items below are complete: ## Current automated guard -`PerlRuntimePoolingResetContractTest` records the present negative contract: -close is terminal, closed runtimes cannot be rebound or executed, and package, -regex-cache, and execution settings retained by the terminal object differ from -a fresh runtime. The test prevents a future pool from treating `close()` as a -reset without first replacing this negative proof with the full equivalence -suite above. +`PerlRuntimePoolingResetContractTest` preserves the negative `close()` contract. +`PerlRuntimeResetTest` proves the positive transition: representative package, +CODE, `%INC`, regex, execution, thread-option, and standard-I/O state matches a +fresh runtime on both backends; reset rejects bindings, child threads, and +shared locks; pending END work drains; failed reset poisons the runtime; and the +same Java runtime identity executes again after successful reset. ## Related documents diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/PerlRuntime.java b/src/main/java/org/perlonjava/runtime/runtimetypes/PerlRuntime.java index 48d331b142..d36c7b23fa 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/PerlRuntime.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/PerlRuntime.java @@ -33,6 +33,7 @@ import java.lang.ref.WeakReference; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; /** @@ -48,8 +49,14 @@ public final class PerlRuntime implements AutoCloseable { public final long pid = ProcessHandle.current().pid(); String currentDirectory = System.getProperty("user.dir"); private final ReentrantLock executionLock = new ReentrantLock(); + private final Object lifecycleMonitor = new Object(); + private final AtomicInteger activeBindings = new AtomicInteger(); + private final AtomicInteger activeSharedLocks = new AtomicInteger(); + private final AtomicInteger activeSharedWaiters = new AtomicInteger(); private volatile boolean initialized; private volatile boolean closed; + private volatile boolean resetting; + private volatile Thread resetOwner; private final PerlThreadRegistry threadRegistry; private final long perlThreadId; private volatile int perlThreadContext = RuntimeContextType.SCALAR; @@ -58,29 +65,29 @@ public final class PerlRuntime implements AutoCloseable { private volatile boolean perlThreadExitOnly; private volatile boolean defaultPerlThreadExitOnly; - public final ExecutionRuntimeState executionState = new ExecutionRuntimeState(); - public final RuntimeRegexState regexState = new RuntimeRegexState(); - public final MroRuntimeState mroState = new MroRuntimeState(); - public final GlobalRuntimeState globalState = new GlobalRuntimeState(); - public final RuntimeCodeRuntimeState runtimeCodeState = new RuntimeCodeRuntimeState(); - public final CompilationRuntimeState compilationState = new CompilationRuntimeState(); - public final ByteCodeSourceMapper.State sourceMapperState = new ByteCodeSourceMapper.State(); - public final FilterRuntimeState filterState = new FilterRuntimeState(); - public final Time.State timeState = new Time.State(); - public final PerlSignalQueue.State signalState = new PerlSignalQueue.State(); - public final Random.State randomState = new Random.State(); - public final DataSection.State dataSectionState = new DataSection.State(); + public ExecutionRuntimeState executionState = new ExecutionRuntimeState(); + public RuntimeRegexState regexState = new RuntimeRegexState(); + public MroRuntimeState mroState = new MroRuntimeState(); + public GlobalRuntimeState globalState = new GlobalRuntimeState(); + public RuntimeCodeRuntimeState runtimeCodeState = new RuntimeCodeRuntimeState(); + public CompilationRuntimeState compilationState = new CompilationRuntimeState(); + public ByteCodeSourceMapper.State sourceMapperState = new ByteCodeSourceMapper.State(); + public FilterRuntimeState filterState = new FilterRuntimeState(); + public Time.State timeState = new Time.State(); + public PerlSignalQueue.State signalState = new PerlSignalQueue.State(); + public Random.State randomState = new Random.State(); + public DataSection.State dataSectionState = new DataSection.State(); public final Map flipFlopState = new HashMap<>(); public final Map scalarGlobState = new HashMap<>(); public final Map pointerPackState = new HashMap<>(); final Map> bObjectState = new HashMap<>(); - public final IORuntimeRegistryState ioRegistryState = new IORuntimeRegistryState(); - public final FileTestOperator.State fileTestState = new FileTestOperator.State(); - public final DebugRuntimeState debugState = new DebugRuntimeState(); - public final DiamondIO.State diamondIOState = new DiamondIO.State(); - public final ExtendedNativeUtils.State nativeState = new ExtendedNativeUtils.State(); + public IORuntimeRegistryState ioRegistryState = new IORuntimeRegistryState(); + public FileTestOperator.State fileTestState = new FileTestOperator.State(); + public DebugRuntimeState debugState = new DebugRuntimeState(); + public DiamondIO.State diamondIOState = new DiamondIO.State(); + public ExtendedNativeUtils.State nativeState = new ExtendedNativeUtils.State(); public final Deque netSslErrorQueue = new ArrayDeque<>(); - public final NetSSLeay.State netSslState = new NetSSLeay.State(); + public NetSSLeay.State netSslState = new NetSSLeay.State(); public ForkOpenState.PendingForkOpen pendingForkOpen; public RuntimeArray libOriginalInc; public boolean storableLastOpInNetorder; @@ -111,8 +118,8 @@ protected boolean removeEldestEntry(Map.Entry eldest) { private final Map standardIOGlobs = new HashMap<>(); private final Set hiddenStandardIOGlobs = new HashSet<>(); final Map stateVariableInitialized = new HashMap<>(); - final LifecycleRuntimeState lifecycleState = new LifecycleRuntimeState(); - final NameNormalizer.State nameNormalizerState = new NameNormalizer.State(); + LifecycleRuntimeState lifecycleState = new LifecycleRuntimeState(); + NameNormalizer.State nameNormalizerState = new NameNormalizer.State(); public PerlRuntime() { this(new PerlThreadRegistry(), 0); @@ -194,12 +201,18 @@ Map snapshotReferenceAddresses() { /** Bind this runtime until the returned scope is closed. */ public Binding bind() { - if (closed) { - throw new IllegalStateException("PerlRuntime is closed"); + synchronized (lifecycleMonitor) { + if (closed) { + throw new IllegalStateException("PerlRuntime is closed"); + } + if (resetting && resetOwner != Thread.currentThread()) { + throw new IllegalStateException("PerlRuntime is resetting"); + } + activeBindings.incrementAndGet(); } BindingFrame frame = new BindingFrame(this, CURRENT.get()); CURRENT.set(frame); - return new Binding(frame, Thread.currentThread()); + return new Binding(frame, Thread.currentThread(), this); } /** Convenience form for {@code runtime.bind()}. */ @@ -318,6 +331,61 @@ public boolean isClosed() { return closed; } + /** + * Return this runtime to fresh-initialized state without changing its Java identity. + * Reset is rejected rather than delayed whenever observable work is still active. + * A failure after the transition starts permanently poisons the runtime. + */ + public PerlRuntime reset() { + if (!executionLock.tryLock()) { + throw new IllegalStateException("PerlRuntime reset requires exclusive execution ownership"); + } + try { + synchronized (lifecycleMonitor) { + if (closed) throw new IllegalStateException("PerlRuntime is closed"); + if (resetting) throw new IllegalStateException("PerlRuntime is already resetting"); + if (activeBindings.get() != 0) { + throw new IllegalStateException("PerlRuntime reset requires all bindings to be closed"); + } + if (threadRegistry.size() != 0) { + throw new IllegalStateException("PerlRuntime reset requires all child threads to finish"); + } + if (activeSharedLocks.get() != 0 || activeSharedWaiters.get() != 0) { + throw new IllegalStateException( + "PerlRuntime reset requires shared locks and waiters to be quiescent"); + } + if (org.perlonjava.app.scriptengine.PerlLanguageProvider.COMPILE_LOCK.isLocked()) { + throw new IllegalStateException("PerlRuntime reset requires compilation to be quiescent"); + } + resetting = true; + resetOwner = Thread.currentThread(); + } + + try { + try (Binding ignored = bind()) { + releaseResettableResources(); + } + replaceRuntimeState(); + initialized = false; + initialize(); + threadRegistry.clearTerminalStateForReset(); + return this; + } catch (Throwable failure) { + closed = true; + if (failure instanceof RuntimeException runtime) throw runtime; + if (failure instanceof Error error) throw error; + throw new IllegalStateException("PerlRuntime reset failed", failure); + } finally { + synchronized (lifecycleMonitor) { + resetting = false; + resetOwner = null; + } + } + } finally { + executionLock.unlock(); + } + } + /** * Clone this interpreter's package graph for a new ithread. Execution, * lifecycle, alarm, signal, native and I/O state starts fresh in the child. @@ -444,6 +512,97 @@ public void close() { } } + private void releaseResettableResources() { + MortalList.flush(); + MortalList.flushDeferredCapturesBeforeEnd(); + try { + SpecialBlock.runEndBlocks(false); + } finally { + MortalList.flushDeferredCaptures(); + org.perlonjava.runtime.regex.RuntimeRegex.emitCurrentRuntimeDebugFreeTraces(); + } + GlobalDestruction.runGlobalDestruction(); + Time.cancelCurrentAlarm(); + PerlSignalQueue.clearSignals(); + RuntimeIO.closeAllHandles(); + NetSSLeay.resetState(); + MortalList.clearCurrentRuntimeState(); + } + + private void replaceRuntimeState() { + executionState = new ExecutionRuntimeState(); + regexState = new RuntimeRegexState(); + mroState = new MroRuntimeState(); + globalState = new GlobalRuntimeState(); + runtimeCodeState = new RuntimeCodeRuntimeState(); + compilationState = new CompilationRuntimeState(); + sourceMapperState = new ByteCodeSourceMapper.State(); + filterState = new FilterRuntimeState(); + timeState = new Time.State(); + signalState = new PerlSignalQueue.State(); + randomState = new Random.State(); + dataSectionState = new DataSection.State(); + ioRegistryState = new IORuntimeRegistryState(); + fileTestState = new FileTestOperator.State(); + debugState = new DebugRuntimeState(); + diamondIOState = new DiamondIO.State(); + nativeState = new ExtendedNativeUtils.State(); + lifecycleState = new LifecycleRuntimeState(); + nameNormalizerState = new NameNormalizer.State(); + + flipFlopState.clear(); + scalarGlobState.clear(); + pointerPackState.clear(); + bObjectState.clear(); + netSslErrorQueue.clear(); + netSslState = new NetSSLeay.State(); + xsShimLoadingInProgress.clear(); + referenceAddresses.clear(); + stateVariableInitialized.clear(); + pendingForkOpen = null; + libOriginalInc = null; + storableLastOpInNetorder = false; + currentDirectory = System.getProperty("user.dir"); + perlThreadContext = RuntimeContextType.SCALAR; + perlThreadStackSize = 0; + defaultPerlThreadStackSize = 0; + perlThreadExitOnly = false; + defaultPerlThreadExitOnly = false; + resetStandardIOState(); + } + + private void resetStandardIOState() { + ioOpenHandles.clear(); + standardIOGlobs.clear(); + hiddenStandardIOGlobs.clear(); + ioStdout = new RuntimeIO(new StandardIO(System.out, true)); + ioStderr = new RuntimeIO(new StandardIO(System.err, false)); + ioStderr.autoFlush = true; + ioStdin = new RuntimeIO(new StandardIO(System.in)); + ioSelectedHandle = ioStdout; + ioLastWrittenHandle = ioStdout; + ioLastAccessedHandle = null; + ioLastReadlineHandleName = null; + installInitialStandardGlob("main::STDOUT", ioStdout); + installInitialStandardGlob("main::stdout", ioStdout); + installInitialStandardGlob("main::STDERR", ioStderr); + installInitialStandardGlob("main::stderr", ioStderr); + installInitialStandardGlob("main::STDIN", ioStdin); + installInitialStandardGlob("main::stdin", ioStdin); + ioStdout.globName = "main::STDOUT"; + ioStderr.globName = "main::STDERR"; + ioStdin.globName = "main::STDIN"; + } + + private void releaseBinding() { + activeBindings.decrementAndGet(); + } + + void sharedLockAcquired() { activeSharedLocks.incrementAndGet(); } + void sharedLockReleased() { activeSharedLocks.decrementAndGet(); } + void sharedWaiterEntered() { activeSharedWaiters.incrementAndGet(); } + void sharedWaiterExited() { activeSharedWaiters.decrementAndGet(); } + RuntimeGlob standardIOGlob(String name) { return standardIOGlobs.get(name); } @@ -526,11 +685,13 @@ private record BindingFrame(PerlRuntime runtime, BindingFrame previous) { public static final class Binding implements AutoCloseable { private final BindingFrame frame; private final Thread owner; + private final PerlRuntime runtime; private boolean closed; - private Binding(BindingFrame frame, Thread owner) { + private Binding(BindingFrame frame, Thread owner, PerlRuntime runtime) { this.frame = frame; this.owner = owner; + this.runtime = runtime; } @Override @@ -550,6 +711,7 @@ public void close() { } else { CURRENT.set(frame.previous); } + runtime.releaseBinding(); } } } diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/PerlThreadRegistry.java b/src/main/java/org/perlonjava/runtime/runtimetypes/PerlThreadRegistry.java index 89e2897c97..6f2d876750 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/PerlThreadRegistry.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/PerlThreadRegistry.java @@ -57,6 +57,14 @@ public int size() { return threads.size(); } + void clearTerminalStateForReset() { + if (!threads.isEmpty() || !userUnicodeProperties.isEmpty()) { + throw new IllegalStateException("Thread registry is not quiescent"); + } + terminalThreads.clear(); + nextId.set(1); + } + /** Format Perl's process-exit diagnostic for attached, unjoined children. */ public String activeThreadExitWarning() { int running = 0; diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/SharedPerlStorage.java b/src/main/java/org/perlonjava/runtime/runtimetypes/SharedPerlStorage.java index b43f5d9745..b30fc336a1 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/SharedPerlStorage.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/SharedPerlStorage.java @@ -117,8 +117,10 @@ public static RuntimeScalar sharedClone(RuntimeScalar reference) { /** Acquire a recursive advisory lock until the surrounding Perl scope exits. */ public static RuntimeBase lock(RuntimeScalar reference) { RuntimeBase root = requireShared(reference, "lock"); + PerlRuntime owner = PerlRuntime.current(); LockState state = lockState(root); state.lock.lock(); + owner.sharedLockAcquired(); DynamicVariableManager.pushLocalVariable(new DynamicState() { @Override public void dynamicSaveState() { @@ -127,7 +129,11 @@ public void dynamicSaveState() { @Override public void dynamicRestoreState() { - state.lock.unlock(); + try { + state.lock.unlock(); + } finally { + owner.sharedLockReleased(); + } } @Override @@ -201,6 +207,8 @@ private static boolean conditionWait(RuntimeScalar conditionReference, } int holds = lock.getHoldCount(); + PerlRuntime owner = PerlRuntime.current(); + owner.sharedWaiterEntered(); for (int i = 0; i < holds; i++) lock.unlock(); boolean signalled = false; try { @@ -226,6 +234,7 @@ private static boolean conditionWait(RuntimeScalar conditionReference, } } for (int i = 0; i < holds; i++) lock.lock(); + owner.sharedWaiterExited(); } return signalled; } diff --git a/src/test/java/org/perlonjava/runtime/runtimetypes/PerlRuntimeResetTest.java b/src/test/java/org/perlonjava/runtime/runtimetypes/PerlRuntimeResetTest.java new file mode 100644 index 0000000000..d4ed4d2de2 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/runtimetypes/PerlRuntimeResetTest.java @@ -0,0 +1,164 @@ +package org.perlonjava.runtime.runtimetypes; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.perlonjava.app.cli.CompilerOptions; +import org.perlonjava.app.scriptengine.PerlLanguageProvider; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.*; + +@Tag("unit") +class PerlRuntimeResetTest { + @Test + void resetRestoresRepresentativeDomainsAndRemainsExecutable() throws Exception { + PerlRuntime runtime = new PerlRuntime().initialize(); + PerlRuntime fresh = new PerlRuntime().initialize(); + + runtime.globalState.scalarValues().put("ResetContract::tenant", new RuntimeScalar("first")); + runtime.regexState.optimizedRegexCache.put(7, new RuntimeScalar("compiled")); + runtime.executionState.taintMode = true; + runtime.setDefaultPerlThreadStackSize(12345); + RuntimeIO oldStdout = runtime.ioStdout; + + assertSame(runtime, runtime.reset()); + + assertTrue(runtime.isInitialized()); + assertFalse(runtime.isClosed()); + assertFalse(runtime.globalState.scalarValues().containsKey("ResetContract::tenant")); + assertEquals(fresh.globalState.scalarValues().keySet(), runtime.globalState.scalarValues().keySet()); + assertTrue(runtime.regexState.optimizedRegexCache.isEmpty()); + assertFalse(runtime.executionState.taintMode); + assertEquals(0, runtime.defaultPerlThreadStackSize()); + assertNotSame(oldStdout, runtime.ioStdout); + assertSame(runtime.ioStdout, runtime.ioSelectedHandle); + assertEquals("reused", runtime.execute(() -> "reused")); + } + + @Test + void resetRejectsAnOutstandingBindingWithoutPoisoningRuntime() { + PerlRuntime runtime = new PerlRuntime().initialize(); + try (PerlRuntime.Binding ignored = runtime.bind()) { + IllegalStateException failure = assertThrows(IllegalStateException.class, runtime::reset); + assertTrue(failure.getMessage().contains("bindings")); + } + assertFalse(runtime.isClosed()); + assertSame(runtime, runtime.reset()); + } + + @Test + void perlWorkloadAfterResetMatchesFreshRuntimeOnBothBackends() throws Exception { + for (boolean interpreter : new boolean[]{false, true}) { + PerlRuntime used = new PerlRuntime(); + PerlRuntime fresh = new PerlRuntime(); + + run(used, + "package ResetTenant; our $value = 41; sub answer { 42 };" + + " $INC{'Reset/Tenant.pm'} = __FILE__; 'abc' =~ /(b)/; 1", + interpreter); + used.reset(); + + String probe = "no strict 'refs'; join ':'," + + " defined($ResetTenant::value) ? 1 : 0," + + " defined(&ResetTenant::answer) ? 1 : 0," + + " exists($INC{'Reset/Tenant.pm'}) ? 1 : 0," + + " defined($1) ? 1 : 0"; + assertEquals(run(fresh, probe, interpreter), run(used, probe, interpreter)); + assertEquals("0:0:0:0", run(used, probe, interpreter)); + } + } + + @Test + void resetRejectsActiveChildAndSucceedsAfterJoin() throws Exception { + PerlRuntime runtime = new PerlRuntime().initialize(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + PerlThreadControlBlock child; + try (PerlRuntime.Binding ignored = runtime.bind()) { + child = PerlThreadControlBlock.create(runtime, childRuntime -> { + entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); + return new RuntimeScalar(1); + }).start(); + } + + try { + assertTrue(entered.await(5, TimeUnit.SECONDS)); + IllegalStateException failure = assertThrows(IllegalStateException.class, runtime::reset); + assertTrue(failure.getMessage().contains("child threads")); + } finally { + release.countDown(); + child.join(); + } + assertSame(runtime, runtime.reset()); + assertEquals(0, runtime.threadRegistry().size()); + } + + @Test + void resetRejectsAnOutstandingSharedLock() { + PerlRuntime runtime = new PerlRuntime().initialize(); + int level; + try (PerlRuntime.Binding ignored = runtime.bind()) { + RuntimeScalar shared = new RuntimeScalar(1); + SharedPerlStorage.shareValue(shared); + level = DynamicVariableManager.getLocalLevel(); + SharedPerlStorage.lock(shared.createReference()); + } + IllegalStateException failure = assertThrows(IllegalStateException.class, runtime::reset); + assertTrue(failure.getMessage().contains("shared locks")); + try (PerlRuntime.Binding ignored = runtime.bind()) { + DynamicVariableManager.popToLocalLevel(level); + } + assertSame(runtime, runtime.reset()); + } + + @Test + void resetDrainsPendingEndWorkBeforeReinitializing() { + PerlRuntime runtime = new PerlRuntime().initialize(); + AtomicInteger endRuns = new AtomicInteger(); + try (PerlRuntime.Binding ignored = runtime.bind()) { + RuntimeCode end = new RuntimeCode((args, context) -> { + endRuns.incrementAndGet(); + return new RuntimeList(); + }, null); + SpecialBlock.saveEndBlock(new RuntimeScalar(end)); + } + + runtime.reset(); + + assertEquals(1, endRuns.get()); + try (PerlRuntime.Binding ignored = runtime.bind()) { + assertTrue(SpecialBlock.getEndBlocks().isEmpty()); + } + } + + @Test + void resetFailurePoisonsRuntime() { + PerlRuntime runtime = new PerlRuntime().initialize(); + try (PerlRuntime.Binding ignored = runtime.bind()) { + RuntimeCode failingEnd = new RuntimeCode((args, context) -> { + throw new IllegalStateException("reset END failed"); + }, null); + SpecialBlock.saveEndBlock(new RuntimeScalar(failingEnd)); + } + + IllegalStateException failure = assertThrows(IllegalStateException.class, runtime::reset); + assertEquals("reset END failed", failure.getMessage()); + assertTrue(runtime.isClosed()); + assertThrows(IllegalStateException.class, runtime::bind); + } + + private static String run(PerlRuntime runtime, String source, boolean interpreter) + throws Exception { + return runtime.execute(() -> { + CompilerOptions options = new CompilerOptions(); + options.fileName = ""; + options.code = source; + options.useInterpreter = interpreter; + return PerlLanguageProvider.executePerlCode(options, false).scalar().toString(); + }); + } +} From b77c16f69306e101218999dd6ea0042e5313613c Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Sat, 15 Aug 2026 00:07:10 +0200 Subject: [PATCH 5/5] fix: bound regex thread snapshot resources Run regexp_qr_embed_thr in the resource-sensitive serial lane with a 600-second minimum deadline. Avoid quadratic ithread graph growth by cloning only observable named IO handles, while leaving inert parser glob placeholders to be vivified on demand in each child runtime. Add coverage that open named handle aliases still inherit with independent wrapper state while undef placeholders do not inflate snapshots. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/concurrency.md | 14 +++++--- dev/tools/perl_test_runner.pl | 2 ++ .../runtimetypes/GlobalRuntimeState.java | 13 ++++++- ...imeGraphClonerResourceInheritanceTest.java | 35 +++++++++++++++++++ 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/dev/design/concurrency.md b/dev/design/concurrency.md index f9faee533b..c7d3cedf26 100644 --- a/dev/design/concurrency.md +++ b/dev/design/concurrency.md @@ -973,10 +973,16 @@ request history. The core differential runner now reserves an exclusive serial lane for the resource-sensitive `gv.t`, advanced-regex, regex-speed, GH7094 benchmark, and -Abigail JAPH tests. The thread wrappers for `pat.t`, `pat_psycho.t`, and -`speed.t` use that same lane and a 600-second minimum outer deadline because -runtime snapshot startup plus the upstream watchdogs exceed the normal -300-second budget under parallel load. These tests have internal watchdogs or +Abigail JAPH tests. The thread wrappers `pat_thr.t`, `pat_psycho_thr.t`, +`regexp_qr_embed_thr.t`, and `speed_thr.t` use that same lane and a 600-second +minimum outer deadline because runtime snapshot startup plus the upstream +watchdogs exceed the normal 300-second budget under parallel load. The +`regexp_qr_embed_thr.t` classification also prevents a full-corpus memory spike +from exhausting its child runtime near the end of the 2,210-case matrix. Thread +snapshots inherit named IO slots only when they contain a real handle; inert +parser placeholders are child-vivified on demand instead of being copied +quadratically across thousands of eval-created runtimes. These tests have +internal watchdogs or timing assertions whose TAP totals changed when they competed with the normal parallel corpus; they retain stable original indices, and `gv.t` receives the upstream timeout factor. This is test scheduling policy, not a relaxation of diff --git a/dev/tools/perl_test_runner.pl b/dev/tools/perl_test_runner.pl index b5e906be2e..d100e0a00f 100755 --- a/dev/tools/perl_test_runner.pl +++ b/dev/tools/perl_test_runner.pl @@ -457,6 +457,7 @@ sub timeout_for_test { | (?:^|/)perl5_t/t/re/pat_psycho(?:_thr)?\.t$ | (?:^|/)perl5_t/t/op/gv\.t$ | (?:^|/)perl5_t/t/re/pat_advanced(?:_thr)?\.t$ + | (?:^|/)perl5_t/t/re/regexp_qr_embed_thr\.t$ | (?:^|/)perl5_t/t/re/speed(?:_thr)?\.t$ | (?:^|/)perl5_t/t/benchmark/gh7094-speed-up-keys-on-empty-hash\.t$ | (?:^|/)perl5_t/t/japh/abigail\.t$ @@ -471,6 +472,7 @@ sub requires_exclusive_slot { | (?:^|/)perl5_t/t/re/pat(?:_thr)?\.t$ | (?:^|/)perl5_t/t/re/pat_psycho(?:_thr)?\.t$ | (?:^|/)perl5_t/t/re/pat_advanced(?:_thr)?\.t$ + | (?:^|/)perl5_t/t/re/regexp_qr_embed_thr\.t$ | (?:^|/)perl5_t/t/re/speed(?:_thr)?\.t$ | (?:^|/)perl5_t/t/benchmark/gh7094-speed-up-keys-on-empty-hash\.t$ | (?:^|/)perl5_t/t/japh/abigail\.t$ diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeState.java b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeState.java index aaaf36a86d..c3de64ba10 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeState.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeState.java @@ -282,7 +282,18 @@ synchronized void snapshotInto(GlobalRuntimeState target, RuntimeGraphCloner clo cloneMap(codeRefs, target.codeRefs, cloner, RuntimeScalar.class); cloneMap(pseudoConstants, target.pseudoConstants, cloner, RuntimeScalar.class); cloneMap(pinnedCodeRefs, target.pinnedCodeRefs, cloner, RuntimeScalar.class); - cloneMap(ioSlots, target.ioSlots, cloner, RuntimeGlob.class); + // Parsers register many inert glob placeholders (notably through eval). + // Cloning all of them into every ithread makes snapshot cost quadratic + // for regex matrices that compile thousands of evals. Only an IO slot + // with an actual RuntimeIO is observable as an inherited filehandle; + // undef placeholders are recreated on demand in the child. + for (Map.Entry entry : ioSlots.entrySet()) { + RuntimeGlob glob = entry.getValue(); + if (glob != null && glob.IO != null && glob.IO.value instanceof RuntimeIO) { + target.ioSlots.put(entry.getKey(), + (RuntimeGlob) cloner.cloneValue(glob)); + } + } for (Map.Entry entry : compiledCodeRefs.entrySet()) { target.compiledCodeRefs.put(entry.getKey(), (RuntimeScalar) cloner.cloneValue(entry.getValue())); diff --git a/src/test/java/org/perlonjava/runtime/runtimetypes/RuntimeGraphClonerResourceInheritanceTest.java b/src/test/java/org/perlonjava/runtime/runtimetypes/RuntimeGraphClonerResourceInheritanceTest.java index fc67600409..7d95f201d9 100644 --- a/src/test/java/org/perlonjava/runtime/runtimetypes/RuntimeGraphClonerResourceInheritanceTest.java +++ b/src/test/java/org/perlonjava/runtime/runtimetypes/RuntimeGraphClonerResourceInheritanceTest.java @@ -16,7 +16,9 @@ import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -98,4 +100,37 @@ void scalarBackedHandleClonesBackingAliasAndPosition() { assertEquals("x", childIO.ioHandle.doRead(1, StandardCharsets.ISO_8859_1).toString()); assertEquals("x", parentIO.ioHandle.doRead(1, StandardCharsets.ISO_8859_1).toString()); } + + @Test + void snapshotSkipsInertNamedIoPlaceholdersAndInheritsOpenAliases() { + PerlRuntime parent = new PerlRuntime(); + parent.initialize(); + PerlRuntime child; + RuntimeGlob parentHandle; + RuntimeScalar backing = new RuntimeScalar("wxyz"); + + try (PerlRuntime.Binding ignored = parent.bind()) { + GlobalVariable.getGlobalIO("Snapshot::PLACEHOLDER"); + ScalarBackedIO scalarHandle = new ScalarBackedIO(backing); + assertEquals("w", scalarHandle.doRead(1, StandardCharsets.ISO_8859_1).toString()); + parentHandle = GlobalVariable.getGlobalIO("Snapshot::DATA") + .setIO(new RuntimeIO(scalarHandle)); + GlobalVariable.globalIORefs.put("Snapshot::ALIAS", parentHandle); + child = parent.snapshotClone(); + } + + try (PerlRuntime.Binding ignored = child.bind()) { + assertNull(GlobalVariable.peekGlobalIO("Snapshot::PLACEHOLDER")); + RuntimeGlob childHandle = GlobalVariable.peekGlobalIO("Snapshot::DATA"); + assertNotNull(childHandle); + assertNotSame(parentHandle, childHandle); + assertSame(childHandle, GlobalVariable.peekGlobalIO("Snapshot::ALIAS")); + assertEquals("x", childHandle.getRuntimeIO().ioHandle + .doRead(1, StandardCharsets.ISO_8859_1).toString()); + } + try (PerlRuntime.Binding ignored = parent.bind()) { + assertEquals("x", parentHandle.getRuntimeIO().ioHandle + .doRead(1, StandardCharsets.ISO_8859_1).toString()); + } + } }