From 7bda35cf4bb8f5e7eb4011619894341c07de098f Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Fri, 29 Aug 2025 20:52:46 +0300 Subject: [PATCH 01/10] Renamed DefaultLuceneQueryTransformer class to DefaultSearchQueryTransformer --- .../java/org/jabref/logic/importer/fetcher/DOAJFetcher.java | 4 ++-- ...eryTransformer.java => DefaultSearchQueryTransformer.java} | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/{DefaultLuceneQueryTransformer.java => DefaultSearchQueryTransformer.java} (93%) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java index 9c2e314f91e..1bafb1dddf5 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java @@ -15,7 +15,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformers.DefaultLuceneQueryTransformer; +import org.jabref.logic.importer.fetcher.transformers.DefaultSearchQueryTransformer; import org.jabref.logic.os.OS; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; @@ -186,7 +186,7 @@ public Optional getHelpPage() { @Override public URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - DOAJFetcher.addPath(uriBuilder, new DefaultLuceneQueryTransformer().transformSearchQuery(queryNode).orElse("")); + DOAJFetcher.addPath(uriBuilder, new DefaultSearchQueryTransformer().transformSearchQuery(queryNode).orElse("")); // Number of results uriBuilder.addParameter("pageSize", "30"); // Page (not needed so far) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/DefaultLuceneQueryTransformer.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/DefaultSearchQueryTransformer.java similarity index 93% rename from jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/DefaultLuceneQueryTransformer.java rename to jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/DefaultSearchQueryTransformer.java index b13aba1b1b3..2605f75f79d 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/DefaultLuceneQueryTransformer.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/DefaultSearchQueryTransformer.java @@ -3,7 +3,7 @@ /** * Transforms the query to a lucene query string */ -public class DefaultLuceneQueryTransformer extends AbstractQueryTransformer { +public class DefaultSearchQueryTransformer extends AbstractQueryTransformer { @Override protected String getLogicalAndOperator() { From d4c4ac9e431cece60a74fb919acaf407202161d6 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Mon, 1 Sep 2025 21:16:13 +0300 Subject: [PATCH 02/10] Changed the highlighting logic for web search to use the new ANTLR based parser syntax --- .../fetcher/WebSearchPaneViewModel.java | 49 ++++++++++++------- .../main/resources/l10n/JabRef_en.properties | 3 +- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index 9057d8078b8..c42edc8aea0 100644 --- a/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -22,20 +22,23 @@ import org.jabref.logic.importer.WebFetchers; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; +import org.jabref.model.search.ThrowingErrorListener; import org.jabref.model.strings.StringUtil; import org.jabref.model.util.OptionalUtil; +import org.jabref.search.SearchLexer; +import org.jabref.search.SearchParser; import com.tobiasdiez.easybind.EasyBind; import de.saxsys.mvvmfx.utils.validation.FunctionBasedValidator; import de.saxsys.mvvmfx.utils.validation.ValidationMessage; import de.saxsys.mvvmfx.utils.validation.ValidationStatus; import de.saxsys.mvvmfx.utils.validation.Validator; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; -import org.apache.lucene.queryparser.flexible.standard.parser.ParseException; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; - -import static org.jabref.logic.importer.fetcher.transformers.AbstractQueryTransformer.NO_EXPLICIT_FIELD; +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.misc.ParseCancellationException; public class WebSearchPaneViewModel { @@ -47,7 +50,6 @@ public class WebSearchPaneViewModel { private final StateManager stateManager; private final Validator searchQueryValidator; - private final SyntaxParser parser = new StandardSyntaxParser(); public WebSearchPaneViewModel(GuiPreferences preferences, DialogService dialogService, StateManager stateManager) { this.dialogService = dialogService; @@ -85,18 +87,31 @@ public WebSearchPaneViewModel(GuiPreferences preferences, DialogService dialogSe } try { - parser.parse(queryText, NO_EXPLICIT_FIELD); + SearchLexer lexer = new SearchLexer(CharStreams.fromString(queryText)); + lexer.removeErrorListeners(); // no infos on file system + lexer.addErrorListener(ThrowingErrorListener.INSTANCE); + SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); + parser.removeErrorListeners(); // no infos on file system + parser.addErrorListener(ThrowingErrorListener.INSTANCE); + parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors + parser.start(); return null; - } catch (ParseException e) { - String element = e.currentToken.image; - int position = e.currentToken.beginColumn; - if (element == null) { - return ValidationMessage.error(Localization.lang("Invalid query. Check position %0.", position)); - } else { - return ValidationMessage.error(Localization.lang("Invalid query element '%0' at position %1", element, position)); + } catch (ParseCancellationException e) { + // RecognitionException can point out the exact error + if (e.getCause() instanceof RecognitionException) { + RecognitionException recEx = (RecognitionException) e.getCause(); + Token offendingToken = recEx.getOffendingToken(); + + // The character position is 0-based, so we add 1 for user-friendliness. + int line = offendingToken.getLine(); + int charPositionInLine = offendingToken.getCharPositionInLine() + 1; + + String errorMessage = String.format("Invalid syntax at line %d, position %d", line, charPositionInLine); + return ValidationMessage.error(Localization.lang(errorMessage)); } - } catch (QueryNodeParseException e) { - return ValidationMessage.error(""); + + // Fallback for other cancellation reasons + return ValidationMessage.error(Localization.lang("Invalid query")); } }); } diff --git a/jablib/src/main/resources/l10n/JabRef_en.properties b/jablib/src/main/resources/l10n/JabRef_en.properties index 391abcfd0b7..1b2eb47a896 100644 --- a/jablib/src/main/resources/l10n/JabRef_en.properties +++ b/jablib/src/main/resources/l10n/JabRef_en.properties @@ -993,8 +993,7 @@ You\ must\ restart\ JabRef\ for\ this\ to\ come\ into\ effect.=You must restart Could\ not\ find\ fetcher\ '%0'=Could not find fetcher '%0' Running\ query\ '%0'\ with\ fetcher\ '%1'.=Running query '%0' with fetcher '%1'. -Invalid\ query.\ Check\ position\ %0.=Invalid query. Check position %0. -Invalid\ query\ element\ '%0'\ at\ position\ %1=Invalid query element '%0' at position %1 +Invalid\ query=Invalid query Move\ file=Move file Rename\ file=Rename file From 7c5e623b0b3f6fc69528dc34b435d1297e09bce0 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Tue, 2 Sep 2025 21:28:41 +0300 Subject: [PATCH 03/10] Ran rewrite --- .../org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index ea782a1ccc0..e8972aa1c52 100644 --- a/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -106,7 +106,7 @@ public WebSearchPaneViewModel(GuiPreferences preferences, DialogService dialogSe int line = offendingToken.getLine(); int charPositionInLine = offendingToken.getCharPositionInLine() + 1; - String errorMessage = String.format("Invalid syntax at line %d, position %d", line, charPositionInLine); + String errorMessage = "Invalid syntax at line %d, position %d".formatted(line, charPositionInLine); return ValidationMessage.error(Localization.lang(errorMessage)); } From cdfd9315ef8d20c0787a682d18e6b8dcab10bcdf Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Wed, 3 Sep 2025 21:07:49 +0300 Subject: [PATCH 04/10] Removed invalid DOI check as the new parser doesn't support checking for validity of DOI --- .../gui/importer/fetcher/WebSearchPaneViewModelTest.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/jabgui/src/test/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModelTest.java b/jabgui/src/test/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModelTest.java index dd8dec30616..5f9578f8780 100644 --- a/jabgui/src/test/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModelTest.java +++ b/jabgui/src/test/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModelTest.java @@ -70,12 +70,6 @@ void canExtractDOIFromQueryText() { assertTrue(viewModel.queryValidationStatus().validProperty().getValue()); } - @Test - void queryConsistingOfInvalidDOIIsInvalid() { - viewModel.queryProperty().setValue("101.1007/JHEP02(2023)082"); - assertFalse(viewModel.queryValidationStatus().validProperty().getValue()); - } - @Test void queryConsistingOfISBNIsValid() { viewModel.queryProperty().setValue("9780134685991"); @@ -90,7 +84,7 @@ void canExtractISBNFromQueryText() { @Test void queryConsistingOfArXivIdIsValid() { - viewModel.queryProperty().setValue("arXiv:2110.02957"); + viewModel.queryProperty().setValue("arXiv=2110.02957"); assertTrue(viewModel.queryValidationStatus().validProperty().getValue()); } From 3aa0bfefd22968f92c106a6723ebe33ef49c2135 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sun, 7 Sep 2025 21:35:03 +0300 Subject: [PATCH 05/10] Reused the existing l10n for error message --- .../jabref/gui/importer/fetcher/WebSearchPaneViewModel.java | 3 +-- jablib/src/main/resources/l10n/JabRef_en.properties | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index e8972aa1c52..07993b9f5ec 100644 --- a/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -106,8 +106,7 @@ public WebSearchPaneViewModel(GuiPreferences preferences, DialogService dialogSe int line = offendingToken.getLine(); int charPositionInLine = offendingToken.getCharPositionInLine() + 1; - String errorMessage = "Invalid syntax at line %d, position %d".formatted(line, charPositionInLine); - return ValidationMessage.error(Localization.lang(errorMessage)); + return ValidationMessage.error(Localization.lang("Invalid query element '%0' at position %1", line, charPositionInLine)); } // Fallback for other cancellation reasons diff --git a/jablib/src/main/resources/l10n/JabRef_en.properties b/jablib/src/main/resources/l10n/JabRef_en.properties index 32601476ef5..511eb8565fe 100644 --- a/jablib/src/main/resources/l10n/JabRef_en.properties +++ b/jablib/src/main/resources/l10n/JabRef_en.properties @@ -1002,6 +1002,7 @@ You\ must\ restart\ JabRef\ for\ this\ to\ come\ into\ effect.=You must restart Could\ not\ find\ fetcher\ '%0'=Could not find fetcher '%0' Running\ query\ '%0'\ with\ fetcher\ '%1'.=Running query '%0' with fetcher '%1'. Invalid\ query=Invalid query +Invalid\ query\ element\ '%0'\ at\ position\ %1=Invalid query element '%0' at position %1 Move\ file=Move file Rename\ file=Rename file From 2d0c95ae0196eb8513953f7b50e6b762b04da54e Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 8 Sep 2025 13:39:40 +0200 Subject: [PATCH 06/10] Re-use existing code --- .../fetcher/WebSearchPaneViewModel.java | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index 07993b9f5ec..7136f797725 100644 --- a/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -22,20 +22,15 @@ import org.jabref.logic.importer.WebFetchers; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; -import org.jabref.model.search.ThrowingErrorListener; +import org.jabref.model.search.query.SearchQuery; import org.jabref.model.strings.StringUtil; import org.jabref.model.util.OptionalUtil; -import org.jabref.search.SearchLexer; -import org.jabref.search.SearchParser; import com.tobiasdiez.easybind.EasyBind; import de.saxsys.mvvmfx.utils.validation.FunctionBasedValidator; import de.saxsys.mvvmfx.utils.validation.ValidationMessage; import de.saxsys.mvvmfx.utils.validation.ValidationStatus; import de.saxsys.mvvmfx.utils.validation.Validator; -import org.antlr.v4.runtime.BailErrorStrategy; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.misc.ParseCancellationException; @@ -87,14 +82,8 @@ public WebSearchPaneViewModel(GuiPreferences preferences, DialogService dialogSe } try { - SearchLexer lexer = new SearchLexer(CharStreams.fromString(queryText)); - lexer.removeErrorListeners(); // no infos on file system - lexer.addErrorListener(ThrowingErrorListener.INSTANCE); - SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); - parser.removeErrorListeners(); // no infos on file system - parser.addErrorListener(ThrowingErrorListener.INSTANCE); - parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors - parser.start(); + // The result is ignored because we just check for validity + SearchQuery.getStartContext(queryText); return null; } catch (ParseCancellationException e) { // RecognitionException can point out the exact error From a7cf98fcc5092a6675f003fcfd3693ca307e8f23 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 8 Sep 2025 13:41:46 +0200 Subject: [PATCH 07/10] Modern Java --- .../jabref/gui/importer/fetcher/WebSearchPaneViewModel.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index 7136f797725..31a10571d98 100644 --- a/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -87,8 +87,7 @@ public WebSearchPaneViewModel(GuiPreferences preferences, DialogService dialogSe return null; } catch (ParseCancellationException e) { // RecognitionException can point out the exact error - if (e.getCause() instanceof RecognitionException) { - RecognitionException recEx = (RecognitionException) e.getCause(); + if (e.getCause() instanceof RecognitionException recEx) { Token offendingToken = recEx.getOffendingToken(); // The character position is 0-based, so we add 1 for user-friendliness. From 8c4693d4fae9c3dec5c1b34b41f01f669bcb1a5f Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 8 Sep 2025 15:59:02 +0200 Subject: [PATCH 08/10] Fix comment --- .../org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java b/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java index 31a10571d98..118a0d6a6dd 100644 --- a/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModel.java @@ -97,7 +97,7 @@ public WebSearchPaneViewModel(GuiPreferences preferences, DialogService dialogSe return ValidationMessage.error(Localization.lang("Invalid query element '%0' at position %1", line, charPositionInLine)); } - // Fallback for other cancellation reasons + // Fallback for other failing reasons return ValidationMessage.error(Localization.lang("Invalid query")); } }); From 252a944c78a27ff3270cc7a77f27336e244dccc9 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 8 Sep 2025 16:00:57 +0200 Subject: [PATCH 09/10] Re-add test --- .../gui/importer/fetcher/WebSearchPaneViewModelTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jabgui/src/test/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModelTest.java b/jabgui/src/test/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModelTest.java index 5f9578f8780..5dcde0e530e 100644 --- a/jabgui/src/test/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModelTest.java +++ b/jabgui/src/test/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModelTest.java @@ -82,6 +82,13 @@ void canExtractISBNFromQueryText() { assertTrue(viewModel.queryValidationStatus().validProperty().getValue()); } + @Test + void queryConsistingOfInvalidDOIIsValid() { + viewModel.queryProperty().setValue("101.1007/JHEP02(2023)082"); + // There is currently no interpretation of nearly-valid identifiers, therefore, this is concidered as "regular" search term + assertTrue(viewModel.queryValidationStatus().validProperty().getValue()); + } + @Test void queryConsistingOfArXivIdIsValid() { viewModel.queryProperty().setValue("arXiv=2110.02957"); From c12355c6c7f3830aabce9cea8c1f474fbd1783b7 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 8 Sep 2025 16:02:02 +0200 Subject: [PATCH 10/10] Fix position --- .../fetcher/WebSearchPaneViewModelTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/jabgui/src/test/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModelTest.java b/jabgui/src/test/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModelTest.java index 5dcde0e530e..1d7f94d11d1 100644 --- a/jabgui/src/test/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModelTest.java +++ b/jabgui/src/test/java/org/jabref/gui/importer/fetcher/WebSearchPaneViewModelTest.java @@ -71,21 +71,21 @@ void canExtractDOIFromQueryText() { } @Test - void queryConsistingOfISBNIsValid() { - viewModel.queryProperty().setValue("9780134685991"); + void queryConsistingOfInvalidDOIIsValid() { + viewModel.queryProperty().setValue("101.1007/JHEP02(2023)082"); + // There is currently no interpretation of nearly-valid identifiers, therefore, this is concidered as "regular" search term assertTrue(viewModel.queryValidationStatus().validProperty().getValue()); } @Test - void canExtractISBNFromQueryText() { - viewModel.queryProperty().setValue(";:isbn (9780134685991), text2"); + void queryConsistingOfISBNIsValid() { + viewModel.queryProperty().setValue("9780134685991"); assertTrue(viewModel.queryValidationStatus().validProperty().getValue()); } @Test - void queryConsistingOfInvalidDOIIsValid() { - viewModel.queryProperty().setValue("101.1007/JHEP02(2023)082"); - // There is currently no interpretation of nearly-valid identifiers, therefore, this is concidered as "regular" search term + void canExtractISBNFromQueryText() { + viewModel.queryProperty().setValue(";:isbn (9780134685991), text2"); assertTrue(viewModel.queryValidationStatus().validProperty().getValue()); }