From c10431062f58f9ee885d0cdf0d9f6cd0e3d04137 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Sat, 10 Apr 2021 14:16:07 -0400 Subject: [PATCH 1/2] LUCENE-9922: checksum files should use a deterministic sort order This way the files don't unnecessarily change, depending on filesystem order or anything else. --- gradle/generation/regenerate.gradle | 4 ++-- .../src/generated/checksums/generateClassicTokenizer.json | 4 ++-- .../generated/checksums/generateHTMLStripCharFilter.json | 4 ++-- .../common/src/generated/checksums/generateTlds.json | 2 +- .../generated/checksums/generateUAX29URLEmailTokenizer.json | 6 +++--- .../src/generated/checksums/generateWikipediaTokenizer.json | 4 ++-- .../java/org/apache/lucene/analysis/email/ASCIITLD.jflex | 4 ++-- .../src/generated/checksums/generateStandardTokenizer.json | 4 ++-- .../src/generated/checksums/javaccParserClassic.json | 4 ++-- .../src/generated/checksums/javaccParserFlexible.json | 4 ++-- .../src/generated/checksums/javaccParserSurround.json | 4 ++-- 11 files changed, 22 insertions(+), 22 deletions(-) diff --git a/gradle/generation/regenerate.gradle b/gradle/generation/regenerate.gradle index 72dfa36fe2d6..df97a41e30db 100644 --- a/gradle/generation/regenerate.gradle +++ b/gradle/generation/regenerate.gradle @@ -144,7 +144,7 @@ configure([ } checksumsFile.setText( - JsonOutput.prettyPrint(JsonOutput.toJson(actualChecksums)), "UTF-8") + JsonOutput.prettyPrint(JsonOutput.toJson(new TreeMap(actualChecksums))), "UTF-8") logger.warn("Updated generated file checksums for task ${sourceTask.path}.") } @@ -192,4 +192,4 @@ configure([ return conditionalTask } } -} \ No newline at end of file +} diff --git a/lucene/analysis/common/src/generated/checksums/generateClassicTokenizer.json b/lucene/analysis/common/src/generated/checksums/generateClassicTokenizer.json index 9f534469c27c..2ee9a3e1993c 100644 --- a/lucene/analysis/common/src/generated/checksums/generateClassicTokenizer.json +++ b/lucene/analysis/common/src/generated/checksums/generateClassicTokenizer.json @@ -1,5 +1,5 @@ { - "lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.jflex": "958b028ef3f0aec36488fb2bb033cdec5858035f", "gradle/generation/jflex/skeleton.default.txt": "ca1043249c0eefdf2623a785e2b91f5608bfc3f1", - "lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.java": "21c2cf7ba0a0cdeb43ebe624101e259c9348f6b0" + "lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.java": "21c2cf7ba0a0cdeb43ebe624101e259c9348f6b0", + "lucene/analysis/common/src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.jflex": "958b028ef3f0aec36488fb2bb033cdec5858035f" } \ No newline at end of file diff --git a/lucene/analysis/common/src/generated/checksums/generateHTMLStripCharFilter.json b/lucene/analysis/common/src/generated/checksums/generateHTMLStripCharFilter.json index f9f6ea36956c..14daa0a22cb5 100644 --- a/lucene/analysis/common/src/generated/checksums/generateHTMLStripCharFilter.json +++ b/lucene/analysis/common/src/generated/checksums/generateHTMLStripCharFilter.json @@ -1,5 +1,5 @@ { - "lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex": "71760e2f7abe078109545a0c68aeac9125508d7c", "gradle/generation/jflex/skeleton.default.txt": "ca1043249c0eefdf2623a785e2b91f5608bfc3f1", - "lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java": "78f5208455706d60a9ce4b63624ed04b0fd32573" + "lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java": "78f5208455706d60a9ce4b63624ed04b0fd32573", + "lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex": "71760e2f7abe078109545a0c68aeac9125508d7c" } \ No newline at end of file diff --git a/lucene/analysis/common/src/generated/checksums/generateTlds.json b/lucene/analysis/common/src/generated/checksums/generateTlds.json index de26638a9ead..654973d06e8d 100644 --- a/lucene/analysis/common/src/generated/checksums/generateTlds.json +++ b/lucene/analysis/common/src/generated/checksums/generateTlds.json @@ -1,4 +1,4 @@ { - "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex": "29b05df80da071a249ff68067d5833fd7e4a9ff4", + "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex": "b0a75dc29e472c7e8667c4972e897b2f9527793d", "lucene/analysis/common/src/test/org/apache/lucene/analysis/email/TLDs.txt": "b346a80f511b64a59d556eb3ef58cf396e98c631" } \ No newline at end of file diff --git a/lucene/analysis/common/src/generated/checksums/generateUAX29URLEmailTokenizer.json b/lucene/analysis/common/src/generated/checksums/generateUAX29URLEmailTokenizer.json index 772c127df36a..b96bc1b6aa9b 100644 --- a/lucene/analysis/common/src/generated/checksums/generateUAX29URLEmailTokenizer.json +++ b/lucene/analysis/common/src/generated/checksums/generateUAX29URLEmailTokenizer.json @@ -1,6 +1,6 @@ { - "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.jflex": "56a751d27e481fb55388f91ebf34f5a0cb8cb1b2", "gradle/generation/jflex/skeleton.disable.buffer.expansion.txt": "68263ff0a014904c6e89b040d868d8f399408908", - "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex": "29b05df80da071a249ff68067d5833fd7e4a9ff4", - "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.java": "e437900d9570ca007f9c02c9ea286222b644c329" + "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex": "b0a75dc29e472c7e8667c4972e897b2f9527793d", + "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.java": "e437900d9570ca007f9c02c9ea286222b644c329", + "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.jflex": "56a751d27e481fb55388f91ebf34f5a0cb8cb1b2" } \ No newline at end of file diff --git a/lucene/analysis/common/src/generated/checksums/generateWikipediaTokenizer.json b/lucene/analysis/common/src/generated/checksums/generateWikipediaTokenizer.json index 2bc03343f22a..63ca19a1bc83 100644 --- a/lucene/analysis/common/src/generated/checksums/generateWikipediaTokenizer.json +++ b/lucene/analysis/common/src/generated/checksums/generateWikipediaTokenizer.json @@ -1,5 +1,5 @@ { - "lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex": "a23a4b7cbcdba1fc864c0b85bc2784c8893a0f9f", "gradle/generation/jflex/skeleton.default.txt": "ca1043249c0eefdf2623a785e2b91f5608bfc3f1", - "lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java": "10b391af6953d2f7bcca86da835a1037705509ec" + "lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java": "10b391af6953d2f7bcca86da835a1037705509ec", + "lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex": "a23a4b7cbcdba1fc864c0b85bc2784c8893a0f9f" } \ No newline at end of file diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex b/lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex index 03e05285686b..cd657adc1480 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex @@ -15,8 +15,8 @@ * limitations under the License. */ // Generated from IANA Root Zone Database -// file version from 2021 Apr 6, Tue 17:13:00 Coordinated Universal Time -// generated on 2021 Apr 7, Wed 08:24:24 Coordinated Universal Time +// file version from 2021 Apr 10, Sat 17:37:00 Coordinated Universal Time +// generated on 2021 Apr 10, Sat 17:55:26 Coordinated Universal Time // by org.apache.lucene.analysis.standard.GenerateJflexTLDMacros // LUCENE-8278: None of the TLDs in {ASCIITLD} is a 1-character-shorter prefix of another TLD diff --git a/lucene/core/src/generated/checksums/generateStandardTokenizer.json b/lucene/core/src/generated/checksums/generateStandardTokenizer.json index 1497072a2326..7d1b8db47eda 100644 --- a/lucene/core/src/generated/checksums/generateStandardTokenizer.json +++ b/lucene/core/src/generated/checksums/generateStandardTokenizer.json @@ -1,5 +1,5 @@ { - "lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex": "6158aeb8dd11cd9100623608b2dcce51b2df9d0b", "gradle/generation/jflex/skeleton.disable.buffer.expansion.txt": "68263ff0a014904c6e89b040d868d8f399408908", - "lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java": "8e33c2698446c1c7a9479796a41316d1932ceda9" + "lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java": "8e33c2698446c1c7a9479796a41316d1932ceda9", + "lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex": "6158aeb8dd11cd9100623608b2dcce51b2df9d0b" } \ No newline at end of file diff --git a/lucene/queryparser/src/generated/checksums/javaccParserClassic.json b/lucene/queryparser/src/generated/checksums/javaccParserClassic.json index 48707e9e9e98..b150037beb0b 100644 --- a/lucene/queryparser/src/generated/checksums/javaccParserClassic.json +++ b/lucene/queryparser/src/generated/checksums/javaccParserClassic.json @@ -1,9 +1,9 @@ { - "lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj": "c9584bbe50c3c7479f72ea84145ebbf034a201ea", + "lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java": "0f421768d8a964a00a6566180fe26547ff2f3e1e", "lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java": "e0f1cced0f9448dea63b03931a5287e701b8b8cd", + "lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj": "c9584bbe50c3c7479f72ea84145ebbf034a201ea", "lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserConstants.java": "e59a3fd38b66a3d56779c55955c1e014225a1f50", "lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java": "cba572aa235f3098383a26c369b5585c708647d8", - "lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java": "0f421768d8a964a00a6566180fe26547ff2f3e1e", "lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/Token.java": "310665ba37d982327fcb55cc3523d629ef29ef54", "lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/TokenMgrError.java": "9927caf69afb3a06bc679995cabb14f3b56e16c7" } \ No newline at end of file diff --git a/lucene/queryparser/src/generated/checksums/javaccParserFlexible.json b/lucene/queryparser/src/generated/checksums/javaccParserFlexible.json index b1d7035dac37..7b35647be77f 100644 --- a/lucene/queryparser/src/generated/checksums/javaccParserFlexible.json +++ b/lucene/queryparser/src/generated/checksums/javaccParserFlexible.json @@ -1,9 +1,9 @@ { - "lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj": "08b62ed73607b1646af5dadb81c8bb34e381daee", + "lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/ParseException.java": "3d5f272a6d56b3f4962b252267ce2662e734414e", "lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java": "75e9d84f424bb697f899fe3adacc0094bac00672", + "lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj": "08b62ed73607b1646af5dadb81c8bb34e381daee", "lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserConstants.java": "e73933bff38a62d90dab64f72a1a0deadfff246f", "lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java": "6e503b48ffa9f4648798e5394f7baeec366d1f07", - "lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/ParseException.java": "3d5f272a6d56b3f4962b252267ce2662e734414e", "lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/Token.java": "f4cb9d01587279dba30e549ce4867e4381bbd9d7", "lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/TokenMgrError.java": "cdfa99af5fcf6b1e50691a1c1370ba60bf0d2d2d" } \ No newline at end of file diff --git a/lucene/queryparser/src/generated/checksums/javaccParserSurround.json b/lucene/queryparser/src/generated/checksums/javaccParserSurround.json index edb0713530d2..130a1024df1a 100644 --- a/lucene/queryparser/src/generated/checksums/javaccParserSurround.json +++ b/lucene/queryparser/src/generated/checksums/javaccParserSurround.json @@ -1,9 +1,9 @@ { - "lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj": "21b38627431747c741e2ec24be1e7aef38dc70c9", + "lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/ParseException.java": "37613d8e8557bd17e4af1b0b0279e75094f409fb", "lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.java": "6e5f595be9084b24f0025ccd4b7b4cb11b0bf4b8", + "lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj": "21b38627431747c741e2ec24be1e7aef38dc70c9", "lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParserConstants.java": "8feb77878890c27e874be457d839eba48192c40f", "lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParserTokenManager.java": "7b47f2a971aa94339831b8ab17e1cfe401add06c", - "lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/ParseException.java": "37613d8e8557bd17e4af1b0b0279e75094f409fb", "lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/Token.java": "a5eea2a3043e0aa2781f4a43b9ab9c5d59add80e", "lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/TokenMgrError.java": "77350c188e18ff8338088b1e14b6b34c9d0089eb" } \ No newline at end of file From 877d6f2fa3ad4e8b42aa8d9e92e7fc4492ef5142 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Sat, 10 Apr 2021 14:47:48 -0400 Subject: [PATCH 2/2] LUCENE-9923: remove always-changing timestamp from ASCIITLD.jflex generation This makes regenerate idempotent by removing the new Date() from the output. We already have the root.zone's Last-Modified date, which is the one that matters and only changes when the root.zone changes. --- .../common/src/generated/checksums/generateTlds.json | 2 +- .../generated/checksums/generateUAX29URLEmailTokenizer.json | 2 +- .../java/org/apache/lucene/analysis/email/ASCIITLD.jflex | 3 +-- .../lucene/analysis/standard/GenerateJflexTLDMacros.java | 6 +----- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/lucene/analysis/common/src/generated/checksums/generateTlds.json b/lucene/analysis/common/src/generated/checksums/generateTlds.json index 654973d06e8d..113d0e1a4e83 100644 --- a/lucene/analysis/common/src/generated/checksums/generateTlds.json +++ b/lucene/analysis/common/src/generated/checksums/generateTlds.json @@ -1,4 +1,4 @@ { - "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex": "b0a75dc29e472c7e8667c4972e897b2f9527793d", + "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex": "b1af8dc8532d853fcf1acde1f6c629750b296b40", "lucene/analysis/common/src/test/org/apache/lucene/analysis/email/TLDs.txt": "b346a80f511b64a59d556eb3ef58cf396e98c631" } \ No newline at end of file diff --git a/lucene/analysis/common/src/generated/checksums/generateUAX29URLEmailTokenizer.json b/lucene/analysis/common/src/generated/checksums/generateUAX29URLEmailTokenizer.json index b96bc1b6aa9b..b21231eafc8b 100644 --- a/lucene/analysis/common/src/generated/checksums/generateUAX29URLEmailTokenizer.json +++ b/lucene/analysis/common/src/generated/checksums/generateUAX29URLEmailTokenizer.json @@ -1,6 +1,6 @@ { "gradle/generation/jflex/skeleton.disable.buffer.expansion.txt": "68263ff0a014904c6e89b040d868d8f399408908", - "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex": "b0a75dc29e472c7e8667c4972e897b2f9527793d", + "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex": "b1af8dc8532d853fcf1acde1f6c629750b296b40", "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.java": "e437900d9570ca007f9c02c9ea286222b644c329", "lucene/analysis/common/src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.jflex": "56a751d27e481fb55388f91ebf34f5a0cb8cb1b2" } \ No newline at end of file diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex b/lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex index cd657adc1480..1d9b4b317883 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex @@ -16,8 +16,7 @@ */ // Generated from IANA Root Zone Database // file version from 2021 Apr 10, Sat 17:37:00 Coordinated Universal Time -// generated on 2021 Apr 10, Sat 17:55:26 Coordinated Universal Time -// by org.apache.lucene.analysis.standard.GenerateJflexTLDMacros +// generated by org.apache.lucene.analysis.standard.GenerateJflexTLDMacros // LUCENE-8278: None of the TLDs in {ASCIITLD} is a 1-character-shorter prefix of another TLD ASCIITLD = "." ( diff --git a/lucene/analysis/common/src/tools/java/org/apache/lucene/analysis/standard/GenerateJflexTLDMacros.java b/lucene/analysis/common/src/tools/java/org/apache/lucene/analysis/standard/GenerateJflexTLDMacros.java index 00a60b50f397..c8c230259223 100644 --- a/lucene/analysis/common/src/tools/java/org/apache/lucene/analysis/standard/GenerateJflexTLDMacros.java +++ b/lucene/analysis/common/src/tools/java/org/apache/lucene/analysis/standard/GenerateJflexTLDMacros.java @@ -30,7 +30,6 @@ import java.text.DateFormat; import java.util.ArrayList; import java.util.Comparator; -import java.util.Date; import java.util.List; import java.util.Locale; import java.util.SortedMap; @@ -245,10 +244,7 @@ private void writeOutput() throws IOException { writer.write(dateFormat.format(tldFileLastModified)); writer.write(NL); } - writer.write("// generated on "); - writer.write(dateFormat.format(new Date())); - writer.write(NL); - writer.write("// by "); + writer.write("// generated by "); writer.write(this.getClass().getName()); writer.write(NL); writer.write(NL);