From 3f85beae30e75e982d28dce11ba4c0121254fcf7 Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Fri, 30 Oct 2015 19:10:13 -0400 Subject: [PATCH 01/16] MINDEXER-94: Temp file cleanup on errors (cherry picked from commit 7a0e7ad) --- .../main/java/org/apache/maven/index/updater/WagonHelper.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java b/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java index 0e450c96..40ea1d94 100644 --- a/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java +++ b/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java @@ -198,6 +198,7 @@ public InputStream retrieve( String name ) throws IOException, FileNotFoundException { final File target = File.createTempFile( name, "" ); + target.deleteOnExit(); retrieve( name, target ); return new FileInputStream( target ) { @@ -220,6 +221,7 @@ public void retrieve( final String name, final File targetFile ) } catch ( AuthorizationException e ) { + targetFile.delete(); String msg = "Authorization exception retrieving " + name; logError( msg, e ); IOException ioException = new IOException( msg ); @@ -228,6 +230,7 @@ public void retrieve( final String name, final File targetFile ) } catch ( ResourceDoesNotExistException e ) { + targetFile.delete(); String msg = "Resource " + name + " does not exist"; logError( msg, e ); FileNotFoundException fileNotFoundException = new FileNotFoundException( msg ); @@ -236,6 +239,7 @@ public void retrieve( final String name, final File targetFile ) } catch ( WagonException e ) { + targetFile.delete(); String msg = "Transfer for " + name + " failed"; logError( msg, e ); IOException ioException = new IOException( msg + "; " + e.getMessage() ); From 897a6b011945e13d5fdbf34ddcbb2c9d0a0ad345 Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Fri, 30 Oct 2015 20:05:08 -0400 Subject: [PATCH 02/16] Add Idea iml files to rat exclude (cherry picked from commit 0f17245) --- pom.xml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index e60ccd1d..5a048c68 100644 --- a/pom.xml +++ b/pom.xml @@ -238,7 +238,7 @@ under the License. aether-util ${aether.version} - + org.apache.maven @@ -445,6 +445,27 @@ under the License. check + + + README.md + .gitignore + NOTICE + .git/** + .idea/** + **/*.iml + + src/test/**/*.sha1 + src/test/**/*.md5 + src/test/**/*.xml + src/test/**/*.pom + src/test/**/*.asc + src/test/**/*.properties + src/test/**/*.swc + src/test/**/*.txt + src/test/**/*.filename + src/test/**/.placeholder + + @@ -501,7 +522,7 @@ under the License. - org.eclipse.m2e From d4fb8373171bc343e98cf3c7dbbc7bef37ca8cec Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Fri, 30 Oct 2015 20:15:45 -0400 Subject: [PATCH 03/16] MINDEXER-95: Suboptimal indexing execution in updater (cherry picked from commit c193888) (sesuncedu@gmail.com: Removed call to optimize in NexusIndexWriter. Merged config setup in NexusIndexWriter. Kept increased gzip block size in IndexDataReader. --- .../maven/index/context/NexusIndexWriter.java | 29 ++++++++++++------- .../maven/index/updater/IndexDataReader.java | 9 +++--- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java b/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java index 038eaa83..130ee88f 100644 --- a/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java +++ b/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.SerialMergeScheduler; import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockObtainFailedException; @@ -38,16 +39,27 @@ public class NexusIndexWriter extends IndexWriter { + public interface IndexWriterConfigFactory { + IndexWriterConfig create(Analyzer analyzer); + } + + public static IndexWriterConfigFactory CONFIG_FACTORY = new IndexWriterConfigFactory() { + public IndexWriterConfig create(final Analyzer analyzer) { + IndexWriterConfig config = new IndexWriterConfig( Version.LUCENE_46, analyzer ); + config.setRAMBufferSizeMB( 2.0 ); // old default + config.setMergeScheduler( new SerialMergeScheduler() ); // merging serially + config.setWriteLockTimeout(IndexWriterConfig.WRITE_LOCK_TIMEOUT); + return config; + } + }; + @Deprecated public NexusIndexWriter( final Directory directory, final Analyzer analyzer, boolean create ) throws CorruptIndexException, LockObtainFailedException, IOException { - //super( directory, analyzer, create, MaxFieldLength.LIMITED ); - this(directory, new IndexWriterConfig(Version.LUCENE_46, analyzer)); - - // setSimilarity( new NexusSimilarity() ); + this(directory, CONFIG_FACTORY.create(analyzer).setOpenMode(create ? OpenMode.CREATE : OpenMode.APPEND)); } - + public NexusIndexWriter( final Directory directory, final IndexWriterConfig config ) throws CorruptIndexException, LockObtainFailedException, IOException { @@ -58,11 +70,6 @@ public NexusIndexWriter( final Directory directory, final IndexWriterConfig conf public static IndexWriterConfig defaultConfig() { - final IndexWriterConfig config = new IndexWriterConfig( Version.LUCENE_46, new NexusAnalyzer() ); - // default open mode is CreateOrAppend which suits us - config.setRAMBufferSizeMB( 2.0 ); // old default - config.setMergeScheduler( new SerialMergeScheduler() ); // merging serially - config.setWriteLockTimeout(IndexWriterConfig.WRITE_LOCK_TIMEOUT); - return config; + return CONFIG_FACTORY.create(new NexusAnalyzer()); } } diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java index f76200cc..2d676a7c 100644 --- a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java +++ b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java @@ -48,11 +48,10 @@ public class IndexDataReader { private final DataInputStream dis; - public IndexDataReader( InputStream is ) + public IndexDataReader( final InputStream is ) throws IOException { - BufferedInputStream bis = new BufferedInputStream( is, 1024 * 8 ); - + BufferedInputStream bis = new BufferedInputStream(is, 1024 * 8); // MINDEXER-13 // LightweightHttpWagon may have performed automatic decompression // Handle it transparently @@ -61,7 +60,7 @@ public IndexDataReader( InputStream is ) if ( bis.read() == 0x1f && bis.read() == 0x8b ) // GZIPInputStream.GZIP_MAGIC { bis.reset(); - data = new GZIPInputStream( bis, 2 * 1024 ); + data = new GZIPInputStream( bis, 1024 * 8 ); } else { @@ -97,7 +96,7 @@ public IndexDataReadResult readIndex( IndexWriter w, IndexingContext context ) } w.commit(); - + w.forceMerge(1); w.commit(); From 5645ce868e3aadbff44a6a86e44b6ce5b2549123 Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Fri, 30 Oct 2015 20:29:30 -0400 Subject: [PATCH 04/16] MINDEXER-96: Indexer reader (cherry picked from commit af8783d) --- indexer-reader/README.md | 8 + indexer-reader/header.txt | 17 + indexer-reader/pom.xml | 48 ++ .../maven/index/reader/ChunkReader.java | 454 ++++++++++++++++++ .../maven/index/reader/IndexReader.java | 281 +++++++++++ .../org/apache/maven/index/reader/Record.java | 247 ++++++++++ .../maven/index/reader/ResourceHandler.java | 45 ++ .../index/reader/WritableResourceHandler.java | 44 ++ .../index/reader/CachingResourceHandler.java | 60 +++ .../maven/index/reader/ChunkReaderTest.java | 66 +++ .../reader/DirectoryResourceHandler.java | 72 +++ .../index/reader/HttpResourceHandler.java | 57 +++ .../maven/index/reader/IndexReaderTest.java | 102 ++++ .../resources/nexus-maven-repository-index.gz | Bin 0 -> 319 bytes .../nexus-maven-repository-index.properties | 6 + pom.xml | 1 + 16 files changed, 1508 insertions(+) create mode 100644 indexer-reader/README.md create mode 100644 indexer-reader/header.txt create mode 100644 indexer-reader/pom.xml create mode 100644 indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java create mode 100644 indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java create mode 100644 indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java create mode 100644 indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java create mode 100644 indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java create mode 100644 indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java create mode 100644 indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java create mode 100644 indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java create mode 100644 indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java create mode 100644 indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java create mode 100644 indexer-reader/src/test/resources/nexus-maven-repository-index.gz create mode 100644 indexer-reader/src/test/resources/nexus-maven-repository-index.properties diff --git a/indexer-reader/README.md b/indexer-reader/README.md new file mode 100644 index 00000000..95aa1b82 --- /dev/null +++ b/indexer-reader/README.md @@ -0,0 +1,8 @@ +Indexer Reader Notes +================== + +Indexer Reader is a minimal dep-less library that is able to read published (remote) +index with incremental update support, making this library user able to integrate +published Maven Indexes into any engine without depending on maven-indexer-core +and it's transitive dependencies. + diff --git a/indexer-reader/header.txt b/indexer-reader/header.txt new file mode 100644 index 00000000..1a2ef734 --- /dev/null +++ b/indexer-reader/header.txt @@ -0,0 +1,17 @@ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + diff --git a/indexer-reader/pom.xml b/indexer-reader/pom.xml new file mode 100644 index 00000000..daf364cb --- /dev/null +++ b/indexer-reader/pom.xml @@ -0,0 +1,48 @@ + + + + 4.0.0 + + + org.apache.maven.indexer + maven-indexer + 5.1.2-SNAPSHOT + + + indexer-reader + + Maven :: Indexer Reader + + Indexer Reader is a minimal dep-less library that is able to read published (remote) index with incremental update + support, making user able to integrate published Maven Indexes into any engine without depending on + maven-indexer-core and it's transitive dependencies. + + + + + + junit + junit + test + + + + diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java new file mode 100644 index 00000000..89434fdf --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java @@ -0,0 +1,454 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.Closeable; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.UTFDataFormatException; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.regex.Pattern; +import java.util.zip.GZIPInputStream; + +import org.apache.maven.index.reader.Record.Type; + +/** + * Maven 2 Index published binary chunk reader. + * + * @since 5.1.2 + */ +public class ChunkReader + implements Closeable, Iterable +{ + private static final String FIELD_SEPARATOR = "|"; + + private static final String NOT_AVAILABLE = "NA"; + + private static final String UINFO = "u"; + + private static final String INFO = "i"; + + private static final Pattern FS_PATTERN = Pattern.compile(Pattern.quote(FIELD_SEPARATOR)); + + private final String chunkName; + + private final DataInputStream dataInputStream; + + private final int version; + + private final Date timestamp; + + public ChunkReader(final String chunkName, final InputStream inputStream) throws IOException + { + this.chunkName = chunkName.trim(); + this.dataInputStream = new DataInputStream(new GZIPInputStream(inputStream, 2 * 1024)); + this.version = ((int) dataInputStream.readByte()) & 0xff; + this.timestamp = new Date(dataInputStream.readLong()); + } + + /** + * Returns the chunk name. + */ + public String getName() { + return chunkName; + } + + /** + * Returns index getVersion. All releases so far always returned {@code 1}. + */ + public int getVersion() { + return version; + } + + /** + * Returns the getTimestamp of last update of the index. + */ + public Date getTimestamp() { + return timestamp; + } + + /** + * Returns the {@link Record} iterator. + */ + public Iterator iterator() { + try { + return new IndexIterator(dataInputStream); + } + catch (IOException e) { + throw new RuntimeException("error", e); + } + } + + /** + * Closes this reader and it's underlying input. + */ + public void close() throws IOException { + dataInputStream.close(); + } + + /** + * Low memory footprint index iterator that incrementally parses the underlying stream. + */ + private static class IndexIterator + implements Iterator + { + private final DataInputStream dataInputStream; + + private Record nextRecord; + + public IndexIterator(final DataInputStream dataInputStream) throws IOException { + this.dataInputStream = dataInputStream; + this.nextRecord = readRecord(); + } + + public boolean hasNext() { + return nextRecord != null; + } + + public Record next() { + if (nextRecord == null) { + throw new NoSuchElementException("chunk depleted"); + } + Record result = nextRecord; + try { + nextRecord = readRecord(); + return result; + } + catch (IOException e) { + throw new RuntimeException("read error", e); + } + } + + /** + * Reads and returns next record from the underlying stream, or {@code null} if no more records. + */ + private Record readRecord() + throws IOException + { + int fieldCount; + try { + fieldCount = dataInputStream.readInt(); + } + catch (EOFException ex) { + return null; // no more documents + } + + Map recordMap = new HashMap(); + for (int i = 0; i < fieldCount; i++) { + readField(recordMap); + } + + if (recordMap.containsKey("DESCRIPTOR")) { + return new Record(Type.DESCRIPTOR, recordMap, expandDescriptor(recordMap)); + } + else if (recordMap.containsKey("allGroups")) { + return new Record(Type.ALL_GROUPS, recordMap, expandAllGroups(recordMap)); + } + else if (recordMap.containsKey("rootGroups")) { + return new Record(Type.ROOT_GROUPS, recordMap, expandRootGroups(recordMap)); + } + else if (recordMap.containsKey("del")) { + return new Record(Type.ARTIFACT_REMOVE, recordMap, expandDeletedArtifact(recordMap)); + } + else { + // Fix up UINFO field wrt MINDEXER-41 + final String uinfo = recordMap.get(UINFO); + final String info = recordMap.get(INFO); + if (uinfo != null && !(info == null || info.trim().length() == 0)) { + final String[] splitInfo = FS_PATTERN.split(info); + if (splitInfo.length > 6) { + final String extension = splitInfo[6]; + if (uinfo.endsWith(FIELD_SEPARATOR + NOT_AVAILABLE)) { + recordMap.put(UINFO, uinfo + FIELD_SEPARATOR + extension); + } + } + } + return new Record(Type.ARTIFACT_ADD, recordMap, expandAddedArtifact(recordMap)); + } + } + + private void readField(final Map record) + throws IOException + { + dataInputStream.read(); // flags: neglect them + String name = dataInputStream.readUTF(); + String value = readUTF(); + record.put(name, value); + } + + private String readUTF() + throws IOException + { + int utflen = dataInputStream.readInt(); + + byte[] bytearr; + char[] chararr; + + try { + bytearr = new byte[utflen]; + chararr = new char[utflen]; + } + catch (OutOfMemoryError e) { + IOException ioex = new IOException("Index data content is corrupt"); + ioex.initCause(e); + throw ioex; + } + + int c, char2, char3; + int count = 0; + int chararr_count = 0; + + dataInputStream.readFully(bytearr, 0, utflen); + + while (count < utflen) { + c = bytearr[count] & 0xff; + if (c > 127) { + break; + } + count++; + chararr[chararr_count++] = (char) c; + } + + while (count < utflen) { + c = bytearr[count] & 0xff; + switch (c >> 4) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + /* 0xxxxxxx */ + count++; + chararr[chararr_count++] = (char) c; + break; + + case 12: + case 13: + /* 110x xxxx 10xx xxxx */ + count += 2; + if (count > utflen) { + throw new UTFDataFormatException("malformed input: partial character at end"); + } + char2 = bytearr[count - 1]; + if ((char2 & 0xC0) != 0x80) { + throw new UTFDataFormatException("malformed input around byte " + count); + } + chararr[chararr_count++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F)); + break; + + case 14: + /* 1110 xxxx 10xx xxxx 10xx xxxx */ + count += 3; + if (count > utflen) { + throw new UTFDataFormatException("malformed input: partial character at end"); + } + char2 = bytearr[count - 2]; + char3 = bytearr[count - 1]; + if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) { + throw new UTFDataFormatException("malformed input around byte " + (count - 1)); + } + chararr[chararr_count++] = + (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | (char3 & 0x3F)); + break; + + default: + /* 10xx xxxx, 1111 xxxx */ + throw new UTFDataFormatException("malformed input around byte " + count); + } + } + + // The number of chars produced may be less than utflen + return new String(chararr, 0, chararr_count); + } + + private Map expandDescriptor(final Map raw) { + final Map result = new HashMap(); + String[] r = FS_PATTERN.split(raw.get("IDXINFO")); + result.put(Record.REPOSITORY_ID, r[1]); + return result; + } + + private Map expandAllGroups(final Map raw) { + final Map result = new HashMap(); + putIfNotNullAsList(raw, Record.ALL_GROUPS_LIST, result, "allGroups"); + return result; + } + + private Map expandRootGroups(final Map raw) { + final Map result = new HashMap(); + putIfNotNullAsList(raw, Record.ROOT_GROUPS_LIST, result, "rootGroups"); + return result; + } + + private Map expandDeletedArtifact(final Map raw) { + final Map result = new HashMap(); + putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED); + if (raw.containsKey("del")) { + expandUinfo(raw.get("del"), result); + } + return result; + } + + /** + * Expands the "encoded" Maven Indexer record by splitting the synthetic fields and applying expanded field naming. + */ + private Map expandAddedArtifact(final Map raw) { + final Map result = new HashMap(); + + // Minimal + expandUinfo(raw.get(UINFO), result); + final String info = raw.get(INFO); + if (info != null) { + String[] r = FS_PATTERN.split(info); + result.put(Record.PACKAGING, renvl(r[0])); + result.put(Record.FILE_MODIFIED, Long.valueOf(r[1])); + result.put(Record.FILE_SIZE, Long.valueOf(r[2])); + result.put(Record.HAS_SOURCES, "1".equals(r[3]) ? Boolean.TRUE.toString() : Boolean.FALSE.toString()); + result.put(Record.HAS_JAVADOC, "1".equals(r[4]) ? Boolean.TRUE.toString() : Boolean.FALSE.toString()); + result.put(Record.HAS_SIGNATURE, "1".equals(r[5]) ? Boolean.TRUE.toString() : Boolean.FALSE.toString()); + if (r.length > 6) { + result.put(Record.FILE_EXTENSION, r[6]); + } + else { + final String packaging = raw.get(Record.PACKAGING); + if (raw.get(Record.CLASSIFIER) != null + || "pom".equals(packaging) + || "war".equals(packaging) + || "ear".equals(packaging)) { + result.put(Record.FILE_EXTENSION, packaging); + } + else { + result.put(Record.FILE_EXTENSION, "jar"); // best guess + } + } + } + putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED); + putIfNotNull(raw, "n", result, Record.NAME); + putIfNotNull(raw, "d", result, Record.DESCRIPTION); + putIfNotNull(raw, "1", result, Record.SHA1); + + // Jar file contents (optional) + putIfNotNullAsList(raw, "classnames", result, Record.CLASSNAMES); + + // Maven Plugin (optional) + putIfNotNull(raw, "px", result, Record.PLUGIN_PREFIX); + putIfNotNullAsList(raw, "gx", result, Record.PLUGIN_GOALS); + + // OSGi (optional) + putIfNotNull(raw, "Bundle-SymbolicName", result, "Bundle-SymbolicName"); + putIfNotNull(raw, "Bundle-Version", result, "Bundle-Version"); + putIfNotNull(raw, "Export-Package", result, "Export-Package"); + putIfNotNull(raw, "Export-Service", result, "Export-Service"); + putIfNotNull(raw, "Bundle-Description", result, "Bundle-Description"); + putIfNotNull(raw, "Bundle-Name", result, "Bundle-Name"); + putIfNotNull(raw, "Bundle-License", result, "Bundle-License"); + putIfNotNull(raw, "Bundle-DocURL", result, "Bundle-DocURL"); + putIfNotNull(raw, "Import-Package", result, "Import-Package"); + putIfNotNull(raw, "Require-Bundle", result, "Require-Bundle"); + putIfNotNull(raw, "Bundle-Version", result, "Bundle-Version"); + + return result; + } + + /** + * Expands UINFO synthetic field. Handles {@code null} String inputs. + */ + private void expandUinfo(final String uinfo, final Map result) { + if (uinfo != null) { + String[] r = FS_PATTERN.split(uinfo); + result.put(Record.GROUP_ID, r[0]); + result.put(Record.ARTIFACT_ID, r[1]); + result.put(Record.VERSION, r[2]); + String classifier = renvl(r[3]); + if (classifier != null) { + result.put(Record.CLASSIFIER, classifier); + if (r.length > 4) { + result.put(Record.FILE_EXTENSION, r[4]); + } + } + else if (r.length > 4) { + result.put(Record.PACKAGING, r[4]); + } + } + } + } + + /** + * Helper to put a value from source map into target map, if not null. + */ + private static void putIfNotNull( + final Map source, + final String sourceName, + final Map target, + final String targetName) + { + String value = source.get(sourceName); + if (value != null && value.trim().length() != 0) { + target.put(targetName, value); + } + } + + /** + * Helper to put a {@link Long} value from source map into target map, if not null. + */ + private static void putIfNotNullTS( + final Map source, + final String sourceName, + final Map target, + final String targetName) + { + String value = source.get(sourceName); + if (value != null && value.trim().length() != 0) { + target.put(targetName, Long.valueOf(value)); + } + } + + /** + * Helper to put a collection value from source map into target map as {@link java.util.List}, if not null. + */ + private static void putIfNotNullAsList( + final Map source, + final String sourceName, + final Map target, + final String targetName) + { + String value = source.get(sourceName); + if (value != null && value.trim().length() != 0) { + target.put(targetName, Arrays.asList(FS_PATTERN.split(value))); + } + } + + /** + * Helper to translate the "NA" (not available) input into {@code null} value. + */ + private static String renvl(final String v) { + return NOT_AVAILABLE.equals(v) ? null : v; + } +} diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java new file mode 100644 index 00000000..45514b9b --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java @@ -0,0 +1,281 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; +import java.util.TimeZone; + +/** + * Maven 2 Index reader that handles incremental updates if possible. + * + * @since 5.1.2 + */ +public class IndexReader + implements Iterable, Closeable +{ + private static final String INDEX_FILE_PREFIX = "nexus-maven-repository-index"; + + private static final DateFormat INDEX_DATE_FORMAT; + + static { + INDEX_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss.SSS Z"); + INDEX_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT")); + } + + private final WritableResourceHandler local; + + private final ResourceHandler remote; + + private final Properties localIndexProperties; + + private final Properties remoteIndexProperties; + + private final String indexId; + + private final Date publishedTimestamp; + + private final boolean incremental; + + private final List chunkNames; + + public IndexReader(final WritableResourceHandler local, final ResourceHandler remote) throws IOException { + if (remote == null) { + throw new NullPointerException("remote resource handler null"); + } + this.local = local; + this.remote = remote; + remoteIndexProperties = loadProperties(remote.open(INDEX_FILE_PREFIX + ".properties")); + try { + if (local != null) { + localIndexProperties = loadProperties(local.open(INDEX_FILE_PREFIX + ".properties")); + String remoteIndexId = remoteIndexProperties.getProperty("nexus.index.id"); + String localIndexId = localIndexProperties.getProperty("nexus.index.id"); + if (remoteIndexId == null || localIndexId == null || !remoteIndexId.equals(localIndexId)) { + throw new IllegalArgumentException( + "local and remote index IDs does not match or is null: " + localIndexId + ", " + + remoteIndexId); + } + this.indexId = localIndexId; + this.publishedTimestamp = INDEX_DATE_FORMAT.parse(localIndexProperties.getProperty("nexus.index.timestamp")); + this.incremental = canRetrieveAllChunks(); + this.chunkNames = calculateChunkNames(); + } + else { + localIndexProperties = null; + this.indexId = remoteIndexProperties.getProperty("nexus.index.id"); + this.publishedTimestamp = INDEX_DATE_FORMAT.parse(remoteIndexProperties.getProperty("nexus.index.timestamp")); + this.incremental = false; + this.chunkNames = calculateChunkNames(); + } + } + catch (ParseException e) { + IOException ex = new IOException("Index properties corrupted"); + ex.initCause(e); + throw ex; + } + } + + /** + * Returns the index context ID that published index has set. Usually it is equal to "repository ID" used in {@link + * Record.Type#DESCRIPTOR} but does not have to be. + */ + public String getIndexId() { + return indexId; + } + + /** + * Returns the {@link Date} when remote index was last published. + */ + public Date getPublishedTimestamp() { + return publishedTimestamp; + } + + /** + * Returns {@code true} if incremental update is about to happen. If incremental update, the {@link #iterator()} will + * return only the diff from the last update. + */ + public boolean isIncremental() { + return incremental; + } + + /** + * Returns unmodifiable list of actual chunks that needs to be pulled from remote {@link ResourceHandler}. Those are + * incremental chunks or the big main file, depending on result of {@link #isIncremental()}. Empty list means local + * index is up to date, and {@link #iterator()} will return empty iterator. + */ + public List getChunkNames() { + return chunkNames; + } + + /** + * Closes the underlying {@link ResourceHandler}s. In case of incremental update use, it also assumes that user + * consumed all the iterator and integrated it, hence, it will update the {@link WritableResourceHandler} contents to + * prepare it for future incremental update. If this is not desired (ie. due to aborted update), then this method + * should NOT be invoked, but rather the {@link ResourceHandler}s that caller provided in constructor of + * this class should be closed manually. + */ + public void close() throws IOException { + remote.close(); + if (local != null) { + try { + syncLocalWithRemote(); + } + finally { + local.close(); + } + } + } + + /** + * Returns an {@link Iterator} of {@link ChunkReader}s, that if read in sequence, provide all the (incremental) + * updates from the index. It is caller responsibility to either consume fully this iterator, or to close current + * {@link ChunkReader} if aborting. + */ + public Iterator iterator() { + return new ChunkReaderIterator(remote, chunkNames.iterator()); + } + + /** + * Stores the remote index properties into local index properties, preparing local {@link WritableResourceHandler} + * for future incremental updates. + */ + private void syncLocalWithRemote() throws IOException { + final ByteArrayOutputStream bos = new ByteArrayOutputStream(); + remoteIndexProperties.store(bos, "Maven Indexer Reader"); + local.save(INDEX_FILE_PREFIX + ".properties", new ByteArrayInputStream(bos.toByteArray())); + } + + /** + * Calculates the chunk names that needs to be fetched. + */ + private List calculateChunkNames() { + if (incremental) { + ArrayList chunkNames = new ArrayList(); + int maxCounter = Integer.parseInt(remoteIndexProperties.getProperty("nexus.index.last-incremental")); + int currentCounter = Integer.parseInt(localIndexProperties.getProperty("nexus.index.last-incremental")); + currentCounter++; + while (currentCounter <= maxCounter) { + chunkNames.add(INDEX_FILE_PREFIX + "." + currentCounter++ + ".gz"); + } + return Collections.unmodifiableList(chunkNames); + } + else { + return Collections.singletonList(INDEX_FILE_PREFIX + ".gz"); + } + } + + /** + * Verifies incremental update is possible, as all the diff chunks we need are still enlisted in remote properties. + */ + private boolean canRetrieveAllChunks() + { + String localChainId = localIndexProperties.getProperty("nexus.index.chain-id"); + String remoteChainId = remoteIndexProperties.getProperty("nexus.index.chain-id"); + + // If no chain id, or not the same, do full update + if (localChainId == null || remoteChainId == null || !localChainId.equals(remoteChainId)) { + return false; + } + + try { + int localLastIncremental = Integer.parseInt(localIndexProperties.getProperty("nexus.index.last-incremental")); + String currentLocalCounter = String.valueOf(localLastIncremental); + String nextLocalCounter = String.valueOf(localLastIncremental + 1); + // check remote props for existence of current or next chunk after local + for (Object key : remoteIndexProperties.keySet()) { + String sKey = (String) key; + if (sKey.startsWith("nexus.index.incremental-")) { + String value = remoteIndexProperties.getProperty(sKey); + if (currentLocalCounter.equals(value) || nextLocalCounter.equals(value)) { + return true; + } + } + } + } + catch (NumberFormatException e) { + // fall through + } + return false; + } + + /** + * Internal iterator implementation that lazily opens and closes the returned {@link ChunkReader}s as this iterator + * is being consumed. + */ + private static class ChunkReaderIterator + implements Iterator + { + private final ResourceHandler resourceHandler; + + private final Iterator chunkNamesIterator; + + private ChunkReader currentChunkReader; + + private ChunkReaderIterator(final ResourceHandler resourceHandler, final Iterator chunkNamesIterator) { + this.resourceHandler = resourceHandler; + this.chunkNamesIterator = chunkNamesIterator; + } + + public boolean hasNext() { + return chunkNamesIterator.hasNext(); + } + + public ChunkReader next() { + String chunkName = chunkNamesIterator.next(); + try { + if (currentChunkReader != null) { + currentChunkReader.close(); + } + currentChunkReader = new ChunkReader(chunkName, resourceHandler.open(chunkName)); + return currentChunkReader; + } + catch (IOException e) { + throw new RuntimeException("IO problem while switching chunk readers", e); + } + } + } + + /** + * Creates and loads {@link Properties} from provided {@link InputStream} and closes the stream. + */ + private static Properties loadProperties(final InputStream inputStream) throws IOException { + try { + final Properties properties = new Properties(); + properties.load(inputStream); + return properties; + } + finally { + inputStream.close(); + } + } +} diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java new file mode 100644 index 00000000..3354008e --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java @@ -0,0 +1,247 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.Map; + +/** + * Maven 2 Index record. + * + * @since 5.1.2 + */ +public class Record +{ + /** + * Key of repository ID entry, that contains {@link String}. + */ + public static final String REPOSITORY_ID = "repositoryId"; + + /** + * Key of all groups list entry, that contains {@link java.util.List}. + */ + public static final String ALL_GROUPS_LIST = "allGroupsList"; + + /** + * Key of root groups list entry, that contains {@link java.util.List}. + */ + public static final String ROOT_GROUPS_LIST = "rootGroupsList"; + + /** + * Key of index record modification (added to index or removed from index) timestamp entry, that contains {@link + * Long}. + */ + public static final String REC_MODIFIED = "recordModified"; + + /** + * Key of artifact groupId entry, that contains {@link String}. + */ + public static final String GROUP_ID = "groupId"; + + /** + * Key of artifact artifactId entry, that contains {@link String}. + */ + public static final String ARTIFACT_ID = "artifactId"; + + /** + * Key of artifact version entry, that contains {@link String}. + */ + public static final String VERSION = "version"; + + /** + * Key of artifact classifier entry, that contains {@link String}. + */ + public static final String CLASSIFIER = "classifier"; + + /** + * Key of artifact packaging entry, that contains {@link String}. + */ + public static final String PACKAGING = "packaging"; + + /** + * Key of artifact file extension, that contains {@link String}. + */ + public static final String FILE_EXTENSION = "fileExtension"; + + /** + * Key of artifact file last modified timestamp, that contains {@link Long}. + */ + public static final String FILE_MODIFIED = "fileModified"; + + /** + * Key of artifact file size in bytes, that contains {@link Long}. + */ + public static final String FILE_SIZE = "fileSize"; + + /** + * Key of artifact Sources presence flag, that contains {@link Boolean}. + */ + public static final String HAS_SOURCES = "hasSources"; + + /** + * Key of artifact Javadoc presence flag, that contains {@link Boolean}. + */ + public static final String HAS_JAVADOC = "hasJavadoc"; + + /** + * Key of artifact signature presence flag, that contains {@link Boolean}. + */ + public static final String HAS_SIGNATURE = "hasSignature"; + + /** + * Key of artifact name (as set in POM), that contains {@link String}. + */ + public static final String NAME = "name"; + + /** + * Key of artifact description (as set in POM), that contains {@link String}. + */ + public static final String DESCRIPTION = "description"; + + /** + * Key of artifact SHA1 digest, that contains {@link String}. + */ + public static final String SHA1 = "sha1"; + + /** + * Key of artifact contained class names, that contains {@link java.util.List}. + */ + public static final String CLASSNAMES = "classNames"; + + /** + * Key of plugin artifact prefix, that contains {@link String}. + */ + public static final String PLUGIN_PREFIX = "pluginPrefix"; + + /** + * Key of plugin artifact goals, that contains {@link java.util.List}. + */ + public static final String PLUGIN_GOALS = "pluginGoals"; + + /** + * Types of returned records returned from index. + */ + public enum Type + { + /** + * Descriptor record. Can be safely ignored. + * Contains following entries: + *
    + *
  • {@link #REPOSITORY_ID}
    • + *
+ */ + DESCRIPTOR, + + /** + * Artifact ADD record. Records of this type should be added to your indexing system. + * Contains following entries: + *
    + *
  • {@link #REC_MODIFIED} (when record was added/modified on index)
    • + *
  • {@link #GROUP_ID}
    • + *
  • {@link #ARTIFACT_ID}
    • + *
  • {@link #VERSION}
    • + *
  • {@link #CLASSIFIER} (optional)
    • + *
  • {@link #FILE_EXTENSION}
    • + *
  • {@link #FILE_MODIFIED}
    • + *
  • {@link #FILE_SIZE}
    • + *
  • {@link #PACKAGING}
    • + *
  • {@link #HAS_SOURCES}
    • + *
  • {@link #HAS_JAVADOC}
    • + *
  • {@link #HAS_SIGNATURE}
    • + *
  • {@link #NAME}
    • + *
  • {@link #DESCRIPTION}
    • + *
  • {@link #SHA1}
    • + *
  • {@link #CLASSNAMES} (optional)
    • + *
  • {@link #PLUGIN_PREFIX} (optional, for maven-plugins only)
    • + *
  • {@link #PLUGIN_GOALS} (optional, for maven-plugins only)
    • + *
+ */ + ARTIFACT_ADD, + + /** + * Artifact REMOTE record. In case of incremental updates, notes that this artifact was removed. Records of this + * type should be removed from your indexing system. + * Contains following entries: + *
    + *
  • {@link #REC_MODIFIED} (when record was deleted from index)
    • + *
  • {@link #GROUP_ID}
    • + *
  • {@link #ARTIFACT_ID}
    • + *
  • {@link #VERSION}
    • + *
  • {@link #CLASSIFIER} (optional)
    • + *
  • {@link #FILE_EXTENSION} (if {@link #CLASSIFIER} present)
    • + *
  • {@link #PACKAGING} (optional)
    • + *
+ */ + ARTIFACT_REMOVE, + + /** + * Special record, containing all the Maven "groupId"s that are enlisted on the index. Can be safely ignored. + * Contains following entries: + *
    + *
  • {@link #ALL_GROUPS_LIST}
    • + *
+ */ + ALL_GROUPS, + + /** + * Special record, containing all the root groups of Maven "groupId"s that are enlisted on the index. Can be safely + * ignored. + * Contains following entries: + *
    + *
  • {@link #ROOT_GROUPS_LIST}
    • + *
+ */ + ROOT_GROUPS + } + + private final Type type; + + private final Map raw; + + private final Map expanded; + + public Record(final Type type, final Map raw, final Map expanded) { + this.type = type; + this.raw = raw; + this.expanded = expanded; + } + + /** + * Returns the {@link Type} of this record. Usually users would be interested in {@link Type#ARTIFACT_ADD} and {@link + * Type#ARTIFACT_REMOVE} types only to maintain their own index. Still, indexer offers extra records too, see {@link + * Type} for all existing types. + */ + public Type getType() { + return type; + } + + /** + * Returns the "raw", Maven Indexer specific record as a {@link Map}. + */ + public Map getRaw() { + return raw; + } + + /** + * Returns the expanded (processed and expanded synthetic fields) record as {@link Map} ready for consumption. + */ + public Map getExpanded() { + return expanded; + } +} diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java new file mode 100644 index 00000000..97800674 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java @@ -0,0 +1,45 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; + +/** + * Maven 2 Index resource abstraction, that should be handled as a resource (is {@link Closeable}. That means, that + * implementations could perform any extra activity as FS locking or so (if uses FS as backing store). If the + * implementation plans to fetch from remote, it could implement very simple "cache" mechanism, to fetch only once + * during the lifespan of the instance, as for indexer there is no reason to re-fetch during single session, nor + * yo have any advanced caching (ie. TTLs etc). + * + * @since 5.1.2 + */ +public interface ResourceHandler + extends Closeable +{ + /** + * Returns the {@link InputStream} of resource with {@code name} or {@code null} if no such resource. Closing the + * stream is the responsibility of the caller. + * + * @param name Resource name, guaranteed to be non-{@code null} and is FS name and URL safe string. + */ + InputStream open(String name) throws IOException; +} diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java new file mode 100644 index 00000000..7fe38964 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java @@ -0,0 +1,44 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.io.InputStream; + +/** + * Maven 2 Index writable {@link ResourceHandler}, is capable of saving resources too. Needed only if incremental index + * updates are wanted, to store the index state locally, and be able to calculate incremental diffs on next {@link + * IndexReader} invocation. + * + * @see ResourceHandler + * @since 5.1.2 + */ +public interface WritableResourceHandler + extends ResourceHandler +{ + /** + * Stores (creates or overwrites if resource with name exists) the resource under {@code name} with content provided + * by the stream. The {@link InputStream} should be closed when method returns. + * + * @param name Resource name, guaranteed to be non-{@code null} and is FS name and URL safe string. + * @param inputStream the content of the resource, guaranteed to be non-{@code null}. + */ + void save(final String name, final InputStream inputStream) throws IOException; +} diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java new file mode 100644 index 00000000..6afe2222 --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java @@ -0,0 +1,60 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.io.InputStream; + +/** + * A trivial caching {@link ResourceHandler} that caches forever during single session (existence of the instance). + */ +public class CachingResourceHandler + implements ResourceHandler +{ + private final DirectoryResourceHandler local; + + private final ResourceHandler remote; + + public CachingResourceHandler(final DirectoryResourceHandler local, final ResourceHandler remote) { + if (local == null || remote == null) { + throw new NullPointerException("null resource handler"); + } + this.local = local; + this.remote = remote; + } + + public InputStream open(final String name) throws IOException { + InputStream inputStream = local.open(name); + if (inputStream != null) { + return inputStream; + } + inputStream = remote.open(name); + if (inputStream == null) { + return null; + } + local.save(name, inputStream); + return local.open(name); + } + + public void close() throws IOException { + remote.close(); + local.close(); + } +} diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java new file mode 100644 index 00000000..0d1915de --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java @@ -0,0 +1,66 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.maven.index.reader.Record.Type; +import org.junit.Test; + +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; + +/** + * UT for {@link ChunkReader} + */ +public class ChunkReaderTest +{ + @Test + public void simple() throws IOException { + final Map recordTypes = new HashMap(); + recordTypes.put(Type.DESCRIPTOR, 0); + recordTypes.put(Type.ROOT_GROUPS, 0); + recordTypes.put(Type.ALL_GROUPS, 0); + recordTypes.put(Type.ARTIFACT_ADD, 0); + recordTypes.put(Type.ARTIFACT_REMOVE, 0); + + final ChunkReader chunkReader = new ChunkReader("full", + new FileInputStream("src/test/resources/nexus-maven-repository-index.gz")); + try { + assertThat(chunkReader.getVersion(), equalTo(1)); + assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); + for (Record record : chunkReader) { + recordTypes.put(record.getType(), recordTypes.get(record.getType()) + 1); + } + } + finally { + chunkReader.close(); + } + + assertThat(recordTypes.get(Type.DESCRIPTOR), equalTo(1)); + assertThat(recordTypes.get(Type.ROOT_GROUPS), equalTo(1)); + assertThat(recordTypes.get(Type.ALL_GROUPS), equalTo(1)); + assertThat(recordTypes.get(Type.ARTIFACT_ADD), equalTo(2)); + assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), equalTo(0)); + } +} diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java new file mode 100644 index 00000000..edb614af --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java @@ -0,0 +1,72 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * A trivial {@link File} directory handler that does not perform any locking or extra bits, and just serves up files + * by name from specified existing directory. + */ +public class DirectoryResourceHandler + implements WritableResourceHandler +{ + private final File rootDirectory; + + public DirectoryResourceHandler(final File rootDirectory) { + if (rootDirectory == null) { + throw new NullPointerException("null rootDirectory"); + } + if (!rootDirectory.isDirectory()) { + throw new IllegalArgumentException("rootDirectory exists and is not a directory"); + } + this.rootDirectory = rootDirectory; + } + + public InputStream open(final String name) throws IOException { + return new BufferedInputStream(new FileInputStream(new File(rootDirectory, name))); + } + + public void save(final String name, final InputStream inputStream) throws IOException { + try { + final BufferedOutputStream outputStream = new BufferedOutputStream( + new FileOutputStream(new File(rootDirectory, name))); + int read; + byte[] bytes = new byte[8192]; + while ((read = inputStream.read(bytes)) != -1) { + outputStream.write(bytes, 0, read); + } + outputStream.close(); + } + finally { + inputStream.close(); + } + } + + public void close() throws IOException { + // nop + } +} diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java new file mode 100644 index 00000000..933cfbd8 --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java @@ -0,0 +1,57 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +/** + * A trivial HTTP {@link ResourceHandler} that uses {@link URL} to fetch remote content. This implementation does not + * handle any advanced cases, like redirects, authentication, etc. + */ +public class HttpResourceHandler + implements ResourceHandler +{ + private final URI root; + + public HttpResourceHandler(final URL root) throws URISyntaxException { + if (root == null) { + throw new NullPointerException("root URL null"); + } + this.root = root.toURI(); + } + + public InputStream open(final String name) throws IOException { + URL target = root.resolve(name).toURL(); + HttpURLConnection conn = (HttpURLConnection) target.openConnection(); + conn.setRequestMethod("GET"); + conn.setRequestProperty("User-Agent", "ASF Maven-Indexer-Reader/1.0"); + return new BufferedInputStream(conn.getInputStream()); + } + + public void close() throws IOException { + // nop + } +} diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java new file mode 100644 index 00000000..e5d7df1e --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java @@ -0,0 +1,102 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.File; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.net.URL; +import java.util.Arrays; + +import org.junit.Ignore; +import org.junit.Test; + +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; + +/** + * UT for {@link IndexReader} + */ +public class IndexReaderTest +{ + @Test + public void simple() throws IOException { + final IndexReader indexReader = new IndexReader( + null, + new DirectoryResourceHandler(new File("src/test/resources/"))); + try { + assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local")); + assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(1243533418015L)); + assertThat(indexReader.isIncremental(), equalTo(false)); + assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz"))); + int chunks = 0; + int records = 0; + for (ChunkReader chunkReader : indexReader) { + chunks++; + assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); + assertThat(chunkReader.getVersion(), equalTo(1)); + assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); + for (Record record : chunkReader) { + records++; + } + } + + assertThat(chunks, equalTo(1)); + assertThat(records, equalTo(5)); + } + finally { + indexReader.close(); + } + } + + @Test + @Ignore("Here for example but test depending on external resource is not nice thing to have") + public void central() throws Exception { + final File tempDir = File.createTempFile("index-reader", "tmp"); + tempDir.mkdirs(); + final Writer writer = new OutputStreamWriter(System.out); + final IndexReader indexReader = new IndexReader( + new DirectoryResourceHandler(tempDir), + new HttpResourceHandler(new URL("http://repo1.maven.org/maven2/.index/")) + ); + try { + writer.write("indexRepoId=" + indexReader.getIndexId() + "\n"); + writer.write("indexLastPublished=" + indexReader.getPublishedTimestamp() + "\n"); + writer.write("isIncremental=" + indexReader.isIncremental() + "\n"); + writer.write("indexRequiredChunkNames=" + indexReader.getChunkNames() + "\n"); + for (ChunkReader chunkReader : indexReader) { + writer.write("chunkName=" + chunkReader.getName() + "\n"); + writer.write("chunkVersion=" + chunkReader.getVersion() + "\n"); + writer.write("chunkPublished=" + chunkReader.getTimestamp() + "\n"); + writer.write("= = = = = = \n"); + for (Record record : chunkReader) { + writer.write(record.getExpanded() + "\n"); + writer.write("--------- \n"); + writer.write(record.getRaw() + "\n"); + } + } + } + finally { + indexReader.close(); + writer.close(); + } + } +} diff --git a/indexer-reader/src/test/resources/nexus-maven-repository-index.gz b/indexer-reader/src/test/resources/nexus-maven-repository-index.gz new file mode 100644 index 0000000000000000000000000000000000000000..490b21c3b21b0fc8b5217c258f502b08e0e8f5cf GIT binary patch literal 319 zcmV-F0l@wriwFP!000000EJOqPlGTNEebK3%qJgwakAGXkYPdH<1+jpQCMIPKDjho zvtfV|TFjR82l%tSYzQ-EF-_XubIxtfxm6HSdSCC|YlM)dA*F<~&x)nVSw_Srr)Eai zC6~M1GO;-@B7ZcAo`b3MM=DZM=;&BCHg+(!t-GOv0gM6c<}8&B;b0p}!;*vT0t!?a zs4o5LWj&-hz4S<(Xd1@OJ7HliOAg6Oz*?*4KvC>I9(jORlBU`C70EYqH) zBu*gJf0=IY>r7W4q}b2o45eL10tN&}8agW&6Hf&za}K~>rK0n;c!{~FoP* indexer-core indexer-cli + indexer-reader indexer-examples From 2848b0d2b72b7c1a2ea2df695f9bbebfda53e184 Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Tue, 3 Nov 2015 06:28:41 -0500 Subject: [PATCH 05/16] Added index writer, that writes single chunk for now (cherry picked from commit b9c4d90) --- .../maven/index/reader/ChunkReader.java | 423 +++++------------- .../maven/index/reader/ChunkWriter.java | 177 ++++++++ .../maven/index/reader/IndexReader.java | 59 +-- .../maven/index/reader/IndexWriter.java | 196 ++++++++ .../apache/maven/index/reader/Iterables.java | 204 +++++++++ .../org/apache/maven/index/reader/Record.java | 215 +++++++-- .../maven/index/reader/RecordCompactor.java | 205 +++++++++ .../maven/index/reader/RecordExpander.java | 228 ++++++++++ .../maven/index/reader/ResourceHandler.java | 2 +- .../org/apache/maven/index/reader/Utils.java | 98 ++++ .../index/reader/WritableResourceHandler.java | 11 +- .../index/reader/CachingResourceHandler.java | 22 +- .../maven/index/reader/ChunkReaderTest.java | 71 ++- .../reader/DirectoryResourceHandler.java | 26 +- .../maven/index/reader/IndexReaderTest.java | 66 ++- .../maven/index/reader/IndexWriterTest.java | 91 ++++ .../maven/index/reader/IterablesTest.java | 92 ++++ .../maven/index/reader/TestSupport.java | 163 +++++++ .../nexus-maven-repository-index.gz | Bin .../nexus-maven-repository-index.properties | 0 pom.xml | 2 +- 21 files changed, 1913 insertions(+), 438 deletions(-) create mode 100644 indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java create mode 100644 indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java create mode 100644 indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java create mode 100644 indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java create mode 100644 indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java create mode 100644 indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java create mode 100644 indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java create mode 100644 indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java create mode 100644 indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java rename indexer-reader/src/test/resources/{ => simple}/nexus-maven-repository-index.gz (100%) rename indexer-reader/src/test/resources/{ => simple}/nexus-maven-repository-index.properties (100%) diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java index 89434fdf..dc6ee794 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java @@ -20,40 +20,27 @@ */ import java.io.Closeable; +import java.io.DataInput; import java.io.DataInputStream; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.UTFDataFormatException; -import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.NoSuchElementException; -import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; -import org.apache.maven.index.reader.Record.Type; - /** - * Maven 2 Index published binary chunk reader. + * Maven 2 Index published binary chunk reader, it reads raw Maven Indexer records from the transport binary format. * * @since 5.1.2 */ public class ChunkReader - implements Closeable, Iterable + implements Closeable, Iterable> { - private static final String FIELD_SEPARATOR = "|"; - - private static final String NOT_AVAILABLE = "NA"; - - private static final String UINFO = "u"; - - private static final String INFO = "i"; - - private static final Pattern FS_PATTERN = Pattern.compile(Pattern.quote(FIELD_SEPARATOR)); - private final String chunkName; private final DataInputStream dataInputStream; @@ -78,14 +65,14 @@ public String getName() { } /** - * Returns index getVersion. All releases so far always returned {@code 1}. + * Returns index version. All releases so far always returned {@code 1}. */ public int getVersion() { return version; } /** - * Returns the getTimestamp of last update of the index. + * Returns the index timestamp of last update of the index. */ public Date getTimestamp() { return timestamp; @@ -94,7 +81,7 @@ public Date getTimestamp() { /** * Returns the {@link Record} iterator. */ - public Iterator iterator() { + public Iterator> iterator() { try { return new IndexIterator(dataInputStream); } @@ -114,341 +101,155 @@ public void close() throws IOException { * Low memory footprint index iterator that incrementally parses the underlying stream. */ private static class IndexIterator - implements Iterator + implements Iterator> { private final DataInputStream dataInputStream; - private Record nextRecord; + private Map nextRecord; public IndexIterator(final DataInputStream dataInputStream) throws IOException { this.dataInputStream = dataInputStream; - this.nextRecord = readRecord(); + this.nextRecord = nextRecord(); } public boolean hasNext() { return nextRecord != null; } - public Record next() { + public Map next() { if (nextRecord == null) { throw new NoSuchElementException("chunk depleted"); } - Record result = nextRecord; + Map result = nextRecord; + nextRecord = nextRecord(); + return result; + } + + private Map nextRecord() { try { - nextRecord = readRecord(); - return result; + return readRecord(dataInputStream); } catch (IOException e) { throw new RuntimeException("read error", e); } } + } - /** - * Reads and returns next record from the underlying stream, or {@code null} if no more records. - */ - private Record readRecord() - throws IOException - { - int fieldCount; - try { - fieldCount = dataInputStream.readInt(); - } - catch (EOFException ex) { - return null; // no more documents - } - - Map recordMap = new HashMap(); - for (int i = 0; i < fieldCount; i++) { - readField(recordMap); - } - - if (recordMap.containsKey("DESCRIPTOR")) { - return new Record(Type.DESCRIPTOR, recordMap, expandDescriptor(recordMap)); - } - else if (recordMap.containsKey("allGroups")) { - return new Record(Type.ALL_GROUPS, recordMap, expandAllGroups(recordMap)); - } - else if (recordMap.containsKey("rootGroups")) { - return new Record(Type.ROOT_GROUPS, recordMap, expandRootGroups(recordMap)); - } - else if (recordMap.containsKey("del")) { - return new Record(Type.ARTIFACT_REMOVE, recordMap, expandDeletedArtifact(recordMap)); - } - else { - // Fix up UINFO field wrt MINDEXER-41 - final String uinfo = recordMap.get(UINFO); - final String info = recordMap.get(INFO); - if (uinfo != null && !(info == null || info.trim().length() == 0)) { - final String[] splitInfo = FS_PATTERN.split(info); - if (splitInfo.length > 6) { - final String extension = splitInfo[6]; - if (uinfo.endsWith(FIELD_SEPARATOR + NOT_AVAILABLE)) { - recordMap.put(UINFO, uinfo + FIELD_SEPARATOR + extension); - } - } - } - return new Record(Type.ARTIFACT_ADD, recordMap, expandAddedArtifact(recordMap)); - } + /** + * Reads and returns next record from the underlying stream, or {@code null} if no more records. + */ + private static Map readRecord(final DataInput dataInput) + throws IOException + { + int fieldCount; + try { + fieldCount = dataInput.readInt(); } - - private void readField(final Map record) - throws IOException - { - dataInputStream.read(); // flags: neglect them - String name = dataInputStream.readUTF(); - String value = readUTF(); - record.put(name, value); + catch (EOFException ex) { + return null; // no more documents } - private String readUTF() - throws IOException - { - int utflen = dataInputStream.readInt(); - - byte[] bytearr; - char[] chararr; - - try { - bytearr = new byte[utflen]; - chararr = new char[utflen]; - } - catch (OutOfMemoryError e) { - IOException ioex = new IOException("Index data content is corrupt"); - ioex.initCause(e); - throw ioex; - } - - int c, char2, char3; - int count = 0; - int chararr_count = 0; + Map recordMap = new HashMap(); + for (int i = 0; i < fieldCount; i++) { + readField(recordMap, dataInput); + } + return recordMap; + } - dataInputStream.readFully(bytearr, 0, utflen); + private static void readField(final Map record, final DataInput dataInput) + throws IOException + { + dataInput.readByte(); // flags: neglect them + String name = dataInput.readUTF(); + String value = readUTF(dataInput); + record.put(name, value); + } - while (count < utflen) { - c = bytearr[count] & 0xff; - if (c > 127) { - break; - } - count++; - chararr[chararr_count++] = (char) c; - } + private static String readUTF(final DataInput dataInput) + throws IOException + { + int utflen = dataInput.readInt(); - while (count < utflen) { - c = bytearr[count] & 0xff; - switch (c >> 4) { - case 0: - case 1: - case 2: - case 3: - case 4: - case 5: - case 6: - case 7: - /* 0xxxxxxx */ - count++; - chararr[chararr_count++] = (char) c; - break; + byte[] bytearr; + char[] chararr; - case 12: - case 13: - /* 110x xxxx 10xx xxxx */ - count += 2; - if (count > utflen) { - throw new UTFDataFormatException("malformed input: partial character at end"); - } - char2 = bytearr[count - 1]; - if ((char2 & 0xC0) != 0x80) { - throw new UTFDataFormatException("malformed input around byte " + count); - } - chararr[chararr_count++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F)); - break; - - case 14: - /* 1110 xxxx 10xx xxxx 10xx xxxx */ - count += 3; - if (count > utflen) { - throw new UTFDataFormatException("malformed input: partial character at end"); - } - char2 = bytearr[count - 2]; - char3 = bytearr[count - 1]; - if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) { - throw new UTFDataFormatException("malformed input around byte " + (count - 1)); - } - chararr[chararr_count++] = - (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | (char3 & 0x3F)); - break; - - default: - /* 10xx xxxx, 1111 xxxx */ - throw new UTFDataFormatException("malformed input around byte " + count); - } - } - - // The number of chars produced may be less than utflen - return new String(chararr, 0, chararr_count); + try { + bytearr = new byte[utflen]; + chararr = new char[utflen]; } - - private Map expandDescriptor(final Map raw) { - final Map result = new HashMap(); - String[] r = FS_PATTERN.split(raw.get("IDXINFO")); - result.put(Record.REPOSITORY_ID, r[1]); - return result; + catch (OutOfMemoryError e) { + IOException ioex = new IOException("Index data content is corrupt"); + ioex.initCause(e); + throw ioex; } - private Map expandAllGroups(final Map raw) { - final Map result = new HashMap(); - putIfNotNullAsList(raw, Record.ALL_GROUPS_LIST, result, "allGroups"); - return result; - } + int c, char2, char3; + int count = 0; + int chararr_count = 0; - private Map expandRootGroups(final Map raw) { - final Map result = new HashMap(); - putIfNotNullAsList(raw, Record.ROOT_GROUPS_LIST, result, "rootGroups"); - return result; - } + dataInput.readFully(bytearr, 0, utflen); - private Map expandDeletedArtifact(final Map raw) { - final Map result = new HashMap(); - putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED); - if (raw.containsKey("del")) { - expandUinfo(raw.get("del"), result); + while (count < utflen) { + c = bytearr[count] & 0xff; + if (c > 127) { + break; } - return result; + count++; + chararr[chararr_count++] = (char) c; } - /** - * Expands the "encoded" Maven Indexer record by splitting the synthetic fields and applying expanded field naming. - */ - private Map expandAddedArtifact(final Map raw) { - final Map result = new HashMap(); - - // Minimal - expandUinfo(raw.get(UINFO), result); - final String info = raw.get(INFO); - if (info != null) { - String[] r = FS_PATTERN.split(info); - result.put(Record.PACKAGING, renvl(r[0])); - result.put(Record.FILE_MODIFIED, Long.valueOf(r[1])); - result.put(Record.FILE_SIZE, Long.valueOf(r[2])); - result.put(Record.HAS_SOURCES, "1".equals(r[3]) ? Boolean.TRUE.toString() : Boolean.FALSE.toString()); - result.put(Record.HAS_JAVADOC, "1".equals(r[4]) ? Boolean.TRUE.toString() : Boolean.FALSE.toString()); - result.put(Record.HAS_SIGNATURE, "1".equals(r[5]) ? Boolean.TRUE.toString() : Boolean.FALSE.toString()); - if (r.length > 6) { - result.put(Record.FILE_EXTENSION, r[6]); - } - else { - final String packaging = raw.get(Record.PACKAGING); - if (raw.get(Record.CLASSIFIER) != null - || "pom".equals(packaging) - || "war".equals(packaging) - || "ear".equals(packaging)) { - result.put(Record.FILE_EXTENSION, packaging); + while (count < utflen) { + c = bytearr[count] & 0xff; + switch (c >> 4) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + /* 0xxxxxxx */ + count++; + chararr[chararr_count++] = (char) c; + break; + + case 12: + case 13: + /* 110x xxxx 10xx xxxx */ + count += 2; + if (count > utflen) { + throw new UTFDataFormatException("malformed input: partial character at end"); } - else { - result.put(Record.FILE_EXTENSION, "jar"); // best guess + char2 = bytearr[count - 1]; + if ((char2 & 0xC0) != 0x80) { + throw new UTFDataFormatException("malformed input around byte " + count); } - } - } - putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED); - putIfNotNull(raw, "n", result, Record.NAME); - putIfNotNull(raw, "d", result, Record.DESCRIPTION); - putIfNotNull(raw, "1", result, Record.SHA1); - - // Jar file contents (optional) - putIfNotNullAsList(raw, "classnames", result, Record.CLASSNAMES); - - // Maven Plugin (optional) - putIfNotNull(raw, "px", result, Record.PLUGIN_PREFIX); - putIfNotNullAsList(raw, "gx", result, Record.PLUGIN_GOALS); - - // OSGi (optional) - putIfNotNull(raw, "Bundle-SymbolicName", result, "Bundle-SymbolicName"); - putIfNotNull(raw, "Bundle-Version", result, "Bundle-Version"); - putIfNotNull(raw, "Export-Package", result, "Export-Package"); - putIfNotNull(raw, "Export-Service", result, "Export-Service"); - putIfNotNull(raw, "Bundle-Description", result, "Bundle-Description"); - putIfNotNull(raw, "Bundle-Name", result, "Bundle-Name"); - putIfNotNull(raw, "Bundle-License", result, "Bundle-License"); - putIfNotNull(raw, "Bundle-DocURL", result, "Bundle-DocURL"); - putIfNotNull(raw, "Import-Package", result, "Import-Package"); - putIfNotNull(raw, "Require-Bundle", result, "Require-Bundle"); - putIfNotNull(raw, "Bundle-Version", result, "Bundle-Version"); - - return result; - } + chararr[chararr_count++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F)); + break; - /** - * Expands UINFO synthetic field. Handles {@code null} String inputs. - */ - private void expandUinfo(final String uinfo, final Map result) { - if (uinfo != null) { - String[] r = FS_PATTERN.split(uinfo); - result.put(Record.GROUP_ID, r[0]); - result.put(Record.ARTIFACT_ID, r[1]); - result.put(Record.VERSION, r[2]); - String classifier = renvl(r[3]); - if (classifier != null) { - result.put(Record.CLASSIFIER, classifier); - if (r.length > 4) { - result.put(Record.FILE_EXTENSION, r[4]); + case 14: + /* 1110 xxxx 10xx xxxx 10xx xxxx */ + count += 3; + if (count > utflen) { + throw new UTFDataFormatException("malformed input: partial character at end"); } - } - else if (r.length > 4) { - result.put(Record.PACKAGING, r[4]); - } - } - } - } - - /** - * Helper to put a value from source map into target map, if not null. - */ - private static void putIfNotNull( - final Map source, - final String sourceName, - final Map target, - final String targetName) - { - String value = source.get(sourceName); - if (value != null && value.trim().length() != 0) { - target.put(targetName, value); - } - } - - /** - * Helper to put a {@link Long} value from source map into target map, if not null. - */ - private static void putIfNotNullTS( - final Map source, - final String sourceName, - final Map target, - final String targetName) - { - String value = source.get(sourceName); - if (value != null && value.trim().length() != 0) { - target.put(targetName, Long.valueOf(value)); - } - } + char2 = bytearr[count - 2]; + char3 = bytearr[count - 1]; + if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) { + throw new UTFDataFormatException("malformed input around byte " + (count - 1)); + } + chararr[chararr_count++] = + (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | (char3 & 0x3F)); + break; - /** - * Helper to put a collection value from source map into target map as {@link java.util.List}, if not null. - */ - private static void putIfNotNullAsList( - final Map source, - final String sourceName, - final Map target, - final String targetName) - { - String value = source.get(sourceName); - if (value != null && value.trim().length() != 0) { - target.put(targetName, Arrays.asList(FS_PATTERN.split(value))); + default: + /* 10xx xxxx, 1111 xxxx */ + throw new UTFDataFormatException("malformed input around byte " + count); + } } - } - /** - * Helper to translate the "NA" (not available) input into {@code null} value. - */ - private static String renvl(final String v) { - return NOT_AVAILABLE.equals(v) ? null : v; + // The number of chars produced may be less than utflen + return new String(chararr, 0, chararr_count); } } diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java new file mode 100644 index 00000000..24e4701e --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java @@ -0,0 +1,177 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.Closeable; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Date; +import java.util.Iterator; +import java.util.Map; +import java.util.zip.GZIPOutputStream; + +/** + * Maven 2 Index published binary chunk writer, it writes raw Maven Indexer records to the transport binary format. + * + * @since 5.1.2 + */ +public class ChunkWriter + implements Closeable +{ + private static final int F_INDEXED = 1; + + private static final int F_TOKENIZED = 2; + + private static final int F_STORED = 4; + + private final String chunkName; + + private final DataOutputStream dataOutputStream; + + private final int version; + + private final Date timestamp; + + public ChunkWriter(final String chunkName, final OutputStream outputStream, final int version, final Date timestamp) + throws IOException + { + this.chunkName = chunkName.trim(); + this.dataOutputStream = new DataOutputStream(new GZIPOutputStream(outputStream, 2 * 1024)); + this.version = version; + this.timestamp = timestamp; + + dataOutputStream.writeByte(version); + dataOutputStream.writeLong(timestamp == null ? -1 : timestamp.getTime()); + } + + /** + * Returns the chunk name. + */ + public String getName() { + return chunkName; + } + + /** + * Returns index version. All releases so far always returned {@code 1}. + */ + public int getVersion() { + return version; + } + + /** + * Returns the index timestamp of last update of the index. + */ + public Date getTimestamp() { + return timestamp; + } + + /** + * Writes out the record iterator and returns the written record count. + */ + public int writeChunk(final Iterator> iterator) throws IOException { + int written = 0; + while (iterator.hasNext()) { + writeRecord(iterator.next(), dataOutputStream); + written++; + } + return written; + } + + /** + * Closes this reader and it's underlying input. + */ + public void close() throws IOException { + dataOutputStream.close(); + } + + private static void writeRecord(final Map record, final DataOutput dataOutput) + throws IOException + { + dataOutput.writeInt(record.size()); + for (Map.Entry entry : record.entrySet()) { + writeField(entry.getKey(), entry.getValue(), dataOutput); + } + } + + private static void writeField(final String fieldName, final String fieldValue, final DataOutput dataOutput) + throws IOException + { + boolean isIndexed = !(fieldName.equals("i") || fieldName.equals("m")); + boolean isTokenized = !(fieldName.equals("i") + || fieldName.equals("m") + || fieldName.equals("1") + || fieldName.equals("px")); + int flags = (isIndexed ? F_INDEXED : 0) + (isTokenized ? F_TOKENIZED : 0) + F_STORED; + dataOutput.writeByte(flags); + dataOutput.writeUTF(fieldName); + writeUTF(fieldValue, dataOutput); + } + + private static void writeUTF(final String str, final DataOutput dataOutput) + throws IOException + { + int strlen = str.length(); + int utflen = 0; + int c; + // use charAt instead of copying String to char array + for (int i = 0; i < strlen; i++) { + c = str.charAt(i); + if ((c >= 0x0001) && (c <= 0x007F)) { + utflen++; + } + else if (c > 0x07FF) { + utflen += 3; + } + else { + utflen += 2; + } + } + dataOutput.writeInt(utflen); + byte[] bytearr = new byte[utflen]; + int count = 0; + int i = 0; + for (; i < strlen; i++) { + c = str.charAt(i); + if (!((c >= 0x0001) && (c <= 0x007F))) { + break; + } + bytearr[count++] = (byte) c; + } + for (; i < strlen; i++) { + c = str.charAt(i); + if ((c >= 0x0001) && (c <= 0x007F)) { + bytearr[count++] = (byte) c; + + } + else if (c > 0x07FF) { + bytearr[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F)); + bytearr[count++] = (byte) (0x80 | ((c >> 6) & 0x3F)); + bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F)); + } + else { + bytearr[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F)); + bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F)); + } + } + dataOutput.write(bytearr, 0, utflen); + } +} diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java index 45514b9b..d74c366a 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java @@ -19,39 +19,28 @@ * under the License. */ -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; -import java.text.DateFormat; +import java.io.OutputStream; import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Properties; -import java.util.TimeZone; + +import static org.apache.maven.index.reader.Utils.loadProperties; /** - * Maven 2 Index reader that handles incremental updates if possible. + * Maven 2 Index reader that handles incremental updates if possible and provides one or more {@link ChunkReader}s, to + * read all the required records. * * @since 5.1.2 */ public class IndexReader implements Iterable, Closeable { - private static final String INDEX_FILE_PREFIX = "nexus-maven-repository-index"; - - private static final DateFormat INDEX_DATE_FORMAT; - - static { - INDEX_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss.SSS Z"); - INDEX_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT")); - } - private final WritableResourceHandler local; private final ResourceHandler remote; @@ -74,10 +63,10 @@ public IndexReader(final WritableResourceHandler local, final ResourceHandler re } this.local = local; this.remote = remote; - remoteIndexProperties = loadProperties(remote.open(INDEX_FILE_PREFIX + ".properties")); + remoteIndexProperties = loadProperties(remote.open(Utils.INDEX_FILE_PREFIX + ".properties")); try { if (local != null) { - localIndexProperties = loadProperties(local.open(INDEX_FILE_PREFIX + ".properties")); + localIndexProperties = loadProperties(local.open(Utils.INDEX_FILE_PREFIX + ".properties")); String remoteIndexId = remoteIndexProperties.getProperty("nexus.index.id"); String localIndexId = localIndexProperties.getProperty("nexus.index.id"); if (remoteIndexId == null || localIndexId == null || !remoteIndexId.equals(localIndexId)) { @@ -86,14 +75,16 @@ public IndexReader(final WritableResourceHandler local, final ResourceHandler re remoteIndexId); } this.indexId = localIndexId; - this.publishedTimestamp = INDEX_DATE_FORMAT.parse(localIndexProperties.getProperty("nexus.index.timestamp")); + this.publishedTimestamp = Utils.INDEX_DATE_FORMAT + .parse(localIndexProperties.getProperty("nexus.index.timestamp")); this.incremental = canRetrieveAllChunks(); this.chunkNames = calculateChunkNames(); } else { localIndexProperties = null; this.indexId = remoteIndexProperties.getProperty("nexus.index.id"); - this.publishedTimestamp = INDEX_DATE_FORMAT.parse(remoteIndexProperties.getProperty("nexus.index.timestamp")); + this.publishedTimestamp = Utils.INDEX_DATE_FORMAT + .parse(remoteIndexProperties.getProperty("nexus.index.timestamp")); this.incremental = false; this.chunkNames = calculateChunkNames(); } @@ -170,9 +161,13 @@ public Iterator iterator() { * for future incremental updates. */ private void syncLocalWithRemote() throws IOException { - final ByteArrayOutputStream bos = new ByteArrayOutputStream(); - remoteIndexProperties.store(bos, "Maven Indexer Reader"); - local.save(INDEX_FILE_PREFIX + ".properties", new ByteArrayInputStream(bos.toByteArray())); + final OutputStream outputStream = local.openWrite(Utils.INDEX_FILE_PREFIX + ".properties"); + try { + remoteIndexProperties.store(outputStream, "Maven Indexer Reader"); + } + finally { + outputStream.close(); + } } /** @@ -185,12 +180,12 @@ private List calculateChunkNames() { int currentCounter = Integer.parseInt(localIndexProperties.getProperty("nexus.index.last-incremental")); currentCounter++; while (currentCounter <= maxCounter) { - chunkNames.add(INDEX_FILE_PREFIX + "." + currentCounter++ + ".gz"); + chunkNames.add(Utils.INDEX_FILE_PREFIX + "." + currentCounter++ + ".gz"); } return Collections.unmodifiableList(chunkNames); } else { - return Collections.singletonList(INDEX_FILE_PREFIX + ".gz"); + return Collections.singletonList(Utils.INDEX_FILE_PREFIX + ".gz"); } } @@ -264,18 +259,4 @@ public ChunkReader next() { } } } - - /** - * Creates and loads {@link Properties} from provided {@link InputStream} and closes the stream. - */ - private static Properties loadProperties(final InputStream inputStream) throws IOException { - try { - final Properties properties = new Properties(); - properties.load(inputStream); - return properties; - } - finally { - inputStream.close(); - } - } } diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java new file mode 100644 index 00000000..f52cb298 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java @@ -0,0 +1,196 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.text.ParseException; +import java.util.Date; +import java.util.Iterator; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; + +import static org.apache.maven.index.reader.Utils.loadProperties; + +/** + * Maven 2 Index writer that writes chunk and maintains published property file. + *

+ * Currently no incremental update is supported, as the deleteion states should be maintained by + * caller. Hence, this writer will always produce the "main" chunk only. + * + * @since 5.1.2 + */ +public class IndexWriter + implements Closeable +{ + private static final int INDEX_V1 = 1; + + private final WritableResourceHandler local; + + private final Properties localIndexProperties; + + private final boolean incremental; + + private final String nextChunkCounter; + + private final String nextChunkName; + + public IndexWriter(final WritableResourceHandler local, final String indexId, final boolean incrementalSupported) + throws IOException + { + if (local == null) { + throw new NullPointerException("local resource handler null"); + } + if (indexId == null) { + throw new NullPointerException("indexId null"); + } + this.local = local; + InputStream localIndexPropertiesInputStream = local.open(Utils.INDEX_FILE_PREFIX + ".properties"); + if (incrementalSupported && localIndexPropertiesInputStream != null) { + // existing index, this is incremental publish, and we will add new chunk + this.localIndexProperties = loadProperties(local.open(Utils.INDEX_FILE_PREFIX + ".properties")); + String localIndexId = localIndexProperties.getProperty("nexus.index.id"); + if (localIndexId == null || !localIndexId.equals(indexId)) { + throw new IllegalArgumentException( + "index already exists and indexId mismatch or unreadable: " + localIndexId + ", " + + indexId); + } + this.incremental = true; + this.nextChunkCounter = calculateNextChunkCounter(); + this.nextChunkName = Utils.INDEX_FILE_PREFIX + "." + nextChunkCounter + ".gz"; + } + else { + // non-existing index, create published index from scratch + this.localIndexProperties = new Properties(); + this.localIndexProperties.setProperty("nexus.index.id", indexId); + this.localIndexProperties.setProperty("nexus.index.chain-id", UUID.randomUUID().toString()); + this.incremental = false; + this.nextChunkCounter = null; + this.nextChunkName = Utils.INDEX_FILE_PREFIX + ".gz"; + } + } + + /** + * Returns the index context ID that published index has set. + */ + public String getIndexId() { + return localIndexProperties.getProperty("nexus.index.id"); + } + + /** + * Returns the {@link Date} when index was last published or {@code null} if this is first publishing. In other + * words,returns {@code null} when {@link #isIncremental()} returns {@code false}. After this writer is closed, the + * return value is updated to "now" (in {@link #close() method}. + */ + public Date getPublishedTimestamp() { + try { + String timestamp = localIndexProperties.getProperty("nexus.index.timestamp"); + if (timestamp != null) { + return Utils.INDEX_DATE_FORMAT.parse(timestamp); + } + return null; + } + catch (ParseException e) { + throw new RuntimeException("Corrupt date", e); + } + } + + /** + * Returns {@code true} if incremental publish is about to happen. + */ + public boolean isIncremental() { + return incremental; + } + + /** + * Returns the chain id of published index. If {@link #isIncremental()} is {@code false}, this is the newly generated + * chain ID. + */ + public String getChainId() { + return localIndexProperties.getProperty("nexus.index.chain-id"); + } + + /** + * Returns the next chunk name about to be published. + */ + public String getNextChunkName() { + return nextChunkName; + } + + /** + * Writes out the record iterator and returns the written record count. + */ + public int writeChunk(final Iterator> iterator) throws IOException { + int written; + final ChunkWriter chunkWriter = new ChunkWriter(nextChunkName, local.openWrite(nextChunkName), INDEX_V1, new Date()); + try { + written = chunkWriter.writeChunk(iterator); + } + finally { + chunkWriter.close(); + } + if (incremental) { + // TODO: update main gz file + } + return written; + } + + /** + * Closes the underlying {@link ResourceHandler} and synchronizes published index properties, so remote clients + * becomes able to consume newly published index. If sync is not desired (ie. due to aborted publish), then this + * method should NOT be invoked, but rather the {@link ResourceHandler} that caller provided in constructor of + * this class should be closed manually. + */ + public void close() throws IOException { + try { + if (incremental) { + localIndexProperties.setProperty("nexus.index.last-incremental", nextChunkCounter); + } + localIndexProperties.setProperty("nexus.index.timestamp", Utils.INDEX_DATE_FORMAT.format(new Date())); + + final OutputStream outputStream = local.openWrite(Utils.INDEX_FILE_PREFIX + ".properties"); + try { + localIndexProperties.store(outputStream, "Maven Indexer Writer"); + } + finally { + outputStream.close(); + } + } + finally { + local.close(); + } + } + + /** + * Calculates the chunk names that needs to be fetched. + */ + private String calculateNextChunkCounter() { + String lastChunkCounter = localIndexProperties.getProperty("nexus.index.last-incremental"); + if (lastChunkCounter != null) { + return String.valueOf(Integer.parseInt(lastChunkCounter) + 1); + } + else { + return "1"; + } + } +} diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java new file mode 100644 index 00000000..78941dcd --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java @@ -0,0 +1,204 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.TreeSet; + +import org.apache.maven.index.reader.Record.EntryKey; +import org.apache.maven.index.reader.Record.Type; + +/** + * Helpers to transform {@link Iterable}. + * + * @since 5.1.2 + */ +public final class Iterables +{ + private Iterables() { + // nothing + } + + /** + * Transforming function. + */ + public interface Function + { + O apply(I rec); + } + + /** + * Applies {@link Function} to an {@link Iterable} on the fly. + */ + public static Iterable transform(final Iterable iterable, final Function function) { + return new Iterable() + { + public Iterator iterator() { + return new TransformIterator(iterable.iterator(), function); + } + }; + } + + /** + * Helper method, that "decorates" the stream of records to be written out with "special" Maven Indexer records, so + * all the caller is needed to provide {@Link Iterable} or {@link Record}s to be on the index, with + * record type {@link Record.Type#ARTIFACT_ADD}. This method will create the output as "proper" Maven Indexer record + * streeam, by adding the {@link Type#DESCRIPTOR}, {@link Type#ROOT_GROUPS} and {@link Type#ALL_GROUPS} special + * records. + */ + public static Iterable> decorateAndTransform(final Iterable iterable, + final String repoId) + { + final RecordCompactor recordCompactor = new RecordCompactor(); + final TreeSet allGroups = new TreeSet(); + final TreeSet rootGroups = new TreeSet(); + final ArrayList> iterators = new ArrayList>(); + iterators.add(Collections.singletonList(descriptor(repoId)).iterator()); + iterators.add(iterable.iterator()); + iterators.add(Collections.singletonList(allGroups(allGroups)).iterator()); + iterators.add(Collections.singletonList(rootGroups(rootGroups)).iterator()); + return transform( + new Iterable() + { + public Iterator iterator() { + return new ConcatIterator(iterators.iterator()); + } + }, + new Function>() + { + public Map apply(final Record rec) { + if (Type.DESCRIPTOR == rec.getType()) { + return recordCompactor.apply(descriptor(repoId)); + } + else if (Type.ALL_GROUPS == rec.getType()) { + return recordCompactor.apply(allGroups(allGroups)); + } + else if (Type.ROOT_GROUPS == rec.getType()) { + return recordCompactor.apply(rootGroups(rootGroups)); + } + else { + final String groupId = rec.get(Record.GROUP_ID); + if (groupId != null) { + allGroups.add(groupId); + rootGroups.add(Utils.rootGroup(groupId)); + } + return recordCompactor.apply(rec); + } + } + } + ); + } + + private static Record descriptor(final String repoId) { + HashMap entries = new HashMap(); + entries.put(Record.REPOSITORY_ID, repoId); + return new Record(Type.DESCRIPTOR, entries); + } + + private static Record allGroups(final Collection allGroups) { + HashMap entries = new HashMap(); + entries.put(Record.ALL_GROUPS, allGroups.toArray(new String[allGroups.size()])); + return new Record(Type.ALL_GROUPS, entries); + } + + private static Record rootGroups(final Collection rootGroups) { + HashMap entries = new HashMap(); + entries.put(Record.ROOT_GROUPS, rootGroups.toArray(new String[rootGroups.size()])); + return new Record(Type.ROOT_GROUPS, entries); + } + + // == + + private static final class TransformIterator + implements Iterator + { + private final Iterator iterator; + + private final Function function; + + private TransformIterator(final Iterator iterator, final Function function) { + this.iterator = iterator; + this.function = function; + } + + public boolean hasNext() { + return iterator.hasNext(); + } + + public O next() { + return function.apply(iterator.next()); + } + } + + private static final class ConcatIterator + implements Iterator + { + private final Iterator> iterators; + + private Iterator current; + + private T nextElement; + + private ConcatIterator(final Iterator> iterators) { + this.iterators = iterators; + this.nextElement = getNextElement(); + } + + public boolean hasNext() { + return nextElement != null; + } + + public T next() { + if (nextElement == null) { + throw new NoSuchElementException(); + } + T result = nextElement; + nextElement = getNextElement(); + return result; + } + + protected T getNextElement() { + if ((current == null || !current.hasNext()) && iterators.hasNext()) { + current = iterators.next(); + } + while (current != null && !current.hasNext()) { + if (!iterators.hasNext()) { + current = null; + break; + } + current = iterators.next(); + } + if (current != null) { + return current.next(); + } + else { + return null; + } + } + } +} diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java index 3354008e..17468457 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java @@ -26,113 +26,225 @@ * * @since 5.1.2 */ -public class Record +public final class Record { + public static final class EntryKey + { + private final String name; + + private final Class proto; + + public EntryKey(final String name, final Class proto) { + if (name == null) { + throw new NullPointerException("name is null"); + } + if (proto == null) { + throw new NullPointerException("proto is null"); + } + this.name = name; + this.proto = proto; + } + + public T coerce(final Object object) { + return (T) proto.cast(object); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof EntryKey)) { + return false; + } + EntryKey entryKey = (EntryKey) o; + return name.equals(entryKey.name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public String toString() { + return "Key{" + + "name='" + name + '\'' + + ", type=" + proto.getSimpleName() + + '}'; + } + } + /** * Key of repository ID entry, that contains {@link String}. */ - public static final String REPOSITORY_ID = "repositoryId"; + public static final EntryKey REPOSITORY_ID = new EntryKey("repositoryId", String.class); /** * Key of all groups list entry, that contains {@link java.util.List}. */ - public static final String ALL_GROUPS_LIST = "allGroupsList"; + public static final EntryKey ALL_GROUPS = new EntryKey("allGroups", String[].class); /** * Key of root groups list entry, that contains {@link java.util.List}. */ - public static final String ROOT_GROUPS_LIST = "rootGroupsList"; + public static final EntryKey ROOT_GROUPS = new EntryKey("rootGroups", String[].class); /** * Key of index record modification (added to index or removed from index) timestamp entry, that contains {@link * Long}. */ - public static final String REC_MODIFIED = "recordModified"; + public static final EntryKey REC_MODIFIED = new EntryKey("recordModified", Long.class); /** * Key of artifact groupId entry, that contains {@link String}. */ - public static final String GROUP_ID = "groupId"; + public static final EntryKey GROUP_ID = new EntryKey("groupId", String.class); /** * Key of artifact artifactId entry, that contains {@link String}. */ - public static final String ARTIFACT_ID = "artifactId"; + public static final EntryKey ARTIFACT_ID = new EntryKey("artifactId", String.class); /** * Key of artifact version entry, that contains {@link String}. */ - public static final String VERSION = "version"; + public static final EntryKey VERSION = new EntryKey("version", String.class); /** * Key of artifact classifier entry, that contains {@link String}. */ - public static final String CLASSIFIER = "classifier"; + public static final EntryKey CLASSIFIER = new EntryKey("classifier", String.class); /** * Key of artifact packaging entry, that contains {@link String}. */ - public static final String PACKAGING = "packaging"; + public static final EntryKey PACKAGING = new EntryKey("packaging", String.class); /** * Key of artifact file extension, that contains {@link String}. */ - public static final String FILE_EXTENSION = "fileExtension"; + public static final EntryKey FILE_EXTENSION = new EntryKey("fileExtension", String.class); /** * Key of artifact file last modified timestamp, that contains {@link Long}. */ - public static final String FILE_MODIFIED = "fileModified"; + public static final EntryKey FILE_MODIFIED = new EntryKey("fileModified", Long.class); /** * Key of artifact file size in bytes, that contains {@link Long}. */ - public static final String FILE_SIZE = "fileSize"; + public static final EntryKey FILE_SIZE = new EntryKey("fileSize", Long.class); /** * Key of artifact Sources presence flag, that contains {@link Boolean}. */ - public static final String HAS_SOURCES = "hasSources"; + public static final EntryKey HAS_SOURCES = new EntryKey("hasSources", Boolean.class); /** * Key of artifact Javadoc presence flag, that contains {@link Boolean}. */ - public static final String HAS_JAVADOC = "hasJavadoc"; + public static final EntryKey HAS_JAVADOC = new EntryKey("hasJavadoc", Boolean.class); /** * Key of artifact signature presence flag, that contains {@link Boolean}. */ - public static final String HAS_SIGNATURE = "hasSignature"; + public static final EntryKey HAS_SIGNATURE = new EntryKey("hasSignature", Boolean.class); /** * Key of artifact name (as set in POM), that contains {@link String}. */ - public static final String NAME = "name"; + public static final EntryKey NAME = new EntryKey("name", String.class); /** * Key of artifact description (as set in POM), that contains {@link String}. */ - public static final String DESCRIPTION = "description"; + public static final EntryKey DESCRIPTION = new EntryKey("description", String.class); /** * Key of artifact SHA1 digest, that contains {@link String}. */ - public static final String SHA1 = "sha1"; + public static final EntryKey SHA1 = new EntryKey("sha1", String.class); + + /** + * Key of artifact contained class names, that contains {@link java.util.List}. Extracted by {@code + * JarFileContentsIndexCreator}. + */ + public static final EntryKey CLASSNAMES = new EntryKey("classNames", String[].class); + + /** + * Key of plugin artifact prefix, that contains {@link String}. Extracted by {@code + * MavenPluginArtifactInfoIndexCreator}. + */ + public static final EntryKey PLUGIN_PREFIX = new EntryKey("pluginPrefix", String.class); + + /** + * Key of plugin artifact goals, that contains {@link java.util.List}. Extracted by {@code + * MavenPluginArtifactInfoIndexCreator}. + */ + public static final EntryKey PLUGIN_GOALS = new EntryKey("pluginGoals", String[].class); + + /** + * Key of OSGi "Bundle-SymbolicName" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey OSGI_BUNDLE_SYMBOLIC_NAME = new EntryKey("Bundle-SymbolicName", + String.class); + + /** + * Key of OSGi "Bundle-Version" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey OSGI_BUNDLE_VERSION = new EntryKey("Bundle-Version", String.class); + + /** + * Key of OSGi "Export-Package" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey OSGI_EXPORT_PACKAGE = new EntryKey("Export-Package", String.class); + + /** + * Key of OSGi "Export-Service" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey OSGI_EXPORT_SERVICE = new EntryKey("Export-Service", String.class); /** - * Key of artifact contained class names, that contains {@link java.util.List}. + * Key of OSGi "Bundle-Description" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. */ - public static final String CLASSNAMES = "classNames"; + public static final EntryKey OSGI_BUNDLE_DESCRIPTION = new EntryKey("Bundle-Description", + String.class); /** - * Key of plugin artifact prefix, that contains {@link String}. + * Key of OSGi "Bundle-Name" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. */ - public static final String PLUGIN_PREFIX = "pluginPrefix"; + public static final EntryKey OSGI_BUNDLE_NAME = new EntryKey("Bundle-Name", String.class); /** - * Key of plugin artifact goals, that contains {@link java.util.List}. + * Key of OSGi "Bundle-License" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. */ - public static final String PLUGIN_GOALS = "pluginGoals"; + public static final EntryKey OSGI_BUNDLE_LICENSE = new EntryKey("Bundle-License", String.class); + + /** + * Key of OSGi "Bundle-DocURL" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey OSGI_EXPORT_DOCURL = new EntryKey("Bundle-DocURL", String.class); + + /** + * Key of OSGi "Import-Package" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey OSGI_IMPORT_PACKAGE = new EntryKey("Import-Package", String.class); + + /** + * Key of OSGi "Require-Bundle" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey OSGI_REQUIRE_BUNDLE = new EntryKey("Require-Bundle", String.class); /** * Types of returned records returned from index. @@ -175,7 +287,7 @@ public enum Type ARTIFACT_ADD, /** - * Artifact REMOTE record. In case of incremental updates, notes that this artifact was removed. Records of this + * Artifact REMOVE record. In case of incremental updates, signals that this artifact was removed. Records of this * type should be removed from your indexing system. * Contains following entries: *

    @@ -194,7 +306,7 @@ public enum Type * Special record, containing all the Maven "groupId"s that are enlisted on the index. Can be safely ignored. * Contains following entries: *
      - *
    • {@link #ALL_GROUPS_LIST}
      • + *
    • {@link #ALL_GROUPS}
      • *
    */ ALL_GROUPS, @@ -204,7 +316,7 @@ public enum Type * ignored. * Contains following entries: *
      - *
    • {@link #ROOT_GROUPS_LIST}
      • + *
    • {@link #ROOT_GROUPS}
      • *
    */ ROOT_GROUPS @@ -212,13 +324,10 @@ public enum Type private final Type type; - private final Map raw; + private final Map expanded; - private final Map expanded; - - public Record(final Type type, final Map raw, final Map expanded) { + public Record(final Type type, final Map expanded) { this.type = type; - this.raw = raw; this.expanded = expanded; } @@ -232,16 +341,44 @@ public Type getType() { } /** - * Returns the "raw", Maven Indexer specific record as a {@link Map}. + * Returns the expanded (processed and expanded synthetic fields) record as {@link Map} ready for consumption. */ - public Map getRaw() { - return raw; + public Map getExpanded() { + return expanded; } /** - * Returns the expanded (processed and expanded synthetic fields) record as {@link Map} ready for consumption. + * Returns {@code true} if this record contains given {@link EntryKey}. */ - public Map getExpanded() { - return expanded; + boolean containsKey(final EntryKey entryKey) { return expanded.containsKey(entryKey); } + + /** + * Type safe handy method to get value from expanded map. + */ + public T get(final EntryKey entryKey) { + return entryKey.coerce(expanded.get(entryKey)); + } + + /** + * Type safe handy method to put value to expanded map. Accepts {@code null} values, that removes the mapping. + */ + public T put(final EntryKey entryKey, final T value) { + if (value == null) { + return entryKey.coerce(expanded.remove(entryKey)); + } + else { + if (!entryKey.proto.isAssignableFrom(value.getClass())) { + throw new IllegalArgumentException("Key " + entryKey + " does not accepts value " + value); + } + return entryKey.coerce(expanded.put(entryKey, value)); + } + } + + @Override + public String toString() { + return "Record{" + + "type=" + type + + ", expanded=" + expanded + + '}'; } } diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java new file mode 100644 index 00000000..3fcfd250 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java @@ -0,0 +1,205 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.HashMap; +import java.util.Map; + +import org.apache.maven.index.reader.Iterables.Function; +import org.apache.maven.index.reader.Record.Type; + +import static org.apache.maven.index.reader.Utils.FIELD_SEPARATOR; +import static org.apache.maven.index.reader.Utils.INFO; +import static org.apache.maven.index.reader.Utils.UINFO; +import static org.apache.maven.index.reader.Utils.nvl; + +/** + * Maven 2 Index record transformer, that transforms {@link Record}s into "native" Maven Indexer records. + * + * @since 5.1.2 + */ +public class RecordCompactor + implements Function> +{ + public Map apply(final Record record) { + if (Type.DESCRIPTOR == record.getType()) { + return compactDescriptor(record); + } + else if (Type.ALL_GROUPS == record.getType()) { + return compactAllGroups(record); + } + else if (Type.ROOT_GROUPS == record.getType()) { + return compactRootGroups(record); + } + else if (Type.ARTIFACT_REMOVE == record.getType()) { + return compactDeletedArtifact(record); + } + else if (Type.ARTIFACT_ADD == record.getType()) { + return compactAddedArtifact(record); + } + else { + throw new IllegalArgumentException("Unknown record: " + record); + } + } + + private static Map compactDescriptor(final Record record) { + final Map result = new HashMap(); + result.put("DESCRIPTOR", "NexusIndex"); + result.put("IDXINFO", "1.0|" + record.get(Record.REPOSITORY_ID)); + return result; + } + + private static Map compactAllGroups(final Record record) { + final Map result = new HashMap(); + result.put("allGroups", "allGroups"); + putIfNotNullAsStringArray(record.get(Record.ALL_GROUPS), result, "allGroupsList"); + return result; + } + + private static Map compactRootGroups(final Record record) { + final Map result = new HashMap(); + result.put("rootGroups", "allGroups"); + putIfNotNullAsStringArray(record.get(Record.ROOT_GROUPS), result, "rootGroupsList"); + return result; + } + + private static Map compactDeletedArtifact(final Record record) { + final Map result = new HashMap(); + putIfNotNullTS(record.get(Record.REC_MODIFIED), result, "m"); + result.put("del", compactUinfo(record)); + return result; + } + + /** + * Expands the "encoded" Maven Indexer record by splitting the synthetic fields and applying expanded field naming. + */ + private static Map compactAddedArtifact(final Record record) { + final Map result = new HashMap(); + + // Minimal + result.put(UINFO, compactUinfo(record)); + + StringBuilder info = new StringBuilder(); + info.append(nvl(record.get(Record.PACKAGING))); + info.append(FIELD_SEPARATOR); + info.append(record.get(Record.FILE_MODIFIED)); + info.append(FIELD_SEPARATOR); + info.append(record.get(Record.FILE_SIZE)); + info.append(FIELD_SEPARATOR); + info.append(record.get(Record.HAS_SOURCES) ? "1" : "0"); + info.append(FIELD_SEPARATOR); + info.append(record.get(Record.HAS_JAVADOC) ? "1" : "0"); + info.append(FIELD_SEPARATOR); + info.append(record.get(Record.HAS_SIGNATURE) ? "1" : "0"); + info.append(FIELD_SEPARATOR); + info.append(nvl(record.get(Record.FILE_EXTENSION))); + result.put(INFO, info.toString()); + + putIfNotNullTS(record.get(Record.REC_MODIFIED), result, "m"); + putIfNotNull(record.get(Record.NAME), result, "n"); + putIfNotNull(record.get(Record.DESCRIPTION), result, "d"); + putIfNotNull(record.get(Record.SHA1), result, "1"); + + // Jar file contents (optional) + putIfNotNullAsStringArray(record.get(Record.CLASSNAMES), result, "classnames"); + + // Maven Plugin (optional) + putIfNotNull(record.get(Record.PLUGIN_PREFIX), result, "px"); + putIfNotNullAsStringArray(record.get(Record.PLUGIN_GOALS), result, "gx"); + + // OSGi (optional) + putIfNotNull(record.get(Record.OSGI_BUNDLE_SYMBOLIC_NAME), result, "Bundle-SymbolicName"); + putIfNotNull(record.get(Record.OSGI_BUNDLE_VERSION), result, "Bundle-Version"); + putIfNotNull(record.get(Record.OSGI_EXPORT_PACKAGE), result, "Export-Package"); + putIfNotNull(record.get(Record.OSGI_EXPORT_SERVICE), result, "Export-Service"); + putIfNotNull(record.get(Record.OSGI_BUNDLE_DESCRIPTION), result, "Bundle-Description"); + putIfNotNull(record.get(Record.OSGI_BUNDLE_NAME), result, "Bundle-Name"); + putIfNotNull(record.get(Record.OSGI_BUNDLE_LICENSE), result, "Bundle-License"); + putIfNotNull(record.get(Record.OSGI_EXPORT_DOCURL), result, "Bundle-DocURL"); + putIfNotNull(record.get(Record.OSGI_IMPORT_PACKAGE), result, "Import-Package"); + putIfNotNull(record.get(Record.OSGI_REQUIRE_BUNDLE), result, "Require-Bundle"); + + return result; + } + + /** + * Creates UINFO synthetic field. + */ + private static String compactUinfo(final Record record) { + final String classifier = record.get(Record.CLASSIFIER); + StringBuilder sb = new StringBuilder(); + sb.append(record.get(Record.GROUP_ID)) + .append(FIELD_SEPARATOR) + .append(record.get(Record.ARTIFACT_ID)) + .append(FIELD_SEPARATOR) + .append(record.get(Record.VERSION)) + .append(FIELD_SEPARATOR) + .append(nvl(classifier)); + if (classifier != null) { + sb.append(record.get(Record.FILE_EXTENSION)); + } + return sb.toString(); + } + + /** + * Helper to put a value from source map into target map, if not null. + */ + private static void putIfNotNull( + final String source, + final Map target, + final String targetName) + { + if (source != null) { + target.put(targetName, source); + } + } + + /** + * Helper to put a {@link Long} value from source map into target map, if not null. + */ + private static void putIfNotNullTS( + final Long source, + final Map target, + final String targetName) + { + if (source != null) { + target.put(targetName, String.valueOf(source)); + } + } + + /** + * Helper to put a array value from source map into target map joined with {@link Utils#FIELD_SEPARATOR}, if not + * null. + */ + private static void putIfNotNullAsStringArray( + final String[] source, + final Map target, + final String targetName) + { + if (source != null && source.length > 0) { + StringBuilder sb = new StringBuilder(); + sb.append(source[0]); + for (int i = 1; i < source.length; i++) { + sb.append(FIELD_SEPARATOR).append(source[i]); + } + target.put(targetName, sb.toString()); + } + } +} diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java new file mode 100644 index 00000000..83f6ca7c --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java @@ -0,0 +1,228 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.HashMap; +import java.util.Map; + +import org.apache.maven.index.reader.Iterables.Function; +import org.apache.maven.index.reader.Record.EntryKey; +import org.apache.maven.index.reader.Record.Type; + +import static org.apache.maven.index.reader.Utils.FIELD_SEPARATOR; +import static org.apache.maven.index.reader.Utils.FS_PATTERN; +import static org.apache.maven.index.reader.Utils.INFO; +import static org.apache.maven.index.reader.Utils.NOT_AVAILABLE; +import static org.apache.maven.index.reader.Utils.UINFO; +import static org.apache.maven.index.reader.Utils.renvl; + +/** + * Maven 2 Index record transformer, that transforms "native" Maven Indexer records into {@link Record}s. + * + * @since 5.1.2 + */ +public class RecordExpander + implements Function, Record> +{ + public Record apply(final Map recordMap) { + if (recordMap.containsKey("DESCRIPTOR")) { + return expandDescriptor(recordMap); + } + else if (recordMap.containsKey("allGroups")) { + return expandAllGroups(recordMap); + } + else if (recordMap.containsKey("rootGroups")) { + return expandRootGroups(recordMap); + } + else if (recordMap.containsKey("del")) { + return expandDeletedArtifact(recordMap); + } + else { + // Fix up UINFO field wrt MINDEXER-41 + final String uinfo = recordMap.get(UINFO); + final String info = recordMap.get(INFO); + if (uinfo != null && !(info == null || info.trim().length() == 0)) { + final String[] splitInfo = FS_PATTERN.split(info); + if (splitInfo.length > 6) { + final String extension = splitInfo[6]; + if (uinfo.endsWith(FIELD_SEPARATOR + NOT_AVAILABLE)) { + recordMap.put(UINFO, uinfo + FIELD_SEPARATOR + extension); + } + } + } + return expandAddedArtifact(recordMap); + } + } + + private static Record expandDescriptor(final Map raw) { + final Record result = new Record(Type.DESCRIPTOR, new HashMap()); + String[] r = FS_PATTERN.split(raw.get("IDXINFO")); + result.put(Record.REPOSITORY_ID, r[1]); + return result; + } + + private static Record expandAllGroups(final Map raw) { + final Record result = new Record(Type.ALL_GROUPS, new HashMap()); + putIfNotNullAsStringArray(raw, "allGroupsList", result, Record.ALL_GROUPS); + return result; + } + + private static Record expandRootGroups(final Map raw) { + final Record result = new Record(Type.ROOT_GROUPS, new HashMap()); + putIfNotNullAsStringArray(raw, "rootGroupsList", result, Record.ROOT_GROUPS); + return result; + } + + private static Record expandDeletedArtifact(final Map raw) { + final Record result = new Record(Type.ARTIFACT_REMOVE, new HashMap()); + putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED); + if (raw.containsKey("del")) { + expandUinfo(raw.get("del"), result); + } + return result; + } + + /** + * Expands the "encoded" Maven Indexer record by splitting the synthetic fields and applying expanded field naming. + */ + private static Record expandAddedArtifact(final Map raw) { + final Record result = new Record(Type.ARTIFACT_ADD, new HashMap()); + + // Minimal + expandUinfo(raw.get(UINFO), result); + final String info = raw.get(INFO); + if (info != null) { + String[] r = FS_PATTERN.split(info); + result.put(Record.PACKAGING, renvl(r[0])); + result.put(Record.FILE_MODIFIED, Long.valueOf(r[1])); + result.put(Record.FILE_SIZE, Long.valueOf(r[2])); + result.put(Record.HAS_SOURCES, "1".equals(r[3]) ? Boolean.TRUE : Boolean.FALSE); + result.put(Record.HAS_JAVADOC, "1".equals(r[4]) ? Boolean.TRUE : Boolean.FALSE); + result.put(Record.HAS_SIGNATURE, "1".equals(r[5]) ? Boolean.TRUE : Boolean.FALSE); + if (r.length > 6) { + result.put(Record.FILE_EXTENSION, r[6]); + } + else { + final String packaging = Record.PACKAGING.coerce(result.get(Record.PACKAGING)); + if (result.containsKey(Record.CLASSIFIER) + || "pom".equals(packaging) + || "war".equals(packaging) + || "ear".equals(packaging)) { + result.put(Record.FILE_EXTENSION, packaging); + } + else { + result.put(Record.FILE_EXTENSION, "jar"); // best guess + } + } + } + putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED); + putIfNotNull(raw, "n", result, Record.NAME); + putIfNotNull(raw, "d", result, Record.DESCRIPTION); + putIfNotNull(raw, "1", result, Record.SHA1); + + // Jar file contents (optional) + putIfNotNullAsStringArray(raw, "classnames", result, Record.CLASSNAMES); + + // Maven Plugin (optional) + putIfNotNull(raw, "px", result, Record.PLUGIN_PREFIX); + putIfNotNullAsStringArray(raw, "gx", result, Record.PLUGIN_GOALS); + + // OSGi (optional) + putIfNotNull(raw, "Bundle-SymbolicName", result, Record.OSGI_BUNDLE_SYMBOLIC_NAME); + putIfNotNull(raw, "Bundle-Version", result, Record.OSGI_BUNDLE_VERSION); + putIfNotNull(raw, "Export-Package", result, Record.OSGI_EXPORT_PACKAGE); + putIfNotNull(raw, "Export-Service", result, Record.OSGI_EXPORT_SERVICE); + putIfNotNull(raw, "Bundle-Description", result, Record.OSGI_BUNDLE_DESCRIPTION); + putIfNotNull(raw, "Bundle-Name", result, Record.OSGI_BUNDLE_NAME); + putIfNotNull(raw, "Bundle-License", result, Record.OSGI_BUNDLE_LICENSE); + putIfNotNull(raw, "Bundle-DocURL", result, Record.OSGI_EXPORT_DOCURL); + putIfNotNull(raw, "Import-Package", result, Record.OSGI_IMPORT_PACKAGE); + putIfNotNull(raw, "Require-Bundle", result, Record.OSGI_REQUIRE_BUNDLE); + + return result; + } + + /** + * Expands UINFO synthetic field. Handles {@code null} String inputs. + */ + private static void expandUinfo(final String uinfo, final Record result) { + if (uinfo != null) { + String[] r = FS_PATTERN.split(uinfo); + result.put(Record.GROUP_ID, r[0]); + result.put(Record.ARTIFACT_ID, r[1]); + result.put(Record.VERSION, r[2]); + String classifier = renvl(r[3]); + if (classifier != null) { + result.put(Record.CLASSIFIER, classifier); + if (r.length > 4) { + result.put(Record.FILE_EXTENSION, r[4]); + } + } + else if (r.length > 4) { + result.put(Record.PACKAGING, r[4]); + } + } + } + + /** + * Helper to put a value from source map into target map, if not null. + */ + private static void putIfNotNull( + final Map source, + final String sourceName, + final Record target, + final EntryKey targetName) + { + String value = source.get(sourceName); + if (value != null && value.trim().length() != 0) { + target.put(targetName, value); + } + } + + /** + * Helper to put a {@link Long} value from source map into target map, if not null. + */ + private static void putIfNotNullTS( + final Map source, + final String sourceName, + final Record target, + final EntryKey targetName) + { + String value = source.get(sourceName); + if (value != null && value.trim().length() != 0) { + target.put(targetName, Long.valueOf(value)); + } + } + + /** + * Helper to put a collection value from source map into target map as {@link java.util.List}, if not null. + */ + private static void putIfNotNullAsStringArray( + final Map source, + final String sourceName, + final Record target, + final EntryKey targetName) + { + String value = source.get(sourceName); + if (value != null && value.trim().length() != 0) { + target.put(targetName, FS_PATTERN.split(value)); + } + } +} diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java index 97800674..fa6b6a73 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java @@ -37,7 +37,7 @@ public interface ResourceHandler { /** * Returns the {@link InputStream} of resource with {@code name} or {@code null} if no such resource. Closing the - * stream is the responsibility of the caller. + * stream is the responsibility of the caller. The stream should be buffered if possible. * * @param name Resource name, guaranteed to be non-{@code null} and is FS name and URL safe string. */ diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java new file mode 100644 index 00000000..798c7a17 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java @@ -0,0 +1,98 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.io.InputStream; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Properties; +import java.util.TimeZone; +import java.util.regex.Pattern; + +/** + * Reusable code snippets and constants. + * + * @since 5.1.2 + */ +public final class Utils +{ + private Utils() { + // nothing + } + + static final String INDEX_FILE_PREFIX = "nexus-maven-repository-index"; + + static final DateFormat INDEX_DATE_FORMAT; + + static { + INDEX_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss.SSS Z"); + INDEX_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT")); + } + + static final String FIELD_SEPARATOR = "|"; + + static final String NOT_AVAILABLE = "NA"; + + static final String UINFO = "u"; + + static final String INFO = "i"; + + static final Pattern FS_PATTERN = Pattern.compile(Pattern.quote(FIELD_SEPARATOR)); + + /** + * Creates and loads {@link Properties} from provided {@link InputStream} and closes the stream. + */ + static Properties loadProperties(final InputStream inputStream) throws IOException { + try { + final Properties properties = new Properties(); + properties.load(inputStream); + return properties; + } + finally { + inputStream.close(); + } + } + + /** + * Helper to translate the "NA" (not available) input into {@code null} value. + */ + static String renvl(final String v) { + return NOT_AVAILABLE.equals(v) ? null : v; + } + + /** + * Helper to translate {@code null} into "NA" (not available) value. + */ + static String nvl(final String v) { + return v == null ? NOT_AVAILABLE : v; + } + + /** + * Returns the "root group" of given groupId. + */ + static String rootGroup(final String groupId) { + int n = groupId.indexOf('.'); + if (n > -1) { + return groupId.substring(0, n); + } + return groupId; + } +} diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java index 7fe38964..7d1ff03e 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java @@ -20,7 +20,7 @@ */ import java.io.IOException; -import java.io.InputStream; +import java.io.OutputStream; /** * Maven 2 Index writable {@link ResourceHandler}, is capable of saving resources too. Needed only if incremental index @@ -34,11 +34,10 @@ public interface WritableResourceHandler extends ResourceHandler { /** - * Stores (creates or overwrites if resource with name exists) the resource under {@code name} with content provided - * by the stream. The {@link InputStream} should be closed when method returns. + * Returns the {@link OutputStream} of resource with {@code name}, never {@code null}. Closing the stream is the + * responsibility of the caller. The stream should be buffered if possible. * - * @param name Resource name, guaranteed to be non-{@code null} and is FS name and URL safe string. - * @param inputStream the content of the resource, guaranteed to be non-{@code null}. + * @param name Resource name, guaranteed to be non-{@code null} and is FS name and URL safe string. */ - void save(final String name, final InputStream inputStream) throws IOException; + OutputStream openWrite(final String name) throws IOException; } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java index 6afe2222..d1f7a64f 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; /** * A trivial caching {@link ResourceHandler} that caches forever during single session (existence of the instance). @@ -49,7 +50,7 @@ public InputStream open(final String name) throws IOException { if (inputStream == null) { return null; } - local.save(name, inputStream); + writeLocal(name, inputStream); return local.open(name); } @@ -57,4 +58,23 @@ public void close() throws IOException { remote.close(); local.close(); } + + private void writeLocal(final String name, final InputStream inputStream) throws IOException { + try { + final OutputStream outputStream = local.openWrite(name); + try { + int read; + byte[] bytes = new byte[8192]; + while ((read = inputStream.read(bytes)) != -1) { + outputStream.write(bytes, 0, read); + } + } + finally { + outputStream.close(); + } + } + finally { + inputStream.close(); + } + } } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java index 0d1915de..097d44d4 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java @@ -19,14 +19,18 @@ * under the License. */ +import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; -import java.util.HashMap; +import java.util.Date; +import java.util.List; import java.util.Map; import org.apache.maven.index.reader.Record.Type; import org.junit.Test; +import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.core.IsEqual.equalTo; import static org.junit.Assert.assertThat; @@ -34,33 +38,56 @@ * UT for {@link ChunkReader} */ public class ChunkReaderTest + extends TestSupport { @Test public void simple() throws IOException { - final Map recordTypes = new HashMap(); - recordTypes.put(Type.DESCRIPTOR, 0); - recordTypes.put(Type.ROOT_GROUPS, 0); - recordTypes.put(Type.ALL_GROUPS, 0); - recordTypes.put(Type.ARTIFACT_ADD, 0); - recordTypes.put(Type.ARTIFACT_REMOVE, 0); + final ChunkReader chunkReader = new ChunkReader( + "full", + testResourceHandler("simple").open("nexus-maven-repository-index.gz") + ); + final Map> recordTypes = countRecordTypes(chunkReader); + assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1)); + assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1)); + assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1)); + assertThat(recordTypes.get(Type.ARTIFACT_ADD).size(), equalTo(2)); + assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), nullValue()); + } - final ChunkReader chunkReader = new ChunkReader("full", - new FileInputStream("src/test/resources/nexus-maven-repository-index.gz")); - try { - assertThat(chunkReader.getVersion(), equalTo(1)); - assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); - for (Record record : chunkReader) { - recordTypes.put(record.getType(), recordTypes.get(record.getType()) + 1); + @Test + public void roundtrip() throws IOException { + final Date published; + File tempChunkFile = createTempFile("nexus-maven-repository-index.gz"); + { + final ChunkReader chunkReader = new ChunkReader( + "full", + testResourceHandler("simple").open("nexus-maven-repository-index.gz") + ); + final ChunkWriter chunkWriter = new ChunkWriter( + chunkReader.getName(), + new FileOutputStream(tempChunkFile), 1, new Date() + ); + try { + chunkWriter.writeChunk(chunkReader.iterator()); } - } - finally { - chunkReader.close(); + finally { + chunkWriter.close(); + chunkReader.close(); + } + published = chunkWriter.getTimestamp(); } - assertThat(recordTypes.get(Type.DESCRIPTOR), equalTo(1)); - assertThat(recordTypes.get(Type.ROOT_GROUPS), equalTo(1)); - assertThat(recordTypes.get(Type.ALL_GROUPS), equalTo(1)); - assertThat(recordTypes.get(Type.ARTIFACT_ADD), equalTo(2)); - assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), equalTo(0)); + final ChunkReader chunkReader = new ChunkReader( + "full", + new FileInputStream(tempChunkFile) + ); + assertThat(chunkReader.getVersion(), equalTo(1)); + assertThat(chunkReader.getTimestamp().getTime(), equalTo(published.getTime())); + final Map> recordTypes = countRecordTypes(chunkReader); + assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1)); + assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1)); + assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1)); + assertThat(recordTypes.get(Type.ARTIFACT_ADD).size(), equalTo(2)); + assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), nullValue()); } } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java index edb614af..fb33285e 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java @@ -23,9 +23,11 @@ import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; /** * A trivial {@link File} directory handler that does not perform any locking or extra bits, and just serves up files @@ -46,26 +48,24 @@ public DirectoryResourceHandler(final File rootDirectory) { this.rootDirectory = rootDirectory; } - public InputStream open(final String name) throws IOException { - return new BufferedInputStream(new FileInputStream(new File(rootDirectory, name))); + public File getRootDirectory() { + return rootDirectory; } - public void save(final String name, final InputStream inputStream) throws IOException { + public InputStream open(final String name) throws IOException { try { - final BufferedOutputStream outputStream = new BufferedOutputStream( - new FileOutputStream(new File(rootDirectory, name))); - int read; - byte[] bytes = new byte[8192]; - while ((read = inputStream.read(bytes)) != -1) { - outputStream.write(bytes, 0, read); - } - outputStream.close(); + return new BufferedInputStream(new FileInputStream(new File(rootDirectory, name))); } - finally { - inputStream.close(); + catch (FileNotFoundException e) { + return null; } } + public OutputStream openWrite(final String name) throws IOException { + return new BufferedOutputStream( + new FileOutputStream(new File(rootDirectory, name))); + } + public void close() throws IOException { // nop } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java index e5d7df1e..153391fe 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java @@ -25,6 +25,7 @@ import java.io.Writer; import java.net.URL; import java.util.Arrays; +import java.util.Date; import org.junit.Ignore; import org.junit.Test; @@ -36,12 +37,14 @@ * UT for {@link IndexReader} */ public class IndexReaderTest + extends TestSupport { @Test public void simple() throws IOException { final IndexReader indexReader = new IndexReader( null, - new DirectoryResourceHandler(new File("src/test/resources/"))); + testResourceHandler("simple") + ); try { assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local")); assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(1243533418015L)); @@ -54,7 +57,62 @@ public void simple() throws IOException { assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); assertThat(chunkReader.getVersion(), equalTo(1)); assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); - for (Record record : chunkReader) { + for (Record record : Iterables.transform(chunkReader, new RecordExpander())) { + records++; + } + } + + assertThat(chunks, equalTo(1)); + assertThat(records, equalTo(5)); + } + finally { + indexReader.close(); + } + } + + @Test + public void roundtrip() throws IOException { + WritableResourceHandler writableResourceHandler = createWritableResourceHandler(); + Date published; + { + final IndexReader indexReader = new IndexReader( + null, + testResourceHandler("simple") + ); + final IndexWriter indexWriter = new IndexWriter( + writableResourceHandler, + indexReader.getIndexId(), + false + ); + try { + for (ChunkReader chunkReader : indexReader) { + indexWriter.writeChunk(chunkReader.iterator()); + } + } + finally { + indexWriter.close(); + published = indexWriter.getPublishedTimestamp(); + indexReader.close(); + } + } + + final IndexReader indexReader = new IndexReader( + null, + writableResourceHandler + ); + try { + assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local")); + assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(published.getTime())); + assertThat(indexReader.isIncremental(), equalTo(false)); + assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz"))); + int chunks = 0; + int records = 0; + for (ChunkReader chunkReader : indexReader) { + chunks++; + assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); + assertThat(chunkReader.getVersion(), equalTo(1)); + // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); + for (Record record : Iterables.transform(chunkReader, new RecordExpander())) { records++; } } @@ -87,10 +145,8 @@ public void central() throws Exception { writer.write("chunkVersion=" + chunkReader.getVersion() + "\n"); writer.write("chunkPublished=" + chunkReader.getTimestamp() + "\n"); writer.write("= = = = = = \n"); - for (Record record : chunkReader) { + for (Record record : Iterables.transform(chunkReader, new RecordExpander())) { writer.write(record.getExpanded() + "\n"); - writer.write("--------- \n"); - writer.write(record.getRaw() + "\n"); } } } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java new file mode 100644 index 00000000..0895c9d2 --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java @@ -0,0 +1,91 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.junit.Test; + +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; + +/** + * UT for {@link IndexWriter} + */ +public class IndexWriterTest + extends TestSupport +{ + @Test + public void roundtrip() throws IOException { + IndexReader indexReader; + IndexWriter indexWriter; + WritableResourceHandler writableResourceHandler = createWritableResourceHandler(); + + // write it once + indexReader = new IndexReader( + null, + testResourceHandler("simple") + ); + indexWriter = new IndexWriter( + writableResourceHandler, + indexReader.getIndexId(), + false + ); + try { + for (ChunkReader chunkReader : indexReader) { + indexWriter.writeChunk(chunkReader.iterator()); + } + } + finally { + indexWriter.close(); + indexReader.close(); + } + + // read what we wrote out + indexReader = new IndexReader( + null, + writableResourceHandler + ); + try { + assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local")); + // assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(published.getTime())); + assertThat(indexReader.isIncremental(), equalTo(false)); + assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz"))); + int chunks = 0; + int records = 0; + for (ChunkReader chunkReader : indexReader) { + chunks++; + assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); + assertThat(chunkReader.getVersion(), equalTo(1)); + // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); + for (Record record : Iterables.transform(chunkReader, new RecordExpander())) { + records++; + } + } + + assertThat(chunks, equalTo(1)); + assertThat(records, equalTo(5)); + } + finally { + indexReader.close(); + } + } +} diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java new file mode 100644 index 00000000..c0b1142f --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java @@ -0,0 +1,92 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.maven.index.reader.Record.EntryKey; +import org.apache.maven.index.reader.Record.Type; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; + +/** + * UT for {@link Iterables} + */ +public class IterablesTest + extends TestSupport +{ + @Test + public void decorateAndTransform() throws IOException { + final String indexId = "test"; + final Record r1 = new Record(Type.ARTIFACT_ADD, artifactMap("org.apache")); + final Record r2 = new Record(Type.ARTIFACT_ADD, artifactMap("org.foo")); + final Record r3 = new Record(Type.ARTIFACT_ADD, artifactMap("com.bar")); + Iterable> iterable = Iterables.decorateAndTransform(Arrays.asList(r1, r2, r3), indexId); + + WritableResourceHandler writableResourceHandler = createWritableResourceHandler(); + try { + IndexWriter indexWriter = new IndexWriter( + writableResourceHandler, + indexId, + false + ); + indexWriter.writeChunk(iterable.iterator()); + indexWriter.close(); + } + finally { + writableResourceHandler.close(); + } + + IndexReader indexReader = new IndexReader(null, writableResourceHandler); + assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz"))); + ChunkReader chunkReader = indexReader.iterator().next(); + final Map> recordTypes = countRecordTypes(chunkReader); + assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1)); + assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1)); + assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1)); + assertThat(recordTypes.get(Type.ARTIFACT_ADD).size(), equalTo(3)); + assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), nullValue()); + + assertThat(recordTypes.get(Type.ROOT_GROUPS).get(0).get(Record.ROOT_GROUPS), equalTo(new String[] {"com","org"})); + assertThat(recordTypes.get(Type.ALL_GROUPS).get(0).get(Record.ALL_GROUPS), equalTo(new String[] {"com.bar", "org.apache", "org.foo"})); + } + + private Map artifactMap(final String groupId) { + final HashMap result = new HashMap(); + result.put(Record.GROUP_ID, groupId); + result.put(Record.ARTIFACT_ID, "artifact"); + result.put(Record.VERSION, "1.0"); + result.put(Record.PACKAGING, "jar"); + result.put(Record.FILE_MODIFIED, System.currentTimeMillis()); + result.put(Record.FILE_SIZE, 123L); + result.put(Record.FILE_EXTENSION, "jar"); + result.put(Record.HAS_SOURCES, Boolean.FALSE); + result.put(Record.HAS_JAVADOC, Boolean.FALSE); + result.put(Record.HAS_SIGNATURE, Boolean.FALSE); + return result; + } +} diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java new file mode 100644 index 00000000..bcd465e0 --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java @@ -0,0 +1,163 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.maven.index.reader.Record.Type; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TestName; + +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; + +/** + * Test support. + */ +public class TestSupport +{ + @Rule + public TestName testName = new TestName(); + + private File tempDir; + + private List directoryResourceHandlers; + + /** + * Creates the temp directory and list for resource handlers. + */ + @Before + public void setup() throws IOException { + this.tempDir = new File("target/tmp-" + getClass().getSimpleName()); + this.tempDir.delete(); + this.tempDir.mkdirs(); + this.directoryResourceHandlers = new ArrayList(); + } + + /** + * Closes all the registered resources handlers and deletes the temp directory. + */ + @After + public void cleanup() throws IOException { + for (DirectoryResourceHandler directoryResourceHandler : directoryResourceHandlers) { + directoryResourceHandler.close(); + } + // delete(tempDir); + } + + /** + * Creates a temp file within {@link #tempDir}. + */ + protected File createTempFile() throws IOException { + return File.createTempFile(testName.getMethodName() + "-file", "", tempDir); + } + + + /** + * Creates a temp file within {@link #tempDir} with given name. + */ + protected File createTempFile(final String name) throws IOException { + return new File(tempDir, name); + } + + /** + * Creates a temp directory within {@link #tempDir}. + */ + protected File createTempDirectory() throws IOException { + File result = File.createTempFile(testName.getMethodName() + "-dir", "", tempDir); + result.delete(); + result.mkdirs(); + return result; + } + + /** + * Creates an empty {@link DirectoryResourceHandler}. + */ + protected WritableResourceHandler createWritableResourceHandler() throws IOException { + DirectoryResourceHandler result = new DirectoryResourceHandler(createTempDirectory()); + directoryResourceHandlers.add(result); + return result; + } + + /** + * Creates a "test" {@link ResourceHandler} that contains predefined files, is mapped to test resources under given + * name. + */ + protected ResourceHandler testResourceHandler(final String name) throws IOException { + DirectoryResourceHandler result = new DirectoryResourceHandler(new File("src/test/resources/" + name)); + directoryResourceHandlers.add(result); + return result; + } + + /** + * Consumes {@link ChunkReader} and creates "statistics" of index by record type. + */ + protected Map> countRecordTypes(final ChunkReader chunkReader) throws IOException { + HashMap> stat = new HashMap>(); + try { + assertThat(chunkReader.getVersion(), equalTo(1)); + final RecordExpander recordExpander = new RecordExpander(); + for (Map rec : chunkReader) { + System.out.println(rec); + final Record record = recordExpander.apply(rec); + System.out.println(record); + if (!stat.containsKey(record.getType())) { + stat.put(record.getType(), new ArrayList()); + } + stat.get(record.getType()).add(record); + } + } + finally { + chunkReader.close(); + } + return stat; + } + + /** + * Delete recursively. + */ + private static boolean delete(final File file) { + if (file == null) { + return false; + } + if (!file.exists()) { + return true; + } + if (file.isDirectory()) { + String[] list = file.list(); + if (list != null) { + for (int i = 0; i < list.length; i++) { + File entry = new File(file, list[i]); + if (!delete(entry)) { + return false; + } + } + } + } + return file.delete(); + } +} diff --git a/indexer-reader/src/test/resources/nexus-maven-repository-index.gz b/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.gz similarity index 100% rename from indexer-reader/src/test/resources/nexus-maven-repository-index.gz rename to indexer-reader/src/test/resources/simple/nexus-maven-repository-index.gz diff --git a/indexer-reader/src/test/resources/nexus-maven-repository-index.properties b/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.properties similarity index 100% rename from indexer-reader/src/test/resources/nexus-maven-repository-index.properties rename to indexer-reader/src/test/resources/simple/nexus-maven-repository-index.properties diff --git a/pom.xml b/pom.xml index ddc058e8..8a16cc21 100644 --- a/pom.xml +++ b/pom.xml @@ -341,7 +341,7 @@ under the License. junit junit - 4.11 + 4.12 test From 26cbb0df8d3f358a7456086e9a18ab774725461b Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Tue, 3 Nov 2015 08:23:32 -0500 Subject: [PATCH 06/16] MINDEXER-96: Indexer reader and writer (cherry picked from commit 5c75bba) --- .../maven/index/reader/IndexReader.java | 57 +++++----- .../maven/index/reader/IndexWriter.java | 40 +++---- .../maven/index/reader/ResourceHandler.java | 17 ++- .../org/apache/maven/index/reader/Utils.java | 46 ++++++++ .../index/reader/WritableResourceHandler.java | 19 +++- .../index/reader/CachingResourceHandler.java | 101 +++++++++++++----- .../maven/index/reader/ChunkReaderTest.java | 33 +++--- .../reader/DirectoryResourceHandler.java | 39 ++++--- .../index/reader/HttpResourceHandler.java | 30 ++++-- .../maven/index/reader/IndexReaderTest.java | 36 +++++++ 10 files changed, 309 insertions(+), 109 deletions(-) diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java index d74c366a..878ec6b9 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java @@ -21,7 +21,6 @@ import java.io.Closeable; import java.io.IOException; -import java.io.OutputStream; import java.text.ParseException; import java.util.ArrayList; import java.util.Collections; @@ -30,7 +29,10 @@ import java.util.List; import java.util.Properties; +import org.apache.maven.index.reader.ResourceHandler.Resource; + import static org.apache.maven.index.reader.Utils.loadProperties; +import static org.apache.maven.index.reader.Utils.storeProperties; /** * Maven 2 Index reader that handles incremental updates if possible and provides one or more {@link ChunkReader}s, to @@ -63,31 +65,38 @@ public IndexReader(final WritableResourceHandler local, final ResourceHandler re } this.local = local; this.remote = remote; - remoteIndexProperties = loadProperties(remote.open(Utils.INDEX_FILE_PREFIX + ".properties")); + remoteIndexProperties = loadProperties(remote.locate(Utils.INDEX_FILE_PREFIX + ".properties")); + if (remoteIndexProperties == null) { + throw new NullPointerException("remote index properties null"); + } try { if (local != null) { - localIndexProperties = loadProperties(local.open(Utils.INDEX_FILE_PREFIX + ".properties")); - String remoteIndexId = remoteIndexProperties.getProperty("nexus.index.id"); - String localIndexId = localIndexProperties.getProperty("nexus.index.id"); - if (remoteIndexId == null || localIndexId == null || !remoteIndexId.equals(localIndexId)) { - throw new IllegalArgumentException( - "local and remote index IDs does not match or is null: " + localIndexId + ", " + - remoteIndexId); + Properties localProperties = loadProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties")); + if (localProperties != null) { + this.localIndexProperties = localProperties; + String remoteIndexId = remoteIndexProperties.getProperty("nexus.index.id"); + String localIndexId = localIndexProperties.getProperty("nexus.index.id"); + if (remoteIndexId == null || localIndexId == null || !remoteIndexId.equals(localIndexId)) { + throw new IllegalArgumentException( + "local and remote index IDs does not match or is null: " + localIndexId + ", " + + remoteIndexId); + } + this.indexId = localIndexId; + this.incremental = canRetrieveAllChunks(); + } + else { + localIndexProperties = null; + this.indexId = remoteIndexProperties.getProperty("nexus.index.id"); + this.incremental = false; } - this.indexId = localIndexId; - this.publishedTimestamp = Utils.INDEX_DATE_FORMAT - .parse(localIndexProperties.getProperty("nexus.index.timestamp")); - this.incremental = canRetrieveAllChunks(); - this.chunkNames = calculateChunkNames(); } else { localIndexProperties = null; this.indexId = remoteIndexProperties.getProperty("nexus.index.id"); - this.publishedTimestamp = Utils.INDEX_DATE_FORMAT - .parse(remoteIndexProperties.getProperty("nexus.index.timestamp")); this.incremental = false; - this.chunkNames = calculateChunkNames(); } + this.publishedTimestamp = Utils.INDEX_DATE_FORMAT.parse(remoteIndexProperties.getProperty("nexus.index.timestamp")); + this.chunkNames = calculateChunkNames(); } catch (ParseException e) { IOException ex = new IOException("Index properties corrupted"); @@ -161,13 +170,7 @@ public Iterator iterator() { * for future incremental updates. */ private void syncLocalWithRemote() throws IOException { - final OutputStream outputStream = local.openWrite(Utils.INDEX_FILE_PREFIX + ".properties"); - try { - remoteIndexProperties.store(outputStream, "Maven Indexer Reader"); - } - finally { - outputStream.close(); - } + storeProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties"), remoteIndexProperties); } /** @@ -234,6 +237,8 @@ private static class ChunkReaderIterator private final Iterator chunkNamesIterator; + private Resource currentResource; + private ChunkReader currentChunkReader; private ChunkReaderIterator(final ResourceHandler resourceHandler, final Iterator chunkNamesIterator) { @@ -250,8 +255,10 @@ public ChunkReader next() { try { if (currentChunkReader != null) { currentChunkReader.close(); + currentResource.close(); } - currentChunkReader = new ChunkReader(chunkName, resourceHandler.open(chunkName)); + currentResource = resourceHandler.locate(chunkName); + currentChunkReader = new ChunkReader(chunkName, currentResource.read()); return currentChunkReader; } catch (IOException e) { diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java index f52cb298..68d4abbf 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java @@ -21,8 +21,6 @@ import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.text.ParseException; import java.util.Date; import java.util.Iterator; @@ -30,7 +28,10 @@ import java.util.Properties; import java.util.UUID; +import org.apache.maven.index.reader.WritableResourceHandler.WritableResource; + import static org.apache.maven.index.reader.Utils.loadProperties; +import static org.apache.maven.index.reader.Utils.storeProperties; /** * Maven 2 Index writer that writes chunk and maintains published property file. @@ -65,10 +66,10 @@ public IndexWriter(final WritableResourceHandler local, final String indexId, fi throw new NullPointerException("indexId null"); } this.local = local; - InputStream localIndexPropertiesInputStream = local.open(Utils.INDEX_FILE_PREFIX + ".properties"); - if (incrementalSupported && localIndexPropertiesInputStream != null) { + Properties indexProperties = loadProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties")); + if (incrementalSupported && indexProperties != null) { + this.localIndexProperties = indexProperties; // existing index, this is incremental publish, and we will add new chunk - this.localIndexProperties = loadProperties(local.open(Utils.INDEX_FILE_PREFIX + ".properties")); String localIndexId = localIndexProperties.getProperty("nexus.index.id"); if (localIndexId == null || !localIndexId.equals(indexId)) { throw new IllegalArgumentException( @@ -142,17 +143,23 @@ public String getNextChunkName() { */ public int writeChunk(final Iterator> iterator) throws IOException { int written; - final ChunkWriter chunkWriter = new ChunkWriter(nextChunkName, local.openWrite(nextChunkName), INDEX_V1, new Date()); + WritableResource writableResource = local.locate(nextChunkName); try { - written = chunkWriter.writeChunk(iterator); + final ChunkWriter chunkWriter = new ChunkWriter(nextChunkName, writableResource.write(), INDEX_V1, new Date()); + try { + written = chunkWriter.writeChunk(iterator); + } + finally { + chunkWriter.close(); + } + if (incremental) { + // TODO: update main gz file + } + return written; } finally { - chunkWriter.close(); + writableResource.close(); } - if (incremental) { - // TODO: update main gz file - } - return written; } /** @@ -167,14 +174,7 @@ public void close() throws IOException { localIndexProperties.setProperty("nexus.index.last-incremental", nextChunkCounter); } localIndexProperties.setProperty("nexus.index.timestamp", Utils.INDEX_DATE_FORMAT.format(new Date())); - - final OutputStream outputStream = local.openWrite(Utils.INDEX_FILE_PREFIX + ".properties"); - try { - localIndexProperties.store(outputStream, "Maven Indexer Writer"); - } - finally { - outputStream.close(); - } + storeProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties"), localIndexProperties); } finally { local.close(); diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java index fa6b6a73..f8deb382 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java @@ -35,11 +35,20 @@ public interface ResourceHandler extends Closeable { + interface Resource + extends Closeable + { + /** + * Returns the {@link InputStream} stream of the resource, if exists, {@code null} otherwise. The stream should + * be closed by caller, otherwise resource leaks might be introduced. + */ + InputStream read() throws IOException; + } + /** - * Returns the {@link InputStream} of resource with {@code name} or {@code null} if no such resource. Closing the - * stream is the responsibility of the caller. The stream should be buffered if possible. + * Returns the {@link Resource} of resource with {@code name}. Returned locator should be handled as resource. * - * @param name Resource name, guaranteed to be non-{@code null} and is FS name and URL safe string. + * @param name Resource name, guaranteed to be non-{@code null} and is FS and URL safe string. */ - InputStream open(String name) throws IOException; + Resource locate(String name) throws IOException; } diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java index 798c7a17..2a971d43 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java @@ -21,12 +21,16 @@ import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Properties; import java.util.TimeZone; import java.util.regex.Pattern; +import org.apache.maven.index.reader.ResourceHandler.Resource; +import org.apache.maven.index.reader.WritableResourceHandler.WritableResource; + /** * Reusable code snippets and constants. * @@ -71,6 +75,48 @@ static Properties loadProperties(final InputStream inputStream) throws IOExcepti } } + /** + * Creates and loads {@link Properties} from provided {@link Resource} if exists, and closes the resource. If not + * exists, returns {@code null}. + */ + static Properties loadProperties(final Resource resource) throws IOException { + try { + final InputStream inputStream = resource.read(); + if (inputStream == null) { + return null; + } + return loadProperties(resource.read()); + } + finally { + resource.close(); + } + } + + /** + * Saves {@link Properties} to provided {@link OutputStream} and closes the stream. + */ + static void storeProperties(final OutputStream outputStream, final Properties properties) throws IOException { + try { + properties.store(outputStream, "Maven Indexer Writer"); + } + finally { + outputStream.close(); + } + } + + + /** + * Saves {@link Properties} to provided {@link WritableResource} and closes the resource. + */ + static void storeProperties(final WritableResource writableResource, final Properties properties) throws IOException { + try { + storeProperties(writableResource.write(), properties); + } + finally { + writableResource.close(); + } + } + /** * Helper to translate the "NA" (not available) input into {@code null} value. */ diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java index 7d1ff03e..9fe11270 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java @@ -33,11 +33,22 @@ public interface WritableResourceHandler extends ResourceHandler { + interface WritableResource + extends Resource + { + /** + * Returns the {@link OutputStream} stream of the resource, if exists, it will replace the existing content, or if + * not exists, the resource will be created. The stream should be closed by caller, otherwise resource leaks might + * be introduced. + */ + OutputStream write() throws IOException; + } + /** - * Returns the {@link OutputStream} of resource with {@code name}, never {@code null}. Closing the stream is the - * responsibility of the caller. The stream should be buffered if possible. + * Returns the {@link WritableResource} of resource with {@code name}. Returned locator should be handled as + * resource. * - * @param name Resource name, guaranteed to be non-{@code null} and is FS name and URL safe string. + * @param name Resource name, guaranteed to be non-{@code null} and is FS and URL safe string. */ - OutputStream openWrite(final String name) throws IOException; + WritableResource locate(String name) throws IOException; } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java index d1f7a64f..b6cf436e 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java @@ -22,6 +22,10 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.HashSet; +import java.util.Set; + +import org.apache.maven.index.reader.WritableResourceHandler.WritableResource; /** * A trivial caching {@link ResourceHandler} that caches forever during single session (existence of the instance). @@ -29,52 +33,99 @@ public class CachingResourceHandler implements ResourceHandler { - private final DirectoryResourceHandler local; + private static final Resource NOT_EXISTING_RESOURCE = new Resource() + { + public InputStream read() throws IOException { + return null; + } + + public void close() throws IOException { + // nop + } + }; + + private final WritableResourceHandler local; private final ResourceHandler remote; - public CachingResourceHandler(final DirectoryResourceHandler local, final ResourceHandler remote) { + private final Set notFoundResources; + + public CachingResourceHandler(final WritableResourceHandler local, final ResourceHandler remote) { if (local == null || remote == null) { throw new NullPointerException("null resource handler"); } this.local = local; this.remote = remote; + this.notFoundResources = new HashSet(); } - public InputStream open(final String name) throws IOException { - InputStream inputStream = local.open(name); - if (inputStream != null) { - return inputStream; + public Resource locate(final String name) throws IOException { + if (notFoundResources.contains(name)) { + return NOT_EXISTING_RESOURCE; } - inputStream = remote.open(name); - if (inputStream == null) { - return null; + else { + return new CachingResource(name, local.locate(name)); } - writeLocal(name, inputStream); - return local.open(name); } - public void close() throws IOException { - remote.close(); - local.close(); - } + private class CachingResource + implements Resource + { + private final String name; + + private final WritableResource localResource; - private void writeLocal(final String name, final InputStream inputStream) throws IOException { - try { - final OutputStream outputStream = local.openWrite(name); + private CachingResource(final String name, final WritableResource localResource) { + this.name = name; + this.localResource = localResource; + } + + public InputStream read() throws IOException { + InputStream inputStream = localResource.read(); + if (inputStream != null) { + return inputStream; + } + if (cacheLocally(name)) { + return localResource.read(); + } + notFoundResources.add(name); + return null; + } + + private boolean cacheLocally(final String name) throws IOException { + final Resource remoteResource = remote.locate(name); try { - int read; - byte[] bytes = new byte[8192]; - while ((read = inputStream.read(bytes)) != -1) { - outputStream.write(bytes, 0, read); + final InputStream inputStream = remoteResource.read(); + if (inputStream != null) { + final OutputStream outputStream = localResource.write(); + try { + int read; + byte[] bytes = new byte[8192]; + while ((read = inputStream.read(bytes)) != -1) { + outputStream.write(bytes, 0, read); + } + outputStream.flush(); + return true; + } + finally { + outputStream.close(); + inputStream.close(); + } } + return false; } finally { - outputStream.close(); + remoteResource.close(); } } - finally { - inputStream.close(); + + public void close() throws IOException { + // nop } } + + public void close() throws IOException { + remote.close(); + local.close(); + } } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java index 097d44d4..904ee446 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java @@ -28,6 +28,7 @@ import java.util.Map; import org.apache.maven.index.reader.Record.Type; +import org.apache.maven.index.reader.ResourceHandler.Resource; import org.junit.Test; import static org.hamcrest.CoreMatchers.nullValue; @@ -44,7 +45,7 @@ public class ChunkReaderTest public void simple() throws IOException { final ChunkReader chunkReader = new ChunkReader( "full", - testResourceHandler("simple").open("nexus-maven-repository-index.gz") + testResourceHandler("simple").locate("nexus-maven-repository-index.gz").read() ); final Map> recordTypes = countRecordTypes(chunkReader); assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1)); @@ -59,22 +60,28 @@ public void roundtrip() throws IOException { final Date published; File tempChunkFile = createTempFile("nexus-maven-repository-index.gz"); { - final ChunkReader chunkReader = new ChunkReader( - "full", - testResourceHandler("simple").open("nexus-maven-repository-index.gz") - ); - final ChunkWriter chunkWriter = new ChunkWriter( - chunkReader.getName(), - new FileOutputStream(tempChunkFile), 1, new Date() - ); + final Resource resource = testResourceHandler("simple").locate("nexus-maven-repository-index.gz"); try { - chunkWriter.writeChunk(chunkReader.iterator()); + final ChunkReader chunkReader = new ChunkReader( + "full", + resource.read() + ); + final ChunkWriter chunkWriter = new ChunkWriter( + chunkReader.getName(), + new FileOutputStream(tempChunkFile), 1, new Date() + ); + try { + chunkWriter.writeChunk(chunkReader.iterator()); + } + finally { + chunkWriter.close(); + chunkReader.close(); + } + published = chunkWriter.getTimestamp(); } finally { - chunkWriter.close(); - chunkReader.close(); + resource.close(); } - published = chunkWriter.getTimestamp(); } final ChunkReader chunkReader = new ChunkReader( diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java index fb33285e..aea0ee11 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java @@ -23,7 +23,6 @@ import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -52,21 +51,37 @@ public File getRootDirectory() { return rootDirectory; } - public InputStream open(final String name) throws IOException { - try { - return new BufferedInputStream(new FileInputStream(new File(rootDirectory, name))); + public WritableResource locate(final String name) throws IOException { + return new FileResource(new File(rootDirectory, name)); + } + + public void close() throws IOException { + // nop + } + + private class FileResource + implements WritableResource + { + private final File file; + + private FileResource(final File file) { + this.file = file; } - catch (FileNotFoundException e) { + + public InputStream read() throws IOException { + if (file.isFile()) { + return new BufferedInputStream(new FileInputStream(file)); + } return null; } - } - public OutputStream openWrite(final String name) throws IOException { - return new BufferedOutputStream( - new FileOutputStream(new File(rootDirectory, name))); - } + public OutputStream write() throws IOException { + return new BufferedOutputStream(new FileOutputStream(file)); + } - public void close() throws IOException { - // nop + public void close() throws IOException { + // nop + } } + } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java index 933cfbd8..e9ea16d1 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java @@ -43,12 +43,30 @@ public HttpResourceHandler(final URL root) throws URISyntaxException { this.root = root.toURI(); } - public InputStream open(final String name) throws IOException { - URL target = root.resolve(name).toURL(); - HttpURLConnection conn = (HttpURLConnection) target.openConnection(); - conn.setRequestMethod("GET"); - conn.setRequestProperty("User-Agent", "ASF Maven-Indexer-Reader/1.0"); - return new BufferedInputStream(conn.getInputStream()); + public Resource locate(final String name) throws IOException { + return new HttpResource(name); + } + + private class HttpResource + implements Resource + { + private final String name; + + private HttpResource(final String name) { + this.name = name; + } + + public InputStream read() throws IOException { + URL target = root.resolve(name).toURL(); + HttpURLConnection conn = (HttpURLConnection) target.openConnection(); + conn.setRequestMethod("GET"); + conn.setRequestProperty("User-Agent", "ASF Maven-Indexer-Reader/1.0"); + return new BufferedInputStream(conn.getInputStream()); + } + + public void close() throws IOException { + // nop + } } public void close() throws IOException { diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java index 153391fe..fc086108 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java @@ -31,6 +31,8 @@ import org.junit.Test; import static org.hamcrest.core.IsEqual.equalTo; +import static org.hamcrest.core.IsNot.not; +import static org.hamcrest.core.IsNull.nullValue; import static org.junit.Assert.assertThat; /** @@ -70,6 +72,40 @@ public void simple() throws IOException { } } + @Test + public void simpleWithLocal() throws IOException { + WritableResourceHandler writableResourceHandler = createWritableResourceHandler(); + final IndexReader indexReader = new IndexReader( + writableResourceHandler, + testResourceHandler("simple") + ); + try { + assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local")); + assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(1243533418015L)); + assertThat(indexReader.isIncremental(), equalTo(false)); + assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz"))); + int chunks = 0; + int records = 0; + for (ChunkReader chunkReader : indexReader) { + chunks++; + assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); + assertThat(chunkReader.getVersion(), equalTo(1)); + assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); + for (Record record : Iterables.transform(chunkReader, new RecordExpander())) { + records++; + } + } + + assertThat(chunks, equalTo(1)); + assertThat(records, equalTo(5)); + } + finally { + indexReader.close(); + } + + assertThat(writableResourceHandler.locate("nexus-maven-repository-index.properties").read(), not(nullValue())); + } + @Test public void roundtrip() throws IOException { WritableResourceHandler writableResourceHandler = createWritableResourceHandler(); From db9ab5080495058f50ccb2dcdb4d197feddc8466 Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Tue, 3 Nov 2015 10:38:26 -0500 Subject: [PATCH 07/16] INDEXER-96: Fix iterators (cherry picked from commit be227e2) --- .../java/org/apache/maven/index/reader/ChunkReader.java | 4 ++++ .../java/org/apache/maven/index/reader/IndexReader.java | 4 ++++ .../java/org/apache/maven/index/reader/Iterables.java | 8 ++++++++ 3 files changed, 16 insertions(+) diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java index dc6ee794..a6c75ac1 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java @@ -125,6 +125,10 @@ public Map next() { return result; } + public void remove() { + throw new UnsupportedOperationException("remove"); + } + private Map nextRecord() { try { return readRecord(dataInputStream); diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java index 878ec6b9..cb047b73 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java @@ -265,5 +265,9 @@ public ChunkReader next() { throw new RuntimeException("IO problem while switching chunk readers", e); } } + + public void remove() { + throw new UnsupportedOperationException("remove"); + } } } diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java index 78941dcd..6e65e8f8 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java @@ -153,6 +153,10 @@ public boolean hasNext() { public O next() { return function.apply(iterator.next()); } + + public void remove() { + throw new UnsupportedOperationException("remove"); + } } private static final class ConcatIterator @@ -182,6 +186,10 @@ public T next() { return result; } + public void remove() { + throw new UnsupportedOperationException("remove"); + } + protected T getNextElement() { if ((current == null || !current.hasNext()) && iterators.hasNext()) { current = iterators.next(); From 6c905500423a324750f8c5256e6bd69b0083c640 Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Thu, 5 Nov 2015 04:53:26 -0500 Subject: [PATCH 08/16] Cleanup (cherry picked from commit c3431b6) --- .../maven/index/reader/IndexReader.java | 1 - .../maven/index/reader/ResourceHandler.java | 9 ++-- .../org/apache/maven/index/reader/Utils.java | 13 ++--- .../index/reader/WritableResourceHandler.java | 14 +++-- .../index/reader/CachingResourceHandler.java | 14 +++-- .../maven/index/reader/ChunkReaderTest.java | 33 +++++------- .../maven/index/reader/IndexReaderTest.java | 52 ++++++++++++------- .../maven/index/reader/IterablesTest.java | 2 +- .../maven/index/reader/TestSupport.java | 29 +++++++++-- 9 files changed, 95 insertions(+), 72 deletions(-) diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java index cb047b73..7de1f4f3 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java @@ -255,7 +255,6 @@ public ChunkReader next() { try { if (currentChunkReader != null) { currentChunkReader.close(); - currentResource.close(); } currentResource = resourceHandler.locate(chunkName); currentChunkReader = new ChunkReader(chunkName, currentResource.read()); diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java index f8deb382..244a994c 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java @@ -25,10 +25,8 @@ /** * Maven 2 Index resource abstraction, that should be handled as a resource (is {@link Closeable}. That means, that - * implementations could perform any extra activity as FS locking or so (if uses FS as backing store). If the - * implementation plans to fetch from remote, it could implement very simple "cache" mechanism, to fetch only once - * during the lifespan of the instance, as for indexer there is no reason to re-fetch during single session, nor - * yo have any advanced caching (ie. TTLs etc). + * implementations could perform any extra activity as FS locking or so (if uses FS as backing store). Is used by single + * thread only. * * @since 5.1.2 */ @@ -36,7 +34,6 @@ public interface ResourceHandler extends Closeable { interface Resource - extends Closeable { /** * Returns the {@link InputStream} stream of the resource, if exists, {@code null} otherwise. The stream should @@ -46,7 +43,7 @@ interface Resource } /** - * Returns the {@link Resource} of resource with {@code name}. Returned locator should be handled as resource. + * Returns the {@link Resource} with {@code name}, non {@code null}. * * @param name Resource name, guaranteed to be non-{@code null} and is FS and URL safe string. */ diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java index 2a971d43..c978bd67 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java @@ -80,16 +80,11 @@ static Properties loadProperties(final InputStream inputStream) throws IOExcepti * exists, returns {@code null}. */ static Properties loadProperties(final Resource resource) throws IOException { - try { - final InputStream inputStream = resource.read(); - if (inputStream == null) { - return null; - } - return loadProperties(resource.read()); - } - finally { - resource.close(); + final InputStream inputStream = resource.read(); + if (inputStream == null) { + return null; } + return loadProperties(resource.read()); } /** diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java index 9fe11270..e3f8ab92 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java @@ -19,13 +19,14 @@ * under the License. */ +import java.io.Closeable; import java.io.IOException; import java.io.OutputStream; /** * Maven 2 Index writable {@link ResourceHandler}, is capable of saving resources too. Needed only if incremental index * updates are wanted, to store the index state locally, and be able to calculate incremental diffs on next {@link - * IndexReader} invocation. + * IndexReader} invocation. Is used by single thread only. * * @see ResourceHandler * @since 5.1.2 @@ -34,19 +35,22 @@ public interface WritableResourceHandler extends ResourceHandler { interface WritableResource - extends Resource + extends Resource, Closeable { /** * Returns the {@link OutputStream} stream of the resource, if exists, it will replace the existing content, or if * not exists, the resource will be created. The stream should be closed by caller, otherwise resource leaks might - * be introduced. + * be introduced. How and when content is written is left to implementation, but it is guaranteed that this method + * is called only once, and will be followed by {@link #close()} on the resource itself. Implementation does not + * have to be "read consistent", in a way to worry what subsequent {@link #read()} method will return, as mixed + * calls will not happen on same instance of resource. */ OutputStream write() throws IOException; } /** - * Returns the {@link WritableResource} of resource with {@code name}. Returned locator should be handled as - * resource. + * Returns the {@link WritableResource} with {@code name}. Returned locator should be handled as + * resource, is {@link Closeable}. * * @param name Resource name, guaranteed to be non-{@code null} and is FS and URL safe string. */ diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java index b6cf436e..68e01f63 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java @@ -64,7 +64,7 @@ public Resource locate(final String name) throws IOException { return NOT_EXISTING_RESOURCE; } else { - return new CachingResource(name, local.locate(name)); + return new CachingResource(name); } } @@ -73,20 +73,17 @@ private class CachingResource { private final String name; - private final WritableResource localResource; - - private CachingResource(final String name, final WritableResource localResource) { + private CachingResource(final String name) { this.name = name; - this.localResource = localResource; } public InputStream read() throws IOException { - InputStream inputStream = localResource.read(); + InputStream inputStream = local.locate(name).read(); if (inputStream != null) { return inputStream; } if (cacheLocally(name)) { - return localResource.read(); + return local.locate(name).read(); } notFoundResources.add(name); return null; @@ -94,6 +91,7 @@ public InputStream read() throws IOException { private boolean cacheLocally(final String name) throws IOException { final Resource remoteResource = remote.locate(name); + final WritableResource localResource = local.locate(name); try { final InputStream inputStream = remoteResource.read(); if (inputStream != null) { @@ -115,7 +113,7 @@ private boolean cacheLocally(final String name) throws IOException { return false; } finally { - remoteResource.close(); + localResource.close(); } } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java index 904ee446..7d5cc33e 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java @@ -47,7 +47,7 @@ public void simple() throws IOException { "full", testResourceHandler("simple").locate("nexus-maven-repository-index.gz").read() ); - final Map> recordTypes = countRecordTypes(chunkReader); + final Map> recordTypes = loadRecordsByType(chunkReader); assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1)); assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1)); assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1)); @@ -61,27 +61,22 @@ public void roundtrip() throws IOException { File tempChunkFile = createTempFile("nexus-maven-repository-index.gz"); { final Resource resource = testResourceHandler("simple").locate("nexus-maven-repository-index.gz"); + final ChunkReader chunkReader = new ChunkReader( + "full", + resource.read() + ); + final ChunkWriter chunkWriter = new ChunkWriter( + chunkReader.getName(), + new FileOutputStream(tempChunkFile), 1, new Date() + ); try { - final ChunkReader chunkReader = new ChunkReader( - "full", - resource.read() - ); - final ChunkWriter chunkWriter = new ChunkWriter( - chunkReader.getName(), - new FileOutputStream(tempChunkFile), 1, new Date() - ); - try { - chunkWriter.writeChunk(chunkReader.iterator()); - } - finally { - chunkWriter.close(); - chunkReader.close(); - } - published = chunkWriter.getTimestamp(); + chunkWriter.writeChunk(chunkReader.iterator()); } finally { - resource.close(); + chunkWriter.close(); + chunkReader.close(); } + published = chunkWriter.getTimestamp(); } final ChunkReader chunkReader = new ChunkReader( @@ -90,7 +85,7 @@ public void roundtrip() throws IOException { ); assertThat(chunkReader.getVersion(), equalTo(1)); assertThat(chunkReader.getTimestamp().getTime(), equalTo(published.getTime())); - final Map> recordTypes = countRecordTypes(chunkReader); + final Map> recordTypes = loadRecordsByType(chunkReader); assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1)); assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1)); assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1)); diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java index fc086108..3e7f806f 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java @@ -21,12 +21,14 @@ import java.io.File; import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.Writer; +import java.io.PrintWriter; import java.net.URL; import java.util.Arrays; import java.util.Date; +import java.util.List; +import java.util.Map; +import org.apache.maven.index.reader.Record.Type; import org.junit.Ignore; import org.junit.Test; @@ -161,34 +163,46 @@ public void roundtrip() throws IOException { } } + /** + * This UT is here for demonstration purposes only. Bashing Central is not something you want to do, and risk your + * IP address being banned. You were warned! + */ @Test - @Ignore("Here for example but test depending on external resource is not nice thing to have") + @Ignore("For eyes only") public void central() throws Exception { - final File tempDir = File.createTempFile("index-reader", "tmp"); - tempDir.mkdirs(); - final Writer writer = new OutputStreamWriter(System.out); - final IndexReader indexReader = new IndexReader( - new DirectoryResourceHandler(tempDir), + // local index location, against which we perform incremental updates + final File indexDir = createTempDirectory(); + // cache of remote, to not rely on HTTP transport possible failures, or, to detect them early + final File cacheDir = createTempDirectory(); + + final PrintWriter writer = new PrintWriter(System.out, true); + final WritableResourceHandler local = new DirectoryResourceHandler(indexDir); + final CachingResourceHandler remote = new CachingResourceHandler( + new DirectoryResourceHandler(cacheDir), new HttpResourceHandler(new URL("http://repo1.maven.org/maven2/.index/")) ); + final IndexReader indexReader = new IndexReader(local, remote); try { - writer.write("indexRepoId=" + indexReader.getIndexId() + "\n"); - writer.write("indexLastPublished=" + indexReader.getPublishedTimestamp() + "\n"); - writer.write("isIncremental=" + indexReader.isIncremental() + "\n"); - writer.write("indexRequiredChunkNames=" + indexReader.getChunkNames() + "\n"); + writer.println("indexRepoId=" + indexReader.getIndexId()); + writer.println("indexLastPublished=" + indexReader.getPublishedTimestamp()); + writer.println("isIncremental=" + indexReader.isIncremental()); + writer.println("indexRequiredChunkNames=" + indexReader.getChunkNames()); for (ChunkReader chunkReader : indexReader) { - writer.write("chunkName=" + chunkReader.getName() + "\n"); - writer.write("chunkVersion=" + chunkReader.getVersion() + "\n"); - writer.write("chunkPublished=" + chunkReader.getTimestamp() + "\n"); - writer.write("= = = = = = \n"); - for (Record record : Iterables.transform(chunkReader, new RecordExpander())) { - writer.write(record.getExpanded() + "\n"); + writer.println("chunkName=" + chunkReader.getName()); + writer.println("chunkVersion=" + chunkReader.getVersion()); + writer.println("chunkPublished=" + chunkReader.getTimestamp()); + writer.println("Chunk stats:"); + Map stats = countRecordsByType(chunkReader); + for (Map.Entry entry : stats.entrySet()) { + writer.println(entry.getKey() + " = " + entry.getValue()); } + writer.println("= = = = = ="); } } finally { indexReader.close(); - writer.close(); + remote.close(); + local.close(); } } } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java index c0b1142f..f584eeae 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java @@ -64,7 +64,7 @@ public void decorateAndTransform() throws IOException { IndexReader indexReader = new IndexReader(null, writableResourceHandler); assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz"))); ChunkReader chunkReader = indexReader.iterator().next(); - final Map> recordTypes = countRecordTypes(chunkReader); + final Map> recordTypes = loadRecordsByType(chunkReader); assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1)); assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1)); assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1)); diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java index bcd465e0..8794be0e 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java @@ -114,17 +114,15 @@ protected ResourceHandler testResourceHandler(final String name) throws IOExcept } /** - * Consumes {@link ChunkReader} and creates "statistics" of index by record type. + * Consumes {@link ChunkReader} and creates a map "by type" with records. */ - protected Map> countRecordTypes(final ChunkReader chunkReader) throws IOException { + protected Map> loadRecordsByType(final ChunkReader chunkReader) throws IOException { HashMap> stat = new HashMap>(); try { assertThat(chunkReader.getVersion(), equalTo(1)); final RecordExpander recordExpander = new RecordExpander(); for (Map rec : chunkReader) { - System.out.println(rec); final Record record = recordExpander.apply(rec); - System.out.println(record); if (!stat.containsKey(record.getType())) { stat.put(record.getType(), new ArrayList()); } @@ -137,6 +135,29 @@ protected Map> countRecordTypes(final ChunkReader chunkReader return stat; } + + /** + * Consumes {@link ChunkReader} and creates a map "by type" with record type counts. + */ + protected Map countRecordsByType(final ChunkReader chunkReader) throws IOException { + HashMap stat = new HashMap(); + try { + assertThat(chunkReader.getVersion(), equalTo(1)); + final RecordExpander recordExpander = new RecordExpander(); + for (Map rec : chunkReader) { + final Record record = recordExpander.apply(rec); + if (!stat.containsKey(record.getType())) { + stat.put(record.getType(), 0); + } + stat.put(record.getType(), stat.get(record.getType()) + 1); + } + } + finally { + chunkReader.close(); + } + return stat; + } + /** * Delete recursively. */ From 6fc7a9661f3c5349a74f2f7eedbd25bb2d4eb43c Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Thu, 5 Nov 2015 07:53:34 -0500 Subject: [PATCH 09/16] Make the reader an osgi bundle (cherry picked from commit 4c5d1d6) --- indexer-reader/pom.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/indexer-reader/pom.xml b/indexer-reader/pom.xml index daf364cb..95d821f2 100644 --- a/indexer-reader/pom.xml +++ b/indexer-reader/pom.xml @@ -28,6 +28,7 @@ under the License. indexer-reader + bundle Maven :: Indexer Reader @@ -45,4 +46,15 @@ under the License. + + + + org.apache.felix + maven-bundle-plugin + 3.0.0 + true + + + + From 732fdbf82efe9f9412cd8d47c13e7431af4b9728 Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Thu, 5 Nov 2015 09:36:13 -0500 Subject: [PATCH 10/16] rename helper class to avoid name clash with other utilities (cherry picked from commit 4333789) --- .../java/org/apache/maven/index/reader/IndexReader.java | 2 +- .../org/apache/maven/index/reader/RecordCompactor.java | 2 +- .../org/apache/maven/index/reader/RecordExpander.java | 2 +- .../maven/index/reader/{Iterables.java => Transform.java} | 8 ++++---- .../org/apache/maven/index/reader/IndexReaderTest.java | 7 +++---- .../org/apache/maven/index/reader/IndexWriterTest.java | 2 +- .../reader/{IterablesTest.java => TransformTest.java} | 6 +++--- 7 files changed, 14 insertions(+), 15 deletions(-) rename indexer-reader/src/main/java/org/apache/maven/index/reader/{Iterables.java => Transform.java} (96%) rename indexer-reader/src/test/java/org/apache/maven/index/reader/{IterablesTest.java => TransformTest.java} (96%) diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java index 7de1f4f3..85c5409d 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java @@ -67,7 +67,7 @@ public IndexReader(final WritableResourceHandler local, final ResourceHandler re this.remote = remote; remoteIndexProperties = loadProperties(remote.locate(Utils.INDEX_FILE_PREFIX + ".properties")); if (remoteIndexProperties == null) { - throw new NullPointerException("remote index properties null"); + throw new IllegalArgumentException("Non-existent remote index"); } try { if (local != null) { diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java index 3fcfd250..8cf01d89 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java @@ -22,7 +22,7 @@ import java.util.HashMap; import java.util.Map; -import org.apache.maven.index.reader.Iterables.Function; +import org.apache.maven.index.reader.Transform.Function; import org.apache.maven.index.reader.Record.Type; import static org.apache.maven.index.reader.Utils.FIELD_SEPARATOR; diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java index 83f6ca7c..1de965fc 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java @@ -22,7 +22,7 @@ import java.util.HashMap; import java.util.Map; -import org.apache.maven.index.reader.Iterables.Function; +import org.apache.maven.index.reader.Transform.Function; import org.apache.maven.index.reader.Record.EntryKey; import org.apache.maven.index.reader.Record.Type; diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Transform.java similarity index 96% rename from indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java rename to indexer-reader/src/main/java/org/apache/maven/index/reader/Transform.java index 6e65e8f8..50821bd1 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Transform.java @@ -34,13 +34,13 @@ import org.apache.maven.index.reader.Record.Type; /** - * Helpers to transform {@link Iterable}. + * Helpers to transform records from one to another representation, and, some helpers for publishing. * * @since 5.1.2 */ -public final class Iterables +public final class Transform { - private Iterables() { + private Transform() { // nothing } @@ -66,7 +66,7 @@ public Iterator iterator() { /** * Helper method, that "decorates" the stream of records to be written out with "special" Maven Indexer records, so - * all the caller is needed to provide {@Link Iterable} or {@link Record}s to be on the index, with + * all the caller is needed to provide {@link Iterable} or {@link Record}s to be on the index, with * record type {@link Record.Type#ARTIFACT_ADD}. This method will create the output as "proper" Maven Indexer record * streeam, by adding the {@link Type#DESCRIPTOR}, {@link Type#ROOT_GROUPS} and {@link Type#ALL_GROUPS} special * records. diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java index 3e7f806f..c0913176 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java @@ -25,7 +25,6 @@ import java.net.URL; import java.util.Arrays; import java.util.Date; -import java.util.List; import java.util.Map; import org.apache.maven.index.reader.Record.Type; @@ -61,7 +60,7 @@ public void simple() throws IOException { assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); assertThat(chunkReader.getVersion(), equalTo(1)); assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); - for (Record record : Iterables.transform(chunkReader, new RecordExpander())) { + for (Record record : Transform.transform(chunkReader, new RecordExpander())) { records++; } } @@ -93,7 +92,7 @@ public void simpleWithLocal() throws IOException { assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); assertThat(chunkReader.getVersion(), equalTo(1)); assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); - for (Record record : Iterables.transform(chunkReader, new RecordExpander())) { + for (Record record : Transform.transform(chunkReader, new RecordExpander())) { records++; } } @@ -150,7 +149,7 @@ public void roundtrip() throws IOException { assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); assertThat(chunkReader.getVersion(), equalTo(1)); // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); - for (Record record : Iterables.transform(chunkReader, new RecordExpander())) { + for (Record record : Transform.transform(chunkReader, new RecordExpander())) { records++; } } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java index 0895c9d2..7573e070 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java @@ -76,7 +76,7 @@ public void roundtrip() throws IOException { assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); assertThat(chunkReader.getVersion(), equalTo(1)); // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); - for (Record record : Iterables.transform(chunkReader, new RecordExpander())) { + for (Record record : Transform.transform(chunkReader, new RecordExpander())) { records++; } } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java similarity index 96% rename from indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java rename to indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java index f584eeae..0eb302b9 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java @@ -34,9 +34,9 @@ import static org.junit.Assert.assertThat; /** - * UT for {@link Iterables} + * UT for {@link Transform} */ -public class IterablesTest +public class TransformTest extends TestSupport { @Test @@ -45,7 +45,7 @@ public void decorateAndTransform() throws IOException { final Record r1 = new Record(Type.ARTIFACT_ADD, artifactMap("org.apache")); final Record r2 = new Record(Type.ARTIFACT_ADD, artifactMap("org.foo")); final Record r3 = new Record(Type.ARTIFACT_ADD, artifactMap("com.bar")); - Iterable> iterable = Iterables.decorateAndTransform(Arrays.asList(r1, r2, r3), indexId); + Iterable> iterable = Transform.decorateAndTransform(Arrays.asList(r1, r2, r3), indexId); WritableResourceHandler writableResourceHandler = createWritableResourceHandler(); try { From 62cdc3bd3cc7b18c54ee5b02f92f43ac28bbbac0 Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Wed, 11 Nov 2015 15:11:06 -0500 Subject: [PATCH 11/16] Fix classifier separator (cherry picked from commit ea1205e) --- .../java/org/apache/maven/index/reader/RecordCompactor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java index 8cf01d89..47ca80bb 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java @@ -153,7 +153,7 @@ private static String compactUinfo(final Record record) { .append(FIELD_SEPARATOR) .append(nvl(classifier)); if (classifier != null) { - sb.append(record.get(Record.FILE_EXTENSION)); + sb.append(FIELD_SEPARATOR).append(record.get(Record.FILE_EXTENSION)); } return sb.toString(); } From b680ab2bbc798f378ac7a65c738e0c9a2fe81813 Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Thu, 12 Nov 2015 05:30:50 -0500 Subject: [PATCH 12/16] Remove Transform, let user use any lib it wants to for iterable manipulation Also, UTs got new TestUtils based on Guava (cherry picked from commit e0570bf) --- indexer-reader/pom.xml | 6 + .../maven/index/reader/RecordCompactor.java | 6 +- .../maven/index/reader/RecordExpander.java | 5 +- .../apache/maven/index/reader/Transform.java | 212 ------------------ .../org/apache/maven/index/reader/Utils.java | 59 +++-- .../maven/index/reader/IndexReaderTest.java | 8 +- .../maven/index/reader/IndexWriterTest.java | 4 +- .../apache/maven/index/reader/TestUtils.java | 108 +++++++++ .../maven/index/reader/TransformTest.java | 10 +- 9 files changed, 182 insertions(+), 236 deletions(-) delete mode 100644 indexer-reader/src/main/java/org/apache/maven/index/reader/Transform.java create mode 100644 indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java diff --git a/indexer-reader/pom.xml b/indexer-reader/pom.xml index 95d821f2..49e4c6c8 100644 --- a/indexer-reader/pom.xml +++ b/indexer-reader/pom.xml @@ -44,6 +44,12 @@ under the License. junit test + + com.google.guava + guava + 18.0 + test + diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java index 47ca80bb..76fc1e52 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java @@ -22,7 +22,6 @@ import java.util.HashMap; import java.util.Map; -import org.apache.maven.index.reader.Transform.Function; import org.apache.maven.index.reader.Record.Type; import static org.apache.maven.index.reader.Utils.FIELD_SEPARATOR; @@ -36,8 +35,11 @@ * @since 5.1.2 */ public class RecordCompactor - implements Function> { + /** + * Compacts {@link Record} into low level MI record with all the encoded fields as physically present in MI binary + * chunk. + */ public Map apply(final Record record) { if (Type.DESCRIPTOR == record.getType()) { return compactDescriptor(record); diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java index 1de965fc..6d7f4be1 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java @@ -22,7 +22,6 @@ import java.util.HashMap; import java.util.Map; -import org.apache.maven.index.reader.Transform.Function; import org.apache.maven.index.reader.Record.EntryKey; import org.apache.maven.index.reader.Record.Type; @@ -39,8 +38,10 @@ * @since 5.1.2 */ public class RecordExpander - implements Function, Record> { + /** + * Expands MI low level record into {@link Record}. + */ public Record apply(final Map recordMap) { if (recordMap.containsKey("DESCRIPTOR")) { return expandDescriptor(recordMap); diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Transform.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Transform.java deleted file mode 100644 index 50821bd1..00000000 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Transform.java +++ /dev/null @@ -1,212 +0,0 @@ -package org.apache.maven.index.reader; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.TreeSet; - -import org.apache.maven.index.reader.Record.EntryKey; -import org.apache.maven.index.reader.Record.Type; - -/** - * Helpers to transform records from one to another representation, and, some helpers for publishing. - * - * @since 5.1.2 - */ -public final class Transform -{ - private Transform() { - // nothing - } - - /** - * Transforming function. - */ - public interface Function - { - O apply(I rec); - } - - /** - * Applies {@link Function} to an {@link Iterable} on the fly. - */ - public static Iterable transform(final Iterable iterable, final Function function) { - return new Iterable() - { - public Iterator iterator() { - return new TransformIterator(iterable.iterator(), function); - } - }; - } - - /** - * Helper method, that "decorates" the stream of records to be written out with "special" Maven Indexer records, so - * all the caller is needed to provide {@link Iterable} or {@link Record}s to be on the index, with - * record type {@link Record.Type#ARTIFACT_ADD}. This method will create the output as "proper" Maven Indexer record - * streeam, by adding the {@link Type#DESCRIPTOR}, {@link Type#ROOT_GROUPS} and {@link Type#ALL_GROUPS} special - * records. - */ - public static Iterable> decorateAndTransform(final Iterable iterable, - final String repoId) - { - final RecordCompactor recordCompactor = new RecordCompactor(); - final TreeSet allGroups = new TreeSet(); - final TreeSet rootGroups = new TreeSet(); - final ArrayList> iterators = new ArrayList>(); - iterators.add(Collections.singletonList(descriptor(repoId)).iterator()); - iterators.add(iterable.iterator()); - iterators.add(Collections.singletonList(allGroups(allGroups)).iterator()); - iterators.add(Collections.singletonList(rootGroups(rootGroups)).iterator()); - return transform( - new Iterable() - { - public Iterator iterator() { - return new ConcatIterator(iterators.iterator()); - } - }, - new Function>() - { - public Map apply(final Record rec) { - if (Type.DESCRIPTOR == rec.getType()) { - return recordCompactor.apply(descriptor(repoId)); - } - else if (Type.ALL_GROUPS == rec.getType()) { - return recordCompactor.apply(allGroups(allGroups)); - } - else if (Type.ROOT_GROUPS == rec.getType()) { - return recordCompactor.apply(rootGroups(rootGroups)); - } - else { - final String groupId = rec.get(Record.GROUP_ID); - if (groupId != null) { - allGroups.add(groupId); - rootGroups.add(Utils.rootGroup(groupId)); - } - return recordCompactor.apply(rec); - } - } - } - ); - } - - private static Record descriptor(final String repoId) { - HashMap entries = new HashMap(); - entries.put(Record.REPOSITORY_ID, repoId); - return new Record(Type.DESCRIPTOR, entries); - } - - private static Record allGroups(final Collection allGroups) { - HashMap entries = new HashMap(); - entries.put(Record.ALL_GROUPS, allGroups.toArray(new String[allGroups.size()])); - return new Record(Type.ALL_GROUPS, entries); - } - - private static Record rootGroups(final Collection rootGroups) { - HashMap entries = new HashMap(); - entries.put(Record.ROOT_GROUPS, rootGroups.toArray(new String[rootGroups.size()])); - return new Record(Type.ROOT_GROUPS, entries); - } - - // == - - private static final class TransformIterator - implements Iterator - { - private final Iterator iterator; - - private final Function function; - - private TransformIterator(final Iterator iterator, final Function function) { - this.iterator = iterator; - this.function = function; - } - - public boolean hasNext() { - return iterator.hasNext(); - } - - public O next() { - return function.apply(iterator.next()); - } - - public void remove() { - throw new UnsupportedOperationException("remove"); - } - } - - private static final class ConcatIterator - implements Iterator - { - private final Iterator> iterators; - - private Iterator current; - - private T nextElement; - - private ConcatIterator(final Iterator> iterators) { - this.iterators = iterators; - this.nextElement = getNextElement(); - } - - public boolean hasNext() { - return nextElement != null; - } - - public T next() { - if (nextElement == null) { - throw new NoSuchElementException(); - } - T result = nextElement; - nextElement = getNextElement(); - return result; - } - - public void remove() { - throw new UnsupportedOperationException("remove"); - } - - protected T getNextElement() { - if ((current == null || !current.hasNext()) && iterators.hasNext()) { - current = iterators.next(); - } - while (current != null && !current.hasNext()) { - if (!iterators.hasNext()) { - current = null; - break; - } - current = iterators.next(); - } - if (current != null) { - return current.next(); - } - else { - return null; - } - } - } -} diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java index c978bd67..c5b210a0 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java @@ -24,10 +24,14 @@ import java.io.OutputStream; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.Collection; +import java.util.HashMap; import java.util.Properties; import java.util.TimeZone; import java.util.regex.Pattern; +import org.apache.maven.index.reader.Record.EntryKey; +import org.apache.maven.index.reader.Record.Type; import org.apache.maven.index.reader.ResourceHandler.Resource; import org.apache.maven.index.reader.WritableResourceHandler.WritableResource; @@ -42,29 +46,29 @@ private Utils() { // nothing } - static final String INDEX_FILE_PREFIX = "nexus-maven-repository-index"; + public static final String INDEX_FILE_PREFIX = "nexus-maven-repository-index"; - static final DateFormat INDEX_DATE_FORMAT; + public static final DateFormat INDEX_DATE_FORMAT; static { INDEX_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss.SSS Z"); INDEX_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT")); } - static final String FIELD_SEPARATOR = "|"; + public static final String FIELD_SEPARATOR = "|"; - static final String NOT_AVAILABLE = "NA"; + public static final String NOT_AVAILABLE = "NA"; - static final String UINFO = "u"; + public static final String UINFO = "u"; - static final String INFO = "i"; + public static final String INFO = "i"; - static final Pattern FS_PATTERN = Pattern.compile(Pattern.quote(FIELD_SEPARATOR)); + public static final Pattern FS_PATTERN = Pattern.compile(Pattern.quote(FIELD_SEPARATOR)); /** * Creates and loads {@link Properties} from provided {@link InputStream} and closes the stream. */ - static Properties loadProperties(final InputStream inputStream) throws IOException { + public static Properties loadProperties(final InputStream inputStream) throws IOException { try { final Properties properties = new Properties(); properties.load(inputStream); @@ -79,7 +83,7 @@ static Properties loadProperties(final InputStream inputStream) throws IOExcepti * Creates and loads {@link Properties} from provided {@link Resource} if exists, and closes the resource. If not * exists, returns {@code null}. */ - static Properties loadProperties(final Resource resource) throws IOException { + public static Properties loadProperties(final Resource resource) throws IOException { final InputStream inputStream = resource.read(); if (inputStream == null) { return null; @@ -90,7 +94,7 @@ static Properties loadProperties(final Resource resource) throws IOException { /** * Saves {@link Properties} to provided {@link OutputStream} and closes the stream. */ - static void storeProperties(final OutputStream outputStream, final Properties properties) throws IOException { + public static void storeProperties(final OutputStream outputStream, final Properties properties) throws IOException { try { properties.store(outputStream, "Maven Indexer Writer"); } @@ -103,7 +107,7 @@ static void storeProperties(final OutputStream outputStream, final Properties pr /** * Saves {@link Properties} to provided {@link WritableResource} and closes the resource. */ - static void storeProperties(final WritableResource writableResource, final Properties properties) throws IOException { + public static void storeProperties(final WritableResource writableResource, final Properties properties) throws IOException { try { storeProperties(writableResource.write(), properties); } @@ -112,24 +116,51 @@ static void storeProperties(final WritableResource writableResource, final Prope } } + /** + * Creates a record of type {@link Type#DESCRIPTOR}. + */ + public static Record descriptor(final String repoId) { + HashMap entries = new HashMap(); + entries.put(Record.REPOSITORY_ID, repoId); + return new Record(Type.DESCRIPTOR, entries); + } + + /** + * Creates a record of type {@link Type#ALL_GROUPS}. + */ + public static Record allGroups(final Collection allGroups) { + HashMap entries = new HashMap(); + entries.put(Record.ALL_GROUPS, allGroups.toArray(new String[allGroups.size()])); + return new Record(Type.ALL_GROUPS, entries); + } + + /** + * Creates a record of type {@link Type#ROOT_GROUPS}. + */ + public static Record rootGroups(final Collection rootGroups) { + HashMap entries = new HashMap(); + entries.put(Record.ROOT_GROUPS, rootGroups.toArray(new String[rootGroups.size()])); + return new Record(Type.ROOT_GROUPS, entries); + } + /** * Helper to translate the "NA" (not available) input into {@code null} value. */ - static String renvl(final String v) { + public static String renvl(final String v) { return NOT_AVAILABLE.equals(v) ? null : v; } /** * Helper to translate {@code null} into "NA" (not available) value. */ - static String nvl(final String v) { + public static String nvl(final String v) { return v == null ? NOT_AVAILABLE : v; } /** * Returns the "root group" of given groupId. */ - static String rootGroup(final String groupId) { + public static String rootGroup(final String groupId) { int n = groupId.indexOf('.'); if (n > -1) { return groupId.substring(0, n); diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java index c0913176..8fc4c347 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java @@ -31,6 +31,8 @@ import org.junit.Ignore; import org.junit.Test; +import static org.apache.maven.index.reader.TestUtils.expandFunction; +import static com.google.common.collect.Iterables.transform; import static org.hamcrest.core.IsEqual.equalTo; import static org.hamcrest.core.IsNot.not; import static org.hamcrest.core.IsNull.nullValue; @@ -60,7 +62,7 @@ public void simple() throws IOException { assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); assertThat(chunkReader.getVersion(), equalTo(1)); assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); - for (Record record : Transform.transform(chunkReader, new RecordExpander())) { + for (Record record : transform(chunkReader, expandFunction)) { records++; } } @@ -92,7 +94,7 @@ public void simpleWithLocal() throws IOException { assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); assertThat(chunkReader.getVersion(), equalTo(1)); assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); - for (Record record : Transform.transform(chunkReader, new RecordExpander())) { + for (Record record : transform(chunkReader, expandFunction)) { records++; } } @@ -149,7 +151,7 @@ public void roundtrip() throws IOException { assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); assertThat(chunkReader.getVersion(), equalTo(1)); // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); - for (Record record : Transform.transform(chunkReader, new RecordExpander())) { + for (Record record : transform(chunkReader, expandFunction)) { records++; } } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java index 7573e070..d1e9bc48 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java @@ -24,6 +24,8 @@ import org.junit.Test; +import static org.apache.maven.index.reader.TestUtils.expandFunction; +import static com.google.common.collect.Iterables.transform; import static org.hamcrest.core.IsEqual.equalTo; import static org.junit.Assert.assertThat; @@ -76,7 +78,7 @@ public void roundtrip() throws IOException { assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); assertThat(chunkReader.getVersion(), equalTo(1)); // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); - for (Record record : Transform.transform(chunkReader, new RecordExpander())) { + for (Record record : transform(chunkReader, expandFunction)) { records++; } } diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java new file mode 100644 index 00000000..348d1996 --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java @@ -0,0 +1,108 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.Map; +import java.util.TreeSet; + +import com.google.common.base.Function; +import org.apache.maven.index.reader.Record.Type; + +import static com.google.common.collect.Iterables.concat; +import static com.google.common.collect.Iterables.transform; +import static java.util.Collections.singletonList; +import static org.apache.maven.index.reader.Utils.allGroups; +import static org.apache.maven.index.reader.Utils.descriptor; +import static org.apache.maven.index.reader.Utils.rootGroup; +import static org.apache.maven.index.reader.Utils.rootGroups; + +/** + * Helpers to transform records from one to another representation, and, some helpers for publishing using Guava. + */ +public final class TestUtils +{ + private TestUtils() { + // nothing + } + + private static final RecordCompactor RECORD_COMPACTOR = new RecordCompactor(); + + private static final RecordExpander RECORD_EXPANDER = new RecordExpander(); + + public static Function> compactFunction = + new Function>() + { + public Map apply(final Record input) { + return RECORD_COMPACTOR.apply(input); + } + }; + + public static Function, Record> expandFunction = + new Function, Record>() + { + public Record apply(final Map input) { + return RECORD_EXPANDER.apply(input); + } + }; + + /** + * Helper method, that "decorates" the stream of records to be written out with "special" Maven Indexer records, so + * all the caller is needed to provide {@link Iterable} or {@link Record}s to be on the index, with + * record type {@link Type#ARTIFACT_ADD}. This method will create the output as "proper" Maven Indexer record + * stream, by adding the {@link Type#DESCRIPTOR}, {@link Type#ROOT_GROUPS} and {@link Type#ALL_GROUPS} special + * records. + */ + public static Iterable decorate(final Iterable iterable, + final String repoId) + { + final TreeSet allGroupsSet = new TreeSet(); + final TreeSet rootGroupsSet = new TreeSet(); + return transform( + concat( + singletonList(descriptor(repoId)), + iterable, + singletonList(allGroups(allGroupsSet)), // placeholder, will be recreated at the end with proper content + singletonList(rootGroups(rootGroupsSet)) // placeholder, will be recreated at the end with proper content + ), + new Function() + { + public Record apply(final Record rec) { + if (Type.DESCRIPTOR == rec.getType()) { + return rec; + } + else if (Type.ALL_GROUPS == rec.getType()) { + return allGroups(allGroupsSet); + } + else if (Type.ROOT_GROUPS == rec.getType()) { + return rootGroups(rootGroupsSet); + } + else { + final String groupId = rec.get(Record.GROUP_ID); + if (groupId != null) { + allGroupsSet.add(groupId); + rootGroupsSet.add(rootGroup(groupId)); + } + return rec; + } + } + } + ); + } +} diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java index 0eb302b9..7d48154a 100644 --- a/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java @@ -29,12 +29,15 @@ import org.apache.maven.index.reader.Record.Type; import org.junit.Test; +import static org.apache.maven.index.reader.TestUtils.decorate; +import static org.apache.maven.index.reader.TestUtils.compactFunction; +import static com.google.common.collect.Iterables.transform; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.core.IsEqual.equalTo; import static org.junit.Assert.assertThat; /** - * UT for {@link Transform} + * UT for {@link RecordCompactor} and {@linl RecordExpander}. */ public class TransformTest extends TestSupport @@ -45,7 +48,10 @@ public void decorateAndTransform() throws IOException { final Record r1 = new Record(Type.ARTIFACT_ADD, artifactMap("org.apache")); final Record r2 = new Record(Type.ARTIFACT_ADD, artifactMap("org.foo")); final Record r3 = new Record(Type.ARTIFACT_ADD, artifactMap("com.bar")); - Iterable> iterable = Transform.decorateAndTransform(Arrays.asList(r1, r2, r3), indexId); + Iterable> iterable = transform( + decorate(Arrays.asList(r1, r2, r3), indexId), + compactFunction + ); WritableResourceHandler writableResourceHandler = createWritableResourceHandler(); try { From 4efc41f7310f61c23c1e7b91ce0055d581eb56f9 Mon Sep 17 00:00:00 2001 From: Simon Spero Date: Sun, 19 Mar 2017 17:41:07 -0400 Subject: [PATCH 13/16] Minor build and test-resource fixes. --- ...rg.apache.karaf.features.command-2.2.2.pom | 120 +++++++++++++++++- indexer-reader/pom.xml | 2 +- .../org/apache/maven/index/reader/packageinfo | 1 + pom.xml | 5 +- 4 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 indexer-reader/src/main/java/org/apache/maven/index/reader/packageinfo diff --git a/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom b/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom index 67e206f6..062383c7 100644 --- a/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom +++ b/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom @@ -1,5 +1,115 @@ -#Generated by org.apache.felix.bundleplugin -#Thu Jun 30 15:25:37 NDT 2011 -version=2.2.2 -groupId=org.apache.karaf.features -artifactId=org.apache.karaf.features.command + + + 4.0.0 + + org.apache.karaf.features + features + 2.2.2 + + org.apache.karaf.features.command + bundle + Apache Karaf :: Features :: Command + + This bundle provides the Karaf shell commands to manipulate features. + + + ${basedir}/../../etc/appended-resources + + + + org.osgi + org.osgi.core + provided + + + org.osgi + org.osgi.compendium + provided + + + org.apache.karaf.features + org.apache.karaf.features.core + + + org.apache.felix + org.apache.felix.bundlerepository + + + org.apache.karaf.shell + org.apache.karaf.shell.console + + + org.apache.karaf.shell + org.apache.karaf.shell.obr + + + org.springframework.osgi + spring-osgi-core + test + + + org.apache.servicemix.bundles + org.apache.servicemix.bundles.junit + test + + + org.easymock + easymock + test + + + org.slf4j + slf4j-jdk14 + test + + + + + + ${project.basedir}/src/main/resources + + **/* + + + + ${project.basedir}/src/main/resources + true + + **/*.info + + + + + + org.apache.felix + maven-bundle-plugin + + + + !${project.artifactId}*, javax.management, javax.management.loading, org.apache.felix.service.command, org.apache.felix.gogo.commands, org.apache.karaf.shell.console, * + + !* + + + + + + diff --git a/indexer-reader/pom.xml b/indexer-reader/pom.xml index 49e4c6c8..3f91847d 100644 --- a/indexer-reader/pom.xml +++ b/indexer-reader/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.maven.indexer maven-indexer - 5.1.2-SNAPSHOT + 6.0-SNAPSHOT indexer-reader diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/packageinfo b/indexer-reader/src/main/java/org/apache/maven/index/reader/packageinfo new file mode 100644 index 00000000..9ad81f6f --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/packageinfo @@ -0,0 +1 @@ +version 1.0.0 diff --git a/pom.xml b/pom.xml index 8a16cc21..0bffd839 100644 --- a/pom.xml +++ b/pom.xml @@ -441,8 +441,8 @@ under the License. apache-rat-plugin - rat-verify - test + rat-check + validate check @@ -451,6 +451,7 @@ under the License. README.md .gitignore NOTICE + **/packageinfo .git/** .idea/** **/*.iml From 171f0902f9424c8fed8bc6ca4f62eec6fa1759c6 Mon Sep 17 00:00:00 2001 From: Simon Spero Date: Mon, 20 Mar 2017 17:10:05 -0400 Subject: [PATCH 14/16] MINDEXER-102 Forward port OSGI related index-reader changes to master (Changes relative to MINDEXER-100) --- .../org/apache/maven/index/reader/Record.java | 22 +++++++++++++++++++ .../maven/index/reader/RecordCompactor.java | 4 ++++ .../maven/index/reader/RecordExpander.java | 4 ++++ 3 files changed, 30 insertions(+) diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java index 17468457..677a36f1 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java @@ -246,6 +246,28 @@ public String toString() { */ public static final EntryKey OSGI_REQUIRE_BUNDLE = new EntryKey("Require-Bundle", String.class); + /** + * Key of OSGi "Provide-Capability" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey OSGI_PROVIDE_CAPABILITY = new EntryKey("Provide-Capability", String.class); + /** + * Key of OSGi "Require-Capability" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey OSGI_REQUIRE_CAPABILITY = new EntryKey("Require-Capability", String.class); + /** + * Key of OSGi "Fragment-Host" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey OSGI_FRAGMENT_HOST = new EntryKey("Fragment-Host", String.class); + /** + * Key of OSGi "SHA-256" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey OSGI_SHA_256 = new EntryKey("SHA-256", String.class); + + /** * Types of returned records returned from index. */ diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java index 76fc1e52..07b49374 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java @@ -137,6 +137,10 @@ private static Map compactAddedArtifact(final Record record) { putIfNotNull(record.get(Record.OSGI_EXPORT_DOCURL), result, "Bundle-DocURL"); putIfNotNull(record.get(Record.OSGI_IMPORT_PACKAGE), result, "Import-Package"); putIfNotNull(record.get(Record.OSGI_REQUIRE_BUNDLE), result, "Require-Bundle"); + putIfNotNull(record.get(Record.OSGI_PROVIDE_CAPABILITY), result, "Provide-Capability"); + putIfNotNull(record.get(Record.OSGI_REQUIRE_CAPABILITY), result, "Require-Capability"); + putIfNotNull(record.get(Record.OSGI_FRAGMENT_HOST), result, "Fragment-Host"); + putIfNotNull(record.get(Record.OSGI_SHA_256), result, "SHA-256"); return result; } diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java index 6d7f4be1..da7a1946 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java @@ -156,6 +156,10 @@ private static Record expandAddedArtifact(final Map raw) { putIfNotNull(raw, "Bundle-DocURL", result, Record.OSGI_EXPORT_DOCURL); putIfNotNull(raw, "Import-Package", result, Record.OSGI_IMPORT_PACKAGE); putIfNotNull(raw, "Require-Bundle", result, Record.OSGI_REQUIRE_BUNDLE); + putIfNotNull(raw, "Provide-Capability", result, Record.OSGI_PROVIDE_CAPABILITY); + putIfNotNull(raw, "Require-Capability", result, Record.OSGI_REQUIRE_CAPABILITY); + putIfNotNull(raw, "Fragment-Host", result, Record.OSGI_FRAGMENT_HOST); + putIfNotNull(raw, "SHA-256", result, Record.OSGI_SHA_256); return result; } From b946678b7f4e82712e450c65c34d7ee1d52ccf85 Mon Sep 17 00:00:00 2001 From: Simon Spero Date: Tue, 21 Mar 2017 13:18:22 -0400 Subject: [PATCH 15/16] Set osgi package version of org.apache.maven.index.reader to 5.2.0 to reflect backward compatible changes --- .../src/main/java/org/apache/maven/index/reader/packageinfo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/packageinfo b/indexer-reader/src/main/java/org/apache/maven/index/reader/packageinfo index 9ad81f6f..6824e44a 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/packageinfo +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/packageinfo @@ -1 +1 @@ -version 1.0.0 +version 5.2.0 From 87497bf147422bfb6c5e8203cf8c89e96af56008 Mon Sep 17 00:00:00 2001 From: Simon Spero Date: Tue, 28 Mar 2017 15:37:49 -0400 Subject: [PATCH 16/16] In index-reader, rename SHA-256 key to sha256 / SHA_256. Add Bundle-RequiredExecutionEnvironment. --- .../java/org/apache/maven/index/reader/Record.java | 10 ++++++++-- .../org/apache/maven/index/reader/RecordCompactor.java | 3 ++- .../org/apache/maven/index/reader/RecordExpander.java | 3 ++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java index 677a36f1..fce1a19a 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java @@ -262,10 +262,16 @@ public String toString() { */ public static final EntryKey OSGI_FRAGMENT_HOST = new EntryKey("Fragment-Host", String.class); /** - * Key of OSGi "SHA-256" manifest entry, that contains {@link String}. Extracted by {@code + * Key of deprecated OSGi "Bundle-RequiredExecutionEnvironment" manifest entry, that contains {@link String}. Extracted by {@code * OsgiArtifactIndexCreator}. */ - public static final EntryKey OSGI_SHA_256 = new EntryKey("SHA-256", String.class); + public static final EntryKey OSGI_BREE = new EntryKey("Bundle-RequiredExecutionEnvironment", String.class); + + /** + * Key for SHA-256 checksum needed for OSGI content capability that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey SHA_256 = new EntryKey("sha256", String.class); /** diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java index 07b49374..b005400c 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java @@ -140,7 +140,8 @@ private static Map compactAddedArtifact(final Record record) { putIfNotNull(record.get(Record.OSGI_PROVIDE_CAPABILITY), result, "Provide-Capability"); putIfNotNull(record.get(Record.OSGI_REQUIRE_CAPABILITY), result, "Require-Capability"); putIfNotNull(record.get(Record.OSGI_FRAGMENT_HOST), result, "Fragment-Host"); - putIfNotNull(record.get(Record.OSGI_SHA_256), result, "SHA-256"); + putIfNotNull(record.get(Record.OSGI_BREE), result, "Bundle-RequiredExecutionEnvironment"); + putIfNotNull(record.get(Record.SHA_256), result, "sha256"); return result; } diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java index da7a1946..5d97f2a3 100644 --- a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java @@ -159,7 +159,8 @@ private static Record expandAddedArtifact(final Map raw) { putIfNotNull(raw, "Provide-Capability", result, Record.OSGI_PROVIDE_CAPABILITY); putIfNotNull(raw, "Require-Capability", result, Record.OSGI_REQUIRE_CAPABILITY); putIfNotNull(raw, "Fragment-Host", result, Record.OSGI_FRAGMENT_HOST); - putIfNotNull(raw, "SHA-256", result, Record.OSGI_SHA_256); + putIfNotNull(raw, "Bundle-RequiredExecutionEnvironment", result, Record.OSGI_BREE); + putIfNotNull(raw, "sha256", result, Record.SHA_256); return result; }