diff --git a/Jenkinsfile b/Jenkinsfile index 579c7607b7..87c85a45ba 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,19 +22,6 @@ pipeline { sh 'mvn -B -V verify -Prun-its -Pci' } } - stage('Deploy') { - when { branch 'master' } - steps { - echo "Deploy" - sh 'mvn help:effective-settings -B -V deploy -e' - } - } - stage('Archive') { - steps { - echo "Archive" - archiveArtifacts artifacts: "$artifact", fingerprint: true - } - } stage('Check Image Build Hook') { when { expression { env.IMG_BUILD_HOOKS != null } @@ -54,6 +41,25 @@ pipeline { } } } + stage('Deploy') { + when { + allOf { + expression { img_build_hook != null } + expression { env.CHANGE_ID == null } // Not pull request + branch 'master' + } + } + steps { + echo "Deploy" + sh 'mvn help:effective-settings -B -V deploy -e' + } + } + stage('Archive') { + steps { + echo "Archive" + archiveArtifacts artifacts: "$artifact", fingerprint: true + } + } stage('Build & Push Image') { when { allOf { diff --git a/addons/content-browse/jaxrs/src/main/java/org/commonjava/indy/content/browse/bind/jaxrs/ContentBrowseResource.java b/addons/content-browse/jaxrs/src/main/java/org/commonjava/indy/content/browse/bind/jaxrs/ContentBrowseResource.java index c27cd63c06..84e976d2d2 100644 --- a/addons/content-browse/jaxrs/src/main/java/org/commonjava/indy/content/browse/bind/jaxrs/ContentBrowseResource.java +++ b/addons/content-browse/jaxrs/src/main/java/org/commonjava/indy/content/browse/bind/jaxrs/ContentBrowseResource.java @@ -28,6 +28,7 @@ import org.commonjava.indy.bind.jaxrs.util.JaxRsRequestHelper; import org.commonjava.indy.bind.jaxrs.util.REST; import org.commonjava.indy.bind.jaxrs.util.ResponseHelper; +import org.commonjava.indy.content.ContentDigester; import org.commonjava.indy.content.browse.ContentBrowseController; import org.commonjava.indy.content.browse.model.ContentBrowseResult; import org.commonjava.indy.model.core.PackageTypes; @@ -39,6 +40,7 @@ import org.commonjava.indy.util.ApplicationHeader; import org.commonjava.indy.util.UriFormatter; import org.commonjava.maven.galley.event.EventMetadata; +import org.commonjava.maven.galley.io.checksum.ContentDigest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,7 +60,7 @@ @Api( value = "Indy Directory Content Browse", description = "Browse directory content in indy repository" ) @Path( "/api/browse/{packageType}/{type: (hosted|group|remote)}/{name}" ) -@ApplicationScoped +//@ApplicationScoped @REST public class ContentBrowseResource implements IndyResources @@ -78,43 +80,67 @@ public class ContentBrowseResource @Inject private UriFormatter uriFormatter; + @Inject protected JaxRsRequestHelper jaxRsRequestHelper; @Inject private ResponseHelper responseHelper; + @Inject + ContentDigester contentDigester; + + @Context UriInfo uriInfo; + + @Context HttpServletRequest request; + @ApiOperation( "Retrieve directory content under the given artifact store (type/name) and directory path." ) - @ApiResponses( { @ApiResponse( code = 404, message = "Content is not available" ), - @ApiResponse( code = 200, response = String.class, message = "Rendered content listing" ) } ) + @ApiResponses({@ApiResponse( code = 200, response = String.class, message = "Rendered content listing" ),@ApiResponse( code = 404, message = "Content is not available" )}) @HEAD - @Path( "/{path: (.*)}" ) + @Path( "/{path (.*)}" ) public Response headForDirectory( - final @ApiParam( allowableValues = "maven,npm", required = true ) @PathParam( "packageType" ) - String packageType, - final @ApiParam( allowableValues = "hosted,group,remote", required = true ) @PathParam( "type" ) - String type, final @ApiParam( required = true ) @PathParam( "name" ) String name, - final @PathParam( "path" ) String path, @Context final UriInfo uriInfo, - @Context final HttpServletRequest request ) - { + final @ApiParam( allowableValues = "maven,npm", required = true ) @PathParam( "packageType" ) String packageType, + final @ApiParam( allowableValues = "hosted,group,remote", required = true ) @PathParam( "type" ) String type, + final @ApiParam( required = true ) @PathParam( "name" ) String name, + final @PathParam( "path" ) String path + ) throws IndyWorkflowException { return processHead( packageType, type, name, path, uriInfo, request ); } - @ApiOperation( "Retrieve directory content under the given artifact store (type/name) and directory path." ) - @ApiResponses( { @ApiResponse( code = 404, message = "Content is not available" ), - @ApiResponse( code = 200, response = String.class, message = "Rendered content listing" ) } ) - @HEAD - @Path( "/" ) - public Response headForRoot( - final @ApiParam( allowableValues = "maven,npm", required = true ) @PathParam( "packageType" ) - String packageType, - final @ApiParam( allowableValues = "hosted,group,remote", required = true ) @PathParam( "type" ) - String type, final @ApiParam( required = true ) @PathParam( "name" ) String name, - final @PathParam( "path" ) String path, @Context final UriInfo uriInfo, - @Context final HttpServletRequest request ) - { - return processHead( packageType, type, name, "", uriInfo, request ); - } + + + +// @ApiOperation( "Retrieve directory content under the given artifact store (type/name) and directory path." ) +// @ApiResponses( { @ApiResponse( code = 404, message = "Content is not available" ), +// @ApiResponse( code = 200, response = String.class, message = "Rendered content listing" ) } ) +// @HEAD +// @Path( "/{path: (.*)}" ) +// public Response headForDirectory( +// final @ApiParam( allowableValues = "maven,npm", required = true ) @PathParam( "packageType" ) +// String packageType, +// final @ApiParam( allowableValues = "hosted,group,remote", required = true ) @PathParam( "type" ) +// String type, final @ApiParam( required = true ) @PathParam( "name" ) String name, +// final @PathParam( "path" ) String path, final UriInfo uriInfo, +// @Context final HttpServletRequest request ) +// { +// return processHead( packageType, type, name, path, uriInfo, request ); +// } +// +// @ApiOperation( "Retrieve directory content under the given artifact store (type/name) and directory path." ) +// @ApiResponses( { @ApiResponse( code = 404, message = "Content is not available" ), +// @ApiResponse( code = 200, response = String.class, message = "Rendered content listing" ) } ) +// @HEAD +// @Path( "/" ) +// public Response headForRoot( +// final @ApiParam( allowableValues = "maven,npm", required = true ) @PathParam( "packageType" ) +// String packageType, +// final @ApiParam( allowableValues = "hosted,group,remote", required = true ) @PathParam( "type" ) +// String type, final @ApiParam( required = true ) @PathParam( "name" ) String name, +// final @PathParam( "path" ) String path, @Context final UriInfo uriInfo, +// @Context final HttpServletRequest request ) +// { +// return processHead( packageType, type, name, "", uriInfo, request ); +// } private Response processHead( final String packageType, final String type, final String name, final String path, final UriInfo uriInfo, final HttpServletRequest request ) @@ -138,7 +164,13 @@ private Response processHead( final String packageType, final String type, final .header( ApplicationHeader.content_length.key(), Long.toString( content.length() ) ) .header( ApplicationHeader.last_modified.key(), - HttpUtils.formatDateHeader( new Date() ) ); + HttpUtils.formatDateHeader( new Date() ) ) + .header(ApplicationHeader.md5.key(), + contentDigester.digest(result.getStoreKey(), path, new EventMetadata()).getDigests().get(ContentDigest.MD5).toUpperCase() ) + .header(ApplicationHeader.sha1.key(), + contentDigester.digest(result.getStoreKey(), path, new EventMetadata()).getDigests().get(ContentDigest.SHA_1).toUpperCase() ) + ; + ; response = builder.build(); } diff --git a/addons/content-index/src/main/conf/conf.d/content-index.conf b/addons/content-index/src/main/conf/conf.d/content-index.conf index 2a6a103e43..1cc0bd81e2 100644 --- a/addons/content-index/src/main/conf/conf.d/content-index.conf +++ b/addons/content-index/src/main/conf/conf.d/content-index.conf @@ -1,4 +1,6 @@ [content-index] +#enabled=false + # This property is used to control if authoritative index is enabled. the authoritative index means: # in terms of performance consideration, we will use content-index as the one-time check for the content # access, and if not found in content indexing, will treat it as "no content" and bypass the following access to the storage. @@ -6,4 +8,4 @@ # This property is used to enable content index warmer, which will scan all repos and load all artifacts # into content index when startup. -# index.warmer.enable=true \ No newline at end of file +# index.warmer.enabled=true diff --git a/addons/content-index/src/main/java/org/commonjava/indy/content/index/DefaultContentIndexManager.java b/addons/content-index/src/main/java/org/commonjava/indy/content/index/DefaultContentIndexManager.java index 8e1888266a..181ab1bb8d 100644 --- a/addons/content-index/src/main/java/org/commonjava/indy/content/index/DefaultContentIndexManager.java +++ b/addons/content-index/src/main/java/org/commonjava/indy/content/index/DefaultContentIndexManager.java @@ -17,6 +17,7 @@ import org.commonjava.indy.action.IndyLifecycleException; import org.commonjava.indy.action.ShutdownAction; +import org.commonjava.indy.content.index.conf.ContentIndexConfig; import org.commonjava.indy.data.StoreDataManager; import org.commonjava.indy.measure.annotation.Measure; import org.commonjava.indy.model.core.ArtifactStore; @@ -77,6 +78,9 @@ public class DefaultContentIndexManager @Inject private Instance indexingStrategyComponents; + @Inject + private ContentIndexConfig config; + private Map indexingStrategies; private QueryFactory queryFactory; @@ -98,6 +102,12 @@ public DefaultContentIndexManager( StoreDataManager storeDataManager, SpecialPat @PostConstruct public void constructed() { + if ( !config.isEnabled() ) + { + logger.debug( "Content indexing is disabled." ); + return; + } + if ( indexingStrategyComponents != null ) { Map strats = new HashMap<>(); @@ -126,6 +136,12 @@ public String getId() public void stop() throws IndyLifecycleException { + if ( !config.isEnabled() ) + { + logger.debug( "Content indexing is disabled." ); + return; + } + logger.debug( "Shutdown index cache" ); contentIndex.stop(); } @@ -140,6 +156,12 @@ public int getShutdownPriority() @Measure public boolean removeIndexedStorePath( String rawPath, StoreKey key, Consumer pathConsumer ) { + if ( !config.isEnabled() ) + { + logger.debug( "Content indexing is disabled." ); + return false; + } + String path = getStrategyPath( key, rawPath ); IndexedStorePath topPath = new IndexedStorePath( key, path ); logger.trace( "Attempting to remove indexed path: {}", topPath ); @@ -158,6 +180,12 @@ public boolean removeIndexedStorePath( String rawPath, StoreKey key, Consumer queryFactory.from( IndexedStorePath.class ) @@ -259,6 +317,12 @@ public void clearAllIndexedPathInStore( ArtifactStore store ) @Measure public void clearAllIndexedPathWithOriginalStore( ArtifactStore originalStore ) { + if ( !config.isEnabled() ) + { + logger.debug( "Content indexing is disabled." ); + return; + } + StoreKey osk = originalStore.getKey(); long total = iterateRemove( () -> queryFactory.from( IndexedStorePath.class ) @@ -303,6 +367,12 @@ private long iterateRemove( final Supplier queryFunction ) @Measure public void clearAllIndexedPathInStoreWithOriginal( ArtifactStore store, ArtifactStore originalStore ) { + if ( !config.isEnabled() ) + { + logger.debug( "Content indexing is disabled." ); + return; + } + StoreKey sk = store.getKey(); StoreKey osk = originalStore.getKey(); @@ -337,6 +407,12 @@ public void clearAllIndexedPathInStoreWithOriginal( ArtifactStore store, Artifac @Measure public void clearIndexedPathFrom( String rawPath, Set groups, Consumer pathConsumer ) { + if ( !config.isEnabled() ) + { + logger.debug( "Content indexing is disabled." ); + return; + } + if ( groups == null || groups.isEmpty() ) { return; diff --git a/addons/content-index/src/main/java/org/commonjava/indy/content/index/IndexingContentManagerDecorator.java b/addons/content-index/src/main/java/org/commonjava/indy/content/index/IndexingContentManagerDecorator.java index e8a389458c..0e045a28cb 100644 --- a/addons/content-index/src/main/java/org/commonjava/indy/content/index/IndexingContentManagerDecorator.java +++ b/addons/content-index/src/main/java/org/commonjava/indy/content/index/IndexingContentManagerDecorator.java @@ -66,7 +66,7 @@ public abstract class IndexingContentManagerDecorator implements ContentManager { - private final Logger logger = LoggerFactory.getLogger( this.getClass() ); + private final Logger logger = LoggerFactory.getLogger( IndexingContentManagerDecorator.class.getName() ); @Inject private StoreDataManager storeDataManager; @@ -296,9 +296,10 @@ else if ( StoreType.hosted != type ) // don't track NFC for hosted repos } } + logger.trace( "Delegating retrieve call for concrete repository: {}/{}", store, path ); transfer = delegate.retrieve( store, path, eventMetadata ); - if ( exists( transfer ) ) + if ( exists( transfer ) && indexCfg.isEnabled() ) { logger.debug( "Got transfer from delegate: {} (will index)", transfer ); indexManager.indexTransferIn( transfer, store.getKey() ); @@ -338,8 +339,13 @@ private Transfer getTransferFromConstituents( Collection constituents, if ( exists( transfer ) ) { nfc.clearMissing( resource ); - logger.debug( "Got transfer from constituent: {} (will index)", transfer ); - indexManager.indexTransferIn( transfer, parentStore.getKey() ); + + if ( indexCfg.isEnabled() ) + { + logger.debug( "Got transfer from constituent: {} (will index)", transfer ); + indexManager.indexTransferIn( transfer, parentStore.getKey() ); + } + return transfer; } } @@ -358,11 +364,17 @@ private boolean exists( final Transfer transfer ) return transfer != null && transfer.exists(); } - @Measure( timers = @MetricNamed( DEFAULT ), exceptions = @MetricNamed( DEFAULT ) ) + @Measure public Transfer getIndexedTransfer( final StoreKey storeKey, final StoreKey topKey, final String path, final TransferOperation op, final EventMetadata metadata ) throws IndyWorkflowException { + if ( !indexCfg.isEnabled() ) + { + logger.debug( "Content indexing is disabled. Returning null for indexedTransfer of: {}/{}", storeKey, path ); + return null; + } + logger.debug( "Looking for indexed path: {} in: {} (entry point: {})", path, storeKey, topKey ); try @@ -489,7 +501,7 @@ else if ( isAuthoritativelyMissing( store ) ) transfer = delegate.getTransfer( store, path, op ); // index the transfer only if it exists, it cannot be null at this point - if ( exists( transfer ) ) + if ( exists( transfer ) && indexCfg.isEnabled() ) { indexManager.indexTransferIn( transfer, store.getKey() ); } @@ -541,7 +553,11 @@ public Transfer getIndexedMemberTransfer( final Group group, final String path, if ( transfer != null ) { - indexManager.indexTransferIn( transfer, key, topKey ); + if ( indexCfg.isEnabled() ) + { + indexManager.indexTransferIn( transfer, key, topKey ); + } + return transfer; } else if ( StoreType.group == key.getType() ) @@ -618,7 +634,7 @@ public Transfer getTransfer( final StoreKey storeKey, final String path, final T } transfer = delegate.getTransfer( storeKey, path, op ); - if ( exists( transfer ) ) + if ( exists( transfer ) && indexCfg.isEnabled() ) { logger.debug( "Indexing transfer: {}", transfer ); indexManager.indexTransferIn( transfer, storeKey ); @@ -683,8 +699,11 @@ public Transfer store( final ArtifactStore store, final String path, final Input Transfer transfer = delegate.store( store, path, stream, op, eventMetadata ); if ( transfer != null ) { - logger.trace( "Indexing: {} in: {}", transfer, store.getKey() ); - indexManager.indexTransferIn( transfer, store.getKey() ); + if ( indexCfg.isEnabled() ) + { + logger.trace( "Indexing: {} in: {}", transfer, store.getKey() ); + indexManager.indexTransferIn( transfer, store.getKey() ); + } if ( store instanceof Group ) { @@ -697,7 +716,7 @@ public Transfer store( final ArtifactStore store, final String path, final Input try { Set groups = storeDataManager.query().getGroupsAffectedBy( store.getKey() ); - if ( groups != null && !groups.isEmpty() ) + if ( groups != null && !groups.isEmpty() && indexCfg.isEnabled() ) { groups.forEach( g -> indexManager.deIndexStorePath( g.getKey(), path ) ); } @@ -731,15 +750,22 @@ public Transfer store( final List stores, final StoreKe Transfer transfer = delegate.store( stores, topKey, path, stream, op, eventMetadata ); if ( transfer != null ) { - indexManager.indexTransferIn( transfer, topKey ); + if ( indexCfg.isEnabled() ) + { + indexManager.indexTransferIn( transfer, topKey ); + } try { ArtifactStore topStore = storeDataManager.getArtifactStore( topKey ); nfc.clearMissing( new ConcreteResource( LocationUtils.toLocation( topStore ), path ) ); - // We should deIndex the path for all parent groups because the new content of the path - // may change the content index sequence based on the constituents sequence in parent groups - indexManager.deIndexStorePath( topKey, path ); + + if ( indexCfg.isEnabled() ) + { + // We should deIndex the path for all parent groups because the new content of the path + // may change the content index sequence based on the constituents sequence in parent groups + indexManager.deIndexStorePath( topKey, path ); + } } catch ( IndyDataException e ) { @@ -764,7 +790,7 @@ public boolean delete( final ArtifactStore store, final String path, final Event throws IndyWorkflowException { boolean result = delegate.delete( store, path, eventMetadata ); - if ( result ) + if ( result && indexCfg.isEnabled() ) { indexManager.deIndexStorePath( store.getKey(), path ); } diff --git a/addons/content-index/src/main/java/org/commonjava/indy/content/index/conf/ContentIndexConfig.java b/addons/content-index/src/main/java/org/commonjava/indy/content/index/conf/ContentIndexConfig.java index efd232b11a..113b04306e 100644 --- a/addons/content-index/src/main/java/org/commonjava/indy/content/index/conf/ContentIndexConfig.java +++ b/addons/content-index/src/main/java/org/commonjava/indy/content/index/conf/ContentIndexConfig.java @@ -18,6 +18,8 @@ import org.commonjava.indy.conf.IndyConfigInfo; import org.commonjava.propulsor.config.annotation.ConfigName; import org.commonjava.propulsor.config.annotation.SectionName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.enterprise.context.ApplicationScoped; import java.io.InputStream; @@ -31,16 +33,22 @@ public class ContentIndexConfig public static final String AUTH_INDEX_PARAM = "support.authoritative.indexes"; - public static final String ENABLE_INDEX_WARMER = "index.warmer.enable"; + public static final String ENABLE_INDEX_WARMER = "index.warmer.enabled"; + + private static final String ENABLE = "enabled"; private static final Boolean DEFAULT_AUTHORITATIVE_INDEXES = Boolean.FALSE; private static final Boolean DEFAULT_WARMER_ENABLED = Boolean.FALSE; + private static final Boolean DEFAULT_ENABLED = Boolean.FALSE; + private Boolean authoritativeIndex; private Boolean warmerEnabled; + private Boolean enabled; + public ContentIndexConfig() { } @@ -83,4 +91,20 @@ public InputStream getDefaultConfig() { return Thread.currentThread().getContextClassLoader().getResourceAsStream( "default-content-index.conf" ); } + + public boolean isEnabled() + { + Boolean result = enabled == null ? DEFAULT_ENABLED : enabled; + + Logger logger = LoggerFactory.getLogger( getClass() ); + logger.debug( "Is content indexer enabled? {}", result ); + + return result; + } + + @ConfigName( ContentIndexConfig.ENABLE ) + public void setEnabled( Boolean enabled ) + { + this.enabled = enabled; + } } diff --git a/addons/content-index/src/main/java/org/commonjava/indy/content/index/conf/HostedContentIndexRescanManager.java b/addons/content-index/src/main/java/org/commonjava/indy/content/index/conf/HostedContentIndexRescanManager.java index a639e423b9..3f0a98f82e 100644 --- a/addons/content-index/src/main/java/org/commonjava/indy/content/index/conf/HostedContentIndexRescanManager.java +++ b/addons/content-index/src/main/java/org/commonjava/indy/content/index/conf/HostedContentIndexRescanManager.java @@ -58,6 +58,9 @@ public class HostedContentIndexRescanManager implements ContentIndexRescanManage @Inject private ContentIndexManager contentIndexManager; + @Inject + private ContentIndexConfig indexConfig; + @Inject private DownloadManager downloadManager; @@ -68,6 +71,12 @@ protected HostedContentIndexRescanManager() public void indexPreRescan( @Observes final ArtifactStorePreRescanEvent e ) throws IndyWorkflowException { + if ( !indexConfig.isEnabled() ) + { + LOGGER.debug( "Content index is disabled." ); + return; + } + Collection affectedRepos = e.getStores(); for ( ArtifactStore repo : affectedRepos ) { @@ -89,6 +98,12 @@ public void indexPreRescan( @Observes final ArtifactStorePreRescanEvent e ) public void indexPostRescan( @Observes final ArtifactStorePostRescanEvent e ) throws IndyWorkflowException { + if ( !indexConfig.isEnabled() ) + { + LOGGER.debug( "Content index is disabled." ); + return; + } + Collection hostedStores = e.getStores(); for ( ArtifactStore repo : hostedStores ) { diff --git a/addons/content-index/src/main/resources/default-content-index.conf b/addons/content-index/src/main/resources/default-content-index.conf index c582e6d187..94dfa0ca48 100644 --- a/addons/content-index/src/main/resources/default-content-index.conf +++ b/addons/content-index/src/main/resources/default-content-index.conf @@ -1,9 +1,11 @@ [content-index] +#enabled = false + # This property is used to control if authoritative index is enabled. the authoritative index means: # in terms of performance consideration, we will use content-index as the one-time check for the content # access, and if not found in content indexing, will treat it as "no content" and bypass the following access to the storage. -support.authoritative.indexes=false +#support.authoritative.indexes=false # This property is used to enable content index warmer, which will scan all repos and load all artifacts # into content index when startup. -index.warmer.enable=false \ No newline at end of file +#index.warmer.enabled=false diff --git a/addons/folo/common/src/main/java/org/commonjava/indy/folo/ctl/FoloAdminController.java b/addons/folo/common/src/main/java/org/commonjava/indy/folo/ctl/FoloAdminController.java index f0cbb2fcec..0ee8617333 100644 --- a/addons/folo/common/src/main/java/org/commonjava/indy/folo/ctl/FoloAdminController.java +++ b/addons/folo/common/src/main/java/org/commonjava/indy/folo/ctl/FoloAdminController.java @@ -334,6 +334,7 @@ private TrackedContentEntryDTO constructContentEntryDTO( final TrackedContentEnt entryDTO.setSha1( entry.getSha1() ); entryDTO.setSha256( entry.getSha256() ); entryDTO.setSize( entry.getSize() ); + entryDTO.setTimestamps( entry.getTimestamps() ); return entryDTO; } diff --git a/addons/folo/common/src/main/java/org/commonjava/indy/folo/data/FoloRecordCache.java b/addons/folo/common/src/main/java/org/commonjava/indy/folo/data/FoloRecordCache.java index 1f45776491..073ba5d707 100644 --- a/addons/folo/common/src/main/java/org/commonjava/indy/folo/data/FoloRecordCache.java +++ b/addons/folo/common/src/main/java/org/commonjava/indy/folo/data/FoloRecordCache.java @@ -107,8 +107,20 @@ public synchronized boolean recordArtifact( final TrackedContentEntry entry ) Logger logger = LoggerFactory.getLogger( getClass() ); logger.debug( "Adding tracking entry: {}", entry ); - inProgressRecordCache.put( entry, entry ); - return true; + return inProgressRecordCache.executeCache( (cache)->{ + TrackedContentEntry existing = cache.get( entry ); + if ( existing != null ) + { + existing.merge( entry ); + cache.put( existing, existing ); + } + else + { + cache.put( entry, entry ); + } + + return true; + } ); } @Measure( timers = @MetricNamed( DEFAULT ) ) diff --git a/addons/folo/ftests/src/main/java/org/commonjava/indy/folo/ftest/content/DuplicateStoreAndVerifyTrackedRecordTest.java b/addons/folo/ftests/src/main/java/org/commonjava/indy/folo/ftest/content/DuplicateStoreAndVerifyTrackedRecordTest.java index 0034e5a614..8a3be8080e 100644 --- a/addons/folo/ftests/src/main/java/org/commonjava/indy/folo/ftest/content/DuplicateStoreAndVerifyTrackedRecordTest.java +++ b/addons/folo/ftests/src/main/java/org/commonjava/indy/folo/ftest/content/DuplicateStoreAndVerifyTrackedRecordTest.java @@ -66,7 +66,7 @@ public void run() throws Exception Set uploads = report.getUploads(); uploads.forEach( et -> { System.out.println( ">>> md5: " + et.getMd5() + ", size=" + et.getSize() ); - assertThat( et.getSize(), equalTo( (long) b2.length ) ); + assertThat( "Mismatched size for: " + et.getPath(), et.getSize(), equalTo( (long) b2.length ) ); } ); } diff --git a/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/dto/TrackedContentEntryDTO.java b/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/dto/TrackedContentEntryDTO.java index 810d3565d3..7391244cc4 100644 --- a/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/dto/TrackedContentEntryDTO.java +++ b/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/dto/TrackedContentEntryDTO.java @@ -20,6 +20,8 @@ import org.commonjava.indy.model.core.AccessChannel; import org.commonjava.indy.model.core.StoreKey; +import java.util.Set; + public class TrackedContentEntryDTO implements Comparable { @@ -48,6 +50,8 @@ public class TrackedContentEntryDTO private Long size; + private Set timestamps; + public TrackedContentEntryDTO() { } @@ -238,4 +242,13 @@ public String toString() storeKey, accessChannel, path, originUrl, localUrl, size, md5, sha256 ); } + public Set getTimestamps() + { + return timestamps; + } + + public void setTimestamps( final Set timestamps ) + { + this.timestamps = timestamps; + } } diff --git a/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/model/TrackedContentEntry.java b/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/model/TrackedContentEntry.java index 20650fdb57..4a5e064088 100644 --- a/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/model/TrackedContentEntry.java +++ b/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/model/TrackedContentEntry.java @@ -23,6 +23,9 @@ import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; import static org.commonjava.indy.model.core.AccessChannel.GENERIC_PROXY; import static org.commonjava.indy.pkg.PackageTypeConstants.PKG_TYPE_GENERIC_HTTP; @@ -31,7 +34,9 @@ public class TrackedContentEntry implements Comparable,Externalizable { - private static final int VERSION = 2; + private static final long serialVersionUID = 6469004486206600578L; + + private static final int VERSION = 3; private TrackingKey trackingKey; @@ -55,6 +60,8 @@ public class TrackedContentEntry private long index = System.currentTimeMillis(); + private Set timestamps; + public TrackedContentEntry() { } @@ -74,6 +81,7 @@ public TrackedContentEntry( final TrackingKey trackingKey, final StoreKey storeK this.sha1=sha1; this.sha256=sha256; this.size = size; + this.timestamps = new HashSet<>( Collections.singleton( System.currentTimeMillis() ) ); } public String getOriginUrl() @@ -278,6 +286,7 @@ public void writeExternal( final ObjectOutput out ) out.writeObject( sha256 == null ? "" : sha256 ); out.writeObject( size ); out.writeLong( index ); + out.writeObject( timestamps ); } @Override @@ -357,6 +366,29 @@ else if ( whatIsThis instanceof TrackingKey ) index = in.readLong(); + if ( version > 2 ) + { + final Set tstamps = (Set) in.readObject(); + timestamps = tstamps; + } + } + + public Set getTimestamps() + { + return timestamps; + } + + public void setTimestamps( final Set timestamps ) + { + this.timestamps = timestamps; } + public void merge( TrackedContentEntry from ) + { + this.md5 = from.md5; + this.sha1 = from.sha1; + this.sha256 = from.sha256; + this.size = from.size; + this.timestamps.addAll( from.timestamps ); + } } diff --git a/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryTest.java b/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryTest.java index f7e08d9aae..a707454101 100644 --- a/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryTest.java +++ b/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryTest.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.util.Set; import static org.commonjava.indy.folo.model.StoreEffect.DOWNLOAD; import static org.commonjava.indy.model.core.AccessChannel.GENERIC_PROXY; @@ -33,6 +34,7 @@ import static org.commonjava.indy.pkg.PackageTypeConstants.PKG_TYPE_GENERIC_HTTP; import static org.commonjava.indy.pkg.PackageTypeConstants.PKG_TYPE_MAVEN; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.Assert.assertThat; public class TrackedContentEntryTest @@ -86,6 +88,59 @@ public void externalizeAsV1_readCurrentVersion() assertThat( out.getSha256(), equalTo( test.getSha256() ) ); } + /** + * We have to hack this test a bit in order to test the ability to deserialize this first version of + * TrackedContentEntry. + * @throws IOException + * @throws ClassNotFoundException + */ + @Test + public void externalizeAsV2_readCurrentVersion() + throws IOException, ClassNotFoundException + { + TrackedContentEntryV2 ev2 = new TrackedContentEntryV2( new TrackingKey( "test-key" ), + new StoreKey( PKG_TYPE_GENERIC_HTTP, remote, + "some-upstream" ), GENERIC_PROXY, + "http://some.upstream.url/path/to/file", "path/to/file", + DOWNLOAD, 10101010L, "aaaafffffccccceeeeddd", + "bbbcccceeeedddaaaaa", "aaadadaaaadaeee" ); + + TrackedContentEntry out = new TrackedContentEntry( new TrackingKey( "test-key2" ), + new StoreKey( PKG_TYPE_MAVEN, hosted, + "some-upstream2" ), MAVEN_REPO, + "http://some.upstream.url/path/to/file2", "path/to/file2", + DOWNLOAD, 10101011L, "aaaafffffccccceeeedddfffffff", + "bbbcccceeeedddaaaaaffffffff", "aaadadaaaadaeeeffffffff" ); + + // nullify this to make sure that reading from the old version of the object doesn't set it. + out.setTimestamps( null ); + + TrackedContentEntry test = + new TrackedContentEntry( ev2.getTrackingKey(), ev2.getStoreKey(), ev2.getAccessChannel(), + ev2.getOriginUrl(), ev2.getPath(), ev2.getEffect(), ev2.getSize(), + ev2.getMd5(), ev2.getSha1(), ev2.getSha256() ); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream( baos ); + ev2.writeExternal( oos ); + oos.flush(); + + ObjectInputStream oin = new ObjectInputStream( new ByteArrayInputStream( baos.toByteArray() ) ); + out.readExternal( oin ); + + assertThat( out.getTrackingKey(), equalTo( test.getTrackingKey() ) ); + assertThat( out.getStoreKey(), equalTo( test.getStoreKey() ) ); + assertThat( out.getAccessChannel(), equalTo( test.getAccessChannel() ) ); + assertThat( out.getOriginUrl(), equalTo( test.getOriginUrl() ) ); + assertThat( out.getPath(), equalTo( test.getPath() ) ); + assertThat( out.getEffect(), equalTo( test.getEffect() ) ); + assertThat( out.getSize(), equalTo( test.getSize() ) ); + assertThat( out.getMd5(), equalTo( test.getMd5() ) ); + assertThat( out.getSha1(), equalTo( test.getSha1() ) ); + assertThat( out.getSha256(), equalTo( test.getSha256() ) ); + assertThat( out.getTimestamps(), nullValue() ); + } + @Test public void serializeRoundTrip_CurrentVersion() throws IOException, ClassNotFoundException @@ -120,5 +175,6 @@ public void serializeRoundTrip_CurrentVersion() assertThat( out.getMd5(), equalTo( test.getMd5() ) ); assertThat( out.getSha1(), equalTo( test.getSha1() ) ); assertThat( out.getSha256(), equalTo( test.getSha256() ) ); + assertThat( out.getTimestamps(), equalTo( test.getTimestamps() ) ); } } diff --git a/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryV2.java b/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryV2.java new file mode 100644 index 0000000000..e1a6e83727 --- /dev/null +++ b/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryV2.java @@ -0,0 +1,363 @@ +/** + * Copyright (C) 2011-2019 Red Hat, Inc. (https://github.com/Commonjava/indy) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.commonjava.indy.folo.model; + +import org.commonjava.indy.model.core.AccessChannel; +import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.model.core.StoreType; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.commonjava.indy.model.core.AccessChannel.GENERIC_PROXY; +import static org.commonjava.indy.pkg.PackageTypeConstants.PKG_TYPE_GENERIC_HTTP; +import static org.commonjava.indy.pkg.PackageTypeConstants.PKG_TYPE_MAVEN; + +public class TrackedContentEntryV2 + implements Comparable, Externalizable +{ + private static final int VERSION = 2; + + private TrackingKey trackingKey; + + private StoreKey storeKey; + + private AccessChannel accessChannel; + + private String path; + + private String originUrl; + + private StoreEffect effect; + + private String md5; + + private String sha256; + + private String sha1; + + private Long size; + + private long index = System.currentTimeMillis(); + + public TrackedContentEntryV2() + { + } + + public TrackedContentEntryV2( final TrackingKey trackingKey, final StoreKey storeKey, + final AccessChannel accessChannel, final String originUrl, final String path, + final StoreEffect effect, final Long size, + final String md5, final String sha1, final String sha256 ) + { + this.trackingKey = trackingKey; + this.storeKey = storeKey; + this.accessChannel = accessChannel; + this.path = path; + this.originUrl = originUrl; + this.effect = effect; + this.md5=md5; + this.sha1=sha1; + this.sha256=sha256; + this.size = size; + } + + public String getOriginUrl() + { + return originUrl; + } + + public String getMd5() + { + return md5; + } + + public String getSha256() + { + return sha256; + } + + public String getSha1() + { + return sha1; + } + + public StoreKey getStoreKey() + { + return storeKey; + } + + public AccessChannel getAccessChannel() + { + return accessChannel; + } + + public String getPath() + { + return path; + } + + public TrackingKey getTrackingKey() + { + return trackingKey; + } + + public StoreEffect getEffect() + { + return effect; + } + + public Long getSize() + { + return size; + } + + public long getIndex() + { + return index; + } + + public void setStoreKey( StoreKey storeKey ) + { + this.storeKey = storeKey; + } + + public void setOriginUrl( String originUrl ) + { + this.originUrl = originUrl; + } + + @Override + public int compareTo( final TrackedContentEntryV2 other ) + { + int comp = trackingKey.getId().compareTo( other.getTrackingKey().getId() ); + if ( comp == 0 ) + { + comp = storeKey.compareTo( other.getStoreKey() ); + } + if ( comp == 0 ) + { + comp = accessChannel.compareTo( other.getAccessChannel() ); + } + if ( comp == 0 ) + { + comp = path.compareTo( other.getPath() ); + } + if ( comp == 0 ) + { + comp = effect.compareTo( other.getEffect() ); + } + + return comp; + } + + @Override + public int hashCode() + { + final int prime = 31; + int result = 1; + result = prime * result + ( ( trackingKey == null ) ? 0 : trackingKey.hashCode() ); + result = prime * result + ( ( path == null ) ? 0 : path.hashCode() ); + result = prime * result + ( ( storeKey == null ) ? 0 : storeKey.hashCode() ); + result = prime * result + ( ( accessChannel == null ) ? 0 : accessChannel.hashCode() ); + result = prime * result + ( ( effect == null ) ? 0 : effect.hashCode() ); + return result; + } + + @Override + public boolean equals( final Object obj ) + { + if ( this == obj ) + { + return true; + } + if ( obj == null ) + { + return false; + } + if ( getClass() != obj.getClass() ) + { + return false; + } + final TrackedContentEntryV2 other = (TrackedContentEntryV2) obj; + if ( trackingKey == null ) + { + if ( other.trackingKey != null ) + { + return false; + } + } + else if ( !trackingKey.equals( other.trackingKey ) ) + { + return false; + } + if ( path == null ) + { + if ( other.path != null ) + { + return false; + } + } + else if ( !path.equals( other.path ) ) + { + return false; + } + if ( storeKey == null ) + { + if ( other.storeKey != null ) + { + return false; + } + } + else if ( !storeKey.equals( other.storeKey ) ) + { + return false; + } + if ( accessChannel == null ) + { + if ( other.accessChannel != null ) + { + return false; + } + } + else if ( !accessChannel.equals( other.accessChannel ) ) + { + return false; + } + if ( effect == null ) + { + if ( other.effect != null ) + { + return false; + } + } + else if ( !effect.equals( other.effect ) ) + { + return false; + } + return true; + } + + @Override + public String toString() + { + return String.format( + "TrackedContentEntry [\n trackingKey=%s\n storeKey=%s\n accessChannel=%s\n path=%s\n originUrl=%s\n effect=%s\n md5=%s\n sha1=%s\n sha256=%s\nObject hashcode=%s\n]", + trackingKey, storeKey, accessChannel, path, originUrl, effect, md5, sha1, sha256, super.hashCode() ); + } + + @Override + public void writeExternal( final ObjectOutput out ) + throws IOException + { + out.writeObject( Integer.toString( VERSION ) ); + out.writeObject( trackingKey ); + out.writeObject( storeKey.getPackageType() ); + out.writeObject( storeKey.getName() ); + out.writeObject( storeKey.getType().name() ); + out.writeObject( accessChannel.name() ); + out.writeObject( path == null ? "" : path ); + out.writeObject( originUrl == null ? "" : originUrl ); + out.writeObject( effect == null ? "" : effect.name() ); + out.writeObject( md5 == null ? "" : md5 ); + out.writeObject( sha1 == null ? "" : sha1 ); + out.writeObject( sha256 == null ? "" : sha256 ); + out.writeObject( size ); + out.writeLong( index ); + } + + @Override + public void readExternal( final ObjectInput in ) + throws IOException, ClassNotFoundException + { + // This is a little awkward. The original version didn't have a version constant, so it wasn't possible + // to just read it from the data stream and use it to guide the deserialization process. Instead, + // we have to read the first object and determine whether it's the object version or the tracking key, + // which was the first field in the serialized data stream back in the first version of this class. + Object whatIsThis = in.readObject(); + + int version; + String packageType; + + if ( whatIsThis == null ) + { + // we see NumberFormatException: For input string: "null" in parseInt. It might be because we forget + // to persist the trackingKey for some reason. In this case, we just ignore it and continues + version = 1; + trackingKey = null; + packageType = PKG_TYPE_MAVEN; + } + else if ( whatIsThis instanceof TrackingKey ) + { + version = 1; + trackingKey = (TrackingKey) whatIsThis; + packageType = PKG_TYPE_MAVEN; + } + else + { + version = Integer.parseInt( String.valueOf( whatIsThis ) ); + trackingKey = (TrackingKey) in.readObject(); + packageType = (String) in.readObject(); + } + + // TODO: We should make future versioning / deserialization decisions based on the version we read / infer above + if ( version > VERSION ) + { + throw new IOException( + "This class is of an older version: " + VERSION + " vs. the version read from the data stream: " + + version + ". Cannot deserialize." ); + } + + final String storeKeyName = (String) in.readObject(); + final StoreType storeType = StoreType.get( (String) in.readObject() ); + + final String accessChannelStr = (String) in.readObject(); + accessChannel = "".equals( accessChannelStr ) ? null : AccessChannel.valueOf( accessChannelStr ); + + if ( version == 1 && accessChannel == GENERIC_PROXY ) + { + packageType = PKG_TYPE_GENERIC_HTTP; + } + + storeKey = new StoreKey( packageType, storeType, storeKeyName ); + + final String pathStr = (String) in.readObject(); + path = "".equals( pathStr ) ? null : pathStr; + + final String originUrlStr = (String) in.readObject(); + originUrl = "".equals( originUrlStr ) ? null : originUrlStr; + + final String effectStr = (String) in.readObject(); + effect = "".equals( effectStr ) ? null : StoreEffect.valueOf( effectStr ); + + final String md5Str = (String) in.readObject(); + md5 = "".equals( md5Str ) ? null : md5Str; + + final String sha1Str = (String) in.readObject(); + sha1 = "".equals( sha1Str ) ? null : sha1Str; + + final String sha256Str = (String) in.readObject(); + sha256 = "".equals( sha256Str ) ? null : sha256Str; + + size = (Long) in.readObject(); + + index = in.readLong(); + } +} diff --git a/addons/implied-repos/common/src/main/java/org/commonjava/indy/implrepo/change/ImpliedRepositoryDetector.java b/addons/implied-repos/common/src/main/java/org/commonjava/indy/implrepo/change/ImpliedRepositoryDetector.java index 0b5a01145c..4f498a0c13 100644 --- a/addons/implied-repos/common/src/main/java/org/commonjava/indy/implrepo/change/ImpliedRepositoryDetector.java +++ b/addons/implied-repos/common/src/main/java/org/commonjava/indy/implrepo/change/ImpliedRepositoryDetector.java @@ -54,11 +54,13 @@ import java.net.MalformedURLException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; import static org.commonjava.indy.implrepo.data.ImpliedRepoMetadataManager.IMPLIED_BY_STORES; import static org.commonjava.indy.implrepo.data.ImpliedRepoMetadataManager.IMPLIED_STORES; @@ -399,19 +401,30 @@ else if ( !config.isIncludeSnapshotRepos() && !repo.isReleasesEnabled() ) e ); } + final RemoteRepository ref = creator.createFrom( gav, repo, LoggerFactory.getLogger( creator.getClass() ) ); + if ( ref == null ) + { + logger.warn( + "ImpliedRepositoryCreator didn't create anything for repo: {}, specified in: {}. Skipping.", + repo.getId(), gav ); + continue; + } + + if ( rrs != null && !rrs.isEmpty() ) + { + rrs = rrs.stream() + .filter( rr -> rr.isAllowReleases() == ref.isAllowReleases() ) + .filter( rr -> rr.isAllowSnapshots() == ref.isAllowSnapshots() ) + .filter( rr -> ( isEmpty(rr.getPathMaskPatterns()) && isEmpty( ref.getPathMaskPatterns() ) ) + || rr.getPathMaskPatterns().equals( ref.getPathMaskPatterns() ) ) + .collect( Collectors.toList() ); + } + if ( rrs == null || rrs.isEmpty() ) { logger.debug( "Creating new RemoteRepository for: {}", repo ); - final RemoteRepository rr = creator.createFrom( gav, repo, LoggerFactory.getLogger( creator.getClass() ) ); - if ( rr == null ) - { - logger.warn( - "ImpliedRepositoryCreator didn't create anything for repo: {}, specified in: {}. Skipping.", - repo.getId(), gav ); - continue; - } - + final RemoteRepository rr = ref.copyOf(); rr.setMetadata( METADATA_ORIGIN, IMPLIED_REPO_ORIGIN ); try { @@ -459,11 +472,49 @@ else if ( !config.isIncludeSnapshotRepos() && !repo.isReleasesEnabled() ) else { logger.debug( "Found existing RemoteRepositories: {}", rrs ); + + for ( final RemoteRepository rr : rrs ) + { + rr.setMetadata( METADATA_ORIGIN, IMPLIED_REPO_ORIGIN ); + try + { + metadataManager.updateImpliedBy( rr, job.store ); + } + catch ( ImpliedReposException e ) + { + logger.error( "Failed to set {}", IMPLIED_BY_STORES ); + continue; + } + + final String changelog = String.format( + "Updating the existing remote repository: %s (url: %s, name: %s), which is implied by the POM: %s (at: %s/%s)", + repo.getId(), repo.getUrl(), repo.getName(), gav, job.transfer.getLocation().getUri(), + job.transfer.getPath() ); + final ChangeSummary summary = new ChangeSummary( ChangeSummary.SYSTEM_USER, changelog ); + try + { + final boolean result = storeManager.storeArtifactStore( rr, summary, false, false, + null ); + + logger.debug( "Updated the RemoteRepository: {}. (successful? {})", rr, result ); + job.implied.add( rr ); + } + catch ( final IndyDataException e ) + { + logger.error( String.format( + "Cannot add implied remote repo: %s from: %s (transfer: %s). Failed to update the remote repository.", + repo.getUrl(), gav, job.transfer ), e ); + } + } } } } } + private boolean isEmpty(final Collection coll) { + return coll == null || coll.isEmpty(); + } + public class ImplicationsJob { private final FileStorageEvent event; diff --git a/addons/implied-repos/ftests/src/main/java/org/commonjava/indy/implrepo/skim/ReuseExistingRepoIntoConfiguredGroupTest.java b/addons/implied-repos/ftests/src/main/java/org/commonjava/indy/implrepo/skim/ReuseExistingRepoIntoConfiguredGroupTest.java new file mode 100644 index 0000000000..a4d513c049 --- /dev/null +++ b/addons/implied-repos/ftests/src/main/java/org/commonjava/indy/implrepo/skim/ReuseExistingRepoIntoConfiguredGroupTest.java @@ -0,0 +1,92 @@ +package org.commonjava.indy.implrepo.skim; + +import org.apache.commons.io.IOUtils; +import org.commonjava.indy.implrepo.data.ImpliedRepoMetadataManager; +import org.commonjava.indy.model.core.Group; +import org.commonjava.indy.model.core.RemoteRepository; +import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.model.core.StoreType; +import org.commonjava.indy.pkg.maven.model.MavenPackageTypeDescriptor; +import org.junit.Test; + +import java.io.InputStream; +import java.util.Collections; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; + +/** + * GIVEN: + *
    + *
  • A pre-existing remote repo pr with path p
    • + *
  • A pom in remote repo test with path p same as above existing repo path
    • + *
  • The pom contains declaration of a repo r
    • + *
  • Group pub contains remote repo test
    • + *
  • No remote repo point to repo r contained in group pub at first
    • + *
+ * + *
+ * WHEN: + *
    + *
  • Access pom through path p in group pub
    • + *
+ * + *
+ * THEN: + *
    + *
  • A pre-existing remote repo pr will be added into group pub
    • + *
  • Remote test will have metatada "implied_stores" point to "remote:pr"
    • + *
  • This remote pr will have metatada "implied_by_stores" point to "remote:test"
    • + *
+ */ +public class ReuseExistingRepoIntoConfiguredGroupTest + extends AbstractSkimFunctionalTest +{ + + private static final String REPO = "i-repo-one"; + + @Test + public void skimPomForExistingRepoAndAddItInGroup() throws Exception + { + RemoteRepository repo = new RemoteRepository( REPO, server.formatUrl( REPO ) ); + repo.setAllowReleases( Boolean.TRUE ); + repo.setAllowSnapshots( Boolean.FALSE ); + repo = client.stores().create( repo, "Pre stored remote repo", RemoteRepository.class ); + + final StoreKey remoteRepoKey = repo.getKey(); + final PomRef ref = loadPom( "one-repo", Collections.singletonMap( "one-repo.url", server.formatUrl( REPO ) ) ); + + server.expect( "HEAD", server.formatUrl( REPO, "/" ), 200, (String) null ); + server.expect( server.formatUrl( TEST_REPO, ref.path ), 200, ref.pom ); + + final StoreKey pubGroupKey = new StoreKey( MavenPackageTypeDescriptor.MAVEN_PKG_KEY, StoreType.group, PUBLIC ); + Group g = client.stores().load( pubGroupKey, Group.class ); + assertThat( "Group membership should not contain implied before getting pom.", + g.getConstituents().contains( remoteRepoKey ), equalTo( false ) ); + + logger.debug( "Start fetching pom!" ); + final InputStream stream = client.content().get( pubGroupKey, ref.path ); + final String downloaded = IOUtils.toString( stream ); + IOUtils.closeQuietly( stream ); + + System.out.println( "Waiting 5s for events to run." ); + Thread.sleep( 5000 ); + + g = client.stores().load( pubGroupKey, Group.class ); + assertThat( "Group membership does not contain implied repository", + g.getConstituents().contains( remoteRepoKey ), equalTo( true ) ); + + repo = client.stores() + .load( new StoreKey( MavenPackageTypeDescriptor.MAVEN_PKG_KEY, StoreType.remote, TEST_REPO ), + RemoteRepository.class ); + String metadata = repo.getMetadata( ImpliedRepoMetadataManager.IMPLIED_STORES ); + assertThat( "Reference to repositories implied by POMs in this repo is missing from metadata.", + metadata.contains( "remote:" + REPO ), equalTo( true ) ); + + repo = client.stores().load( remoteRepoKey, RemoteRepository.class ); + metadata = repo.getMetadata( ImpliedRepoMetadataManager.IMPLIED_BY_STORES ); + assertThat( "Backref to repo with pom that implies this repo is missing from metadata.", + metadata.contains( "remote:" + TEST_REPO ), equalTo( true ) ); + } + +} diff --git a/addons/koji/common/src/main/java/org/commonjava/indy/koji/content/KojiContentManagerDecorator.java b/addons/koji/common/src/main/java/org/commonjava/indy/koji/content/KojiContentManagerDecorator.java index d35dd87910..e582da1617 100644 --- a/addons/koji/common/src/main/java/org/commonjava/indy/koji/content/KojiContentManagerDecorator.java +++ b/addons/koji/common/src/main/java/org/commonjava/indy/koji/content/KojiContentManagerDecorator.java @@ -97,7 +97,7 @@ public abstract class KojiContentManagerDecorator implements ContentManager { - private Logger logger = LoggerFactory.getLogger( getClass() ); + private Logger logger = LoggerFactory.getLogger( KojiContentManagerDecorator.class.getName() ); public static final String CREATION_TRIGGER_GAV = "creation-trigger-GAV"; diff --git a/addons/pkg-maven/common/src/main/java/org/commonjava/indy/pkg/maven/content/MavenContentAdvisor.java b/addons/pkg-maven/common/src/main/java/org/commonjava/indy/pkg/maven/content/MavenContentAdvisor.java index 6c32e9fc3c..9169ab7cfa 100644 --- a/addons/pkg-maven/common/src/main/java/org/commonjava/indy/pkg/maven/content/MavenContentAdvisor.java +++ b/addons/pkg-maven/common/src/main/java/org/commonjava/indy/pkg/maven/content/MavenContentAdvisor.java @@ -43,23 +43,21 @@ public MavenContentAdvisor( SpecialPathManager specialPathManager ) @Override public ContentQuality getContentQuality( String path ) { + final SpecialPathInfo info = specialPathManager.getSpecialPathInfo( path ); + if ( info != null && info.isMetadata() ) + { + return ContentQuality.METADATA; + } + final ArtifactPathInfo pathInfo = ArtifactPathInfo.parse( path ); if ( pathInfo != null ) { - final SpecialPathInfo info = specialPathManager.getSpecialPathInfo( path ); - if ( info != null && info.isMetadata() ) - { - return ContentQuality.METADATA; - } - if ( pathInfo.isSnapshot() ) { return ContentQuality.SNAPSHOT; } - return ContentQuality.RELEASE; } - //FIXME: needs further think here if null and RELEASE have the same meaning? return null; } } diff --git a/api/src/main/java/org/commonjava/indy/data/StoreValidator.java b/api/src/main/java/org/commonjava/indy/data/StoreValidator.java index 84d828ca6f..5d59efb089 100644 --- a/api/src/main/java/org/commonjava/indy/data/StoreValidator.java +++ b/api/src/main/java/org/commonjava/indy/data/StoreValidator.java @@ -33,6 +33,5 @@ public interface StoreValidator { /* Validate ArtifactStore instances */ - public ArtifactStoreValidateData validate(ArtifactStore artifactStore) - throws InvalidArtifactStoreException, MalformedURLException; + public ArtifactStoreValidateData validate(ArtifactStore artifactStore); } diff --git a/api/src/main/java/org/commonjava/indy/util/ApplicationHeader.java b/api/src/main/java/org/commonjava/indy/util/ApplicationHeader.java index e73910ded5..88a457d3a0 100644 --- a/api/src/main/java/org/commonjava/indy/util/ApplicationHeader.java +++ b/api/src/main/java/org/commonjava/indy/util/ApplicationHeader.java @@ -32,7 +32,10 @@ public enum ApplicationHeader cache_control( "Cache-Control" ), content_disposition( "Content-Disposition" ), indy_origin( "Indy-Origin" ), - transfer_encoding( "Transfer-Encoding" ); + transfer_encoding( "Transfer-Encoding" ), + md5("INDY-MD5"), + sha1("INDY-SHA1") + ; private final String key; diff --git a/bindings/jaxrs/src/main/java/org/commonjava/indy/core/bind/jaxrs/ContentAccessHandler.java b/bindings/jaxrs/src/main/java/org/commonjava/indy/core/bind/jaxrs/ContentAccessHandler.java index cfe012847e..603c9466df 100644 --- a/bindings/jaxrs/src/main/java/org/commonjava/indy/core/bind/jaxrs/ContentAccessHandler.java +++ b/bindings/jaxrs/src/main/java/org/commonjava/indy/core/bind/jaxrs/ContentAccessHandler.java @@ -20,6 +20,7 @@ import org.commonjava.indy.bind.jaxrs.util.JaxRsRequestHelper; import org.commonjava.indy.bind.jaxrs.util.REST; import org.commonjava.indy.bind.jaxrs.util.ResponseHelper; +import org.commonjava.indy.content.ContentDigester; import org.commonjava.indy.content.ContentManager; import org.commonjava.indy.core.bind.jaxrs.util.RequestUtils; import org.commonjava.indy.core.bind.jaxrs.util.TransferCountingInputStream; @@ -30,12 +31,9 @@ import org.commonjava.indy.model.core.PackageTypes; import org.commonjava.indy.model.core.StoreKey; import org.commonjava.indy.model.core.StoreType; -import org.commonjava.indy.util.AcceptInfo; -import org.commonjava.indy.util.ApplicationContent; -import org.commonjava.indy.util.ApplicationStatus; -import org.commonjava.indy.util.LocationUtils; -import org.commonjava.indy.util.UriFormatter; +import org.commonjava.indy.util.*; import org.commonjava.maven.galley.event.EventMetadata; +import org.commonjava.maven.galley.io.checksum.ContentDigest; import org.commonjava.maven.galley.model.SpecialPathInfo; import org.commonjava.maven.galley.model.Transfer; import org.commonjava.maven.galley.model.TransferOperation; @@ -95,6 +93,9 @@ public class ContentAccessHandler @Inject private ResponseHelper responseHelper; + + @Inject + ContentDigester contentDigester; protected ContentAccessHandler() @@ -307,6 +308,13 @@ public Response doHead( final String packageType, final String type, final Strin responseHelper.setInfoHeaders( builder, item, sk, path, true, contentType, httpMetadata ); + + if(!path.endsWith("/")) { + // Content hashing headers + builder.header(ApplicationHeader.md5.key(), contentDigester.digest(sk, path, new EventMetadata()).getDigests().get(ContentDigest.MD5)); + builder.header(ApplicationHeader.sha1.key(), contentDigester.digest(sk, path, new EventMetadata()).getDigests().get(ContentDigest.SHA_1)); + } + if ( builderModifier != null ) { builderModifier.accept( builder ); diff --git a/clients/core-java/src/main/java/org/commonjava/indy/client/core/module/IndyContentClientModule.java b/clients/core-java/src/main/java/org/commonjava/indy/client/core/module/IndyContentClientModule.java index a0e010a42d..191009c142 100644 --- a/clients/core-java/src/main/java/org/commonjava/indy/client/core/module/IndyContentClientModule.java +++ b/clients/core-java/src/main/java/org/commonjava/indy/client/core/module/IndyContentClientModule.java @@ -65,6 +65,12 @@ public String contentPath( final StoreKey key, final String... path ) return buildUrl( null, aggregatePathParts( key, path ) ); } + /** + * Please do not use this method to get list content, because new content browse is not using html but using json now, + * so this method will return wrong result now. Please use {@link org.commonjava.indy.content.browse.client.IndyContentBrowseClientModule} instead. + * This method will be removed in recent release. + */ + @Deprecated public DirectoryListingDTO listContents( final StoreKey key, final String path ) throws IndyClientException { @@ -77,6 +83,11 @@ public DirectoryListingDTO listContents( final StoreKey key, final String path ) return http.get( contentPath( key, p ), DirectoryListingDTO.class ); } + /** + * Please do not use this method to get list content, because new content browse is not using html but using json now, + * so this method will return wrong result now. Please use {@link org.commonjava.indy.content.browse.client.IndyContentBrowseClientModule} instead. + * This method will be removed in recent release. + */ @Deprecated public DirectoryListingDTO listContents( final StoreType type, final String name, final String path ) throws IndyClientException diff --git a/core/src/main/java/org/commonjava/indy/core/content/DefaultContentManager.java b/core/src/main/java/org/commonjava/indy/core/content/DefaultContentManager.java index a50741f8f2..cace0cc4a5 100644 --- a/core/src/main/java/org/commonjava/indy/core/content/DefaultContentManager.java +++ b/core/src/main/java/org/commonjava/indy/core/content/DefaultContentManager.java @@ -64,7 +64,7 @@ public class DefaultContentManager implements ContentManager { - private final Logger logger = LoggerFactory.getLogger( getClass() ); + private final Logger logger = LoggerFactory.getLogger( DefaultContentManager.class.getName() ); @Inject private ContentGeneratorManager contentGeneratorManager; diff --git a/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java b/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java index 2aeb4fa1b9..aa97876198 100644 --- a/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java +++ b/db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java @@ -25,6 +25,7 @@ import org.commonjava.indy.model.core.ArtifactStore; import org.commonjava.indy.model.core.HostedRepository; import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.model.core.StoreType; import org.commonjava.indy.util.ApplicationStatus; import org.commonjava.maven.galley.event.EventMetadata; import org.slf4j.Logger; @@ -289,37 +290,14 @@ protected boolean store( final ArtifactStore store, final ChangeSummary summary, logger.warn("Storing {} using operation lock: {}", store, opLocks); -// if(configuration != null) { - ArtifactStoreValidateData validateData = null ; - try { - -// if(configuration.isSSLRequired()) { - if (internalFeatureConfig.getStoreValidation()) { - validateData = storeValidator.validate(store); - - if (!validateData.isValid()) { - logger.warn("=> [AbstractStoreDataManager] Disabling Remote Store: " + store.getKey() + - " with name: " + store.getName()); - disableNotValidStore(store, validateData); - } - } -// } - } - catch (MalformedURLException mue) { - logger.warn("=> [AbstractStoreDataManager] MalformedURLException:" + mue.getMessage()); - // Disable Store - disableNotValidStore(store,validateData); - } catch (IndyDataException ide) { - logger.warn("=> [AbstractStoreDataManager] IndyDataException: " + ide.getMessage()); - // Disable Store - disableNotValidStore(store,validateData); - } catch (Exception e) { - logger.warn("=> [AbstractStoreDataManager] Exception:" + e); - // Disable Store -// disableNotValidStore(store,validateData); - } -// } + if (internalFeatureConfig != null && internalFeatureConfig.getStoreValidation() && store.getType() != StoreType.group) { + ArtifactStoreValidateData validateData = storeValidator.validate(store); + if (!validateData.isValid()) { + logger.warn("=> [AbstractStoreDataManager] Disabling Remote Store: " + store.getKey() + " with name: " + store.getName()); + store.getMetadata().putAll(validateData.getErrors()); + } + } Function lockHandler = k -> doStore( k, store, summary, error, skipIfExists, fireEvents, eventMetadata ); diff --git a/embedder/pom.xml b/embedder/pom.xml index 7519effa00..049355d2e8 100644 --- a/embedder/pom.xml +++ b/embedder/pom.xml @@ -67,6 +67,10 @@ org.commonjava.indy indy-subsys-metrics-reporter + + org.commonjava.indy + indy-subsys-metrics-prometheus + org.commonjava.indy indy-subsys-cpool diff --git a/embedder/src/test/resources/logback-test.xml b/embedder/src/test/resources/logback-test.xml index e3cf2048d0..1fb6d129fb 100644 --- a/embedder/src/test/resources/logback-test.xml +++ b/embedder/src/test/resources/logback-test.xml @@ -48,7 +48,7 @@ - + diff --git a/ftests/core/src/main/java/org/commonjava/indy/data/DefaultStoreValidatorTest.java b/ftests/core/src/main/java/org/commonjava/indy/data/DefaultStoreValidatorTest.java index 3a7faa8a3e..896e8bbf59 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/data/DefaultStoreValidatorTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/data/DefaultStoreValidatorTest.java @@ -106,15 +106,9 @@ public void run() throws Exception { RemoteRepository remoteRepository2 = client.stores().create(notValidUrlRepo, changelog, RemoteRepository.class); ArtifactStoreValidateData validateUrl = null; - try { - LOGGER.warn("=> Start Validating RemoteRepository: [" + remoteRepository2.getUrl()+"]"); - validateUrl = validator.validate(remoteRepository2); - LOGGER.warn("=> Returned [Not Valid URL] ArtifactStoreValidateData: " + validateUrl.toString()); - } catch (MalformedURLException mue) { - if(validateUrl != null ) { - LOGGER.warn("=> Returned [Not Valid URL Exception] in ArtifactStoreValidateData: " + validateUrl.toString()); - } - } + LOGGER.warn("=> Start Validating RemoteRepository: [" + remoteRepository2.getUrl()+"]"); + validateUrl = validator.validate(remoteRepository2); + LOGGER.warn("=> Returned [Not Valid URL] ArtifactStoreValidateData: " + validateUrl.toString()); RemoteRepository allowedRemoteRepo = @@ -129,7 +123,7 @@ public void run() throws Exception { assertNotNull( validateAllowedRepo ); assertFalse(validateAllowedRepo.isValid()); - assertNotNull( validateAllowedRepo.getErrors().get("disabled") ); +// assertNotNull( validateAllowedRepo.getErrors().get("disabled") ); diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/HostedContentIndexRescanTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/HostedContentIndexRescanTest.java index daa4c29046..85d8081435 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/HostedContentIndexRescanTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/HostedContentIndexRescanTest.java @@ -80,6 +80,6 @@ protected void initTestConfig( CoreServerFixture fixture ) throws IOException { super.initTestConfig( fixture ); - writeConfigFile( "conf.d/content-index.conf", "[content-index]\nsupport.authoritative.indexes=true" ); + writeConfigFile( "conf.d/content-index.conf", "[content-index]\nenabled=true\nsupport.authoritative.indexes=true" ); } } diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/RepositoryPathMaskMetadataTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/RepositoryPathMaskMetadataTest.java index e9d0397afd..a3e9c09c01 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/RepositoryPathMaskMetadataTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/RepositoryPathMaskMetadataTest.java @@ -103,7 +103,7 @@ public void run() HostedRepository hostedRepo1 = new HostedRepository( hosted1 ); pathMaskPatterns = new HashSet<>(); - pathMaskPatterns.add("org/bar.*"); + pathMaskPatterns.add("r|org/bar.*|"); hostedRepo1.setPathMaskPatterns(pathMaskPatterns); hostedRepo1 = client.stores().create( hostedRepo1, "adding hosted 1", HostedRepository.class ); client.content().store( hosted, hosted1, path_metadata, new ByteArrayInputStream( meta2.getBytes() ) ); diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/StoreSnapshotMetadataTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/StoreSnapshotMetadataTest.java new file mode 100644 index 0000000000..81668fcb7e --- /dev/null +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/StoreSnapshotMetadataTest.java @@ -0,0 +1,65 @@ +/** + * Copyright (C) 2011-2019 Red Hat, Inc. (https://github.com/Commonjava/indy) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.commonjava.indy.ftest.core.content; + +import org.commonjava.indy.ftest.core.AbstractContentManagementTest; +import org.commonjava.indy.model.core.HostedRepository; +import org.commonjava.indy.model.core.StoreKey; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; + +import static org.commonjava.indy.model.core.StoreType.hosted; +import static org.commonjava.indy.pkg.maven.model.MavenPackageTypeDescriptor.MAVEN_PKG_KEY; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; + +public class StoreSnapshotMetadataTest + extends AbstractContentManagementTest +{ + final String store = "local-deployments"; + + @Test + public void storeFileAndVerifyExistence() throws Exception + { + final String changelog = "Create local-deployments"; + + StoreKey storeKey = new StoreKey( MAVEN_PKG_KEY, hosted, store ); + + final HostedRepository hosted = new HostedRepository( MAVEN_PKG_KEY, store ); + hosted.setAllowSnapshots( true ); + hosted.setAllowReleases( false ); + + this.client.stores().create( hosted, changelog, HostedRepository.class ); + + final InputStream stream = new ByteArrayInputStream( ( "This is a test: " + System.nanoTime() ).getBytes() ); + + final String path = "org/commonjava/util/path-mapped-storage/0.1-SNAPSHOT/maven-metadata.xml"; + + assertThat( client.content().exists( storeKey, path ), equalTo( false ) ); + + client.content().store( storeKey, path, stream ); + + assertThat( client.content().exists( storeKey, path ), equalTo( true ) ); + } + + protected boolean createStandardTestStructures() + { + return false; + } + +} diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/AuthoritativeIndexedContentInHostedTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/AuthoritativeIndexedContentInHostedTest.java index 52e93d5da0..c3a08d459a 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/AuthoritativeIndexedContentInHostedTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/AuthoritativeIndexedContentInHostedTest.java @@ -154,6 +154,6 @@ protected void initTestConfig( CoreServerFixture fixture ) throws IOException { super.initTestConfig( fixture ); - writeConfigFile( "conf.d/content-index.conf", "[content-index]\nsupport.authoritative.indexes=true" ); + writeConfigFile( "conf.d/content-index.conf", "[content-index]\nenabled=true\nsupport.authoritative.indexes=true" ); } } diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/AuthoritativeIndexedContentInRemoteTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/AuthoritativeIndexedContentInRemoteTest.java index 96b1ab7efd..05d990d82e 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/AuthoritativeIndexedContentInRemoteTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/AuthoritativeIndexedContentInRemoteTest.java @@ -137,6 +137,6 @@ protected void initTestConfig( CoreServerFixture fixture ) throws IOException { super.initTestConfig( fixture ); - writeConfigFile( "conf.d/content-index.conf", "[content-index]\nsupport.authoritative.indexes=true" ); + writeConfigFile( "conf.d/content-index.conf", "[content-index]\nenabled=true\nsupport.authoritative.indexes=true" ); } } diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexDirLvWithArtifactsTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexDirLvWithArtifactsTest.java index ed7bbd529f..98425a126f 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexDirLvWithArtifactsTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexDirLvWithArtifactsTest.java @@ -22,6 +22,7 @@ import org.commonjava.indy.model.core.Group; import org.commonjava.indy.model.core.RemoteRepository; import org.commonjava.indy.subsys.infinispan.BasicCacheHandle; +import org.commonjava.indy.test.fixture.core.CoreServerFixture; import org.commonjava.test.http.expect.ExpectationServer; import org.infinispan.AdvancedCache; import org.junit.Before; @@ -29,6 +30,7 @@ import org.junit.Test; import javax.enterprise.inject.spi.CDI; +import java.io.IOException; import java.io.InputStream; import java.util.Arrays; import java.util.List; @@ -168,4 +170,11 @@ public void test() } + @Override + protected void initTestConfig( final CoreServerFixture fixture ) + throws IOException + { + super.initTestConfig( fixture ); + writeConfigFile( "conf.d/content-index.conf", "[content-index]\nenabled=true" ); + } } diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexDirLvWithMetadataTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexDirLvWithMetadataTest.java index e2795bc090..5296fa9508 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexDirLvWithMetadataTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexDirLvWithMetadataTest.java @@ -24,6 +24,7 @@ import org.commonjava.indy.model.core.StoreKey; import org.commonjava.indy.subsys.infinispan.BasicCacheHandle; import org.commonjava.indy.subsys.infinispan.CacheHandle; +import org.commonjava.indy.test.fixture.core.CoreServerFixture; import org.commonjava.test.http.expect.ExpectationServer; import org.infinispan.AdvancedCache; import org.junit.Before; @@ -31,6 +32,7 @@ import org.junit.Test; import javax.enterprise.inject.spi.CDI; +import java.io.IOException; import java.io.InputStream; import java.util.Arrays; import java.util.List; @@ -133,4 +135,11 @@ public void test() } + @Override + protected void initTestConfig( final CoreServerFixture fixture ) + throws IOException + { + super.initTestConfig( fixture ); + writeConfigFile( "conf.d/content-index.conf", "[content-index]\nenabled=true" ); + } } diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexGroupUsageTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexGroupUsageTest.java index ab3bfaa604..364ffae0bc 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexGroupUsageTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexGroupUsageTest.java @@ -22,12 +22,14 @@ import org.commonjava.indy.model.core.Group; import org.commonjava.indy.model.core.RemoteRepository; import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.test.fixture.core.CoreServerFixture; import org.commonjava.test.http.expect.ExpectationServer; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import javax.enterprise.inject.spi.CDI; +import java.io.IOException; import java.io.InputStream; import static org.commonjava.indy.pkg.maven.model.MavenPackageTypeDescriptor.MAVEN_PKG_KEY; @@ -78,6 +80,14 @@ public void getIndexManager() indexManager = CDI.current().select( ContentIndexManager.class ).get(); } + @Override + protected void initTestConfig( final CoreServerFixture fixture ) + throws IOException + { + super.initTestConfig( fixture ); + writeConfigFile( "conf.d/content-index.conf", "[content-index]\nenabled=true" ); + } + @Test public void bypassNotIndexedContentWithAuthoritativeIndex() throws Exception diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexNestedGroupAndStoreDeletionTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexNestedGroupAndStoreDeletionTest.java index 546408bda7..409cba974d 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexNestedGroupAndStoreDeletionTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexNestedGroupAndStoreDeletionTest.java @@ -23,12 +23,14 @@ import org.commonjava.indy.model.core.Group; import org.commonjava.indy.model.core.RemoteRepository; import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.test.fixture.core.CoreServerFixture; import org.commonjava.test.http.expect.ExpectationServer; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import javax.enterprise.inject.spi.CDI; +import java.io.IOException; import java.io.InputStream; import static org.commonjava.indy.pkg.maven.model.MavenPackageTypeDescriptor.MAVEN_PKG_KEY; @@ -148,4 +150,12 @@ private void indexExist( ArtifactStore... stores ) assertThat( "Indexed StoreKey not found for " + store.getKey(), indexedStoreKey, notNullValue() ); } } + + @Override + protected void initTestConfig( final CoreServerFixture fixture ) + throws IOException + { + super.initTestConfig( fixture ); + writeConfigFile( "conf.d/content-index.conf", "[content-index]\nenabled=true" ); + } } diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexRemoteRepoUsageTest.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexRemoteRepoUsageTest.java index 233bcf51ab..35bae64157 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexRemoteRepoUsageTest.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/content/contentindex/ContentIndexRemoteRepoUsageTest.java @@ -23,12 +23,14 @@ import org.commonjava.indy.model.core.RemoteRepository; import org.commonjava.indy.model.core.StoreKey; import org.commonjava.indy.subsys.infinispan.CacheHandle; +import org.commonjava.indy.test.fixture.core.CoreServerFixture; import org.commonjava.test.http.expect.ExpectationServer; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import javax.enterprise.inject.spi.CDI; +import java.io.IOException; import java.io.InputStream; import static org.commonjava.indy.pkg.maven.model.MavenPackageTypeDescriptor.MAVEN_PKG_KEY; @@ -124,4 +126,12 @@ public void bypassNotIndexedContentWithAuthoritativeIndex() // object equality should work since we haven't persisted this anywhere yet. assertThat( indexedStoreKey == indexedStoreKey2, equalTo( true ) ); } + + @Override + protected void initTestConfig( final CoreServerFixture fixture ) + throws IOException + { + super.initTestConfig( fixture ); + writeConfigFile( "conf.d/content-index.conf", "[content-index]\nenabled=true" ); + } } diff --git a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/store/HostedAuthIndexWithReadonly.java b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/store/HostedAuthIndexWithReadonly.java index 894b109b92..41eb834120 100644 --- a/ftests/core/src/main/java/org/commonjava/indy/ftest/core/store/HostedAuthIndexWithReadonly.java +++ b/ftests/core/src/main/java/org/commonjava/indy/ftest/core/store/HostedAuthIndexWithReadonly.java @@ -96,6 +96,6 @@ protected void initTestConfig( CoreServerFixture fixture ) throws IOException { super.initTestConfig( fixture ); - writeConfigFile( "conf.d/content-index.conf", "[content-index]\nsupport.authoritative.indexes=true" ); + writeConfigFile( "conf.d/content-index.conf", "[content-index]\nenabled=true\nsupport.authoritative.indexes=true" ); } } diff --git a/ftests/core/src/main/java/org/commonjava/indy/jaxrs/ContentBrowseHeadRequestTest.java b/ftests/core/src/main/java/org/commonjava/indy/jaxrs/ContentBrowseHeadRequestTest.java new file mode 100644 index 0000000000..8dc63c059f --- /dev/null +++ b/ftests/core/src/main/java/org/commonjava/indy/jaxrs/ContentBrowseHeadRequestTest.java @@ -0,0 +1,85 @@ +package org.commonjava.indy.jaxrs; + + +import org.apache.http.client.methods.HttpHead; +import org.commonjava.indy.client.core.IndyClientException; +import org.commonjava.indy.client.core.IndyClientHttp; +import org.commonjava.indy.client.core.helper.HttpResources; +import org.commonjava.indy.client.core.module.IndyRawHttpModule; +import org.commonjava.indy.ftest.core.AbstractIndyFunctionalTest; +import org.commonjava.indy.model.core.RemoteRepository; +import static org.junit.Assert.*; + +import static org.hamcrest.core.Is.*; + +import org.hamcrest.core.Is; +import org.junit.Test; + +import javax.validation.constraints.AssertTrue; + +public class ContentBrowseHeadRequestTest extends AbstractIndyFunctionalTest { + + private static final String REMOTE_MAVEN_CENTRAL = "/browse/maven/remote/central"; + private static final String REMOTE_NPM_CENTRAL = "/browse/npm/remote/npmjs"; + + private static final String REMOTE_MAVEN_CENTRAL_URL = "https://repo.maven.apache.org/maven2/"; + private static final String REMOTE_NPM_CENTRAL_URL = "https://npmjs.com"; + + + private static final String REMOTE_MAVEN_TEST_SUB_URL = "/browse/maven/remote/m-central/test/test"; + private static final String REMOTE_NPM_TEST_SUB_URL = "/browse/npm/remote/n-central/test/test"; + + @Test + public void run() { + + final String changelog = "Create repo validation test structure"; + + try { + RemoteRepository mavenCentral = + new RemoteRepository( "maven", "m-central", REMOTE_MAVEN_CENTRAL_URL ); + RemoteRepository npmCentral = + new RemoteRepository( "npm", "n-central", REMOTE_NPM_CENTRAL_URL ); + + + client.stores().create(mavenCentral,changelog,RemoteRepository.class); + client.stores().create(npmCentral,changelog,RemoteRepository.class); + + + IndyClientHttp client = getHttp(); + + + HttpHead httpMavenHeadReq = new HttpHead(REMOTE_MAVEN_CENTRAL); + HttpResources mavenResponse = client.execute(httpMavenHeadReq); + assertThat(200, is(mavenResponse.getStatusCode())); + + HttpHead httpNpmHeadReq = new HttpHead(REMOTE_NPM_CENTRAL); + HttpResources npmResponse = client.execute(httpNpmHeadReq); + assertThat(200, is(npmResponse.getStatusCode())); + + + HttpHead httpMavenHeadSubReq = new HttpHead(REMOTE_MAVEN_TEST_SUB_URL); + HttpResources mavenSubResponse = client.execute(httpMavenHeadSubReq); + assertThat(200, is(mavenSubResponse.getStatusCode())); + + HttpHead httpNpmHeadSubReq = new HttpHead(REMOTE_NPM_TEST_SUB_URL); + HttpResources npmSubResponse = client.execute(httpNpmHeadSubReq); + assertThat(200, is(npmSubResponse.getStatusCode())); + + + + + } catch (IndyClientException e) { + e.printStackTrace(); + } + + + } + + + protected IndyClientHttp getHttp() + throws IndyClientException + { + return client.module( IndyRawHttpModule.class ) + .getHttp(); + } +} diff --git a/models/core-java/src/main/java/org/commonjava/indy/model/core/dto/DirectoryListingDTO.java b/models/core-java/src/main/java/org/commonjava/indy/model/core/dto/DirectoryListingDTO.java index e36766190e..8c8d384584 100644 --- a/models/core-java/src/main/java/org/commonjava/indy/model/core/dto/DirectoryListingDTO.java +++ b/models/core-java/src/main/java/org/commonjava/indy/model/core/dto/DirectoryListingDTO.java @@ -17,6 +17,10 @@ import java.util.List; +/** + * @deprecated New content browse service is json based now, so this DTO is not used anymore. Will be removed in recent release + */ +@Deprecated public class DirectoryListingDTO { diff --git a/pom.xml b/pom.xml index 4ba78ec4b3..d61222c478 100644 --- a/pom.xml +++ b/pom.xml @@ -722,6 +722,11 @@ indy-subsys-metrics-reporter 1.9.6-SNAPSHOT + + org.commonjava.indy + indy-subsys-metrics-prometheus + 1.9.6-SNAPSHOT + org.commonjava.indy indy-subsys-metrics-reporter @@ -1023,6 +1028,17 @@ 3.0.6 + + io.prometheus + simpleclient_servlet + 0.7.0 + + + io.prometheus + simpleclient_dropwizard + 0.7.0 + + io.github.hengyunabc zabbix-api diff --git a/subsys/jaxrs/src/main/java/org/commonjava/indy/bind/jaxrs/util/ResponseHelper.java b/subsys/jaxrs/src/main/java/org/commonjava/indy/bind/jaxrs/util/ResponseHelper.java index 7127efb37e..3189a71ee1 100644 --- a/subsys/jaxrs/src/main/java/org/commonjava/indy/bind/jaxrs/util/ResponseHelper.java +++ b/subsys/jaxrs/src/main/java/org/commonjava/indy/bind/jaxrs/util/ResponseHelper.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.codec.digest.DigestUtils; import org.commonjava.indy.IndyWorkflowException; +import org.commonjava.indy.content.ContentDigester; import org.commonjava.indy.metrics.IndyMetricsManager; import org.commonjava.indy.metrics.conf.IndyMetricsConfig; import org.commonjava.indy.model.core.StoreKey; @@ -26,6 +27,8 @@ import org.commonjava.indy.util.ApplicationHeader; import org.commonjava.indy.util.ApplicationStatus; import org.commonjava.indy.util.LocationUtils; +import org.commonjava.maven.galley.event.EventMetadata; +import org.commonjava.maven.galley.io.checksum.ContentDigest; import org.commonjava.maven.galley.model.Transfer; import org.commonjava.maven.galley.transport.htcli.model.HttpExchangeMetadata; import org.slf4j.Logger; @@ -62,6 +65,9 @@ public class ResponseHelper @Inject private IndyMetricsConfig metricsConfig; + + @Inject + ContentDigester contentDigester; public Response formatRedirect( final URI uri ) { @@ -171,6 +177,8 @@ public ResponseBuilder setInfoHeaders( final ResponseBuilder builder, final Tran boolean lastModSet = false; boolean lenSet = false; boolean conTypeSet = false; + + if ( exchangeMetadata != null ) { @@ -204,6 +212,8 @@ else if ( ApplicationHeader.content_length.upperKey().equals( key ) ) } } } + + if ( item != null && item.exists() ) { @@ -229,6 +239,8 @@ else if ( ApplicationHeader.content_length.upperKey().equals( key ) ) // Indy origin contains the storeKey of the repository where the content came from builder.header( ApplicationHeader.indy_origin.key(), LocationUtils.getKey( item ).toString() ); + + } else { diff --git a/subsys/metrics/pom.xml b/subsys/metrics/pom.xml index f7eaf53e14..df90099f46 100644 --- a/subsys/metrics/pom.xml +++ b/subsys/metrics/pom.xml @@ -32,6 +32,7 @@ core reporter + prometheus diff --git a/subsys/metrics/prometheus/pom.xml b/subsys/metrics/prometheus/pom.xml new file mode 100644 index 0000000000..82b84f2161 --- /dev/null +++ b/subsys/metrics/prometheus/pom.xml @@ -0,0 +1,83 @@ + + + + + indy-subsys-metrics + org.commonjava.indy + 1.9.6-SNAPSHOT + + 4.0.0 + + indy-subsys-metrics-prometheus + + + + io.prometheus + simpleclient_dropwizard + + + io.prometheus + simpleclient_servlet + + + org.commonjava.indy + indy-subsys-metrics-core + + + org.commonjava.indy + indy-subsys-jaxrs + + + javax.enterprise + cdi-api + provided + + + org.commonjava.indy + indy-subsys-flatfile + + + org.commonjava.util + http-testserver + + + + + \ No newline at end of file diff --git a/subsys/metrics/prometheus/src/main/java/org/commonjava/indy/metrics/prometheus/IndySampleBuilder.java b/subsys/metrics/prometheus/src/main/java/org/commonjava/indy/metrics/prometheus/IndySampleBuilder.java new file mode 100644 index 0000000000..f9e936d27f --- /dev/null +++ b/subsys/metrics/prometheus/src/main/java/org/commonjava/indy/metrics/prometheus/IndySampleBuilder.java @@ -0,0 +1,36 @@ +package org.commonjava.indy.metrics.prometheus; + +import io.prometheus.client.Collector; +import io.prometheus.client.dropwizard.samplebuilder.DefaultSampleBuilder; + +import java.util.ArrayList; +import java.util.List; + +public class IndySampleBuilder + extends DefaultSampleBuilder +{ + private static final String NODE_NAME_LABEL = "node"; + + private String nodeName; + + public IndySampleBuilder( String nodeName ) + { + super(); + this.nodeName = nodeName; + } + + @Override + public Collector.MetricFamilySamples.Sample createSample( final String dropwizardName, final String nameSuffix, + final List additionalLabelNames, + final List additionalLabelValues, + final double value ) + { + List labelNames = new ArrayList( additionalLabelNames ); + labelNames.add( NODE_NAME_LABEL ); + + List labelValues = new ArrayList( additionalLabelValues ); + labelValues.add( nodeName ); + + return super.createSample( dropwizardName, nameSuffix, labelNames, labelValues, value ); + } +} diff --git a/subsys/metrics/prometheus/src/main/java/org/commonjava/indy/metrics/prometheus/PrometheusDeployment.java b/subsys/metrics/prometheus/src/main/java/org/commonjava/indy/metrics/prometheus/PrometheusDeployment.java new file mode 100644 index 0000000000..972b02b38f --- /dev/null +++ b/subsys/metrics/prometheus/src/main/java/org/commonjava/indy/metrics/prometheus/PrometheusDeployment.java @@ -0,0 +1,57 @@ +package org.commonjava.indy.metrics.prometheus; + +import com.codahale.metrics.MetricRegistry; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.dropwizard.DropwizardExports; +import io.prometheus.client.exporter.MetricsServlet; +import io.undertow.servlet.Servlets; +import io.undertow.servlet.api.DeploymentInfo; +import io.undertow.servlet.api.ServletInfo; +import org.commonjava.indy.bind.jaxrs.IndyDeploymentProvider; +import org.commonjava.indy.bind.jaxrs.metrics.IndyHealthCheckServletContextListener; +import org.commonjava.indy.metrics.conf.IndyMetricsConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.enterprise.context.ApplicationScoped; +import javax.inject.Inject; +import javax.ws.rs.core.Application; + +@ApplicationScoped +public class PrometheusDeployment + extends IndyDeploymentProvider +{ + private static final String PROMETHEUS_REPORTER = "prometheus"; + + private final Logger logger = LoggerFactory.getLogger( getClass() ); + + @Inject + private IndyMetricsConfig config; + + @Inject + private MetricRegistry metricRegistry; + + @Override + public DeploymentInfo getDeploymentInfo( String contextRoot, Application application ) + { + if ( !config.isMetricsEnabled() || !config.getReporter().contains( PROMETHEUS_REPORTER ) ) + { + return null; + } + + CollectorRegistry.defaultRegistry.register( new DropwizardExports( metricRegistry, new IndySampleBuilder( config.getNodePrefix() ) ) ); + + final ServletInfo servlet = + Servlets.servlet( "prometheus-metrics", MetricsServlet.class ).addMapping( "/metrics" ); + + final DeploymentInfo di = new DeploymentInfo().addListener( + Servlets.listener( IndyHealthCheckServletContextListener.class ) ) + .setContextPath( contextRoot ) + .addServlet( servlet ) + .setDeploymentName( "Prometheus Metrics Deployment" ) + .setClassLoader( ClassLoader.getSystemClassLoader() ); + + logger.info( "Returning deployment info for Prometheus metrics servlet" ); + return di; + } +}