Master merge 1.7.x

addons/folo/common/src/main/java/org/commonjava/indy/folo/data/idxmodel/TrackingKey2StringMapper.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2013~2019 Red Hat, Inc.
+ * Copyright (C) 2011-2018 Red Hat, Inc. (https://github.com/Commonjava/indy)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

addons/folo/common/src/test/java/org/commonjava/indy/folo/data/TrackingKey2StringMapperTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2013~2019 Red Hat, Inc.
+ * Copyright (C) 2011-2018 Red Hat, Inc. (https://github.com/Commonjava/indy)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryTest.java

@@ -1,3 +1,18 @@
+/**
+ * Copyright (C) 2011-2018 Red Hat, Inc. (https://github.com/Commonjava/indy)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.commonjava.indy.folo.model;
 
 import org.commonjava.indy.model.core.AccessChannel;

addons/promote/common/src/main/java/org/commonjava/indy/promote/data/PromotionManager.java

@@ -58,6 +58,7 @@
 import org.commonjava.maven.galley.spi.nfc.NotFoundCache;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
 
 import javax.annotation.PostConstruct;
 import javax.enterprise.context.ApplicationScoped;
@@ -97,6 +98,18 @@
 public class PromotionManager
 {
 
+    private static final String PROMOTION_TYPE = "promotion-type";
+
+    private static final String PATH_PROMOTION = "path";
+
+    private static final String GROUP_PROMOTION = "group";
+
+    private static final String PROMOTION_SOURCE = "promotion-source";
+
+    private static final String PROMOTION_TARGET = "promotion-target";
+
+    private static final String PROMOTION_CONTENT_PATH = "promotion-content-path";
+
     private final Logger logger = LoggerFactory.getLogger( getClass() );
 
     @Inject
@@ -189,6 +202,10 @@ public PromotionManager( PromotionValidator validator, final ContentManager cont
     public GroupPromoteResult promoteToGroup( GroupPromoteRequest request, String user, String baseUrl )
             throws PromotionException, IndyWorkflowException
     {
+        MDC.put( PROMOTION_TYPE, GROUP_PROMOTION );
+        MDC.put( PROMOTION_SOURCE, request.getSource().toString() );
+        MDC.put( PROMOTION_TARGET, request.getTargetKey().toString() );
+
         if ( !storeManager.hasArtifactStore( request.getSource() ) )
         {
             String error = String.format( "Cannot promote from missing source: %s", request.getSource() );
@@ -485,6 +502,10 @@ private Future<GroupPromoteResult> submitGroupPromoteRequest( final GroupPromote
     public PathsPromoteResult promotePaths( final PathsPromoteRequest request, final String baseUrl )
             throws PromotionException, IndyWorkflowException
     {
+        MDC.put( PROMOTION_TYPE, PATH_PROMOTION );
+        MDC.put( PROMOTION_SOURCE, request.getSource().toString() );
+        MDC.put( PROMOTION_TARGET, request.getTargetKey().toString() );
+
         Future<PathsPromoteResult> future = submitPathsPromoteRequest( request, baseUrl );
         if ( request.isAsync() )
         {
@@ -527,25 +548,67 @@ private PathsPromoteResult doPathsPromotion( PathsPromoteRequest request, String
             pending.add( transfer.getPath() );
         }
 
-        ValidationResult validation = new ValidationResult();
-        ValidationRequest validationRequest = validator.validate( request, validation, baseUrl );
-        if ( request.isDryRun() )
-        {
-            return new PathsPromoteResult( request, pending, Collections.emptySet(), Collections.emptySet(),
-                                           validation );
-        }
-        else if ( validation.isValid() )
-        {
-            PathsPromoteResult result = runPathPromotions( request, pending, Collections.emptySet(), Collections.emptySet(), contents,
-                                                           validation, validationRequest );
-            doPathPromoteMetrics( contents.size(), result );
-            return result;
-        }
-        else
+        AtomicReference<Exception> ex = new AtomicReference<>();
+
+        PathsPromoteResult promoteResult = byPathTargetLocks.lockAnd( request.getTargetKey(), config.getLockTimeoutSeconds(), k -> {
+            try
+            {
+                ValidationResult validation = new ValidationResult();
+                ValidationRequest validationRequest = validator.validate( request, validation, baseUrl );
+                if ( request.isDryRun() )
+                {
+                    return new PathsPromoteResult( request, pending, Collections.emptySet(), Collections.emptySet(),
+                                                   validation );
+                }
+                else if ( validation.isValid() )
+                {
+                    PathsPromoteResult result = runPathPromotions( request, pending, Collections.emptySet(), Collections.emptySet(),
+                                                                   contents, validation, validationRequest );
+                    doPathPromoteMetrics( contents.size(), result );
+                    return result;
+                }
+                else
+                {
+                    return new PathsPromoteResult( request, pending, Collections.emptySet(), Collections.emptySet(),
+                                                   validation );
+                }
+            }
+            catch ( PromotionValidationException e )
+            {
+                ex.set( e );
+            }
+            catch ( IndyWorkflowException e )
+            {
+                ex.set( e );
+            }
+
+            return null;
+        }, ( k, lock ) -> {
+            String error = String.format( "Failed to acquire promotion lock on target: %s in %d seconds.",
+                                          request.getTargetKey(), config.getLockTimeoutSeconds() );
+
+            logger.error( error );
+            ex.set( new IndyWorkflowException( error ) );
+
+            return false;
+        } );
+
+        Exception e = ex.get();
+        if ( e != null )
         {
-            return new PathsPromoteResult( request, pending, Collections.emptySet(), Collections.emptySet(),
-                                           validation );
+            if ( e instanceof PromotionValidationException )
+            {
+                throw (PromotionValidationException) e;
+            }
+            else if ( e instanceof IndyWorkflowException )
+            {
+                throw (IndyWorkflowException) e;
+            }
+
+            throw new IndyWorkflowException( "Promotion failed.", e );
         }
+
+        return promoteResult;
     }
 
     private void doPathPromoteMetrics( int total, PathsPromoteResult result )
@@ -888,69 +951,49 @@ private PathsPromoteResult runPathPromotions( final PathsPromoteRequest request,
 
             AtomicReference<IndyWorkflowException> wfError = new AtomicReference<>();
 
-            Set<PathTransferResult> results =
-                    byPathTargetLocks.lockAnd( targetKey, config.getLockTimeoutSeconds(), k -> {
-                        logger.info( "Running promotions from: {} (key: {})\n  to: {} (key: {})", src,
-                                     request.getSource(), tgt, request.getTarget() );
-
-                        DrainingExecutorCompletionService<PathTransferResult> svc =
-                                new DrainingExecutorCompletionService<>( transferService );
-
-                        try
-                        {
-                            detectOverloadVoid( () -> contents.forEach( ( transfer ) -> svc.submit(
-                                    newPathsPromoteTransfer( transfer, src, tgt, request ) ) ) );
+            logger.info( "Running promotions from: {} (key: {})\n  to: {} (key: {})", src, request.getSource(), tgt,
+                         request.getTarget() );
 
-                            Set<PathTransferResult> pathResults = new HashSet<>();
-                            try
-                            {
-                                svc.drain( ptr -> pathResults.add( ptr ) );
-                            }
-                            catch ( InterruptedException | ExecutionException e )
-                            {
-                                Set<String> paths;
-                                try
-                                {
-                                    paths = validationRequest.getSourcePaths();
-                                }
-                                catch ( PromotionValidationException e1 )
-                                {
-                                    paths = contents.stream()
-                                                    .map( txfr -> txfr.getPath() )
-                                                    .collect( Collectors.toSet() );
-                                }
+            DrainingExecutorCompletionService<PathTransferResult> svc =
+                    new DrainingExecutorCompletionService<>( transferService );
 
-                                logger.error(
-                                        String.format( "Error waiting for promotion of: %s to: %s\nPaths:\n\n%s\n\n",
-                                                       request.getSource(), targetKey, paths ), e );
-                            }
+            Set<PathTransferResult> results = new HashSet<>();
+            detectOverloadVoid( () -> contents.forEach( ( transfer ) -> svc.submit(
+                    newPathsPromoteTransfer( transfer, src, tgt, request ) ) ) );
 
-                            try
-                            {
-                                clearStoreNFC( validationRequest.getSourcePaths(), tgt );
-                            }
-                            catch ( IndyDataException | PromotionValidationException e )
-                            {
-                                String msg = String.format( "Failed to promote to: %s. Reason: %s", tgt, e.getMessage() );
-                                errors.add( msg );
-                                logger.error( msg, e );
-                            }
-                            return pathResults;
-                        }
-                        catch ( IndyWorkflowException e )
-                        {
-                            wfError.set( e );
-                            return null;
-                        }
-                    }, ( k, lock ) -> {
-                        String error = String.format( "Failed to acquire promotion lock on target: %s in %d seconds.",
-                                                      targetKey, config.getLockTimeoutSeconds() );
+            try
+            {
+                svc.drain( ptr -> results.add( ptr ) );
+            }
+            catch ( InterruptedException | ExecutionException e )
+            {
+                Set<String> paths;
+                try
+                {
+                    paths = validationRequest.getSourcePaths();
+                }
+                catch ( PromotionValidationException e1 )
+                {
+                    paths = contents.stream()
+                                    .map( txfr -> txfr.getPath() )
+                                    .collect( Collectors.toSet() );
+                }
 
-                        errors.add( error );
-                        logger.warn( error );
+                logger.error(
+                        String.format( "Error waiting for promotion of: %s to: %s\nPaths:\n\n%s\n\n",
+                                       request.getSource(), targetKey, paths ), e );
+            }
 
-                        return false;
-                    } );
+            try
+            {
+                clearStoreNFC( validationRequest.getSourcePaths(), tgt );
+            }
+            catch ( IndyDataException | PromotionValidationException e )
+            {
+                String msg = String.format( "Failed to promote to: %s. Reason: %s", tgt, e.getMessage() );
+                errors.add( msg );
+                logger.error( msg, e );
+            }
 
             if ( wfError.get() != null )
             {
@@ -997,6 +1040,7 @@ private PathsPromoteResult runPathPromotions( final PathsPromoteRequest request,
             PathsPromoteCompleteEvent evt = new PathsPromoteCompleteEvent( result );
             fireEvent( promoteCompleteEvent, evt );
         }
+
         return result;
     }
 
@@ -1023,6 +1067,8 @@ private Callable<PathTransferResult> newPathsPromoteTransfer( final Transfer tra
                                                                   final PathsPromoteRequest request )
     {
         return () -> {
+            MDC.put( PROMOTION_CONTENT_PATH, transfer.getPath() );
+
             PathTransferResult result = new PathTransferResult( transfer.getPath() );
             final String path = transfer.getPath();
 

addons/promote/common/src/main/java/org/commonjava/indy/promote/validate/PromoteValidationsManager.java

@@ -212,6 +212,7 @@ public ValidationRuleSet getRuleSetMatching( StoreKey storeKey )
     {
         if ( ruleSets == null )
         {
+            logger.debug( "No rule sets to match against. No validations will be executed for: {}", storeKey );
             return null;
         }
 
@@ -220,13 +221,15 @@ public ValidationRuleSet getRuleSetMatching( StoreKey storeKey )
                                                  String.format( "%s:%s", storeKey.getType().singularEndpointName(),
                                                                 storeKey.getName() ) );
 
-        for ( ValidationRuleSet set : ruleSets.values() )
+        for ( Map.Entry<String, ValidationRuleSet> entry : ruleSets.entrySet() )
         {
             for ( String keyStr : keyStrings )
             {
-                if ( set.matchesKey( keyStr ) )
+                logger.debug( "Checking for rule-set match. Key='{}', rule-set: '{}'", keyStr, entry.getKey() );
+                if ( entry.getValue().matchesKey( keyStr ) )
                 {
-                    return set;
+                    logger.debug( "Rule set '{}' matches key: '{}'", entry.getKey(), keyStr );
+                    return entry.getValue();
                 }
             }
         }

addons/promote/common/src/main/java/org/commonjava/indy/promote/validate/PromotionValidationTools.java

@@ -17,6 +17,7 @@
 
 import groovy.lang.Closure;
 import org.commonjava.cdi.util.weft.ExecutorConfig;
+import org.commonjava.cdi.util.weft.ThreadContext;
 import org.commonjava.cdi.util.weft.WeftManaged;
 import org.commonjava.indy.IndyWorkflowException;
 import org.commonjava.indy.content.ContentDigester;
@@ -54,6 +55,7 @@
 import org.commonjava.maven.galley.transport.htcli.model.HttpExchangeMetadata;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
 
 import javax.inject.Inject;
 import java.net.URI;
@@ -68,6 +70,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -89,6 +92,10 @@ public class PromotionValidationTools
 
     private static final int DEFAULT_RULE_PARALLEL_WAIT_TIME_MINS = 30;
 
+    private static final String ITERATION_DEPTH = "promotion-validation-parallel-depth";
+
+    private static final String ITERATION_ITEM = "promotion-validation-parallel-item";
+
     @Inject
     private ContentManager contentManager;
 
@@ -600,7 +607,20 @@ private <T> void runParallelInBatchAndWait( Collection<Collection<T>> batches, C
             try
             {
                 logger.trace( "The paralleled exe on batch {}", batch );
-                batch.forEach( e -> closure.call( e ) );
+                batch.forEach( e -> {
+                    String depthStr = MDC.get( ITERATION_DEPTH );
+                    MDC.put( ITERATION_DEPTH, depthStr == null ? "0" : String.valueOf( Integer.parseInt( depthStr ) + 1 ) );
+                    MDC.put( ITERATION_ITEM, String.valueOf( e ) );
+                    try
+                    {
+                        closure.call( e );
+                    }
+                    finally
+                    {
+                        MDC.remove( ITERATION_ITEM );
+                        MDC.remove( ITERATION_DEPTH );
+                    }
+                } );
             }
             finally
             {
@@ -639,6 +659,10 @@ private <T> void runParallelAndWait( Collection<T> runCollection, Closure closur
         Set<T> todo = new HashSet<>( runCollection );
         final CountDownLatch latch = new CountDownLatch( todo.size() );
         todo.forEach( e -> ruleParallelExecutor.execute( () -> {
+            String depthStr = MDC.get( ITERATION_DEPTH );
+            MDC.put( ITERATION_DEPTH, depthStr == null ? "0" : String.valueOf( Integer.parseInt( depthStr ) + 1 ) );
+            MDC.put( ITERATION_ITEM, String.valueOf( e ) );
+
             try
             {
                 logger.trace( "The paralleled exe on element {}", e );
@@ -647,6 +671,8 @@ private <T> void runParallelAndWait( Collection<T> runCollection, Closure closur
             finally
             {
                 latch.countDown();
+                MDC.remove( ITERATION_ITEM );
+                MDC.remove( ITERATION_DEPTH );
             }
         } ) );