Skip to content

Add MDC entries to mark progress in content promotion #1216

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jdcasey merged 2 commits into from
Apr 16, 2019
Merged

Add MDC entries to mark progress in content promotion #1216

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b2354bc
Add MDC entries to mark progress in content promotion
jdcasey Apr 15, 2019
62aca5d
fix iteration depth MDC injection
jdcasey Apr 16, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions addons/promote/common/src/main/java/org/commonjava/indy/promote/data/PromotionManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
import org.commonjava.maven.galley.spi.nfc.NotFoundCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;

import javax.enterprise.context.ApplicationScoped;
import javax.enterprise.event.Event;
Expand Down Expand Up @@ -94,6 +95,18 @@
public class PromotionManager
{

private static final String PROMOTION_TYPE = "promotion-type";

private static final String PATH_PROMOTION = "path";

private static final String GROUP_PROMOTION = "group";

private static final String PROMOTION_SOURCE = "promotion-source";

private static final String PROMOTION_TARGET = "promotion-target";

private static final String PROMOTION_CONTENT_PATH = "promotion-content-path";

private final Logger logger = LoggerFactory.getLogger( getClass() );

@Inject
Expand Down Expand Up @@ -170,6 +183,10 @@ public PromotionManager( PromotionValidator validator, final ContentManager cont
public GroupPromoteResult promoteToGroup( GroupPromoteRequest request, String user, String baseUrl )
throws PromotionException, IndyWorkflowException
{
MDC.put( PROMOTION_TYPE, GROUP_PROMOTION );
MDC.put( PROMOTION_SOURCE, request.getSource().toString() );
MDC.put( PROMOTION_TARGET, request.getTargetKey().toString() );

if ( !storeManager.hasArtifactStore( request.getSource() ) )
{
String error = String.format( "Cannot promote from missing source: %s", request.getSource() );
Expand Down Expand Up @@ -466,6 +483,10 @@ private Future<GroupPromoteResult> submitGroupPromoteRequest( final GroupPromote
public PathsPromoteResult promotePaths( final PathsPromoteRequest request, final String baseUrl )
throws PromotionException, IndyWorkflowException
{
MDC.put( PROMOTION_TYPE, PATH_PROMOTION );
MDC.put( PROMOTION_SOURCE, request.getSource().toString() );
MDC.put( PROMOTION_TARGET, request.getTargetKey().toString() );

Future<PathsPromoteResult> future = submitPathsPromoteRequest( request, baseUrl );
if ( request.isAsync() )
{
Expand Down Expand Up @@ -1013,6 +1034,8 @@ private Callable<PathTransferResult> newPathsPromoteTransfer( final Transfer tra
final PathsPromoteRequest request )
{
return () -> {
MDC.put( PROMOTION_CONTENT_PATH, transfer.getPath() );

PathTransferResult result = new PathTransferResult( transfer.getPath() );
final String path = transfer.getPath();

Expand Down
9 changes: 6 additions & 3 deletions .../common/src/main/java/org/commonjava/indy/promote/validate/PromoteValidationsManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,7 @@ public ValidationRuleSet getRuleSetMatching( StoreKey storeKey )
{
if ( ruleSets == null )
{
logger.debug( "No rule sets to match against. No validations will be executed for: {}", storeKey );
return null;
}

Expand All @@ -220,13 +221,15 @@ public ValidationRuleSet getRuleSetMatching( StoreKey storeKey )
String.format( "%s:%s", storeKey.getType().singularEndpointName(),
storeKey.getName() ) );

for ( ValidationRuleSet set : ruleSets.values() )
for ( Map.Entry<String, ValidationRuleSet> entry : ruleSets.entrySet() )
{
for ( String keyStr : keyStrings )
{
if ( set.matchesKey( keyStr ) )
logger.debug( "Checking for rule-set match. Key='{}', rule-set: '{}'", keyStr, entry.getKey() );
if ( entry.getValue().matchesKey( keyStr ) )
{
return set;
logger.debug( "Rule set '{}' matches key: '{}'", entry.getKey(), keyStr );
return entry.getValue();
}
}
}
Expand Down
109 changes: 82 additions & 27 deletions ...e/common/src/main/java/org/commonjava/indy/promote/validate/PromotionValidationTools.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import groovy.lang.Closure;
import org.commonjava.cdi.util.weft.ExecutorConfig;
import org.commonjava.cdi.util.weft.ThreadContext;
import org.commonjava.cdi.util.weft.WeftManaged;
import org.commonjava.indy.IndyWorkflowException;
import org.commonjava.indy.content.ContentDigester;
Expand Down Expand Up @@ -54,6 +55,7 @@
import org.commonjava.maven.galley.transport.htcli.model.HttpExchangeMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;

import javax.inject.Inject;
import java.net.URI;
Expand All @@ -68,6 +70,7 @@
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand All @@ -89,6 +92,10 @@ public class PromotionValidationTools

private static final int DEFAULT_RULE_PARALLEL_WAIT_TIME_MINS = 30;

private static final String ITERATION_DEPTH = "promotion-validation-parallel-depth";

private static final String ITERATION_ITEM = "promotion-validation-parallel-item";

@Inject
private ContentManager contentManager;

Expand Down Expand Up @@ -595,20 +602,47 @@ public <K, V> void paralleledInBatch( Map<K, V> map, Closure closure )

private <T> void runParallelInBatchAndWait( Collection<Collection<T>> batches, Closure closure, Logger logger )
{
final CountDownLatch latch = new CountDownLatch( batches.size() );
batches.forEach( batch -> ruleParallelExecutor.execute( () -> {
try
{
logger.trace( "The paralleled exe on batch {}", batch );
batch.forEach( e -> closure.call( e ) );
}
finally
{
latch.countDown();
}
} ) );
ThreadContext ctx = ThreadContext.getContext( true );
AtomicInteger depth = (AtomicInteger) ctx.computeIfAbsent( ITERATION_DEPTH, k -> new AtomicInteger( -1 ) );
int nextDepth = depth.incrementAndGet();
if ( nextDepth > 0 )
{
logger.warn( "Nested parallel iteration detected in promotion validation rule!!" );
}

waitForCompletion( latch );
try
{
final CountDownLatch latch = new CountDownLatch( batches.size() );
batches.forEach( batch -> ruleParallelExecutor.execute( () -> {
try
{
logger.trace( "The paralleled exe on batch {}", batch );
batch.forEach( e -> {
MDC.put( ITERATION_ITEM, String.valueOf( e ) );
MDC.put( ITERATION_DEPTH, String.valueOf( depth.get() ) );
try
{
closure.call( e );
}
finally
{
MDC.remove( ITERATION_ITEM );
MDC.remove( ITERATION_DEPTH );
}
} );
}
finally
{
latch.countDown();
}
} ) );

waitForCompletion( latch );
}
finally
{
depth.decrementAndGet();
}
}

private <T> Collection<Collection<T>> batch( Collection<T> collection, int batchSize )
Expand Down Expand Up @@ -636,21 +670,42 @@ private <T> Collection<Collection<T>> batch( Collection<T> collection, int batch

private <T> void runParallelAndWait( Collection<T> runCollection, Closure closure, Logger logger )
{
Set<T> todo = new HashSet<>( runCollection );
final CountDownLatch latch = new CountDownLatch( todo.size() );
todo.forEach( e -> ruleParallelExecutor.execute( () -> {
try
{
logger.trace( "The paralleled exe on element {}", e );
closure.call( e );
}
finally
{
latch.countDown();
}
} ) );
ThreadContext ctx = ThreadContext.getContext( true );
AtomicInteger depth = (AtomicInteger) ctx.computeIfAbsent( ITERATION_DEPTH, k -> new AtomicInteger( -1 ) );
int nextDepth = depth.incrementAndGet();
if ( nextDepth > 0 )
{
logger.warn( "Nested parallel iteration detected in promotion validation rule!!" );
}

try
{
Set<T> todo = new HashSet<>( runCollection );
final CountDownLatch latch = new CountDownLatch( todo.size() );
todo.forEach( e -> ruleParallelExecutor.execute( () -> {
MDC.put( ITERATION_ITEM, String.valueOf( e ) );
MDC.put( ITERATION_DEPTH, String.valueOf( depth.get() ) );

try
{
logger.trace( "The paralleled exe on element {}", e );
closure.call( e );
}
finally
{
latch.countDown();
MDC.remove( ITERATION_ITEM );
MDC.remove( ITERATION_DEPTH );
}
} ) );

waitForCompletion( latch );
}
finally
{
depth.decrementAndGet();
}

waitForCompletion( latch );
}

private void waitForCompletion( CountDownLatch latch )
Expand Down
5 changes: 5 additions & 0 deletions ...promote/common/src/main/java/org/commonjava/indy/promote/validate/PromotionValidator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import org.commonjava.maven.galley.model.Transfer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;

import javax.inject.Inject;
import java.io.File;
Expand All @@ -67,6 +68,8 @@ public class PromotionValidator
{
private static final String PROMOTE_REPO_PREFIX = "Promote_";

private static final String PROMOTION_VALIDATION_RULE = "promotion-validation-rule";

@Inject
private PromoteValidationsManager validationsManager;

Expand Down Expand Up @@ -157,6 +160,8 @@ public ValidationRequest validate( PromoteRequest request, ValidationResult resu
for ( String ruleRef : ruleNames )
{
svc.submit( () -> {
MDC.put( PROMOTION_VALIDATION_RULE, ruleRef );

Exception err = null;
try
{
Expand Down