Skip to content

Commit

Permalink
[PDI-15928]Get Repository Names step hangs when working with the Pentaho
Browse files Browse the repository at this point in the history 
Repository - added tests and behaviour for mask
  • Loading branch information
Sergey Travin authored and Sergey Travin committed Mar 1, 2017
1 parent 73cdee8 commit 3dc8f4f
Show file tree
Hide file tree
Showing 3 changed files with 223 additions and 76 deletions.
44 changes: 29 additions & 15 deletions engine/src/org/pentaho/di/trans/steps/getrepositorynames/GetRepositoryNames.java
View file
Expand Up @@ -164,26 +164,25 @@ private List<RepositoryElementMetaInterface> getRepositoryObjects() throws Kettl
RepositoryExtended extendedRep = (RepositoryExtended) repository; RepositoryExtended extendedRep = (RepositoryExtended) repository;
for ( int i = 0; i < meta.getDirectory().length; i++ ) { for ( int i = 0; i < meta.getDirectory().length; i++ ) {
String directoryPath = environmentSubstitute( meta.getDirectory()[i] ); String directoryPath = environmentSubstitute( meta.getDirectory()[i] );
String nameMaskFromMeta = environmentSubstitute( meta.getNameMask()[i] );
//if we have the name mask from user than we applied the name mask for job and transformation
//otherwise we use the * to search all pdi objects
String actualNameMask = Utils.isEmpty( nameMaskFromMeta ) ? "*" : nameMaskFromMeta;
String filter = null; String filter = null;
// by default we look for current level
int depth = 0; int depth = 0;
if ( meta.getObjectTypeSelection().areTransformationsSelected() ) { if ( meta.getObjectTypeSelection().areTransformationsSelected() ) {
filter = actualNameMask + ".ktr"; filter = "*.ktr";
} }
if ( meta.getObjectTypeSelection().areJobsSelected() ) { if ( meta.getObjectTypeSelection().areJobsSelected() ) {
//if we have selected the job and transformation than we have applied filter with both condition // if we have selected the job and transformation than we have applied filter with both condition
filter = Utils.isEmpty( filter ) ? actualNameMask + ".kjb" : filter + "|" + actualNameMask + ".kjb"; filter = Utils.isEmpty( filter ) ? "*.kjb" : filter + "|*.kjb";
} }
// should include unlimited subfolder
if ( meta.getIncludeSubFolders()[i] ) { if ( meta.getIncludeSubFolders()[i] ) {
depth = -1; depth = -1;
} }
RepositoryDirectoryInterface directory = RepositoryDirectoryInterface directory =
extendedRep.loadRepositoryDirectoryTree( directoryPath, filter, depth, true, false, false ); extendedRep.loadRepositoryDirectoryTree( directoryPath, filter, depth, true, false, false );


list.addAll( getRepositoryObjects( directory ) ); list.addAll( getRepositoryObjects( directory, environmentSubstitute( meta.getNameMask()[i] ),
environmentSubstitute( meta.getExcludeNameMask()[i] ) ) );
} }
} else { } else {
// Get the repository directory tree. // Get the repository directory tree.
Expand Down Expand Up @@ -244,14 +243,29 @@ private List<RepositoryElementMetaInterface> getRepositoryObjects( Repository re
return list; return list;
} }


private List<RepositoryElementMetaInterface> getRepositoryObjects( RepositoryDirectoryInterface directory ) private List<RepositoryElementMetaInterface> getRepositoryObjects( RepositoryDirectoryInterface directory,
throws KettleException { String nameMask, String excludeNameMask ) throws KettleException {
List<RepositoryElementMetaInterface> objects = new ArrayList<RepositoryElementMetaInterface>(); List<RepositoryElementMetaInterface> list = new ArrayList<RepositoryElementMetaInterface>();
objects.addAll( directory.getRepositoryObjects() ); if ( directory.getRepositoryObjects() != null ) {
for ( RepositoryDirectoryInterface subdir : directory.getChildren() ) { for ( RepositoryElementMetaInterface object : directory.getRepositoryObjects() ) {
objects.addAll( getRepositoryObjects( subdir ) ); boolean add = false;
if ( Utils.isEmpty( nameMask ) || object.getName().matches( nameMask ) ) {
add = true;
}
if ( !Utils.isEmpty( excludeNameMask ) && object.getName().matches( excludeNameMask ) ) {
add = false;
}
if ( add ) {
list.add( object );
}
}
} }
return objects; if ( directory.getChildren() != null ) {
for ( RepositoryDirectoryInterface subdir : directory.getChildren() ) {
list.addAll( getRepositoryObjects( subdir, nameMask, excludeNameMask ) );
}
}
return list;
} }


} }
250 changes: 189 additions & 61 deletions engine/test-src/org/pentaho/di/trans/steps/getrepositorynames/GetRepositoryNamesTest.java
View file
Expand Up @@ -2,7 +2,7 @@
* *
* Pentaho Data Integration * Pentaho Data Integration
* *
* Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
* *
******************************************************************************* *******************************************************************************
* *
Expand All @@ -24,123 +24,251 @@


import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
import static org.pentaho.di.trans.steps.getrepositorynames.ObjectTypeSelection.*;
import static org.mockito.Mockito.*;


import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID; import java.util.UUID;


import org.apache.commons.vfs2.FileObject; import org.apache.commons.io.FileUtils;
import org.apache.commons.vfs2.FileSystemException; import org.junit.AfterClass;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.pentaho.di.core.KettleClientEnvironment; import org.pentaho.di.core.KettleClientEnvironment;
import org.pentaho.di.core.exception.KettleException; import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.variables.VariableSpace; import org.pentaho.di.core.variables.VariableSpace;
import org.pentaho.di.core.variables.Variables; import org.pentaho.di.core.variables.Variables;
import org.pentaho.di.job.JobMeta;
import org.pentaho.di.repository.Repository; import org.pentaho.di.repository.Repository;
import org.pentaho.di.repository.RepositoryDirectory; import org.pentaho.di.repository.RepositoryDirectory;
import org.pentaho.di.repository.RepositoryDirectoryInterface; import org.pentaho.di.repository.RepositoryDirectoryInterface;
import org.pentaho.di.repository.RepositoryElementMetaInterface;
import org.pentaho.di.repository.RepositoryExtended;
import org.pentaho.di.repository.RepositoryMeta; import org.pentaho.di.repository.RepositoryMeta;
import org.pentaho.di.repository.RepositoryObject;
import org.pentaho.di.repository.RepositoryObjectType;
import org.pentaho.di.repository.filerep.KettleFileRepository; import org.pentaho.di.repository.filerep.KettleFileRepository;
import org.pentaho.di.repository.filerep.KettleFileRepositoryMeta; import org.pentaho.di.repository.filerep.KettleFileRepositoryMeta;
import org.pentaho.di.trans.Trans; import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.StepMeta; import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.utils.TestUtils;


public class GetRepositoryNamesTest { public class GetRepositoryNamesTest {


String baseDirName; static Path baseDirName;
RepositoryMeta repoMeta; static Repository repo;
Repository repo; static RepositoryExtended repoExtended;


@BeforeClass @BeforeClass
public static void setUpBeforeClass() throws KettleException { public static void setUpBeforeClass() throws KettleException, IOException {
KettleClientEnvironment.init(); KettleClientEnvironment.init();
prepareFileRepository();
prepareExtendedRepository();
} }


@Before private static void prepareFileRepository() throws IOException, KettleException {
public void setup() throws KettleException { baseDirName = Files.createTempDirectory( "GetRepositoryNamesIT" );
baseDirName = TestUtils.createTempDir(); RepositoryMeta repoMeta =
repoMeta = new KettleFileRepositoryMeta( UUID.randomUUID().toString(), UUID.randomUUID().toString(), new KettleFileRepositoryMeta( UUID.randomUUID().toString(), UUID.randomUUID().toString(), UUID.randomUUID()
UUID.randomUUID().toString(), baseDirName ); .toString(), baseDirName.toString() );
repo = new KettleFileRepository(); repo = new KettleFileRepository();
repo.init( repoMeta ); repo.init( repoMeta );
repo.connect( null, null ); repo.connect( null, null );


// Populate // Populate
RepositoryDirectoryInterface ROOT_DIR = repo.findDirectory( "/" ); RepositoryDirectoryInterface rootDir = repo.findDirectory( "/" );
RepositoryDirectoryInterface DIR1 = new RepositoryDirectory( ROOT_DIR, "dir1" );
repo.saveRepositoryDirectory( DIR1 ); RepositoryDirectoryInterface subdir1 = new RepositoryDirectory( rootDir, "subdir1" );
TransMeta transMeta = new TransMeta(); repo.saveRepositoryDirectory( subdir1 );
transMeta.setName( "Trans1" );
transMeta.setRepositoryDirectory( DIR1 ); TransMeta transMeta1 = new TransMeta();
repo.save( transMeta, null, null ); transMeta1.setName( "Trans1" );
transMeta1.setRepositoryDirectory( subdir1 );
repo.save( transMeta1, null, null );

JobMeta jobMeta1 = new JobMeta();
jobMeta1.setName( "Job1" );
jobMeta1.setRepositoryDirectory( subdir1 );
repo.save( jobMeta1, null, null );

RepositoryDirectoryInterface subdir2 = new RepositoryDirectory( subdir1, "subdir2" );
repo.saveRepositoryDirectory( subdir2 );

TransMeta transMeta2 = new TransMeta();
transMeta2.setName( "Trans2" );
transMeta2.setRepositoryDirectory( subdir2 );
repo.save( transMeta2, null, null );

JobMeta jobMeta2 = new JobMeta();
jobMeta2.setName( "Job2" );
jobMeta2.setRepositoryDirectory( subdir2 );
repo.save( jobMeta2, null, null );
} }


@After @SuppressWarnings( "deprecation" )
public void cleanup() { private static void prepareExtendedRepository() throws KettleException {
repoExtended = mock( RepositoryExtended.class );
when( repoExtended.loadRepositoryDirectoryTree( anyString(), anyString(), anyInt(), anyBoolean(), anyBoolean(),
anyBoolean() ) ).then( new Answer<RepositoryDirectoryInterface>() {
@Override
public RepositoryDirectoryInterface answer( InvocationOnMock invocation ) throws Throwable {
Object[] args = invocation.getArguments();

RepositoryDirectoryInterface root = new RepositoryDirectory();
root.setName( "/" );
RepositoryDirectoryInterface subdir1 = new RepositoryDirectory( root, "subdir1" );
RepositoryDirectoryInterface subdir2 = new RepositoryDirectory( subdir1, "subdir2" );
RepositoryElementMetaInterface trans1 =
new RepositoryObject( null, "Trans1", subdir1, "user", null, RepositoryObjectType.TRANSFORMATION, "",
false );
RepositoryElementMetaInterface trans2 =
new RepositoryObject( null, "Trans2", subdir2, "user", null, RepositoryObjectType.TRANSFORMATION, "",
false );
RepositoryElementMetaInterface job1 =
new RepositoryObject( null, "Job1", subdir1, "user", null, RepositoryObjectType.JOB, "", false );
RepositoryElementMetaInterface job2 =
new RepositoryObject( null, "Job2", subdir2, "user", null, RepositoryObjectType.JOB, "", false );

List<RepositoryElementMetaInterface> list1 = new ArrayList<RepositoryElementMetaInterface>();
List<RepositoryElementMetaInterface> list2 = new ArrayList<RepositoryElementMetaInterface>();
if ( ( (String) args[1] ).contains( "ktr" ) ) {
list1.add( trans1 );
list2.add( trans2 );
}
if ( ( (String) args[1] ).contains( "kjb" ) ) {
list1.add( job1 );
list2.add( job2 );
}
subdir1.setRepositoryObjects( list1 );
subdir2.setRepositoryObjects( list2 );

if ( ( (Integer) args[2] ) == -1 ) {
subdir1.addSubdirectory( subdir2 );
root.addSubdirectory( subdir1 );
}
String actualPath = ( (String) args[0] );
if ( actualPath.equals( "/" ) ) {
return root;
} else if ( actualPath.equals( subdir1.getPath() ) ) {
return subdir1;
} else if ( actualPath.equals( subdir2.getPath() ) ) {
return subdir2;
} else {
return null;
}
}
} );
}

@AfterClass
public static void tearDownAfterClass() throws KettleException, IOException {
if ( repo != null ) { if ( repo != null ) {
repo.disconnect(); repo.disconnect();
} }
FileObject baseDir = TestUtils.getFileObject( baseDirName ); FileUtils.forceDelete( new File( baseDirName.toString() ) );
if ( baseDir != null ) {
try {
baseDir.delete();
} catch ( FileSystemException ignored ) {
// Ignore, couldn't cleanup after tests
}
}
} }


@Test @Test
public void testGetRepoList() throws KettleException { public void testGetRepoList_includeSubfolders() throws KettleException {
GetRepositoryNamesMeta meta = new GetRepositoryNamesMeta(); init( repo, "/", true, ".*", "", All, 4 );
meta.setDirectory( new String[]{ "/" } ); }
meta.setNameMask( new String[]{ ".*" } );
meta.setExcludeNameMask( new String[]{ "" } );
meta.setIncludeSubFolders( new boolean[]{ true } );
StepMeta stepMeta = new StepMeta( "GetRepoNamesStep", meta );


TransMeta transMeta = new TransMeta(); @Test
transMeta.setRepository( repo ); public void testGetRepoList_excludeSubfolders() throws KettleException {
transMeta.addStep( stepMeta ); init( repo, "/", false, ".*", "", All, 0 );
}


GetRepositoryNamesData data = (GetRepositoryNamesData) meta.getStepData(); @Test
GetRepositoryNames step = new GetRepositoryNames( stepMeta, data, 0, transMeta, new Trans( transMeta ) ); public void testGetRepoList_transOnly() throws KettleException {
step.init( meta, data ); init( repo, "/", true, ".*", "", Transformations, 2 );
assertNotNull( data.list ); }
assertEquals( 1, data.list.size() );
assertEquals( "Trans1", data.list.get( 0 ).getName() ); @Test
assertEquals( "/dir1", data.list.get( 0 ).getRepositoryDirectory().getPath() ); public void testGetRepoList_jobsOnly() throws KettleException {
init( repo, "/", true, ".*", "", Jobs, 2 );
}

@Test
public void testGetRepoList_nameMask() throws KettleException {
init( repo, "/", true, "Trans.*", "", All, 2 );
} }


@Test @Test
public void testGetRepoListVariables() throws KettleException { public void testGetRepoList_withoutNameMask() throws KettleException {
init( repo, "/", true, "", "", All, 4 );
}

@Test
public void testGetRepoList_excludeNameMask() throws KettleException {
init( repo, "/", true, ".*", "Trans1.*", All, 3 );
}

@Test
public void testGetRepoList_includeSubfolders_Extended() throws KettleException {
init( repoExtended, "/", true, ".*", "", All, 4 );
}

@Test
public void testGetRepoList_excludeSubfolders_Extended() throws KettleException {
init( repoExtended, "/", false, ".*", "", All, 0 );
}

@Test
public void testGetRepoList_transOnly_Extended() throws KettleException {
init( repoExtended, "/", true, ".*", "", Transformations, 2 );
}

@Test
public void testGetRepoList_jobsOnly_Extended() throws KettleException {
init( repoExtended, "/", true, ".*", "", Jobs, 2 );
}

@Test
public void testGetRepoList_nameMask_Extended() throws KettleException {
init( repoExtended, "/", true, "Trans.*", "", All, 2 );
}

@Test
public void testGetRepoList_withoutNameMask_Extended() throws KettleException {
init( repoExtended, "/", true, "", "", All, 4 );
}

@Test
public void testGetRepoList_excludeNameMask_Extended() throws KettleException {
init( repoExtended, "/", true, ".*", "Trans1.*", All, 3 );
}

private void init( Repository repository, String directoryName, boolean includeSubFolders, String nameMask, String exludeNameMask,
ObjectTypeSelection typeSelection, int itemCount ) throws KettleException {

VariableSpace vars = new Variables(); VariableSpace vars = new Variables();
vars.setVariable( "DirName", "/dir1" ); vars.setVariable( "DirName", "/subdir1" );
vars.setVariable( "IncludeMask", ".*" ); vars.setVariable( "IncludeMask", ".*" );
vars.setVariable( "ExcludeMask", "" ); vars.setVariable( "ExcludeMask", "" );


GetRepositoryNamesMeta meta = new GetRepositoryNamesMeta(); GetRepositoryNamesMeta meta = new GetRepositoryNamesMeta();
meta.setDirectory( new String[]{ "${DirName}" } ); meta.setDirectory( new String[] { directoryName } );
meta.setNameMask( new String[]{ "${IncludeMask}" } ); meta.setNameMask( new String[] { nameMask } );
meta.setExcludeNameMask( new String[]{ "${ExcludeMask}" } ); meta.setExcludeNameMask( new String[] { exludeNameMask } );
meta.setIncludeSubFolders( new boolean[]{ true } ); meta.setIncludeSubFolders( new boolean[] { includeSubFolders } );
meta.setObjectTypeSelection( typeSelection );
StepMeta stepMeta = new StepMeta( "GetRepoNamesStep", meta ); StepMeta stepMeta = new StepMeta( "GetRepoNamesStep", meta );


TransMeta transMeta = new TransMeta( vars ); TransMeta transMeta = new TransMeta( vars );
transMeta.setRepository( repo ); transMeta.setRepository( repository );
transMeta.addStep( stepMeta ); transMeta.addStep( stepMeta );


GetRepositoryNamesData data = (GetRepositoryNamesData) meta.getStepData(); GetRepositoryNamesData data = (GetRepositoryNamesData) meta.getStepData();
GetRepositoryNames step = new GetRepositoryNames( stepMeta, data, 0, transMeta, new Trans( transMeta ) ); GetRepositoryNames step = new GetRepositoryNames( stepMeta, data, 0, transMeta, new Trans( transMeta ) );
step.shareVariablesWith( vars ); // Required, as we're calling init() directly, instead of using Trans
step.init( meta, data ); step.init( meta, data );

assertNotNull( data.list ); assertNotNull( data.list );
assertEquals( 1, data.list.size() ); assertEquals( itemCount, data.list.size() );
assertEquals( "Trans1", data.list.get( 0 ).getName() );
assertEquals( "/dir1", data.list.get( 0 ).getRepositoryDirectory().getPath() );
} }
} }

0 comments on commit 3dc8f4f

Please sign in to comment.