Skip to content

Commit

Permalink
[PDI-15928]Get Repository Names step hangs when working with the Pentaho
Browse files Browse the repository at this point in the history 
Repository - added tests and behaviour for mask
  • Loading branch information
Sergey Travin authored and Sergey Travin committed Mar 1, 2017
1 parent 73cdee8 commit 3dc8f4f
Show file tree
Hide file tree
Showing 3 changed files with 223 additions and 76 deletions.
44 changes: 29 additions & 15 deletions engine/src/org/pentaho/di/trans/steps/getrepositorynames/GetRepositoryNames.java
View file
Expand Up @@ -164,26 +164,25 @@ private List<RepositoryElementMetaInterface> getRepositoryObjects() throws Kettl
RepositoryExtended extendedRep = (RepositoryExtended) repository;
for ( int i = 0; i < meta.getDirectory().length; i++ ) {
String directoryPath = environmentSubstitute( meta.getDirectory()[i] );
String nameMaskFromMeta = environmentSubstitute( meta.getNameMask()[i] );
//if we have the name mask from user than we applied the name mask for job and transformation
//otherwise we use the * to search all pdi objects
String actualNameMask = Utils.isEmpty( nameMaskFromMeta ) ? "*" : nameMaskFromMeta;
String filter = null;
// by default we look for current level
int depth = 0;
if ( meta.getObjectTypeSelection().areTransformationsSelected() ) {
filter = actualNameMask + ".ktr";
filter = "*.ktr";
}
if ( meta.getObjectTypeSelection().areJobsSelected() ) {
//if we have selected the job and transformation than we have applied filter with both condition
filter = Utils.isEmpty( filter ) ? actualNameMask + ".kjb" : filter + "|" + actualNameMask + ".kjb";
// if we have selected the job and transformation than we have applied filter with both condition
filter = Utils.isEmpty( filter ) ? "*.kjb" : filter + "|*.kjb";
}
// should include unlimited subfolder
if ( meta.getIncludeSubFolders()[i] ) {
depth = -1;
}
RepositoryDirectoryInterface directory =
extendedRep.loadRepositoryDirectoryTree( directoryPath, filter, depth, true, false, false );

list.addAll( getRepositoryObjects( directory ) );
list.addAll( getRepositoryObjects( directory, environmentSubstitute( meta.getNameMask()[i] ),
environmentSubstitute( meta.getExcludeNameMask()[i] ) ) );
}
} else {
// Get the repository directory tree.
Expand Down Expand Up @@ -244,14 +243,29 @@ private List<RepositoryElementMetaInterface> getRepositoryObjects( Repository re
return list;
}

private List<RepositoryElementMetaInterface> getRepositoryObjects( RepositoryDirectoryInterface directory )
throws KettleException {
List<RepositoryElementMetaInterface> objects = new ArrayList<RepositoryElementMetaInterface>();
objects.addAll( directory.getRepositoryObjects() );
for ( RepositoryDirectoryInterface subdir : directory.getChildren() ) {
objects.addAll( getRepositoryObjects( subdir ) );
private List<RepositoryElementMetaInterface> getRepositoryObjects( RepositoryDirectoryInterface directory,
String nameMask, String excludeNameMask ) throws KettleException {
List<RepositoryElementMetaInterface> list = new ArrayList<RepositoryElementMetaInterface>();
if ( directory.getRepositoryObjects() != null ) {
for ( RepositoryElementMetaInterface object : directory.getRepositoryObjects() ) {
boolean add = false;
if ( Utils.isEmpty( nameMask ) || object.getName().matches( nameMask ) ) {
add = true;
}
if ( !Utils.isEmpty( excludeNameMask ) && object.getName().matches( excludeNameMask ) ) {
add = false;
}
if ( add ) {
list.add( object );
}
}
}
return objects;
if ( directory.getChildren() != null ) {
for ( RepositoryDirectoryInterface subdir : directory.getChildren() ) {
list.addAll( getRepositoryObjects( subdir, nameMask, excludeNameMask ) );
}
}
return list;
}

}
250 changes: 189 additions & 61 deletions engine/test-src/org/pentaho/di/trans/steps/getrepositorynames/GetRepositoryNamesTest.java
View file
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com
* Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
*
*******************************************************************************
*
Expand All @@ -24,123 +24,251 @@

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.pentaho.di.trans.steps.getrepositorynames.ObjectTypeSelection.*;
import static org.mockito.Mockito.*;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
import org.junit.After;
import org.junit.Before;
import org.apache.commons.io.FileUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.pentaho.di.core.KettleClientEnvironment;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.variables.VariableSpace;
import org.pentaho.di.core.variables.Variables;
import org.pentaho.di.job.JobMeta;
import org.pentaho.di.repository.Repository;
import org.pentaho.di.repository.RepositoryDirectory;
import org.pentaho.di.repository.RepositoryDirectoryInterface;
import org.pentaho.di.repository.RepositoryElementMetaInterface;
import org.pentaho.di.repository.RepositoryExtended;
import org.pentaho.di.repository.RepositoryMeta;
import org.pentaho.di.repository.RepositoryObject;
import org.pentaho.di.repository.RepositoryObjectType;
import org.pentaho.di.repository.filerep.KettleFileRepository;
import org.pentaho.di.repository.filerep.KettleFileRepositoryMeta;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.utils.TestUtils;

public class GetRepositoryNamesTest {

String baseDirName;
RepositoryMeta repoMeta;
Repository repo;
static Path baseDirName;
static Repository repo;
static RepositoryExtended repoExtended;

@BeforeClass
public static void setUpBeforeClass() throws KettleException {
public static void setUpBeforeClass() throws KettleException, IOException {
KettleClientEnvironment.init();
prepareFileRepository();
prepareExtendedRepository();
}

@Before
public void setup() throws KettleException {
baseDirName = TestUtils.createTempDir();
repoMeta = new KettleFileRepositoryMeta( UUID.randomUUID().toString(), UUID.randomUUID().toString(),
UUID.randomUUID().toString(), baseDirName );
private static void prepareFileRepository() throws IOException, KettleException {
baseDirName = Files.createTempDirectory( "GetRepositoryNamesIT" );
RepositoryMeta repoMeta =
new KettleFileRepositoryMeta( UUID.randomUUID().toString(), UUID.randomUUID().toString(), UUID.randomUUID()
.toString(), baseDirName.toString() );
repo = new KettleFileRepository();
repo.init( repoMeta );
repo.connect( null, null );

// Populate
RepositoryDirectoryInterface ROOT_DIR = repo.findDirectory( "/" );
RepositoryDirectoryInterface DIR1 = new RepositoryDirectory( ROOT_DIR, "dir1" );
repo.saveRepositoryDirectory( DIR1 );
TransMeta transMeta = new TransMeta();
transMeta.setName( "Trans1" );
transMeta.setRepositoryDirectory( DIR1 );
repo.save( transMeta, null, null );
RepositoryDirectoryInterface rootDir = repo.findDirectory( "/" );

RepositoryDirectoryInterface subdir1 = new RepositoryDirectory( rootDir, "subdir1" );
repo.saveRepositoryDirectory( subdir1 );

TransMeta transMeta1 = new TransMeta();
transMeta1.setName( "Trans1" );
transMeta1.setRepositoryDirectory( subdir1 );
repo.save( transMeta1, null, null );

JobMeta jobMeta1 = new JobMeta();
jobMeta1.setName( "Job1" );
jobMeta1.setRepositoryDirectory( subdir1 );
repo.save( jobMeta1, null, null );

RepositoryDirectoryInterface subdir2 = new RepositoryDirectory( subdir1, "subdir2" );
repo.saveRepositoryDirectory( subdir2 );

TransMeta transMeta2 = new TransMeta();
transMeta2.setName( "Trans2" );
transMeta2.setRepositoryDirectory( subdir2 );
repo.save( transMeta2, null, null );

JobMeta jobMeta2 = new JobMeta();
jobMeta2.setName( "Job2" );
jobMeta2.setRepositoryDirectory( subdir2 );
repo.save( jobMeta2, null, null );
}

@After
public void cleanup() {
@SuppressWarnings( "deprecation" )
private static void prepareExtendedRepository() throws KettleException {
repoExtended = mock( RepositoryExtended.class );
when( repoExtended.loadRepositoryDirectoryTree( anyString(), anyString(), anyInt(), anyBoolean(), anyBoolean(),
anyBoolean() ) ).then( new Answer<RepositoryDirectoryInterface>() {
@Override
public RepositoryDirectoryInterface answer( InvocationOnMock invocation ) throws Throwable {
Object[] args = invocation.getArguments();

RepositoryDirectoryInterface root = new RepositoryDirectory();
root.setName( "/" );
RepositoryDirectoryInterface subdir1 = new RepositoryDirectory( root, "subdir1" );
RepositoryDirectoryInterface subdir2 = new RepositoryDirectory( subdir1, "subdir2" );
RepositoryElementMetaInterface trans1 =
new RepositoryObject( null, "Trans1", subdir1, "user", null, RepositoryObjectType.TRANSFORMATION, "",
false );
RepositoryElementMetaInterface trans2 =
new RepositoryObject( null, "Trans2", subdir2, "user", null, RepositoryObjectType.TRANSFORMATION, "",
false );
RepositoryElementMetaInterface job1 =
new RepositoryObject( null, "Job1", subdir1, "user", null, RepositoryObjectType.JOB, "", false );
RepositoryElementMetaInterface job2 =
new RepositoryObject( null, "Job2", subdir2, "user", null, RepositoryObjectType.JOB, "", false );

List<RepositoryElementMetaInterface> list1 = new ArrayList<RepositoryElementMetaInterface>();
List<RepositoryElementMetaInterface> list2 = new ArrayList<RepositoryElementMetaInterface>();
if ( ( (String) args[1] ).contains( "ktr" ) ) {
list1.add( trans1 );
list2.add( trans2 );
}
if ( ( (String) args[1] ).contains( "kjb" ) ) {
list1.add( job1 );
list2.add( job2 );
}
subdir1.setRepositoryObjects( list1 );
subdir2.setRepositoryObjects( list2 );

if ( ( (Integer) args[2] ) == -1 ) {
subdir1.addSubdirectory( subdir2 );
root.addSubdirectory( subdir1 );
}
String actualPath = ( (String) args[0] );
if ( actualPath.equals( "/" ) ) {
return root;
} else if ( actualPath.equals( subdir1.getPath() ) ) {
return subdir1;
} else if ( actualPath.equals( subdir2.getPath() ) ) {
return subdir2;
} else {
return null;
}
}
} );
}

@AfterClass
public static void tearDownAfterClass() throws KettleException, IOException {
if ( repo != null ) {
repo.disconnect();
}
FileObject baseDir = TestUtils.getFileObject( baseDirName );
if ( baseDir != null ) {
try {
baseDir.delete();
} catch ( FileSystemException ignored ) {
// Ignore, couldn't cleanup after tests
}
}
FileUtils.forceDelete( new File( baseDirName.toString() ) );
}

@Test
public void testGetRepoList() throws KettleException {
GetRepositoryNamesMeta meta = new GetRepositoryNamesMeta();
meta.setDirectory( new String[]{ "/" } );
meta.setNameMask( new String[]{ ".*" } );
meta.setExcludeNameMask( new String[]{ "" } );
meta.setIncludeSubFolders( new boolean[]{ true } );
StepMeta stepMeta = new StepMeta( "GetRepoNamesStep", meta );
public void testGetRepoList_includeSubfolders() throws KettleException {
init( repo, "/", true, ".*", "", All, 4 );
}

TransMeta transMeta = new TransMeta();
transMeta.setRepository( repo );
transMeta.addStep( stepMeta );
@Test
public void testGetRepoList_excludeSubfolders() throws KettleException {
init( repo, "/", false, ".*", "", All, 0 );
}

GetRepositoryNamesData data = (GetRepositoryNamesData) meta.getStepData();
GetRepositoryNames step = new GetRepositoryNames( stepMeta, data, 0, transMeta, new Trans( transMeta ) );
step.init( meta, data );
assertNotNull( data.list );
assertEquals( 1, data.list.size() );
assertEquals( "Trans1", data.list.get( 0 ).getName() );
assertEquals( "/dir1", data.list.get( 0 ).getRepositoryDirectory().getPath() );
@Test
public void testGetRepoList_transOnly() throws KettleException {
init( repo, "/", true, ".*", "", Transformations, 2 );
}

@Test
public void testGetRepoList_jobsOnly() throws KettleException {
init( repo, "/", true, ".*", "", Jobs, 2 );
}

@Test
public void testGetRepoList_nameMask() throws KettleException {
init( repo, "/", true, "Trans.*", "", All, 2 );
}

@Test
public void testGetRepoListVariables() throws KettleException {
public void testGetRepoList_withoutNameMask() throws KettleException {
init( repo, "/", true, "", "", All, 4 );
}

@Test
public void testGetRepoList_excludeNameMask() throws KettleException {
init( repo, "/", true, ".*", "Trans1.*", All, 3 );
}

@Test
public void testGetRepoList_includeSubfolders_Extended() throws KettleException {
init( repoExtended, "/", true, ".*", "", All, 4 );
}

@Test
public void testGetRepoList_excludeSubfolders_Extended() throws KettleException {
init( repoExtended, "/", false, ".*", "", All, 0 );
}

@Test
public void testGetRepoList_transOnly_Extended() throws KettleException {
init( repoExtended, "/", true, ".*", "", Transformations, 2 );
}

@Test
public void testGetRepoList_jobsOnly_Extended() throws KettleException {
init( repoExtended, "/", true, ".*", "", Jobs, 2 );
}

@Test
public void testGetRepoList_nameMask_Extended() throws KettleException {
init( repoExtended, "/", true, "Trans.*", "", All, 2 );
}

@Test
public void testGetRepoList_withoutNameMask_Extended() throws KettleException {
init( repoExtended, "/", true, "", "", All, 4 );
}

@Test
public void testGetRepoList_excludeNameMask_Extended() throws KettleException {
init( repoExtended, "/", true, ".*", "Trans1.*", All, 3 );
}

private void init( Repository repository, String directoryName, boolean includeSubFolders, String nameMask, String exludeNameMask,
ObjectTypeSelection typeSelection, int itemCount ) throws KettleException {

VariableSpace vars = new Variables();
vars.setVariable( "DirName", "/dir1" );
vars.setVariable( "DirName", "/subdir1" );
vars.setVariable( "IncludeMask", ".*" );
vars.setVariable( "ExcludeMask", "" );

GetRepositoryNamesMeta meta = new GetRepositoryNamesMeta();
meta.setDirectory( new String[]{ "${DirName}" } );
meta.setNameMask( new String[]{ "${IncludeMask}" } );
meta.setExcludeNameMask( new String[]{ "${ExcludeMask}" } );
meta.setIncludeSubFolders( new boolean[]{ true } );
meta.setDirectory( new String[] { directoryName } );
meta.setNameMask( new String[] { nameMask } );
meta.setExcludeNameMask( new String[] { exludeNameMask } );
meta.setIncludeSubFolders( new boolean[] { includeSubFolders } );
meta.setObjectTypeSelection( typeSelection );
StepMeta stepMeta = new StepMeta( "GetRepoNamesStep", meta );

TransMeta transMeta = new TransMeta( vars );
transMeta.setRepository( repo );
transMeta.setRepository( repository );
transMeta.addStep( stepMeta );

GetRepositoryNamesData data = (GetRepositoryNamesData) meta.getStepData();
GetRepositoryNames step = new GetRepositoryNames( stepMeta, data, 0, transMeta, new Trans( transMeta ) );
step.shareVariablesWith( vars ); // Required, as we're calling init() directly, instead of using Trans
step.init( meta, data );

assertNotNull( data.list );
assertEquals( 1, data.list.size() );
assertEquals( "Trans1", data.list.get( 0 ).getName() );
assertEquals( "/dir1", data.list.get( 0 ).getRepositoryDirectory().getPath() );
assertEquals( itemCount, data.list.size() );
}
}

0 comments on commit 3dc8f4f

Please sign in to comment.