Skip to content

Commit

Permalink
[PDI-13901] - Export Linked Resources: Metadata Injection step not wo…
Browse files Browse the repository at this point in the history
…rking
  • Loading branch information
AndreyBurikhin committed Dec 30, 2015
1 parent 7ed6eed commit cea7b09
Show file tree
Hide file tree
Showing 4 changed files with 218 additions and 8 deletions.
Expand Up @@ -710,7 +710,7 @@ public String exportResources( VariableSpace space, Map<String, ResourceDefiniti
// So let's change the filename from relative to absolute by grabbing the file object...
// In case the name of the file comes from previous steps, forget about this!
//
if ( Const.isEmpty( filenameField ) ) {
if ( Const.isEmpty( filenameField ) && !Const.isEmpty( filename ) ) {
// From : ${Internal.Transformation.Filename.Directory}/../foo/bar.csv
// To : /home/matt/test/files/foo/bar.csv
//
Expand Down
103 changes: 96 additions & 7 deletions engine/src/org/pentaho/di/trans/steps/metainject/MetaInjectMeta.java
Expand Up @@ -37,13 +37,20 @@
import org.pentaho.di.core.exception.KettleXMLException;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.value.ValueMetaFactory;
import org.pentaho.di.core.util.CurrentDirectoryResolver;
import org.pentaho.di.core.variables.VariableSpace;
import org.pentaho.di.core.xml.XMLHandler;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.repository.ObjectId;
import org.pentaho.di.repository.Repository;
import org.pentaho.di.repository.RepositoryDirectory;
import org.pentaho.di.repository.RepositoryDirectoryInterface;
import org.pentaho.di.repository.StringObjectId;
import org.pentaho.di.resource.ResourceDefinition;
import org.pentaho.di.resource.ResourceEntry;
import org.pentaho.di.resource.ResourceNamingInterface;
import org.pentaho.di.resource.ResourceReference;
import org.pentaho.di.resource.ResourceEntry.ResourceType;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransHopMeta;
import org.pentaho.di.trans.TransMeta;
Expand Down Expand Up @@ -426,19 +433,24 @@ public static final synchronized TransMeta loadTransformationMeta( MetaInjectMet
return loadTransformationMeta( mappingMeta, rep, null, space );
}

public static final synchronized TransMeta loadTransformationMeta( MetaInjectMeta mappingMeta, Repository rep,
public static final synchronized TransMeta loadTransformationMeta( MetaInjectMeta injectMeta, Repository rep,
IMetaStore metaStore, VariableSpace space ) throws KettleException {
TransMeta mappingTransMeta = null;

switch ( mappingMeta.getSpecificationMethod() ) {
CurrentDirectoryResolver resolver = new CurrentDirectoryResolver();
VariableSpace tmpSpace =
resolver.resolveCurrentDirectory( injectMeta.getSpecificationMethod(), space, rep, injectMeta.getParentStepMeta(),
injectMeta.getFileName() );

switch ( injectMeta.getSpecificationMethod() ) {
case FILENAME:
String realFilename = space.environmentSubstitute( mappingMeta.getFileName() );
String realFilename = tmpSpace.environmentSubstitute( injectMeta.getFileName() );
try {
// OK, load the meta-data from file...
//
// Don't set internal variables: they belong to the parent thread!
//
mappingTransMeta = new TransMeta( realFilename, metaStore, rep, false, space, null );
mappingTransMeta = new TransMeta( realFilename, metaStore, rep, false, tmpSpace, null );
mappingTransMeta.getLogChannel().logDetailed(
"Loading Mapping from repository",
"Mapping transformation was loaded from XML file [" + realFilename + "]" );
Expand All @@ -449,8 +461,8 @@ public static final synchronized TransMeta loadTransformationMeta( MetaInjectMet
break;

case REPOSITORY_BY_NAME:
String realTransname = space.environmentSubstitute( mappingMeta.getTransName() );
String realDirectory = space.environmentSubstitute( mappingMeta.getDirectoryPath() );
String realTransname = tmpSpace.environmentSubstitute( injectMeta.getTransName() );
String realDirectory = tmpSpace.environmentSubstitute( injectMeta.getDirectoryPath() );

if ( !Const.isEmpty( realTransname ) && !Const.isEmpty( realDirectory ) && rep != null ) {
RepositoryDirectoryInterface repdir = rep.findDirectory( realDirectory );
Expand Down Expand Up @@ -478,7 +490,7 @@ public static final synchronized TransMeta loadTransformationMeta( MetaInjectMet

case REPOSITORY_BY_REFERENCE:
// Read the last revision by reference...
mappingTransMeta = rep.loadTransformation( mappingMeta.getTransObjectId(), null );
mappingTransMeta = rep.loadTransformation( injectMeta.getTransObjectId(), null );
break;
default:
break;
Expand All @@ -493,6 +505,83 @@ public static final synchronized TransMeta loadTransformationMeta( MetaInjectMet
return mappingTransMeta;
}

/**
* package-local visibility for testing purposes
*/
TransMeta loadTransformationMeta( Repository rep, VariableSpace space ) throws KettleException {
return MetaInjectMeta.loadTransformationMeta( this, repository, null, space );
}

@Override
public List<ResourceReference> getResourceDependencies( TransMeta transMeta, StepMeta stepInfo ) {
List<ResourceReference> references = new ArrayList<ResourceReference>( 5 );
String realFilename = transMeta.environmentSubstitute( fileName );
String realTransname = transMeta.environmentSubstitute( transName );
ResourceReference reference = new ResourceReference( stepInfo );
references.add( reference );

if ( !Const.isEmpty( realFilename ) ) {
// Add the filename to the references, including a reference to this step
// meta data.
//
reference.getEntries().add( new ResourceEntry( realFilename, ResourceType.ACTIONFILE ) );
} else if ( !Const.isEmpty( realTransname ) ) {
// Add the filename to the references, including a reference to this step
// meta data.
//
reference.getEntries().add( new ResourceEntry( realTransname, ResourceType.ACTIONFILE ) );
}
return references;
}

@Override
public String exportResources( VariableSpace space, Map<String, ResourceDefinition> definitions,
ResourceNamingInterface resourceNamingInterface, Repository repository, IMetaStore metaStore )
throws KettleException {
try {
// Try to load the transformation from repository or file.
// Modify this recursively too...
//
// NOTE: there is no need to clone this step because the caller is
// responsible for this.
//
// First load the executor transformation metadata...
//
TransMeta executorTransMeta = loadTransformationMeta( repository, space );

// Also go down into the mapping transformation and export the files
// there. (mapping recursively down)
//
String proposedNewFilename =
executorTransMeta.exportResources( executorTransMeta, definitions, resourceNamingInterface, repository,
metaStore );

// To get a relative path to it, we inject
// ${Internal.Transformation.Filename.Directory}
//
String newFilename =
"${" + Const.INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_DIRECTORY + "}/" + proposedNewFilename;

// Set the correct filename inside the XML.
//
executorTransMeta.setFilename( newFilename );

// exports always reside in the root directory, in case we want to turn
// this into a file repository...
//
executorTransMeta.setRepositoryDirectory( new RepositoryDirectory() );

// change it in the entry
//
fileName = newFilename;

return proposedNewFilename;
} catch ( Exception e ) {
throw new KettleException( BaseMessages.getString( PKG, "MetaInjectMeta.Exception.UnableToLoadTrans",
fileName ) );
}
}

@Override
public boolean excludeFromCopyDistributeVerification() {
return true;
Expand Down
Expand Up @@ -69,3 +69,4 @@ MetaInjectDialog.ColumnInfo.Length = Length
MetaInjectDialog.ColumnInfo.Precision = Precision

MetaInjectMeta.ReferencedObjectAfterInjection.Description=Transformation template after injection
MetaInjectMeta.Exception.UnableToLoadTrans=It was not possible to load the specified transformation
@@ -0,0 +1,120 @@
/*! ******************************************************************************
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com
*
*******************************************************************************
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
******************************************************************************/

package org.pentaho.di.trans.steps.metainject;

import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;

import java.util.Collections;
import java.util.List;
import java.util.Map;

import org.junit.Before;
import org.junit.Test;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.variables.VariableSpace;
import org.pentaho.di.repository.Repository;
import org.pentaho.di.resource.ResourceDefinition;
import org.pentaho.di.resource.ResourceNamingInterface;
import org.pentaho.di.resource.ResourceReference;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.metastore.api.IMetaStore;

public class MetaInjectMetaTest {

private static final String TEST_FILE_NAME = "TEST_FILE_NAME";

private static final String EXPORTED_FILE_NAME =
"${" + Const.INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_DIRECTORY + "}/" + TEST_FILE_NAME;

private MetaInjectMeta metaInjectMeta;

@Before
public void before() {
metaInjectMeta = new MetaInjectMeta();
}

@Test
public void getResourceDependencies() {
TransMeta transMeta = mock( TransMeta.class );
StepMeta stepMeta = mock( StepMeta.class );

List<ResourceReference> actualResult = metaInjectMeta.getResourceDependencies( transMeta, stepMeta );
assertEquals( 1, actualResult.size() );
ResourceReference reference = actualResult.iterator().next();
assertEquals( 0, reference.getEntries().size() );
}

@Test
public void getResourceDependencies_with_defined_fileName() {
TransMeta transMeta = mock( TransMeta.class );
StepMeta stepMeta = mock( StepMeta.class );
metaInjectMeta.setFileName( "FILE_NAME" );
doReturn( "FILE_NAME_WITH_SUBSTITUTIONS" ).when( transMeta ).environmentSubstitute( "FILE_NAME" );

List<ResourceReference> actualResult = metaInjectMeta.getResourceDependencies( transMeta, stepMeta );
assertEquals( 1, actualResult.size() );
ResourceReference reference = actualResult.iterator().next();
assertEquals( 1, reference.getEntries().size() );
}

@Test
public void getResourceDependencies_with_defined_transName() {
TransMeta transMeta = mock( TransMeta.class );
StepMeta stepMeta = mock( StepMeta.class );
metaInjectMeta.setTransName( "TRANS_NAME" );
doReturn( "TRANS_NAME_WITH_SUBSTITUTIONS" ).when( transMeta ).environmentSubstitute( "TRANS_NAME" );

List<ResourceReference> actualResult = metaInjectMeta.getResourceDependencies( transMeta, stepMeta );
assertEquals( 1, actualResult.size() );
ResourceReference reference = actualResult.iterator().next();
assertEquals( 1, reference.getEntries().size() );
}

@Test
public void exportResources() throws KettleException {
VariableSpace variableSpace = mock( VariableSpace.class );
ResourceNamingInterface resourceNamingInterface = mock( ResourceNamingInterface.class );
Repository repository = mock( Repository.class );
IMetaStore metaStore = mock( IMetaStore.class );

MetaInjectMeta injectMetaSpy = spy( metaInjectMeta );
TransMeta transMeta = mock( TransMeta.class );
Map<String, ResourceDefinition> definitions = Collections.<String, ResourceDefinition> emptyMap();
doReturn( TEST_FILE_NAME ).when( transMeta ).exportResources( transMeta, definitions, resourceNamingInterface,
repository, metaStore );
doReturn( transMeta ).when( injectMetaSpy ).loadTransformationMeta( repository, variableSpace );

String actualExportedFileName =
injectMetaSpy.exportResources( variableSpace, definitions, resourceNamingInterface, repository, metaStore );
assertEquals( TEST_FILE_NAME, actualExportedFileName );
assertEquals( EXPORTED_FILE_NAME, injectMetaSpy.getFileName() );
verify( transMeta ).exportResources( transMeta, definitions, resourceNamingInterface, repository, metaStore );
}

}

0 comments on commit cea7b09

Please sign in to comment.