From 7b0146500eb594bf168517b4d412818ce4479e0a Mon Sep 17 00:00:00 2001 From: Yury Bakhmutski Date: Thu, 2 Feb 2017 13:17:14 +0300 Subject: [PATCH] [CDA-91] - As a developer I can't invoke Jobs or Transformations from "Initial Transformation" - JUnit tests were provided --- .../pentaho/di/trans/StepWithMappingMeta.java | 239 ++++++++++++++++++ .../di/trans/steps/mapping/MappingMeta.java | 172 +------------ .../simplemapping/SimpleMappingMeta.java | 179 +------------ .../singlethreader/SingleThreaderMeta.java | 161 +----------- .../steps/transexecutor/TransExecutor.java | 4 +- .../transexecutor/TransExecutorMeta.java | 215 +--------------- .../di/trans/StepWithMappingMetaTest.java | 83 ++++++ 7 files changed, 357 insertions(+), 696 deletions(-) create mode 100644 engine/src/org/pentaho/di/trans/StepWithMappingMeta.java create mode 100644 engine/test-src/org/pentaho/di/trans/StepWithMappingMetaTest.java diff --git a/engine/src/org/pentaho/di/trans/StepWithMappingMeta.java b/engine/src/org/pentaho/di/trans/StepWithMappingMeta.java new file mode 100644 index 000000000000..e2e16f7c0900 --- /dev/null +++ b/engine/src/org/pentaho/di/trans/StepWithMappingMeta.java @@ -0,0 +1,239 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans; + +import org.pentaho.di.core.Const; +import org.pentaho.di.core.ObjectLocationSpecificationMethod; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.logging.LogChannel; +import org.pentaho.di.core.util.CurrentDirectoryResolver; +import org.pentaho.di.core.util.Utils; +import org.pentaho.di.core.variables.VariableSpace; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.repository.ObjectId; +import org.pentaho.di.repository.Repository; +import org.pentaho.di.repository.RepositoryDirectoryInterface; +import org.pentaho.di.trans.step.BaseStepMeta; +import org.pentaho.metastore.api.IMetaStore; + +/** + * This class is supposed to use in steps where the mapping to sub transformations takes place + * + * @since 02-jan-2017 + * @author Yury Bakhmutski + */ +public abstract class StepWithMappingMeta extends BaseStepMeta { + //default value + protected static Class PKG = StepWithMappingMeta.class; + + protected ObjectLocationSpecificationMethod specificationMethod; + protected String transName; + protected String fileName; + protected String directoryPath; + protected ObjectId transObjectId; + + public static TransMeta loadMappingMeta( StepWithMappingMeta mappingMeta, Repository rep, + IMetaStore metaStore, VariableSpace space ) throws KettleException { + return loadMappingMeta( mappingMeta, rep, metaStore, space, true ); + } + + public static synchronized TransMeta loadMappingMeta( StepWithMappingMeta executorMeta, Repository rep, + IMetaStore metaStore, VariableSpace space, boolean share ) throws KettleException { + TransMeta mappingTransMeta = null; + + CurrentDirectoryResolver r = new CurrentDirectoryResolver(); + VariableSpace tmpSpace = + r.resolveCurrentDirectory( executorMeta.getSpecificationMethod(), space, rep, executorMeta.getParentStepMeta(), + executorMeta.getFileName() ); + + switch ( executorMeta.getSpecificationMethod() ) { + case FILENAME: + String realFilename = tmpSpace.environmentSubstitute( executorMeta.getFileName() ); + try { + // OK, load the meta-data from file... + // Don't set internal variables: they belong to the parent thread! + if ( rep != null ) { + // need to try to load from the repository + realFilename = r.normalizeSlashes( realFilename ); + try { + String dirStr = realFilename.substring( 0, realFilename.lastIndexOf( "/" ) ); + String tmpFilename = realFilename.substring( realFilename.lastIndexOf( "/" ) + 1 ); + RepositoryDirectoryInterface dir = rep.findDirectory( dirStr ); + mappingTransMeta = rep.loadTransformation( tmpFilename, dir, null, true, null ); + } catch ( KettleException ke ) { + // try without extension + if ( realFilename.endsWith( Const.STRING_TRANS_DEFAULT_EXT ) ) { + try { + String tmpFilename = + realFilename.substring( realFilename.lastIndexOf( "/" ) + 1, realFilename.indexOf( "." + + Const.STRING_TRANS_DEFAULT_EXT ) ); + String dirStr = realFilename.substring( 0, realFilename.lastIndexOf( "/" ) ); + RepositoryDirectoryInterface dir = rep.findDirectory( dirStr ); + mappingTransMeta = rep.loadTransformation( tmpFilename, dir, null, true, null ); + } catch ( KettleException ke2 ) { + // fall back to try loading from file system (transMeta is going to be null) + } + } + } + } + if ( mappingTransMeta == null ) { + mappingTransMeta = new TransMeta( realFilename, metaStore, rep, true, tmpSpace, null ); + LogChannel.GENERAL.logDetailed( "Loading transformation from repository", + "Transformation was loaded from XML file [" + realFilename + "]" ); + } + } catch ( Exception e ) { + throw new KettleException( BaseMessages.getString( PKG, "TransExecutorMeta.Exception.UnableToLoadTrans" ), + e ); + } + break; + + case REPOSITORY_BY_NAME: + String realTransname = tmpSpace.environmentSubstitute( executorMeta.getTransName() ); + String realDirectory = tmpSpace.environmentSubstitute( executorMeta.getDirectoryPath() ); + + if ( rep != null ) { + if ( !Utils.isEmpty( realTransname ) && !Utils.isEmpty( realDirectory ) ) { + realDirectory = r.normalizeSlashes( realDirectory ); + RepositoryDirectoryInterface repdir = rep.findDirectory( realDirectory ); + if ( repdir != null ) { + try { + // reads the last revision in the repository... + mappingTransMeta = rep.loadTransformation( realTransname, repdir, null, true, null ); + // TODO: FIXME: pass in metaStore to repository? + + LogChannel.GENERAL.logDetailed( "Loading transformation from repository", "Executor transformation [" + + realTransname + "] was loaded from the repository" ); + } catch ( Exception e ) { + throw new KettleException( "Unable to load transformation [" + realTransname + "]", e ); + } + } + } + } else { + // rep is null, let's try loading by filename + try { + mappingTransMeta = + new TransMeta( realDirectory + "/" + realTransname, metaStore, rep, true, tmpSpace, null ); + } catch ( KettleException ke ) { + try { + // add .ktr extension and try again + mappingTransMeta = + new TransMeta( realDirectory + "/" + realTransname + "." + Const.STRING_TRANS_DEFAULT_EXT, metaStore, + rep, true, tmpSpace, null ); + } catch ( KettleException ke2 ) { + throw new KettleException( BaseMessages.getString( PKG, "TransExecutorMeta.Exception.UnableToLoadTrans", + realTransname ) + realDirectory ); + } + } + } + break; + + case REPOSITORY_BY_REFERENCE: + // Read the last revision by reference... + mappingTransMeta = rep.loadTransformation( executorMeta.getTransObjectId(), null ); + break; + default: + break; + } + + // Pass some important information to the mapping transformation metadata: + if ( share ) { + mappingTransMeta.copyVariablesFrom( space ); + } + mappingTransMeta.setRepository( rep ); + mappingTransMeta.setMetaStore( metaStore ); + mappingTransMeta.setFilename( mappingTransMeta.getFilename() ); + + return mappingTransMeta; + } + + /** + * @return the specificationMethod + */ + public ObjectLocationSpecificationMethod getSpecificationMethod() { + return specificationMethod; + } + + /** + * @param specificationMethod the specificationMethod to set + */ + public void setSpecificationMethod( ObjectLocationSpecificationMethod specificationMethod ) { + this.specificationMethod = specificationMethod; + } + + /** + * @return the directoryPath + */ + public String getDirectoryPath() { + return directoryPath; + } + + /** + * @param directoryPath the directoryPath to set + */ + public void setDirectoryPath( String directoryPath ) { + this.directoryPath = directoryPath; + } + + /** + * @return the fileName + */ + public String getFileName() { + return fileName; + } + + /** + * @param fileName the fileName to set + */ + public void setFileName( String fileName ) { + this.fileName = fileName; + } + + /** + * @return the transName + */ + public String getTransName() { + return transName; + } + + /** + * @param transName the transName to set + */ + public void setTransName( String transName ) { + this.transName = transName; + } + + /** + * @return the transObjectId + */ + public ObjectId getTransObjectId() { + return transObjectId; + } + + /** + * @param transObjectId the transObjectId to set + */ + public void setTransObjectId( ObjectId transObjectId ) { + this.transObjectId = transObjectId; + } + +} diff --git a/engine/src/org/pentaho/di/trans/steps/mapping/MappingMeta.java b/engine/src/org/pentaho/di/trans/steps/mapping/MappingMeta.java index 11dae558d389..d9240709b561 100644 --- a/engine/src/org/pentaho/di/trans/steps/mapping/MappingMeta.java +++ b/engine/src/org/pentaho/di/trans/steps/mapping/MappingMeta.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com + * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com * ******************************************************************************* * @@ -56,10 +56,10 @@ import org.pentaho.di.resource.ResourceEntry.ResourceType; import org.pentaho.di.resource.ResourceNamingInterface; import org.pentaho.di.resource.ResourceReference; +import org.pentaho.di.trans.StepWithMappingMeta; import org.pentaho.di.trans.Trans; import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.TransMeta.TransformationType; -import org.pentaho.di.trans.step.BaseStepMeta; import org.pentaho.di.trans.step.StepDataInterface; import org.pentaho.di.trans.step.StepIOMeta; import org.pentaho.di.trans.step.StepIOMetaInterface; @@ -82,13 +82,11 @@ * */ -public class MappingMeta extends BaseStepMeta implements StepMetaInterface, HasRepositoryInterface { - private static Class PKG = MappingMeta.class; // for i18n purposes, needed by Translator2!! - private String transName; - private String fileName; - private String directoryPath; - private ObjectId transObjectId; - private ObjectLocationSpecificationMethod specificationMethod; +public class MappingMeta extends StepWithMappingMeta implements StepMetaInterface, HasRepositoryInterface { + + static { + PKG = MappingMeta.class; // for i18n purposes, needed by Translator2!! + } private List inputMappings; private List outputMappings; @@ -618,101 +616,6 @@ public static final synchronized TransMeta loadMappingMeta( MappingMeta mappingM return loadMappingMeta( mappingMeta, rep, null, space ); } - public static final synchronized TransMeta loadMappingMeta( MappingMeta mappingMeta, Repository rep, - IMetaStore metaStore, VariableSpace space ) throws KettleException { - return loadMappingMeta( mappingMeta, rep, metaStore, space, true ); - } - - public static final synchronized TransMeta loadMappingMeta( MappingMeta mappingMeta, Repository rep, - IMetaStore metaStore, VariableSpace space, boolean share ) throws KettleException { - TransMeta mappingTransMeta = null; - - switch ( mappingMeta.getSpecificationMethod() ) { - case FILENAME: - String realFilename = space.environmentSubstitute( mappingMeta.getFileName() ); - try { - // OK, load the meta-data from file... - // - // Don't set internal variables: they belong to the parent thread! - // PDI-3064 do not share with parent variable space - mappingTransMeta = new TransMeta( realFilename, metaStore, rep, true, null, null ); - mappingTransMeta.getLogChannel().logDetailed( - "Loading Mapping from repository", - "Mapping transformation was loaded from XML file [" + realFilename + "]" ); - } catch ( Exception e ) { - throw new KettleException( BaseMessages.getString( PKG, "MappingMeta.Exception.UnableToLoadMapping" ), e ); - } - break; - - case REPOSITORY_BY_NAME: - String realTransname = space.environmentSubstitute( mappingMeta.getTransName() ); - String realDirectory = space.environmentSubstitute( mappingMeta.getDirectoryPath() ); - - if ( rep == null ) { // hardening because TransMeta.setRepositoryOnMappingSteps(); might be missing in special - // situations - throw new KettleException( BaseMessages.getString( - PKG, "MappingMeta.Exception.InternalErrorRepository.Message" ) ); - } - - if ( !Utils.isEmpty( realTransname ) && !Utils.isEmpty( realDirectory ) && rep != null ) { - RepositoryDirectoryInterface repdir = rep.findDirectory( realDirectory ); - if ( repdir != null ) { - try { - // reads the last revision in the repository... - // - mappingTransMeta = rep.loadTransformation( realTransname, repdir, null, true, null ); // TODO: FIXME: - // Should we pass in - // external - // MetaStore into - // Repository - // methods? - - mappingTransMeta.getLogChannel().logDetailed( - "Loading Mapping from repository", - "Mapping transformation [" + realTransname + "] was loaded from the repository" ); - } catch ( Exception e ) { - throw new KettleException( "Unable to load transformation [" + realTransname + "]", e ); - } - } else { - throw new KettleException( BaseMessages.getString( - PKG, "MappingMeta.Exception.UnableToLoadTransformation", realTransname ) - + realDirectory ); - } - } else { - throw new KettleException( BaseMessages.getString( - PKG, "MappingMeta.Exception.UnableToLoadTransformationNameOrDirNotGiven" ) ); - } - break; - - case REPOSITORY_BY_REFERENCE: - // Read the last revision by reference... - if ( rep == null ) { // hardening because TransMeta.setRepositoryOnMappingSteps(); might be missing in special - // situations - throw new KettleException( BaseMessages.getString( - PKG, "MappingMeta.Exception.InternalErrorRepository.Message" ) ); - } - mappingTransMeta = rep.loadTransformation( mappingMeta.getTransObjectId(), null ); - break; - default: - break; - } - - // Pass some important information to the mapping transformation metadata: - // - if ( mappingTransMeta == null ) { // hardening because TransMeta might have issues in special situations - throw new KettleException( BaseMessages.getString( - PKG, "MappingMeta.Exception.InternalErrorTransMetaIsNULL.Message" ) ); - } - if ( share ) { - mappingTransMeta.copyVariablesFrom( space ); - } - mappingTransMeta.setRepository( rep ); - mappingTransMeta.setMetaStore( metaStore ); - mappingTransMeta.setFilename( mappingTransMeta.getFilename() ); - - return mappingTransMeta; - } - public void check( List remarks, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, VariableSpace space, Repository repository, IMetaStore metaStore ) { @@ -802,51 +705,6 @@ public StepDataInterface getStepData() { return new MappingData(); } - /** - * @return the directoryPath - */ - public String getDirectoryPath() { - return directoryPath; - } - - /** - * @param directoryPath - * the directoryPath to set - */ - public void setDirectoryPath( String directoryPath ) { - this.directoryPath = directoryPath; - } - - /** - * @return the fileName - */ - public String getFileName() { - return fileName; - } - - /** - * @param fileName - * the fileName to set - */ - public void setFileName( String fileName ) { - this.fileName = fileName; - } - - /** - * @return the transName - */ - public String getTransName() { - return transName; - } - - /** - * @param transName - * the transName to set - */ - public void setTransName( String transName ) { - this.transName = transName; - } - /** * @return the inputMappings */ @@ -978,21 +836,6 @@ public void setRepository( Repository repository ) { this.repository = repository; } - /** - * @return the transObjectId - */ - public ObjectId getTransObjectId() { - return transObjectId; - } - - /** - * @param transObjectId - * the transObjectId to set - */ - public void setTransObjectId( ObjectId transObjectId ) { - this.transObjectId = transObjectId; - } - /** * @return the specificationMethod */ @@ -1144,4 +987,5 @@ public IMetaStore getMetaStore() { public void setMetaStore( IMetaStore metaStore ) { this.metaStore = metaStore; } + } diff --git a/engine/src/org/pentaho/di/trans/steps/simplemapping/SimpleMappingMeta.java b/engine/src/org/pentaho/di/trans/steps/simplemapping/SimpleMappingMeta.java index 14c960143067..a7926cfd09b1 100644 --- a/engine/src/org/pentaho/di/trans/steps/simplemapping/SimpleMappingMeta.java +++ b/engine/src/org/pentaho/di/trans/steps/simplemapping/SimpleMappingMeta.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com + * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com * ******************************************************************************* * @@ -56,10 +56,10 @@ import org.pentaho.di.resource.ResourceEntry.ResourceType; import org.pentaho.di.resource.ResourceNamingInterface; import org.pentaho.di.resource.ResourceReference; +import org.pentaho.di.trans.StepWithMappingMeta; import org.pentaho.di.trans.Trans; import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.TransMeta.TransformationType; -import org.pentaho.di.trans.step.BaseStepMeta; import org.pentaho.di.trans.step.StepDataInterface; import org.pentaho.di.trans.step.StepIOMeta; import org.pentaho.di.trans.step.StepIOMetaInterface; @@ -82,13 +82,11 @@ * */ -public class SimpleMappingMeta extends BaseStepMeta implements StepMetaInterface, HasRepositoryInterface { - private static Class PKG = SimpleMappingMeta.class; // for i18n purposes, needed by Translator2!! - private String transName; - private String fileName; - private String directoryPath; - private ObjectId transObjectId; - private ObjectLocationSpecificationMethod specificationMethod; +public class SimpleMappingMeta extends StepWithMappingMeta implements StepMetaInterface, HasRepositoryInterface { + + static { + PKG = SimpleMappingMeta.class; // for i18n purposes, needed by Translator2!! + } private MappingIODefinition inputMapping; private MappingIODefinition outputMapping; @@ -404,94 +402,7 @@ public String[] getTargetSteps() { return null; } - public static final synchronized TransMeta loadMappingMeta( SimpleMappingMeta mappingMeta, Repository rep, - IMetaStore metaStore, VariableSpace space ) throws KettleException { - TransMeta mappingTransMeta = null; - - switch ( mappingMeta.getSpecificationMethod() ) { - case FILENAME: - String realFilename = space.environmentSubstitute( mappingMeta.getFileName() ); - try { - // OK, load the meta-data from file... - // - // Don't set internal variables: they belong to the parent thread! - // - mappingTransMeta = new TransMeta( realFilename, metaStore, rep, true, space, null ); - mappingTransMeta.getLogChannel().logDetailed( - "Loading Mapping from repository", - "Mapping transformation was loaded from XML file [" + realFilename + "]" ); - } catch ( Exception e ) { - throw new KettleException( BaseMessages.getString( - PKG, "SimpleMappingMeta.Exception.UnableToLoadMapping" ), e ); - } - break; - - case REPOSITORY_BY_NAME: - String realTransname = space.environmentSubstitute( mappingMeta.getTransName() ); - String realDirectory = space.environmentSubstitute( mappingMeta.getDirectoryPath() ); - - if ( rep == null ) { // hardening because TransMeta.setRepositoryOnMappingSteps(); might be missing in special - // situations - throw new KettleException( BaseMessages.getString( - PKG, "SimpleMappingMeta.Exception.InternalErrorRepository.Message" ) ); - } - if ( !Utils.isEmpty( realTransname ) && !Utils.isEmpty( realDirectory ) && rep != null ) { - RepositoryDirectoryInterface repdir = rep.findDirectory( realDirectory ); - if ( repdir != null ) { - try { - // reads the last revision in the repository... - // - mappingTransMeta = rep.loadTransformation( realTransname, repdir, null, true, null ); // TODO: FIXME: - // Should we pass in - // external - // MetaStore into - // Repository - // methods? - - mappingTransMeta.getLogChannel().logDetailed( - "Loading Mapping from repository", - "Mapping transformation [" + realTransname + "] was loaded from the repository" ); - } catch ( Exception e ) { - throw new KettleException( "Unable to load transformation [" + realTransname + "]", e ); - } - } else { - throw new KettleException( BaseMessages.getString( - PKG, "SimpleMappingMeta.Exception.UnableToLoadTransformation", realTransname ) - + realDirectory ); - } - } else { - throw new KettleException( BaseMessages.getString( - PKG, "SimpleMappingMeta.Exception.UnableToLoadTransformationNameOrDirNotGiven" ) ); - } - break; - - case REPOSITORY_BY_REFERENCE: - // Read the last revision by reference... - if ( rep == null ) { // hardening because TransMeta.setRepositoryOnMappingSteps(); might be missing in special - // situations - throw new KettleException( BaseMessages.getString( - PKG, "SimpleMappingMeta.Exception.InternalErrorRepository.Message" ) ); - } - mappingTransMeta = rep.loadTransformation( mappingMeta.getTransObjectId(), null ); - break; - default: - break; - } - - // Pass some important information to the mapping transformation metadata: - // - if ( mappingTransMeta == null ) { // hardening because TransMeta might have issues in special situations - throw new KettleException( BaseMessages.getString( - PKG, "SimpleMappingMeta.Exception.InternalErrorTransMetaIsNULL.Message" ) ); - } - mappingTransMeta.copyVariablesFrom( space ); - mappingTransMeta.setRepository( rep ); - mappingTransMeta.setMetaStore( metaStore ); - mappingTransMeta.setFilename( mappingTransMeta.getFilename() ); - - return mappingTransMeta; - } public void check( List remarks, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, VariableSpace space, @@ -532,51 +443,6 @@ public StepDataInterface getStepData() { return new SimpleMappingData(); } - /** - * @return the directoryPath - */ - public String getDirectoryPath() { - return directoryPath; - } - - /** - * @param directoryPath - * the directoryPath to set - */ - public void setDirectoryPath( String directoryPath ) { - this.directoryPath = directoryPath; - } - - /** - * @return the fileName - */ - public String getFileName() { - return fileName; - } - - /** - * @param fileName - * the fileName to set - */ - public void setFileName( String fileName ) { - this.fileName = fileName; - } - - /** - * @return the transName - */ - public String getTransName() { - return transName; - } - - /** - * @param transName - * the transName to set - */ - public void setTransName( String transName ) { - this.transName = transName; - } - /** * @return the mappingParameters */ @@ -677,36 +543,6 @@ public void setRepository( Repository repository ) { this.repository = repository; } - /** - * @return the transObjectId - */ - public ObjectId getTransObjectId() { - return transObjectId; - } - - /** - * @param transObjectId - * the transObjectId to set - */ - public void setTransObjectId( ObjectId transObjectId ) { - this.transObjectId = transObjectId; - } - - /** - * @return the specificationMethod - */ - public ObjectLocationSpecificationMethod getSpecificationMethod() { - return specificationMethod; - } - - /** - * @param specificationMethod - * the specificationMethod to set - */ - public void setSpecificationMethod( ObjectLocationSpecificationMethod specificationMethod ) { - this.specificationMethod = specificationMethod; - } - @Override public StepIOMetaInterface getStepIOMeta() { if ( ioMeta == null ) { @@ -810,4 +646,5 @@ public MappingIODefinition getOutputMapping() { public void setOutputMapping( MappingIODefinition outputMapping ) { this.outputMapping = outputMapping; } + } diff --git a/engine/src/org/pentaho/di/trans/steps/singlethreader/SingleThreaderMeta.java b/engine/src/org/pentaho/di/trans/steps/singlethreader/SingleThreaderMeta.java index d9a24de64193..4d26a9801691 100644 --- a/engine/src/org/pentaho/di/trans/steps/singlethreader/SingleThreaderMeta.java +++ b/engine/src/org/pentaho/di/trans/steps/singlethreader/SingleThreaderMeta.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com + * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com * ******************************************************************************* * @@ -53,10 +53,10 @@ import org.pentaho.di.resource.ResourceEntry.ResourceType; import org.pentaho.di.resource.ResourceNamingInterface; import org.pentaho.di.resource.ResourceReference; +import org.pentaho.di.trans.StepWithMappingMeta; import org.pentaho.di.trans.Trans; import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.TransMeta.TransformationType; -import org.pentaho.di.trans.step.BaseStepMeta; import org.pentaho.di.trans.step.StepDataInterface; import org.pentaho.di.trans.step.StepInterface; import org.pentaho.di.trans.step.StepMeta; @@ -72,14 +72,11 @@ * */ -public class SingleThreaderMeta extends BaseStepMeta implements StepMetaInterface, HasRepositoryInterface { - private static Class PKG = SingleThreaderMeta.class; // for i18n purposes, needed by Translator2!! +public class SingleThreaderMeta extends StepWithMappingMeta implements StepMetaInterface, HasRepositoryInterface { - private String transName; - private String fileName; - private String directoryPath; - private ObjectId transObjectId; - private ObjectLocationSpecificationMethod specificationMethod; + static { + PKG = SingleThreaderMeta.class; // for i18n purposes, needed by Translator2!! + } private String batchSize; private String batchTime; @@ -298,72 +295,7 @@ public void getFields( RowMetaInterface row, String origin, RowMetaInterface[] i public static final synchronized TransMeta loadSingleThreadedTransMeta( SingleThreaderMeta mappingMeta, Repository rep, VariableSpace space ) throws KettleException { - return loadSingleThreadedTransMeta( mappingMeta, rep, null, space ); - } - - public static final synchronized TransMeta loadSingleThreadedTransMeta( SingleThreaderMeta mappingMeta, - Repository rep, IMetaStore metaStore, VariableSpace space ) throws KettleException { - TransMeta mappingTransMeta = null; - - switch ( mappingMeta.getSpecificationMethod() ) { - case FILENAME: - String realFilename = space.environmentSubstitute( mappingMeta.getFileName() ); - try { - // OK, load the meta-data from file... - // - // Don't set internal variables: they belong to the parent thread! - // - mappingTransMeta = new TransMeta( realFilename, false ); - mappingTransMeta.getLogChannel().logDetailed( - "Loading Mapping from repository", - "Mapping transformation was loaded from XML file [" + realFilename + "]" ); - } catch ( Exception e ) { - throw new KettleException( BaseMessages.getString( - PKG, "SingleThreaderMeta.Exception.UnableToLoadMapping" ), e ); - } - break; - - case REPOSITORY_BY_NAME: - String realTransname = space.environmentSubstitute( mappingMeta.getTransName() ); - String realDirectory = space.environmentSubstitute( mappingMeta.getDirectoryPath() ); - - if ( !Utils.isEmpty( realTransname ) && !Utils.isEmpty( realDirectory ) && rep != null ) { - RepositoryDirectoryInterface repdir = rep.findDirectory( realDirectory ); - if ( repdir != null ) { - try { - // reads the last revision in the repository... - // - mappingTransMeta = rep.loadTransformation( realTransname, repdir, null, true, null ); - mappingTransMeta.getLogChannel().logDetailed( - "Loading Mapping from repository", - "Mapping transformation [" + realTransname + "] was loaded from the repository" ); - } catch ( Exception e ) { - throw new KettleException( "Unable to load transformation [" + realTransname + "]", e ); - } - } else { - throw new KettleException( BaseMessages.getString( - PKG, "SingleThreaderMeta.Exception.UnableToLoadTransformation", realTransname ) - + realDirectory ); - } - } - break; - - case REPOSITORY_BY_REFERENCE: - // Read the last revision by reference... - mappingTransMeta = rep.loadTransformation( mappingMeta.getTransObjectId(), null ); - break; - default: - break; - } - - // Pass some important information to the mapping transformation metadata: - // - mappingTransMeta.copyVariablesFrom( space ); - mappingTransMeta.setRepository( rep ); - mappingTransMeta.setMetaStore( metaStore ); - mappingTransMeta.setFilename( mappingTransMeta.getFilename() ); - - return mappingTransMeta; + return loadMappingMeta( mappingMeta, rep, null, space ); } public void check( List remarks, TransMeta transMeta, StepMeta stepMeta, @@ -406,51 +338,6 @@ public StepDataInterface getStepData() { return new SingleThreaderData(); } - /** - * @return the directoryPath - */ - public String getDirectoryPath() { - return directoryPath; - } - - /** - * @param directoryPath - * the directoryPath to set - */ - public void setDirectoryPath( String directoryPath ) { - this.directoryPath = directoryPath; - } - - /** - * @return the fileName - */ - public String getFileName() { - return fileName; - } - - /** - * @param fileName - * the fileName to set - */ - public void setFileName( String fileName ) { - this.fileName = fileName; - } - - /** - * @return the transName - */ - public String getTransName() { - return transName; - } - - /** - * @param transName - * the transName to set - */ - public void setTransName( String transName ) { - this.transName = transName; - } - @Override public List getResourceDependencies( TransMeta transMeta, StepMeta stepInfo ) { List references = new ArrayList( 5 ); @@ -536,36 +423,6 @@ public void setRepository( Repository repository ) { this.repository = repository; } - /** - * @return the transObjectId - */ - public ObjectId getTransObjectId() { - return transObjectId; - } - - /** - * @param transObjectId - * the transObjectId to set - */ - public void setTransObjectId( ObjectId transObjectId ) { - this.transObjectId = transObjectId; - } - - /** - * @return the specificationMethod - */ - public ObjectLocationSpecificationMethod getSpecificationMethod() { - return specificationMethod; - } - - /** - * @param specificationMethod - * the specificationMethod to set - */ - public void setSpecificationMethod( ObjectLocationSpecificationMethod specificationMethod ) { - this.specificationMethod = specificationMethod; - } - public TransformationType[] getSupportedTransformationTypes() { return new TransformationType[] { TransformationType.Normal, }; } @@ -708,8 +565,6 @@ public boolean[] isReferencedObjectEnabled() { /** * Load the referenced object * - * @param meta - * The metadata that references * @param index * the object index to load * @param rep @@ -725,7 +580,7 @@ public Object loadReferencedObject( int index, Repository rep, VariableSpace spa } public Object loadReferencedObject( int index, Repository rep, IMetaStore metaStore, VariableSpace space ) throws KettleException { - return loadSingleThreadedTransMeta( this, rep, metaStore, space ); + return loadMappingMeta( this, rep, metaStore, space ); } @Override diff --git a/engine/src/org/pentaho/di/trans/steps/transexecutor/TransExecutor.java b/engine/src/org/pentaho/di/trans/steps/transexecutor/TransExecutor.java index 50e0fe4769ab..5beeaa0ac984 100644 --- a/engine/src/org/pentaho/di/trans/steps/transexecutor/TransExecutor.java +++ b/engine/src/org/pentaho/di/trans/steps/transexecutor/TransExecutor.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com + * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com * ******************************************************************************* * @@ -435,7 +435,7 @@ public boolean init( StepMetaInterface smi, StepDataInterface sdi ) { @VisibleForTesting TransMeta loadExecutorTransMeta() throws KettleException { - return TransExecutorMeta.loadTransMeta( meta, meta.getRepository(), meta.getMetaStore(), this ); + return TransExecutorMeta.loadMappingMeta( meta, meta.getRepository(), meta.getMetaStore(), this ); } public void dispose( StepMetaInterface smi, StepDataInterface sdi ) { diff --git a/engine/src/org/pentaho/di/trans/steps/transexecutor/TransExecutorMeta.java b/engine/src/org/pentaho/di/trans/steps/transexecutor/TransExecutorMeta.java index 0f5037781a75..7a8b8944af5e 100644 --- a/engine/src/org/pentaho/di/trans/steps/transexecutor/TransExecutorMeta.java +++ b/engine/src/org/pentaho/di/trans/steps/transexecutor/TransExecutorMeta.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com + * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com * ******************************************************************************* * @@ -36,11 +36,9 @@ import org.pentaho.di.core.exception.KettlePluginException; import org.pentaho.di.core.exception.KettleStepException; import org.pentaho.di.core.exception.KettleXMLException; -import org.pentaho.di.core.logging.LogChannel; import org.pentaho.di.core.row.RowMetaInterface; import org.pentaho.di.core.row.ValueMetaInterface; import org.pentaho.di.core.row.value.ValueMetaFactory; -import org.pentaho.di.core.util.CurrentDirectoryResolver; import org.pentaho.di.core.variables.VariableSpace; import org.pentaho.di.core.xml.XMLHandler; import org.pentaho.di.i18n.BaseMessages; @@ -58,10 +56,10 @@ import org.pentaho.di.resource.ResourceEntry.ResourceType; import org.pentaho.di.resource.ResourceNamingInterface; import org.pentaho.di.resource.ResourceReference; +import org.pentaho.di.trans.StepWithMappingMeta; import org.pentaho.di.trans.Trans; import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.TransMeta.TransformationType; -import org.pentaho.di.trans.step.BaseStepMeta; import org.pentaho.di.trans.step.StepDataInterface; import org.pentaho.di.trans.step.StepIOMeta; import org.pentaho.di.trans.step.StepIOMetaInterface; @@ -81,23 +79,16 @@ * @author Matt * @since 18-mar-2013 */ -public class TransExecutorMeta extends BaseStepMeta implements StepMetaInterface, HasRepositoryInterface { - private static final Class PKG = TransExecutorMeta.class; // for i18n purposes, needed by Translator2!! +public class TransExecutorMeta extends StepWithMappingMeta implements StepMetaInterface, HasRepositoryInterface { + + static { + PKG = TransExecutorMeta.class; // for i18n purposes, needed by Translator2!! + } static final String F_EXECUTION_RESULT_TARGET_STEP = "execution_result_target_step"; static final String F_RESULT_FILE_TARGET_STEP = "result_files_target_step"; static final String F_EXECUTOR_OUTPUT_STEP = "executors_output_step"; - private String transName; - - private String fileName; - - private String directoryPath; - - private ObjectId transObjectId; - - private ObjectLocationSpecificationMethod specificationMethod; - /** * The number of input rows that are sent as result rows to the job in one go, defaults to "1" */ @@ -608,120 +599,7 @@ public String[] getInfoSteps() { @Deprecated public static synchronized TransMeta loadTransMeta( TransExecutorMeta executorMeta, Repository rep, VariableSpace space ) throws KettleException { - return loadTransMeta( executorMeta, rep, null, space ); - } - - public static synchronized TransMeta loadTransMeta( TransExecutorMeta executorMeta, Repository rep, - IMetaStore metaStore, VariableSpace space ) throws KettleException { - TransMeta mappingTransMeta = null; - - CurrentDirectoryResolver r = new CurrentDirectoryResolver(); - VariableSpace tmpSpace = - r.resolveCurrentDirectory( executorMeta.getSpecificationMethod(), space, rep, executorMeta.getParentStepMeta(), - executorMeta.getFileName() ); - - switch ( executorMeta.getSpecificationMethod() ) { - case FILENAME: - String realFilename = tmpSpace.environmentSubstitute( executorMeta.getFileName() ); - try { - // OK, load the meta-data from file... - // - // Don't set internal variables: they belong to the parent thread! - // - if ( rep != null ) { - // need to try to load from the repository - realFilename = r.normalizeSlashes( realFilename ); - try { - String dirStr = realFilename.substring( 0, realFilename.lastIndexOf( "/" ) ); - String tmpFilename = realFilename.substring( realFilename.lastIndexOf( "/" ) + 1 ); - RepositoryDirectoryInterface dir = rep.findDirectory( dirStr ); - mappingTransMeta = rep.loadTransformation( tmpFilename, dir, null, true, null ); - } catch ( KettleException ke ) { - // try without extension - if ( realFilename.endsWith( Const.STRING_TRANS_DEFAULT_EXT ) ) { - try { - String tmpFilename = - realFilename.substring( realFilename.lastIndexOf( "/" ) + 1, realFilename.indexOf( "." - + Const.STRING_TRANS_DEFAULT_EXT ) ); - String dirStr = realFilename.substring( 0, realFilename.lastIndexOf( "/" ) ); - RepositoryDirectoryInterface dir = rep.findDirectory( dirStr ); - mappingTransMeta = rep.loadTransformation( tmpFilename, dir, null, true, null ); - } catch ( KettleException ke2 ) { - // fall back to try loading from file system (transMeta is going to be null) - } - } - } - } - if ( mappingTransMeta == null ) { - mappingTransMeta = new TransMeta( realFilename, metaStore, rep, true, tmpSpace, null ); - LogChannel.GENERAL.logDetailed( "Loading transformation from repository", - "Transformation was loaded from XML file [" + realFilename + "]" ); - } - } catch ( Exception e ) { - throw new KettleException( BaseMessages.getString( PKG, "TransExecutorMeta.Exception.UnableToLoadTrans" ), - e ); - } - break; - - case REPOSITORY_BY_NAME: - String realTransname = tmpSpace.environmentSubstitute( executorMeta.getTransName() ); - String realDirectory = tmpSpace.environmentSubstitute( executorMeta.getDirectoryPath() ); - - if ( rep != null ) { - if ( !Utils.isEmpty( realTransname ) && !Utils.isEmpty( realDirectory ) ) { - realDirectory = r.normalizeSlashes( realDirectory ); - RepositoryDirectoryInterface repdir = rep.findDirectory( realDirectory ); - if ( repdir != null ) { - try { - // reads the last revision in the repository... - // - mappingTransMeta = rep.loadTransformation( realTransname, repdir, null, true, null ); // TODO: FIXME: - // pass - // in metaStore to - // repository? - - LogChannel.GENERAL.logDetailed( "Loading transformation from repository", "Executor transformation [" - + realTransname + "] was loaded from the repository" ); - } catch ( Exception e ) { - throw new KettleException( "Unable to load transformation [" + realTransname + "]", e ); - } - } - } - } else { - // rep is null, let's try loading by filename - try { - mappingTransMeta = - new TransMeta( realDirectory + "/" + realTransname, metaStore, rep, true, tmpSpace, null ); - } catch ( KettleException ke ) { - try { - // add .ktr extension and try again - mappingTransMeta = - new TransMeta( realDirectory + "/" + realTransname + "." + Const.STRING_TRANS_DEFAULT_EXT, metaStore, - rep, true, tmpSpace, null ); - } catch ( KettleException ke2 ) { - throw new KettleException( BaseMessages.getString( PKG, "TransExecutorMeta.Exception.UnableToLoadTrans", - realTransname ) + realDirectory ); - } - } - } - break; - - case REPOSITORY_BY_REFERENCE: - // Read the last revision by reference... - mappingTransMeta = rep.loadTransformation( executorMeta.getTransObjectId(), null ); - break; - default: - break; - } - - // Pass some important information to the mapping transformation metadata: - // - mappingTransMeta.copyVariablesFrom( space ); - mappingTransMeta.setRepository( rep ); - mappingTransMeta.setMetaStore( metaStore ); - mappingTransMeta.setFilename( mappingTransMeta.getFilename() ); - - return mappingTransMeta; + return loadMappingMeta( executorMeta, rep, null, space ); } public void check( List remarks, TransMeta transMeta, StepMeta stepinfo, RowMetaInterface prev, @@ -914,36 +792,6 @@ public void lookupRepositoryReferences( Repository repository ) throws KettleExc transObjectId = repository.getTransformationID( transName, repositoryDirectoryInterface ); } - /** - * @return the directoryPath - */ - public String getDirectoryPath() { - return directoryPath; - } - - /** - * @param directoryPath - * the directoryPath to set - */ - public void setDirectoryPath( String directoryPath ) { - this.directoryPath = directoryPath; - } - - /** - * @return the fileName - */ - public String getFileName() { - return fileName; - } - - /** - * @param fileName - * the fileName to set - */ - public void setFileName( String fileName ) { - this.fileName = fileName; - } - /** * @return the mappingParameters */ @@ -974,51 +822,6 @@ public void setRepository( Repository repository ) { this.repository = repository; } - /** - * @return the specificationMethod - */ - public ObjectLocationSpecificationMethod getSpecificationMethod() { - return specificationMethod; - } - - /** - * @param specificationMethod - * the specificationMethod to set - */ - public void setSpecificationMethod( ObjectLocationSpecificationMethod specificationMethod ) { - this.specificationMethod = specificationMethod; - } - - /** - * @return the transName - */ - public String getTransName() { - return transName; - } - - /** - * @param transName - * the transName to set - */ - public void setTransName( String transName ) { - this.transName = transName; - } - - /** - * @return the transObjectId - */ - public ObjectId getTransObjectId() { - return transObjectId; - } - - /** - * @param transObjectId - * the transObjectId to set - */ - public void setTransObjectId( ObjectId transObjectId ) { - this.transObjectId = transObjectId; - } - /** * @return the parameters */ @@ -1369,7 +1172,7 @@ public boolean[] isReferencedObjectEnabled() { */ public Object loadReferencedObject( int index, Repository rep, IMetaStore metaStore, VariableSpace space ) throws KettleException { - return loadTransMeta( this, rep, metaStore, space ); + return loadMappingMeta( this, rep, metaStore, space ); } public IMetaStore getMetaStore() { diff --git a/engine/test-src/org/pentaho/di/trans/StepWithMappingMetaTest.java b/engine/test-src/org/pentaho/di/trans/StepWithMappingMetaTest.java new file mode 100644 index 000000000000..ee26d476b81b --- /dev/null +++ b/engine/test-src/org/pentaho/di/trans/StepWithMappingMetaTest.java @@ -0,0 +1,83 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ +package org.pentaho.di.trans; + +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.pentaho.di.core.ObjectLocationSpecificationMethod; +import org.pentaho.di.core.ProgressMonitorListener; +import org.pentaho.di.core.variables.VariableSpace; +import org.pentaho.di.core.variables.Variables; +import org.pentaho.di.repository.Repository; +import org.pentaho.di.repository.RepositoryDirectoryInterface; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.*; + +/** + * Created by Yury_Bakhmutski on 2/8/2017. + */ +public class StepWithMappingMetaTest { + + @Test + public void loadMappingMeta() throws Exception { + String variablePath = "Internal.Entry.Current.Directory"; + String virtualDir = "/testFolder/CDA-91"; + String fileName = "testTrans.ktr"; + + VariableSpace variables = new Variables(); + variables.setVariable( variablePath, virtualDir ); + + StepWithMappingMeta mappingMetaMock = mock( StepWithMappingMeta.class ); + when( mappingMetaMock.getSpecificationMethod() ).thenReturn( ObjectLocationSpecificationMethod.FILENAME ); + when( mappingMetaMock.getFileName() ).thenReturn( "${" + variablePath + "}/" + fileName ); + + // mock repo and answers + Repository rep = mock( Repository.class ); + + Mockito.doAnswer( new Answer() { + @Override + public TransMeta answer( final InvocationOnMock invocation ) throws Throwable { + final String originalArgument = (String) ( invocation.getArguments() )[ 0 ]; + // be sure that the variable was replaced by real path + assertEquals( originalArgument, virtualDir ); + return null; + } + } ).when( rep ).findDirectory( anyString() ); + + Mockito.doAnswer( new Answer() { + @Override + public TransMeta answer( final InvocationOnMock invocation ) throws Throwable { + final String originalArgument = (String) ( invocation.getArguments() )[ 0 ]; + // be sure that transformation name was resolved correctly + assertEquals( originalArgument, fileName ); + return mock( TransMeta.class ); + } + } ).when( rep ).loadTransformation( anyString(), any( RepositoryDirectoryInterface.class ), + any( ProgressMonitorListener.class ), anyBoolean(), anyString() ); + + StepWithMappingMeta.loadMappingMeta( mappingMetaMock, rep, null, variables, true ); + } + +}