From ae5cc84adedcf0dec4759354ae4be7d44eae02f7 Mon Sep 17 00:00:00 2001 From: Matt Casters Date: Mon, 10 May 2021 15:59:55 +0200 Subject: [PATCH 1/6] HOP-2858 : Allow metadata injection through metadata property annotations --- .../hop/metadata/api/HopMetadataProperty.java | 35 ++ .../injection/bean/BeanInjectionInfo.java | 461 +++++++++++++---- .../core/injection/bean/BeanLevelInfo.java | 268 +++++----- .../serialization/TransformMetaProps.java | 309 +++++------ .../MetaAnnotationInjectionTest.java | 237 +++++---- .../transforms/0011-calculator-basics.hpl | 214 ++------ .../transforms/calculator/CalculatorMeta.java | 8 +- .../calculator/CalculatorMetaFunction.java | 52 +- .../messages/messages_en_US.properties | 19 +- .../transforms/metainject/MetaInject.java | 487 ++++++++++-------- .../metainject/MetaInjectDialog.java | 52 +- 11 files changed, 1217 insertions(+), 925 deletions(-) diff --git a/core/src/main/java/org/apache/hop/metadata/api/HopMetadataProperty.java b/core/src/main/java/org/apache/hop/metadata/api/HopMetadataProperty.java index df4cd146df..7a08a3be3d 100644 --- a/core/src/main/java/org/apache/hop/metadata/api/HopMetadataProperty.java +++ b/core/src/main/java/org/apache/hop/metadata/api/HopMetadataProperty.java @@ -17,6 +17,9 @@ package org.apache.hop.metadata.api; +import org.apache.hop.core.injection.DefaultInjectionTypeConverter; +import org.apache.hop.core.injection.InjectionTypeConverter; + import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -52,4 +55,36 @@ /** @return The default value to return for a non-existing boolean value */ boolean defaultBoolean() default false; + + /** + * @return The metadata key for this property. Don't specify any key if you want this to be the + * same as key(); + */ + String injectionKey() default ""; + + /** @return The metadata description for this property. (i18n) */ + String injectionKeyDescription() default ""; + + /** + * @return The metadata group key to which this property belongs. Don't specify any key if you + * want this to be the same as key(); + */ + String injectionGroupKey() default ""; + + /** @return A description of the metadata group key to which this property belongs. (i18n) */ + String injectionGroupDescription() default ""; + + /** + * A description of the field. Right now this is used only for metadata injection purposes + * + * @return The description of the property + */ + String description() default ""; + + /** + * @return The class to instantiate to convert metadata properly for this property (dates, + * numbers, ...) + */ + Class injectionConverter() default + DefaultInjectionTypeConverter.class; } diff --git a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjectionInfo.java b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjectionInfo.java index 3b718c0324..cd5cc0cc6d 100644 --- a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjectionInfo.java +++ b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjectionInfo.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,11 +23,16 @@ import org.apache.hop.core.logging.HopLogStore; import org.apache.hop.core.logging.ILogChannel; import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.util.ReflectionUtil; import org.apache.hop.pipeline.transform.ITransformMeta; +import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.lang.reflect.TypeVariable; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -35,72 +40,289 @@ import java.util.Map; import java.util.Set; -/** - * Storage for bean annotations info for Metadata Injection and Load/Save. - */ +/** Storage for bean annotations info for Metadata Injection and Load/Save. */ public class BeanInjectionInfo { - private static ILogChannel LOG = HopLogStore.getLogChannelFactory().create( BeanInjectionInfo.class ); + private static ILogChannel LOG = + HopLogStore.getLogChannelFactory().create(BeanInjectionInfo.class); protected final Class clazz; private final InjectionSupported clazzAnnotation; private Map properties = new HashMap<>(); private List groupsList = new ArrayList<>(); - /** - * Used only for fast group search during initialize. - */ + + /** Used only for fast group search during initialize. */ private Map groupsMap = new HashMap<>(); + private Set hideProperties = new HashSet<>(); - public static boolean isInjectionSupported( Class clazz ) { - InjectionSupported annotation = clazz.getAnnotation( InjectionSupported.class ); - return annotation != null; + public static boolean isInjectionSupported(Class clazz) { + InjectionSupported annotation = clazz.getAnnotation(InjectionSupported.class); + if (annotation != null) { + return true; + } + + // See if there are any Hop metadata properties we can use... + // + for (Field field : ReflectionUtil.findAllFields(clazz)) { + HopMetadataProperty property = field.getAnnotation(HopMetadataProperty.class); + if (property != null) { + return true; + } + } + + return false; } - public BeanInjectionInfo( Class clazz ) { + public BeanInjectionInfo(Class clazz) { if (LOG.isDebug()) { - LOG.logDebug( "Collect bean injection info for " + clazz ); + LOG.logDebug("Collect bean injection info for " + clazz); } try { this.clazz = clazz; - clazzAnnotation = clazz.getAnnotation( InjectionSupported.class ); - if ( clazzAnnotation == null ) { - throw new RuntimeException( "Injection not supported in " + clazz ); + clazzAnnotation = clazz.getAnnotation(InjectionSupported.class); + if (!isInjectionSupported(clazz)) { + throw new RuntimeException("Injection not supported in " + clazz); } - Map parameterTypesMap = new HashMap<>( ); - TypeVariable>[] typeParameters = clazz.getTypeParameters(); - for (TypeVariable> typeParameter : typeParameters) { - String parameterTypeName = typeParameter.getName(); - Type parameterType = typeParameter.getBounds()[0]; - parameterTypesMap.put(parameterTypeName, parameterType); + if (clazzAnnotation == null) { + extractMetadataProperties(clazz); + } else { + extractInjectionInfo(clazz); } + } catch (Throwable ex) { + LOG.logError( + "Error bean injection info collection for " + clazz + ": " + ex.getMessage(), ex); + throw ex; + } + } - Group gr0 = new Group( "" ); - groupsList.add( gr0 ); - groupsMap.put( gr0.getName(), gr0 ); - for ( String group : clazzAnnotation.groups() ) { - Group gr = new Group( group ); - groupsList.add( gr ); - groupsMap.put( gr.getName(), gr ); + private void extractMetadataProperties(Class clazz) { + Map propertyFields = new HashMap<>(); + for (Field field : ReflectionUtil.findAllFields(clazz)) { + HopMetadataProperty property = field.getAnnotation(HopMetadataProperty.class); + if (property != null) { + propertyFields.put(field, property); } - for ( String p : clazzAnnotation.hide() ) { - hideProperties.add( p ); + } + + if (propertyFields.isEmpty()) { + throw new RuntimeException("Injection not supported in " + clazz); + } + + // The root Bean Level Info + // + BeanLevelInfo classLevelInfo = new BeanLevelInfo(); + classLevelInfo.leafClass = clazz; + + // Add an empty group... + // + Group gr0 = new Group("", ""); + groupsList.add(gr0); + groupsMap.put(gr0.getKey(), gr0); + + for (Field field : propertyFields.keySet()) { + Class fieldType = field.getType(); + + HopMetadataProperty property = propertyFields.get(field); + + String injectionKey = calculateInjectionKey(field, property); + String injectionKeyDescription = calculateInjectionKeyDescription(property); + String injectionGroupKey = calculateInjectionGroupKey(property); + String injectionGroupDescription = calculateInjectionGroupDescription(property); + + // Class Bean Level Info... + // + BeanLevelInfo fieldLevelInfo = + getLevelInfo(clazz, classLevelInfo, field, fieldType, property, injectionKey); + + Group group = null; + if (StringUtils.isNotEmpty(injectionGroupKey)) { + group = new Group(injectionGroupKey, injectionGroupDescription); + + groupsList.add(group); + groupsMap.put(injectionGroupKey, group); } - BeanLevelInfo root = new BeanLevelInfo(); - root.leafClass = clazz; - if (parameterTypesMap.isEmpty()) { - root.init( this ); + if (StringUtils.isNotEmpty(injectionGroupKey)) { + // Find child properties of the field class... + // If not a POJO we simply have the one property + // + + // If it's a List: get that type... + // + if (fieldType.equals(java.util.List.class)) { + ParameterizedType parameterizedType = (ParameterizedType) field.getGenericType(); + // Take the generic class to pick the child properties from. + // + fieldType = (Class) parameterizedType.getActualTypeArguments()[0]; + fieldLevelInfo.dim = BeanLevelInfo.DIMENSION.LIST; + fieldLevelInfo.leafClass = fieldType; // Not List but the listed type + } + + for (Field childField : ReflectionUtil.findAllFields(fieldType)) { + Class childFieldType = childField.getType(); + HopMetadataProperty childProperty = childField.getAnnotation(HopMetadataProperty.class); + if (childProperty != null) { + String childInjectionKey = calculateInjectionKey(childField, childProperty); + String childInjectionKeyDescription = calculateInjectionKeyDescription(childProperty); + + // Child bean level info... + // + BeanLevelInfo childLevelInfo = + getLevelInfo( + fieldType, + fieldLevelInfo, + childField, + childFieldType, + childProperty, + childInjectionKey); + + List path = + Arrays.asList(classLevelInfo, fieldLevelInfo, childLevelInfo); + Property p = + new Property( + childInjectionKey, childInjectionKeyDescription, injectionGroupKey, path); + group.properties.add(p); + properties.put(childInjectionKey, p); + } + } } else { - root.init( this, parameterTypesMap ); + // Normal property: add it to the root group + // + Property p = + new Property( + injectionKey, + injectionKeyDescription, + "", + Arrays.asList(classLevelInfo, fieldLevelInfo)); + gr0.properties.add(p); + properties.put(injectionKey, p); } - properties = Collections.unmodifiableMap( properties ); - groupsList = Collections.unmodifiableList( groupsList ); - groupsMap = null; - } catch ( Throwable ex ) { - LOG.logError( "Error bean injection info collection for " + clazz + ": " + ex.getMessage(), ex ); - throw ex; } + properties = Collections.unmodifiableMap(properties); + groupsList = Collections.unmodifiableList(groupsList); + } + + private BeanLevelInfo getLevelInfo( + Class parentClass, + BeanLevelInfo parent, + Field field, + Class fieldType, + HopMetadataProperty property, + String injectionKey) { + BeanLevelInfo fieldLevelInfo = new BeanLevelInfo(); + fieldLevelInfo.parent = parent; + fieldLevelInfo.leafClass = fieldType; + try { + fieldLevelInfo.converter = property.injectionConverter().newInstance(); + } catch (Exception e) { + throw new RuntimeException( + "Unable to instantiate injection metadata converter class " + + property.injectionConverter().getName(), + e); + } + fieldLevelInfo.nameKey = injectionKey; + fieldLevelInfo.field = field; + if (field != null) { + fieldLevelInfo.field.setAccessible(true); + } + fieldLevelInfo.dim = BeanLevelInfo.DIMENSION.NONE; + boolean isBoolean = Boolean.class.equals(fieldType) || boolean.class.equals(fieldType); + try { + fieldLevelInfo.getter = + parentClass.getMethod(ReflectionUtil.getGetterMethodName(field.getName(), isBoolean)); + } catch (Exception e) { + throw new RuntimeException( + "Unable to find getter for field " + + field.getName() + + " in class " + + parentClass.getName(), + e); + } + try { + fieldLevelInfo.setter = + parentClass.getMethod(ReflectionUtil.getSetterMethodName(field.getName()), fieldType); + } catch (Exception e) { + throw new RuntimeException( + "Unable to find setter for field " + + field.getName() + + " in class " + + parentClass.getName(), + e); + } + return fieldLevelInfo; + } + + private String calculateInjectionGroupDescription(HopMetadataProperty property) { + String injectionGroupDescription = property.injectionGroupDescription(); + if (StringUtils.isEmpty(injectionGroupDescription)) { + injectionGroupDescription = ""; + } + return injectionGroupDescription; + } + + private String calculateInjectionGroupKey(HopMetadataProperty property) { + String injectionGroupKey = property.injectionGroupKey(); + if (StringUtils.isEmpty(injectionGroupKey)) { + injectionGroupKey = property.groupKey(); + } + return injectionGroupKey; + } + + private String calculateInjectionKeyDescription(HopMetadataProperty property) { + String injectionKeyDescription = property.injectionKeyDescription(); + if (StringUtils.isEmpty(injectionKeyDescription)) { + injectionKeyDescription = ""; + } + return injectionKeyDescription; + } + + private String calculateInjectionKey(Field field, HopMetadataProperty property) { + String injectionKey = property.injectionKey(); + if (StringUtils.isEmpty(injectionKey)) { + injectionKey = property.key(); + } + if (StringUtils.isEmpty(injectionKey)) { + injectionKey = field.getName(); + } + return injectionKey; + } + + private void extractInjectionInfo(Class clazz) { + Map parameterTypesMap = new HashMap<>(); + TypeVariable>[] typeParameters = clazz.getTypeParameters(); + for (TypeVariable> typeParameter : typeParameters) { + String parameterTypeName = typeParameter.getName(); + Type parameterType = typeParameter.getBounds()[0]; + parameterTypesMap.put(parameterTypeName, parameterType); + } + + // Add an empty group + // + Group gr0 = new Group("", ""); + groupsList.add(gr0); + groupsMap.put(gr0.getKey(), gr0); + + for (String group : clazzAnnotation.groups()) { + String groupDescription = clazzAnnotation.localizationPrefix() + group; + Group gr = new Group(group, groupDescription); + groupsList.add(gr); + groupsMap.put(gr.getKey(), gr); + } + for (String p : clazzAnnotation.hide()) { + hideProperties.add(p); + } + + BeanLevelInfo root = new BeanLevelInfo(); + root.leafClass = clazz; + if (parameterTypesMap.isEmpty()) { + root.init(this); + } else { + root.init(this, parameterTypesMap); + } + properties = Collections.unmodifiableMap(properties); + groupsList = Collections.unmodifiableList(groupsList); + groupsMap = null; } public String getLocalizationPrefix() { @@ -115,106 +337,130 @@ public List getGroups() { return groupsList; } - protected void addInjectionProperty( Injection metaInj, BeanLevelInfo leaf ) { - if ( StringUtils.isBlank( metaInj.name() ) ) { - throw new RuntimeException( "Property name shouldn't be blank in the " + clazz ); + protected void addInjectionProperty(Injection metaInj, BeanLevelInfo leaf) { + if (StringUtils.isBlank(metaInj.name())) { + throw new RuntimeException("Property name shouldn't be blank in the " + clazz); } - String propertyName = calcPropertyName( metaInj, leaf ); - if ( properties.containsKey( propertyName ) ) { - throw new RuntimeException( "Property '" + propertyName + "' already defined for " + clazz ); + String propertyName = calcPropertyName(metaInj, leaf); + if (properties.containsKey(propertyName)) { + throw new RuntimeException("Property '" + propertyName + "' already defined for " + clazz); } - // probably hided - if ( hideProperties.contains( propertyName ) ) { + // probably hidden + if (hideProperties.contains(propertyName)) { return; } - Property prop = new Property( propertyName, metaInj.group(), leaf.createCallStack() ); - properties.put( prop.name, prop ); - Group gr = groupsMap.get( metaInj.group() ); - if ( gr == null ) { - throw new RuntimeException( "Group '" + metaInj.group() + "' for property '" + metaInj.name() + "' is not defined " + clazz ); + String injectionKeyDescription = clazzAnnotation.localizationPrefix() + propertyName; + + Property prop = + new Property( + propertyName, injectionKeyDescription, metaInj.group(), leaf.createCallStack()); + properties.put(prop.key, prop); + Group gr = groupsMap.get(metaInj.group()); + if (gr == null) { + throw new RuntimeException( + "Group '" + + metaInj.group() + + "' for property '" + + metaInj.name() + + "' is not defined " + + clazz); } - gr.groupProperties.add( prop ); + gr.properties.add(prop); } - public String getDescription( String name ) { - String description = BaseMessages.getString( clazz, clazzAnnotation.localizationPrefix() + name ); - if ( description != null && description.startsWith( "!" ) && description.endsWith( "!" ) ) { + public String getDescription(String description) { + if (StringUtils.isEmpty(description)) { + return ""; + } + String translated = BaseMessages.getString(clazz, description); + if (translated != null && translated.startsWith("!") && translated.endsWith("!")) { Class baseClass = clazz.getSuperclass(); - while ( baseClass != null ) { - InjectionSupported baseAnnotation = (InjectionSupported) baseClass.getAnnotation( InjectionSupported.class ); - if ( baseAnnotation != null ) { - description = BaseMessages.getString( baseClass, baseAnnotation.localizationPrefix() + name ); - if ( description != null && !description.startsWith( "!" ) && !description.endsWith( "!" ) ) { - return description; + while (baseClass != null) { + InjectionSupported baseAnnotation = baseClass.getAnnotation(InjectionSupported.class); + if (baseAnnotation != null) { + translated = BaseMessages.getString(baseClass, description); + if (translated != null && !translated.startsWith("!") && !translated.endsWith("!")) { + return translated; } } baseClass = baseClass.getSuperclass(); } } - return description; + return translated; } - private String calcPropertyName( Injection metaInj, BeanLevelInfo leaf ) { + private String calcPropertyName(Injection metaInj, BeanLevelInfo leaf) { String name = metaInj.name(); - while ( leaf != null ) { - if ( StringUtils.isNotBlank( leaf.prefix ) ) { - name = leaf.prefix + '.' + name; + while (leaf != null) { + if (StringUtils.isNotBlank(leaf.nameKey)) { + name = leaf.nameKey; } leaf = leaf.parent; } - if ( !name.equals( metaInj.name() ) && !metaInj.group().isEmpty() ) { + if (!name.equals(metaInj.name()) && !metaInj.group().isEmpty()) { // group exist with prefix - throw new RuntimeException( "Group shouldn't be declared with prefix in " + clazz ); + throw new RuntimeException("Group shouldn't be declared with prefix in " + clazz); } return name; } public class Property { - private final String name; - private final String groupName; + private final String key; + private final String description; + private final String groupKey; protected final List path; public final int pathArraysCount; - public Property( String name, String groupName, List path ) { - this.name = name; - this.groupName = groupName; + public Property(String key, String description, String groupKey, List path) { + this.key = key; + this.description = description; + this.groupKey = groupKey; this.path = path; int ac = 0; - for ( BeanLevelInfo level : path ) { - if ( level.dim != BeanLevelInfo.DIMENSION.NONE ) { + for (BeanLevelInfo level : path) { + if (level.dim != BeanLevelInfo.DIMENSION.NONE) { ac++; } } pathArraysCount = ac; } - public String getName() { - return name; + public String getKey() { + return key; } - public String getGroupName() { - return groupName; + /** + * Gets description + * + * @return value of description + */ + public String getDescription() { + return description; } - public String getDescription() { - return BeanInjectionInfo.this.getDescription( name ); + public String getGroupKey() { + return groupKey; + } + + public String getTranslatedDescription() { + return BeanInjectionInfo.this.getDescription(description); } public Class getPropertyClass() { - return path.get( path.size() - 1 ).leafClass; + return path.get(path.size() - 1).leafClass; } - public boolean hasMatch( String filterString ) { + public boolean hasMatch(String filterString) { if (StringUtils.isEmpty(filterString)) { return true; } - if (getName().toUpperCase().contains( filterString.toUpperCase() )) { + if (getKey().toUpperCase().contains(filterString.toUpperCase())) { return true; } - if (getDescription().toUpperCase().contains( filterString.toUpperCase() )) { + if (getTranslatedDescription().toUpperCase().contains(filterString.toUpperCase())) { return true; } return false; @@ -222,36 +468,47 @@ public boolean hasMatch( String filterString ) { } public class Group { - private final String name; - protected final List groupProperties = new ArrayList<>(); + private final String key; + private final String description; + protected final List properties = new ArrayList<>(); + + public Group(String key, String description) { + this.key = key; + this.description = description; + } - public Group( String name ) { - this.name = name; + public String getKey() { + return key; } - public String getName() { - return name; + /** + * Gets groupDescription + * + * @return value of groupDescription + */ + public String getDescription() { + return description; } - public List getGroupProperties() { - return Collections.unmodifiableList( groupProperties ); + public List getProperties() { + return Collections.unmodifiableList(properties); } - public String getDescription() { - return BeanInjectionInfo.this.getDescription( name ); + public String getTranslatedDescription() { + return BeanInjectionInfo.this.getDescription(description); } public boolean hasMatchingProperty(String filterString) { // Empty string always matches - if (StringUtils.isEmpty( filterString )) { + if (StringUtils.isEmpty(filterString)) { return true; } // The group name also matches // - if (name.toUpperCase().contains( filterString.toUpperCase() )) { + if (key.toUpperCase().contains(filterString.toUpperCase())) { return true; } - for (Property property : groupProperties) { + for (Property property : properties) { if (property.hasMatch(filterString)) { return true; } diff --git a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java index 3eb641a80b..5f52afa297 100644 --- a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java +++ b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,6 +17,7 @@ package org.apache.hop.core.injection.bean; +import org.apache.commons.lang.StringUtils; import org.apache.hop.core.injection.Injection; import org.apache.hop.core.injection.InjectionDeep; import org.apache.hop.core.injection.InjectionTypeConverter; @@ -35,67 +36,48 @@ import java.util.Map; import java.util.TreeMap; -/** - * Storage for one transform on the bean deep level. - */ +/** Storage for one transform on the bean deep level. */ class BeanLevelInfo { enum DIMENSION { - NONE, ARRAY, LIST - } + NONE, + ARRAY, + LIST + }; - ; - - /** - * Parent transform or null for root. - */ + /** Parent transform or null for root. */ public BeanLevelInfo parent; - /** - * Class for transform from field or methods. - */ + /** Class for transform from field or methods. */ public Class leafClass; - /** - * Field of transform, or null if bean has getter/setter. - */ + /** Field of transform, or null if bean has getter/setter. */ public Field field; - /** - * Getter and setter. - */ + /** Getter and setter. */ public Method getter, setter; - /** - * Dimension of level. - */ + /** Dimension of level. */ public DIMENSION dim = DIMENSION.NONE; - /** - * Values converter. - */ + /** Values converter. */ public InjectionTypeConverter converter; - /** - * False if source empty value shoudn't affect on target field. - */ + + /** False if source empty value shouldn't affect on target field. */ public boolean convertEmpty; - /** - * Name prefix on the path. - */ - public String prefix; + /** Name on the path : prefix + key usually. */ + public String nameKey; - public void init( BeanInjectionInfo info ) { - introspect( info, leafClass, new TreeMap<>() ); + public void init(BeanInjectionInfo info) { + introspect(info, leafClass, new TreeMap<>()); } - public void init( BeanInjectionInfo info, Map ownerGenericsInfo ) { - introspect( info, leafClass, ownerGenericsInfo ); + public void init(BeanInjectionInfo info, Map ownerGenericsInfo) { + introspect(info, leafClass, ownerGenericsInfo); } - /** - * Introspect class and all interfaces and ancestors recursively. - */ - private void introspect( BeanInjectionInfo info, Type type, Map ownerGenericsInfo ) { - Map genericsInfo = new TreeMap<>( ownerGenericsInfo ); + /** Introspect class and all interfaces and ancestors recursively. */ + private void introspect(BeanInjectionInfo info, Type type, Map ownerGenericsInfo) { + Map genericsInfo = new TreeMap<>(ownerGenericsInfo); - while ( type != null ) { + while (type != null) { Class clazz; ParameterizedType pt; - if ( type instanceof ParameterizedType ) { + if (type instanceof ParameterizedType) { pt = (ParameterizedType) type; clazz = (Class) pt.getRawType(); } else { @@ -104,191 +86,201 @@ private void introspect( BeanInjectionInfo info, Type type, Map ow } // introspect generics TypeVariable[] tps = clazz.getTypeParameters(); - if ( tps.length > 0 ) { - if ( pt == null ) { - throw new RuntimeException( "Can't introspect class with parameters on the high level" ); + if (tps.length > 0) { + if (pt == null) { + throw new RuntimeException("Can't introspect class with parameters on the high level"); } Type[] args = pt.getActualTypeArguments(); - if ( tps.length != args.length ) { - throw new RuntimeException( "Wrong generics declaration" ); + if (tps.length != args.length) { + throw new RuntimeException("Wrong generics declaration"); } Map prevGenerics = genericsInfo; genericsInfo = new TreeMap<>(); - for ( int i = 0; i < tps.length; i++ ) { - if ( args[ i ] instanceof TypeVariable ) { - TypeVariable argsi = (TypeVariable) args[ i ]; - Type prev = prevGenerics.get( argsi.getName() ); - if ( prev == null ) { - throw new RuntimeException( "Generic '" + args[ i ] + "' was not declared yet" ); + for (int i = 0; i < tps.length; i++) { + if (args[i] instanceof TypeVariable) { + TypeVariable argsi = (TypeVariable) args[i]; + Type prev = prevGenerics.get(argsi.getName()); + if (prev == null) { + throw new RuntimeException("Generic '" + args[i] + "' was not declared yet"); } - genericsInfo.put( tps[ i ].getName(), prev ); + genericsInfo.put(tps[i].getName(), prev); } else { - genericsInfo.put( tps[ i ].getName(), args[ i ] ); + genericsInfo.put(tps[i].getName(), args[i]); } } } - introspect( info, clazz.getDeclaredFields(), clazz.getDeclaredMethods(), genericsInfo ); - for ( Type intf : clazz.getGenericInterfaces() ) { - introspect( info, intf, genericsInfo ); + introspect(info, clazz.getDeclaredFields(), clazz.getDeclaredMethods(), genericsInfo); + for (Type intf : clazz.getGenericInterfaces()) { + introspect(info, intf, genericsInfo); } type = clazz.getGenericSuperclass(); } } - /** - * Introspect fields and methods of some class. - */ - protected void introspect( BeanInjectionInfo info, Field[] fields, Method[] methods, - Map genericsInfo ) { - for ( Field f : fields ) { - Injection annotationInjection = f.getAnnotation( Injection.class ); - InjectionDeep annotationInjectionDeep = f.getAnnotation( InjectionDeep.class ); - if ( annotationInjection == null && annotationInjectionDeep == null ) { + /** Introspect fields and methods of some class. */ + protected void introspect( + BeanInjectionInfo info, Field[] fields, Method[] methods, Map genericsInfo) { + for (Field f : fields) { + Injection annotationInjection = f.getAnnotation(Injection.class); + InjectionDeep annotationInjectionDeep = f.getAnnotation(InjectionDeep.class); + if (annotationInjection == null && annotationInjectionDeep == null) { // no injection annotations continue; } - if ( annotationInjection != null && annotationInjectionDeep != null ) { + if (annotationInjection != null && annotationInjectionDeep != null) { // both annotations exist - wrong - throw new RuntimeException( "Field can't be annotated twice for injection " + f ); + throw new RuntimeException("Field can't be annotated twice for injection " + f); } - if ( f.isSynthetic() || f.isEnumConstant() || Modifier.isStatic( f.getModifiers() ) ) { + if (f.isSynthetic() || f.isEnumConstant() || Modifier.isStatic(f.getModifiers())) { // fields can't contain real data with such modifier - throw new RuntimeException( "Wrong modifier for anotated field " + f ); + throw new RuntimeException("Wrong modifier for anotated field " + f); } BeanLevelInfo leaf = new BeanLevelInfo(); leaf.parent = this; leaf.field = f; Type t; - if ( f.getType().isArray() ) { + if (f.getType().isArray()) { Type ff = f.getGenericType(); leaf.dim = DIMENSION.ARRAY; - if ( ff instanceof GenericArrayType ) { + if (ff instanceof GenericArrayType) { GenericArrayType ffg = (GenericArrayType) ff; - t = resolveGenericType( ffg.getGenericComponentType(), genericsInfo ); + t = resolveGenericType(ffg.getGenericComponentType(), genericsInfo); } else { t = f.getType().getComponentType(); } - } else if ( List.class.equals( f.getType() ) ) { + } else if (List.class.equals(f.getType())) { leaf.dim = DIMENSION.LIST; Type fieldType = f.getGenericType(); - Type listType = ( (ParameterizedType) fieldType ).getActualTypeArguments()[ 0 ]; + Type listType = ((ParameterizedType) fieldType).getActualTypeArguments()[0]; try { - t = resolveGenericType( listType, genericsInfo ); - } catch ( Throwable ex ) { - throw new RuntimeException( "Can't retrieve type from List for " + f, ex ); + t = resolveGenericType(listType, genericsInfo); + } catch (Throwable ex) { + throw new RuntimeException("Can't retrieve type from List for " + f, ex); } } else { leaf.dim = DIMENSION.NONE; - t = resolveGenericType( f.getGenericType(), genericsInfo ); + t = resolveGenericType(f.getGenericType(), genericsInfo); } - if ( t instanceof ParameterizedType ) { + if (t instanceof ParameterizedType) { ParameterizedType pt = (ParameterizedType) t; leaf.leafClass = (Class) pt.getRawType(); } else { leaf.leafClass = (Class) t; } - if ( annotationInjection != null ) { + if (annotationInjection != null) { try { leaf.converter = annotationInjection.converter().newInstance(); - } catch ( Exception ex ) { - throw new RuntimeException( "Error instantiate converter for " + f, ex ); + } catch (Exception ex) { + throw new RuntimeException("Error instantiate converter for " + f, ex); } leaf.convertEmpty = annotationInjection.convertEmpty(); - info.addInjectionProperty( annotationInjection, leaf ); - } else if ( annotationInjectionDeep != null ) { + info.addInjectionProperty(annotationInjection, leaf); + } else if (annotationInjectionDeep != null) { // introspect deeper - leaf.prefix = annotationInjectionDeep.prefix(); - TreeMap gi = new TreeMap<>( genericsInfo ); - leaf.introspect( info, t, gi ); + leaf.nameKey = + calculateNameKey( + annotationInjectionDeep.prefix(), + annotationInjection != null ? annotationInjection.name() : ""); + TreeMap gi = new TreeMap<>(genericsInfo); + leaf.introspect(info, t, gi); } } - for ( Method m : methods ) { - Injection annotationInjection = m.getAnnotation( Injection.class ); - InjectionDeep annotationInjectionDeep = m.getAnnotation( InjectionDeep.class ); - if ( annotationInjection == null && annotationInjectionDeep == null ) { + for (Method m : methods) { + Injection annotationInjection = m.getAnnotation(Injection.class); + InjectionDeep annotationInjectionDeep = m.getAnnotation(InjectionDeep.class); + if (annotationInjection == null && annotationInjectionDeep == null) { // no injection annotations continue; } - if ( annotationInjection != null && annotationInjectionDeep != null ) { + if (annotationInjection != null && annotationInjectionDeep != null) { // both annotations exist - wrong - throw new RuntimeException( "Method can't be annotated twice for injection " + m ); + throw new RuntimeException("Method can't be annotated twice for injection " + m); } - if ( m.isSynthetic() || Modifier.isStatic( m.getModifiers() ) ) { + if (m.isSynthetic() || Modifier.isStatic(m.getModifiers())) { // method is static - throw new RuntimeException( "Wrong modifier for anotated method " + m ); + throw new RuntimeException("Wrong modifier for anotated method " + m); } BeanLevelInfo leaf = new BeanLevelInfo(); leaf.parent = this; - if ( annotationInjectionDeep != null ) { - Type getterClass = isGetter( m ); - if ( getterClass == null ) { - throw new RuntimeException( "Method should be getter: " + m ); + if (annotationInjectionDeep != null) { + Type getterClass = isGetter(m); + if (getterClass == null) { + throw new RuntimeException("Method should be getter: " + m); } - if ( m.getReturnType() != null && List.class.equals( m.getReturnType() ) ) { + if (m.getReturnType() != null && List.class.equals(m.getReturnType())) { // returns list leaf.dim = DIMENSION.LIST; ParameterizedType getterType = (ParameterizedType) getterClass; - getterClass = getterType.getActualTypeArguments()[ 0 ]; + getterClass = getterType.getActualTypeArguments()[0]; } - Class getter = (Class) resolveGenericType( getterClass, genericsInfo ); - if ( getter.isArray() ) { - throw new RuntimeException( "Method should be getter: " + m ); + Class getter = (Class) resolveGenericType(getterClass, genericsInfo); + if (getter.isArray()) { + throw new RuntimeException("Method should be getter: " + m); } leaf.getter = m; leaf.leafClass = getter; - leaf.prefix = annotationInjectionDeep.prefix(); - leaf.init( info ); + leaf.nameKey = + calculateNameKey( + annotationInjectionDeep.prefix(), + annotationInjection != null ? annotationInjection.name() : ""); + leaf.init(info); } else { - Class setterClass = isSetter( m ); - if ( setterClass == null || setterClass.isArray() ) { - throw new RuntimeException( "Method should be setter: " + m ); + Class setterClass = isSetter(m); + if (setterClass == null || setterClass.isArray()) { + throw new RuntimeException("Method should be setter: " + m); } leaf.setter = m; leaf.leafClass = setterClass; try { leaf.converter = annotationInjection.converter().newInstance(); - } catch ( Exception ex ) { - throw new RuntimeException( "Error instantiate converter for " + m, ex ); + } catch (Exception ex) { + throw new RuntimeException("Error instantiate converter for " + m, ex); } leaf.convertEmpty = annotationInjection.convertEmpty(); - info.addInjectionProperty( annotationInjection, leaf ); + info.addInjectionProperty(annotationInjection, leaf); } } } - /** - * Resolves class by generic map if required(when type is TypeVariable). - */ - private Type resolveGenericType( Type type, Map genericsInfo ) { - if ( type instanceof TypeVariable ) { - String name = ( (TypeVariable) type ).getName(); - type = genericsInfo.get( name ); - if ( type == null ) { - throw new RuntimeException( "Unknown generics for '" + name + "'" ); + private String calculateNameKey(String prefix, String name) { + if (StringUtils.isEmpty(prefix)) { + return name; + } else { + return prefix + "." + name; + } + } + + /** Resolves class by generic map if required(when type is TypeVariable). */ + private Type resolveGenericType(Type type, Map genericsInfo) { + if (type instanceof TypeVariable) { + String name = ((TypeVariable) type).getName(); + type = genericsInfo.get(name); + if (type == null) { + throw new RuntimeException("Unknown generics for '" + name + "'"); } } return type; } - private Type isGetter( Method m ) { - if ( m.getReturnType() == void.class ) { + private Type isGetter(Method m) { + if (m.getReturnType() == void.class) { return null; } - if ( m.getParameterTypes().length == 0 ) { + if (m.getParameterTypes().length == 0) { // getter without parameters return m.getGenericReturnType(); } return null; } - private Class isSetter( Method m ) { - if ( m.getReturnType() != void.class ) { + private Class isSetter(Method m) { + if (m.getReturnType() != void.class) { return null; } - if ( m.getParameterTypes().length == 1 ) { + if (m.getParameterTypes().length == 1) { // setter with one parameter - return m.getParameterTypes()[ 0 ]; + return m.getParameterTypes()[0]; } return null; } @@ -296,21 +288,21 @@ private Class isSetter( Method m ) { protected List createCallStack() { List stack = new ArrayList<>(); BeanLevelInfo p = this; - while ( p != null ) { - if ( p.field != null ) { - p.field.setAccessible( true ); + while (p != null) { + if (p.field != null) { + p.field.setAccessible(true); } - stack.add( p ); + stack.add(p); p = p.parent; } - Collections.reverse( stack ); + Collections.reverse(stack); return stack; } @Override public String toString() { String r = ""; - if ( field != null ) { + if (field != null) { r += "field " + field.getName(); } else { r += ""; diff --git a/engine/src/main/java/org/apache/hop/core/util/serialization/TransformMetaProps.java b/engine/src/main/java/org/apache/hop/core/util/serialization/TransformMetaProps.java index 017011481d..6076c5d78b 100644 --- a/engine/src/main/java/org/apache/hop/core/util/serialization/TransformMetaProps.java +++ b/engine/src/main/java/org/apache/hop/core/util/serialization/TransformMetaProps.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -49,256 +49,270 @@ import static org.apache.hop.core.util.serialization.TransformMetaProps.TRANSFORM_TAG; /** - * A slim representation of ITransformMeta properties, used as a - * way to leverage alternative serialization strategies (e.g. {@link MetaXmlSerializer} - *

- * Public methods allow conversion: - * {@link #from(ITransformMeta)} and - * {@link #to(ITransformMeta)} - *

- *

- * Internally, the TransformMetaProps holds a list of {@link PropGroup} elements, each corresponding - * to a MetadataInjection group. + * A slim representation of ITransformMeta properties, used as a way to leverage alternative + * serialization strategies (e.g. {@link MetaXmlSerializer} + * + *

Public methods allow conversion: {@link #from(ITransformMeta)} and {@link #to(ITransformMeta)} + * *

- * InjectionDeep not yet supported. + * + *

Internally, the TransformMetaProps holds a list of {@link PropGroup} elements, each + * corresponding to a MetadataInjection group. + * + *

InjectionDeep not yet supported. */ -@XmlRootElement( name = TRANSFORM_TAG ) +@XmlRootElement(name = TRANSFORM_TAG) public class TransformMetaProps { static final String TRANSFORM_TAG = "transform-props"; - @XmlAttribute( name = "secure" ) + @XmlAttribute(name = "secure") private List secureFields; private Meta transformMeta; private IVariables variables = new Variables(); - @SuppressWarnings( "unused" ) - private TransformMetaProps() { - } + @SuppressWarnings("unused") + private TransformMetaProps() {} - private TransformMetaProps( Meta smi ) { - secureFields = sensitiveFields( smi.getClass() ); + private TransformMetaProps(Meta smi) { + secureFields = sensitiveFields(smi.getClass()); } - @XmlElement( name = "group" ) + @XmlElement(name = "group") private List groups = new ArrayList<>(); /** - * Retuns an instance of this class with transformMeta properties mapped - * to a list of {@link PropGroup} + * Retuns an instance of this class with transformMeta properties mapped to a list of {@link + * PropGroup} */ - public static TransformMetaProps from( Meta transformMeta ) { - TransformMetaProps propMap = new TransformMetaProps( transformMeta ); + public static TransformMetaProps from(Meta transformMeta) { + TransformMetaProps propMap = new TransformMetaProps(transformMeta); propMap.transformMeta = transformMeta; // use metadata injection to extract properties - BeanInjectionInfo info = new BeanInjectionInfo( transformMeta.getClass() ); - BeanInjector injector = new BeanInjector( info ); + BeanInjectionInfo info = new BeanInjectionInfo(transformMeta.getClass()); + BeanInjector injector = new BeanInjector(info); - propMap.populateGroups( transformMeta, info, injector ); + propMap.populateGroups(transformMeta, info, injector); return propMap; } /** * Sets the properties of this TransformMetaProps on {@param transformMetaInterface} - *

- * This method mutates the transformMeta, as opposed to returning a new instance, to match - * more cleanly to Hop's {@link ITransformMeta#loadXml} design, which loads props into - * an instance. + * + *

This method mutates the transformMeta, as opposed to returning a new instance, to match more + * cleanly to Hop's {@link ITransformMeta#loadXml} design, which loads props into an instance. */ - public Meta to( Meta meta ) { - BeanInjectionInfo info = new BeanInjectionInfo( meta.getClass() ); + public Meta to(Meta meta) { + BeanInjectionInfo info = new BeanInjectionInfo(meta.getClass()); - BeanInjector injector = new BeanInjector( info ); - info.getProperties().values().forEach( property -> assignValueForProp( property, meta, injector ) ); + BeanInjector injector = new BeanInjector(info); + info.getProperties().values().forEach(property -> assignValueForProp(property, meta, injector)); return meta; } /** - * Allows specifying a variable variables to be used when applying property values to - * a transformMeta. + * Allows specifying a variable variables to be used when applying property values to a + * transformMeta. */ - public TransformMetaProps withVariables( IVariables variables ) { - TransformMetaProps propCopy = from( this.transformMeta ); + public TransformMetaProps withVariables(IVariables variables) { + TransformMetaProps propCopy = from(this.transformMeta); propCopy.variables = variables; return propCopy; } - private void populateGroups( Meta meta, BeanInjectionInfo info, BeanInjector injector ) { - groups = info.getGroups().stream() // get metadata injection groups - .flatMap( group -> group.getGroupProperties().stream() ) // expand to all properties - .map( getProp( meta, injector ) ) // map to property/value - .collect( groupingBy( Prop::getGroup ) ).entrySet().stream() // group by group name - .map( entry -> new PropGroup( entry.getKey(), entry.getValue() ) ) // map the set of properties to a group - .collect( Collectors.toList() ); + private void populateGroups(Meta meta, BeanInjectionInfo info, BeanInjector injector) { + groups = + info.getGroups().stream() // get metadata injection groups + .flatMap(group -> group.getProperties().stream()) // expand to all properties + .map(getProp(meta, injector)) // map to property/value + .collect(groupingBy(Prop::getGroup)) + .entrySet() + .stream() // group by group name + .map( + entry -> + new PropGroup( + entry.getKey(), entry.getValue())) // map the set of properties to a group + .collect(Collectors.toList()); } /** - * Collects the list of declared fields with the {@link Sensitive} annotation - * for the class. Values for these fields will be encrypted. - *

- * Checks the top level fields of clazz, and recurses into - * {@link InjectionDeep} classes. + * Collects the list of declared fields with the {@link Sensitive} annotation for the class. + * Values for these fields will be encrypted. + * + *

Checks the top level fields of clazz, and recurses into {@link InjectionDeep} classes. */ @VisibleForTesting - static List sensitiveFields( Class clazz ) { + static List sensitiveFields(Class clazz) { Field[] declaredFields = clazz.getDeclaredFields(); - return concat( stream( declaredFields ), recurseDeep( declaredFields ) ) - .filter( field -> field.getAnnotation( Sensitive.class ) != null ) - .filter( field -> field.getAnnotation( Injection.class ) != null ) - .map( field -> field.getAnnotation( Injection.class ).name() ) - .collect( Collectors.toList() ); + return concat(stream(declaredFields), recurseDeep(declaredFields)) + .filter(field -> field.getAnnotation(Sensitive.class) != null) + .filter(field -> field.getAnnotation(Injection.class) != null) + .map(field -> field.getAnnotation(Injection.class).name()) + .collect(Collectors.toList()); } - private static Stream recurseDeep( Field[] topLevelFields ) { + private static Stream recurseDeep(Field[] topLevelFields) { Stream deepInjectionFields = Stream.empty(); - if ( stream( topLevelFields ).anyMatch( isInjectionDeep() ) ) { - deepInjectionFields = stream( topLevelFields ) - .filter( isInjectionDeep() ) - .flatMap( field -> stream( field.getType().getDeclaredFields() ) ); - List deepFields = deepInjectionFields.collect( Collectors.toList() ); - return concat( deepFields.stream(), recurseDeep( deepFields.toArray( new Field[ 0 ] ) ) ); + if (stream(topLevelFields).anyMatch(isInjectionDeep())) { + deepInjectionFields = + stream(topLevelFields) + .filter(isInjectionDeep()) + .flatMap(field -> stream(field.getType().getDeclaredFields())); + List deepFields = deepInjectionFields.collect(Collectors.toList()); + return concat(deepFields.stream(), recurseDeep(deepFields.toArray(new Field[0]))); } return deepInjectionFields; } private static Predicate isInjectionDeep() { - return field -> field.getAnnotation( InjectionDeep.class ) != null; + return field -> field.getAnnotation(InjectionDeep.class) != null; } - private Function getProp( ITransformMeta transformMeta, - BeanInjector injector ) { + private Function getProp( + ITransformMeta transformMeta, BeanInjector injector) { return prop -> - new Prop( prop.getName(), - getPropVal( transformMeta, injector, prop ), - prop.getGroupName() ); + new Prop(prop.getKey(), getPropVal(transformMeta, injector, prop), prop.getGroupKey()); } - @SuppressWarnings( "unchecked" ) - private List getPropVal( ITransformMeta transformMeta, BeanInjector injector, - BeanInjectionInfo.Property prop ) { + @SuppressWarnings("unchecked") + private List getPropVal( + ITransformMeta transformMeta, BeanInjector injector, BeanInjectionInfo.Property prop) { try { List ret; - Object o = injector.getPropVal( transformMeta, prop.getName() ); - if ( o instanceof List ) { + Object o = injector.getPropVal(transformMeta, prop.getKey()); + if (o instanceof List) { ret = (List) o; - } else if ( o instanceof Object[] ) { - ret = asList( (Object[]) o ); + } else if (o instanceof Object[]) { + ret = asList((Object[]) o); } else { - ret = singletonList( o ); + ret = singletonList(o); } - return maybeEncrypt( prop.getName(), ret ); - } catch ( Exception e ) { - throw new RuntimeException( e ); + return maybeEncrypt(prop.getKey(), ret); + } catch (Exception e) { + throw new RuntimeException(e); } } - private List maybeEncrypt( String name, List ret ) { - if ( secureFields.contains( name ) ) { + private List maybeEncrypt(String name, List ret) { + if (secureFields.contains(name)) { return ret.stream() - .map( val -> - ( val == null ) || ( val.toString().isEmpty() ) - ? "" : Encr.encryptPasswordIfNotUsingVariables( val.toString() ) ) - .collect( Collectors.toList() ); + .map( + val -> + (val == null) || (val.toString().isEmpty()) + ? "" + : Encr.encryptPasswordIfNotUsingVariables(val.toString())) + .collect(Collectors.toList()); } return ret; } - private void assignValueForProp( BeanInjectionInfo.Property beanInfoProp, ITransformMeta transformMetaInterface, - BeanInjector injector ) { - List props = groups.stream() - .filter( group -> beanInfoProp.getGroupName().equals( group.name ) ) - .flatMap( group -> group.props.stream() ) - .filter( prop -> beanInfoProp.getName().equals( prop.name ) ) - .collect( Collectors.toList() ); - - decryptVals( props ); - props.forEach( entry -> injectVal( beanInfoProp, entry, transformMetaInterface, injector ) ); + private void assignValueForProp( + BeanInjectionInfo.Property beanInfoProp, + ITransformMeta transformMetaInterface, + BeanInjector injector) { + List props = + groups.stream() + .filter(group -> beanInfoProp.getGroupKey().equals(group.name)) + .flatMap(group -> group.props.stream()) + .filter(prop -> beanInfoProp.getKey().equals(prop.name)) + .collect(Collectors.toList()); + + decryptVals(props); + props.forEach(entry -> injectVal(beanInfoProp, entry, transformMetaInterface, injector)); } - private void decryptVals( List props ) { + private void decryptVals(List props) { props.stream() - .filter( prop -> secureFields.contains( prop.getName() ) ) - .forEach( prop -> prop.value = prop.value.stream() - .map( Object::toString ) - .map( Encr::decryptPasswordOptionallyEncrypted ) - .collect( Collectors.toList() ) ); + .filter(prop -> secureFields.contains(prop.getName())) + .forEach( + prop -> + prop.value = + prop.value.stream() + .map(Object::toString) + .map(Encr::decryptPasswordOptionallyEncrypted) + .collect(Collectors.toList())); } - private void injectVal( BeanInjectionInfo.Property beanInfoProp, Prop prop, - ITransformMeta transformMetaInterface, - BeanInjector injector ) { + private void injectVal( + BeanInjectionInfo.Property beanInfoProp, + Prop prop, + ITransformMeta transformMetaInterface, + BeanInjector injector) { - if ( prop.value == null || prop.value.size() == 0 ) { - prop.value = singletonList( null ); + if (prop.value == null || prop.value.size() == 0) { + prop.value = singletonList(null); } try { - injector.setProperty( transformMetaInterface, - beanInfoProp.getName(), - prop.value.stream() - .map( value -> { - RowMetaAndData rmad = new RowMetaAndData(); - rmad.addValue( new ValueMetaString( prop.getName() ), envSubs( value ) ); - return rmad; - } ).collect( Collectors.toList() ), - beanInfoProp.getName() ); - } catch ( HopException e ) { - throw new RuntimeException( e ); + injector.setProperty( + transformMetaInterface, + beanInfoProp.getKey(), + prop.value.stream() + .map( + value -> { + RowMetaAndData rmad = new RowMetaAndData(); + rmad.addValue(new ValueMetaString(prop.getName()), envSubs(value)); + return rmad; + }) + .collect(Collectors.toList()), + beanInfoProp.getKey()); + } catch (HopException e) { + throw new RuntimeException(e); } } - private Object envSubs( Object value ) { - if ( value instanceof String ) { - return variables.resolve( value.toString() ); + private Object envSubs(Object value) { + if (value instanceof String) { + return variables.resolve(value.toString()); } return value; } - @Override public String toString() { + @Override + public String toString() { return "TransformMetaProps{" + "groups=" + groups + '}'; } - /** - * Represents a named grouping of properties, corresponding to a metadata injection group. - */ + /** Represents a named grouping of properties, corresponding to a metadata injection group. */ private static class PropGroup { @XmlAttribute String name; - @XmlElement( name = "property" ) List props; + @XmlElement(name = "property") + List props; - @SuppressWarnings( "unused" ) - public PropGroup() { - } // needed for deserialization + @SuppressWarnings("unused") + public PropGroup() {} // needed for deserialization - PropGroup( String name, List propList ) { + PropGroup(String name, List propList) { this.name = name; this.props = propList; } - @Override public String toString() { + @Override + public String toString() { return "PropGroup{" + "name='" + name + '\'' + ", props=" + props + '}'; } } /** - * Represents a single property from a ITransformMeta impl. - * Values are captured as a List to consistently handle both List properties and single items. + * Represents a single property from a ITransformMeta impl. Values are captured as a List + * to consistently handle both List properties and single items. */ private static class Prop { @XmlAttribute String group; @XmlAttribute String name; - @XmlElement( name = "value" ) List value = new ArrayList<>(); + @XmlElement(name = "value") + List value = new ArrayList<>(); - @SuppressWarnings( "unused" ) - public Prop() { - } // needed for deserialization + @SuppressWarnings("unused") + public Prop() {} // needed for deserialization - private Prop( String name, List value, String group ) { + private Prop(String name, List value, String group) { this.group = group; this.name = name; this.value = value; @@ -308,15 +322,22 @@ String getName() { return name; } - String getGroup() { return group; } - @Override public String toString() { - return "\n Prop{" + "group='" + group + '\'' + ", name='" + name + '\'' + ", value=" + value + '}'; + @Override + public String toString() { + return "\n Prop{" + + "group='" + + group + + '\'' + + ", name='" + + name + + '\'' + + ", value=" + + value + + '}'; } } - - } diff --git a/engine/src/test/java/org/apache/hop/core/injection/MetaAnnotationInjectionTest.java b/engine/src/test/java/org/apache/hop/core/injection/MetaAnnotationInjectionTest.java index c532b9fdb5..7592425466 100644 --- a/engine/src/test/java/org/apache/hop/core/injection/MetaAnnotationInjectionTest.java +++ b/engine/src/test/java/org/apache/hop/core/injection/MetaAnnotationInjectionTest.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -58,20 +58,20 @@ public void before() { @Test public void testInjectionDescription() throws Exception { - BeanInjectionInfo ri = new BeanInjectionInfo( MetaBeanLevel1.class ); + BeanInjectionInfo ri = new BeanInjectionInfo(MetaBeanLevel1.class); - assertEquals( 3, ri.getGroups().size() ); - assertEquals( "", ri.getGroups().get( 0 ).getName() ); - assertEquals( "FILENAME_LINES", ri.getGroups().get( 1 ).getName() ); - assertEquals( "FILENAME_LINES2", ri.getGroups().get( 2 ).getName() ); + assertEquals(3, ri.getGroups().size()); + assertEquals("", ri.getGroups().get(0).getKey()); + assertEquals("FILENAME_LINES", ri.getGroups().get(1).getKey()); + assertEquals("FILENAME_LINES2", ri.getGroups().get(2).getKey()); - assertTrue( ri.getProperties().containsKey( "SEPARATOR" ) ); - assertTrue( ri.getProperties().containsKey( "FILENAME" ) ); - assertTrue( ri.getProperties().containsKey( "BASE" ) ); - assertTrue( ri.getProperties().containsKey( "FIRST" ) ); + assertTrue(ri.getProperties().containsKey("SEPARATOR")); + assertTrue(ri.getProperties().containsKey("FILENAME")); + assertTrue(ri.getProperties().containsKey("BASE")); + assertTrue(ri.getProperties().containsKey("FIRST")); - assertEquals( "FILENAME_LINES", ri.getProperties().get( "FILENAME" ).getGroupName() ); - assertEquals( "!DESCRIPTION!", ri.getDescription( "DESCRIPTION" ) ); + assertEquals("FILENAME_LINES", ri.getProperties().get("FILENAME").getGroupKey()); + assertEquals("!DESCRIPTION!", ri.getDescription("DESCRIPTION")); } @Test @@ -79,176 +79,177 @@ public void testInjectionSets() throws Exception { MetaBeanLevel1 obj = new MetaBeanLevel1(); RowMeta meta = new RowMeta(); - meta.addValueMeta( new ValueMetaString( "f1" ) ); - meta.addValueMeta( new ValueMetaString( "f2" ) ); - meta.addValueMeta( new ValueMetaString( "fstrint" ) ); - meta.addValueMeta( new ValueMetaString( "fstrlong" ) ); - meta.addValueMeta( new ValueMetaString( "fstrboolean" ) ); // TODO STLOCALE + meta.addValueMeta(new ValueMetaString("f1")); + meta.addValueMeta(new ValueMetaString("f2")); + meta.addValueMeta(new ValueMetaString("fstrint")); + meta.addValueMeta(new ValueMetaString("fstrlong")); + meta.addValueMeta(new ValueMetaString("fstrboolean")); // TODO STLOCALE List rows = new ArrayList<>(); - rows.add( new RowMetaAndData( meta, "", "/tmp/file.txt", "123", "1234567891213", "y" ) ); - rows.add( new RowMetaAndData( meta, "", "/tmp/file2.txt", "123", "1234567891213", "y" ) ); - - BeanInjector inj = buildBeanInjectorFor( MetaBeanLevel1.class ); - inj.setProperty( obj, "SEPARATOR", rows, "f1" ); - inj.setProperty( obj, "FILENAME", rows, "f2" ); - inj.setProperty( obj, "FILENAME_ARRAY", rows, "f2" ); - inj.setProperty( obj, "FBOOLEAN", rows, "fstrboolean" ); - inj.setProperty( obj, "FINT", rows, "fstrint" ); - inj.setProperty( obj, "FLONG", rows, "fstrlong" ); - inj.setProperty( obj, "FIRST", rows, "fstrint" ); - - assertEquals( "", obj.getSub().getSeparator() ); - assertEquals( "/tmp/file.txt", obj.getSub().getFiles()[ 0 ].getName() ); - assertTrue( obj.fboolean ); - assertEquals( 123, obj.fint ); - assertEquals( 1234567891213L, obj.flong ); - assertEquals( "123", obj.getSub().first() ); - assertArrayEquals( new String[] { "/tmp/file.txt", "/tmp/file2.txt" }, obj.getSub().getFilenames() ); + rows.add(new RowMetaAndData(meta, "", "/tmp/file.txt", "123", "1234567891213", "y")); + rows.add(new RowMetaAndData(meta, "", "/tmp/file2.txt", "123", "1234567891213", "y")); + + BeanInjector inj = buildBeanInjectorFor(MetaBeanLevel1.class); + inj.setProperty(obj, "SEPARATOR", rows, "f1"); + inj.setProperty(obj, "FILENAME", rows, "f2"); + inj.setProperty(obj, "FILENAME_ARRAY", rows, "f2"); + inj.setProperty(obj, "FBOOLEAN", rows, "fstrboolean"); + inj.setProperty(obj, "FINT", rows, "fstrint"); + inj.setProperty(obj, "FLONG", rows, "fstrlong"); + inj.setProperty(obj, "FIRST", rows, "fstrint"); + + assertEquals("", obj.getSub().getSeparator()); + assertEquals("/tmp/file.txt", obj.getSub().getFiles()[0].getName()); + assertTrue(obj.fboolean); + assertEquals(123, obj.fint); + assertEquals(1234567891213L, obj.flong); + assertEquals("123", obj.getSub().first()); + assertArrayEquals( + new String[] {"/tmp/file.txt", "/tmp/file2.txt"}, obj.getSub().getFilenames()); } @Test public void testInjectionConstant() throws Exception { MetaBeanLevel1 obj = new MetaBeanLevel1(); - BeanInjector inj = buildBeanInjectorFor( MetaBeanLevel1.class ); - inj.setProperty( obj, "SEPARATOR", null, "" ); - inj.setProperty( obj, "FINT", null, "123" ); - inj.setProperty( obj, "FLONG", null, "1234567891213" ); - inj.setProperty( obj, "FBOOLEAN", null, "true" ); - inj.setProperty( obj, "FILENAME", null, "f1" ); - inj.setProperty( obj, "FILENAME_ARRAY", null, "f2" ); - - assertEquals( "", obj.getSub().getSeparator() ); - assertTrue( obj.fboolean ); - assertEquals( 123, obj.fint ); - assertEquals( 1234567891213L, obj.flong ); - assertNull( obj.getSub().getFiles() ); - assertNull( obj.getSub().getFilenames() ); - - obj.getSub().files = new MetaBeanLevel3[] { new MetaBeanLevel3(), new MetaBeanLevel3() }; - obj.getSub().filenames = new String[] { "", "", "" }; - inj.setProperty( obj, "FILENAME", null, "f1" ); - inj.setProperty( obj, "FILENAME_ARRAY", null, "f2" ); - assertEquals( 2, obj.getSub().getFiles().length ); - assertEquals( "f1", obj.getSub().getFiles()[ 0 ].getName() ); - assertEquals( "f1", obj.getSub().getFiles()[ 1 ].getName() ); - assertArrayEquals( new String[] { "f2", "f2", "f2" }, obj.getSub().getFilenames() ); + BeanInjector inj = buildBeanInjectorFor(MetaBeanLevel1.class); + inj.setProperty(obj, "SEPARATOR", null, ""); + inj.setProperty(obj, "FINT", null, "123"); + inj.setProperty(obj, "FLONG", null, "1234567891213"); + inj.setProperty(obj, "FBOOLEAN", null, "true"); + inj.setProperty(obj, "FILENAME", null, "f1"); + inj.setProperty(obj, "FILENAME_ARRAY", null, "f2"); + + assertEquals("", obj.getSub().getSeparator()); + assertTrue(obj.fboolean); + assertEquals(123, obj.fint); + assertEquals(1234567891213L, obj.flong); + assertNull(obj.getSub().getFiles()); + assertNull(obj.getSub().getFilenames()); + + obj.getSub().files = new MetaBeanLevel3[] {new MetaBeanLevel3(), new MetaBeanLevel3()}; + obj.getSub().filenames = new String[] {"", "", ""}; + inj.setProperty(obj, "FILENAME", null, "f1"); + inj.setProperty(obj, "FILENAME_ARRAY", null, "f2"); + assertEquals(2, obj.getSub().getFiles().length); + assertEquals("f1", obj.getSub().getFiles()[0].getName()); + assertEquals("f1", obj.getSub().getFiles()[1].getName()); + assertArrayEquals(new String[] {"f2", "f2", "f2"}, obj.getSub().getFilenames()); } @Test - public void testInjectionForArrayPropertyWithoutDefaultConstructor_class_parameter() throws HopException { - BeanInjector beanInjector = buildBeanInjectorFor( MetadataBean.class ); + public void testInjectionForArrayPropertyWithoutDefaultConstructor_class_parameter() + throws HopException { + BeanInjector beanInjector = buildBeanInjectorFor(MetadataBean.class); MetadataBean targetBean = new MetadataBean(); - beanInjector.setProperty( targetBean, COMPLEX_NAME, createRowMetaAndData(), FIELD_ONE ); + beanInjector.setProperty(targetBean, COMPLEX_NAME, createRowMetaAndData(), FIELD_ONE); - assertNotNull( targetBean.getComplexField() ); - assertTrue( targetBean.getComplexField().length == 1 ); - assertEquals( TEST_NAME, targetBean.getComplexField()[ 0 ].getFieldName() ); + assertNotNull(targetBean.getComplexField()); + assertTrue(targetBean.getComplexField().length == 1); + assertEquals(TEST_NAME, targetBean.getComplexField()[0].getFieldName()); } @Test - public void testInjectionForArrayPropertyWithoutDefaultConstructorInterface_parameter() throws HopException { - BeanInjector beanInjector = buildBeanInjectorFor( MetadataBeanImplementsInterface.class ); + public void testInjectionForArrayPropertyWithoutDefaultConstructorInterface_parameter() + throws HopException { + BeanInjector beanInjector = buildBeanInjectorFor(MetadataBeanImplementsInterface.class); MetadataBeanImplementsInterface targetBean = new MetadataBeanImplementsInterface(); - beanInjector.setProperty( targetBean, COMPLEX_NAME, createRowMetaAndData(), FIELD_ONE ); + beanInjector.setProperty(targetBean, COMPLEX_NAME, createRowMetaAndData(), FIELD_ONE); - assertNotNull( targetBean.getComplexField() ); - assertTrue( targetBean.getComplexField().length == 1 ); - assertEquals( TEST_NAME, targetBean.getComplexField()[ 0 ].getFieldName() ); + assertNotNull(targetBean.getComplexField()); + assertTrue(targetBean.getComplexField().length == 1); + assertEquals(TEST_NAME, targetBean.getComplexField()[0].getFieldName()); } @Test public void testWrongDeclarations() throws Exception { try { - new BeanInjectionInfo( MetaBeanWrong1.class ); + new BeanInjectionInfo(MetaBeanWrong1.class); fail(); - } catch ( Exception ex ) { + } catch (Exception ex) { } try { - new BeanInjectionInfo( MetaBeanWrong2.class ); + new BeanInjectionInfo(MetaBeanWrong2.class); fail(); - } catch ( Exception ex ) { + } catch (Exception ex) { } try { - new BeanInjectionInfo( MetaBeanWrong3.class ); + new BeanInjectionInfo(MetaBeanWrong3.class); fail(); - } catch ( Exception ex ) { + } catch (Exception ex) { } try { - new BeanInjectionInfo( MetaBeanWrong4.class ); + new BeanInjectionInfo(MetaBeanWrong4.class); fail(); - } catch ( Exception ex ) { + } catch (Exception ex) { } try { - new BeanInjectionInfo( MetaBeanWrong5.class ); + new BeanInjectionInfo(MetaBeanWrong5.class); fail(); - } catch ( Exception ex ) { + } catch (Exception ex) { } try { - new BeanInjectionInfo( MetaBeanWrong6.class ); + new BeanInjectionInfo(MetaBeanWrong6.class); fail(); - } catch ( Exception ex ) { + } catch (Exception ex) { } try { - new BeanInjectionInfo( MetaBeanWrong7.class ); + new BeanInjectionInfo(MetaBeanWrong7.class); fail(); - } catch ( Exception ex ) { + } catch (Exception ex) { } } @Test public void testGenerics() throws Exception { - BeanInjectionInfo ri = new BeanInjectionInfo( MetaBeanChild.class ); - - assertTrue( ri.getProperties().size() == 7 ); - assertTrue( ri.getProperties().containsKey( "BASE_ITEM_NAME" ) ); - assertTrue( ri.getProperties().containsKey( "ITEM_CHILD_NAME" ) ); - assertTrue( ri.getProperties().containsKey( "A" ) ); - assertTrue( ri.getProperties().containsKey( "ITEM.BASE_ITEM_NAME" ) ); - assertTrue( ri.getProperties().containsKey( "ITEM.ITEM_CHILD_NAME" ) ); - assertTrue( ri.getProperties().containsKey( "SUB.BASE_ITEM_NAME" ) ); - assertTrue( ri.getProperties().containsKey( "SUB.ITEM_CHILD_NAME" ) ); - - assertEquals( String.class, ri.getProperties().get( "A" ).getPropertyClass() ); + BeanInjectionInfo ri = new BeanInjectionInfo(MetaBeanChild.class); + + assertTrue(ri.getProperties().size() == 7); + assertTrue(ri.getProperties().containsKey("BASE_ITEM_NAME")); + assertTrue(ri.getProperties().containsKey("ITEM_CHILD_NAME")); + assertTrue(ri.getProperties().containsKey("A")); + assertTrue(ri.getProperties().containsKey("ITEM.BASE_ITEM_NAME")); + assertTrue(ri.getProperties().containsKey("ITEM.ITEM_CHILD_NAME")); + assertTrue(ri.getProperties().containsKey("SUB.BASE_ITEM_NAME")); + assertTrue(ri.getProperties().containsKey("SUB.ITEM_CHILD_NAME")); + + assertEquals(String.class, ri.getProperties().get("A").getPropertyClass()); } - private static BeanInjector buildBeanInjectorFor( Class clazz ) { - BeanInjectionInfo metaBeanInfo = new BeanInjectionInfo( clazz ); - return new BeanInjector( metaBeanInfo ); + private static BeanInjector buildBeanInjectorFor(Class clazz) { + BeanInjectionInfo metaBeanInfo = new BeanInjectionInfo(clazz); + return new BeanInjector(metaBeanInfo); } private static List createRowMetaAndData() { RowMeta meta = new RowMeta(); - meta.addValueMeta( new ValueMetaString( FIELD_ONE ) ); - return Collections.singletonList( new RowMetaAndData( meta, TEST_NAME ) ); + meta.addValueMeta(new ValueMetaString(FIELD_ONE)); + return Collections.singletonList(new RowMetaAndData(meta, TEST_NAME)); } - private static interface MetadataInterface { - } + private static interface MetadataInterface {} - @InjectionSupported( localizationPrefix = "", groups = "COMPLEX" ) + @InjectionSupported(localizationPrefix = "", groups = "COMPLEX") public static class MetadataBean { - @InjectionDeep - private ComplexField[] complexField; + @InjectionDeep private ComplexField[] complexField; public ComplexField[] getComplexField() { return complexField; } - public void setComplexField( ComplexField[] complexField ) { + public void setComplexField(ComplexField[] complexField) { this.complexField = complexField; } } public static class ComplexField { - @Injection( name = "COMPLEX_NAME", group = "COMPLEX" ) + @Injection(name = "COMPLEX_NAME", group = "COMPLEX") private String fieldName; private final MetadataBean parentMeta; - public ComplexField( MetadataBean parentMeta ) { + public ComplexField(MetadataBean parentMeta) { this.parentMeta = parentMeta; } @@ -256,7 +257,7 @@ public String getFieldName() { return fieldName; } - public void setFieldName( String fieldName ) { + public void setFieldName(String fieldName) { this.fieldName = fieldName; } @@ -265,29 +266,28 @@ public MetadataBean getParentMeta() { } } - @InjectionSupported( localizationPrefix = "", groups = "COMPLEX" ) + @InjectionSupported(localizationPrefix = "", groups = "COMPLEX") public static class MetadataBeanImplementsInterface implements MetadataInterface { - @InjectionDeep - private ComplexFieldWithInterfaceArg[] complexField; + @InjectionDeep private ComplexFieldWithInterfaceArg[] complexField; public ComplexFieldWithInterfaceArg[] getComplexField() { return complexField; } - public void setComplexField( ComplexFieldWithInterfaceArg[] complexField ) { + public void setComplexField(ComplexFieldWithInterfaceArg[] complexField) { this.complexField = complexField; } } public static class ComplexFieldWithInterfaceArg { - @Injection( name = "COMPLEX_NAME", group = "COMPLEX" ) + @Injection(name = "COMPLEX_NAME", group = "COMPLEX") private String fieldName; private final MetadataInterface parentMeta; - public ComplexFieldWithInterfaceArg( MetadataInterface parentMeta ) { + public ComplexFieldWithInterfaceArg(MetadataInterface parentMeta) { this.parentMeta = parentMeta; } @@ -295,7 +295,7 @@ public String getFieldName() { return fieldName; } - public void setFieldName( String fieldName ) { + public void setFieldName(String fieldName) { this.fieldName = fieldName; } @@ -303,5 +303,4 @@ public MetadataInterface getParentMeta() { return parentMeta; } } - } diff --git a/integration-tests/transforms/0011-calculator-basics.hpl b/integration-tests/transforms/0011-calculator-basics.hpl index 828a2a7eb3..2c5bdf3964 100644 --- a/integration-tests/transforms/0011-calculator-basics.hpl +++ b/integration-tests/transforms/0011-calculator-basics.hpl @@ -46,289 +46,189 @@ Y - constStr CONSTANT static - - - String + constStr + N -1 -1 - N - - - - + String - constInt CONSTANT 1 - - - Integer + constInt + N -1 -1 - N - - - - + Integer - ConstSep CONSTANT - - - - String + ConstSep + Y -1 -1 - Y - - - - + String - intAPlusB ADD intA intB - - Integer + intAPlusB + N -1 -1 - N - - - - + Integer - numAPlusB ADD numA numB - - Number + numAPlusB + N -1 -1 - N - - - - + Number - strAPlusBTmp ADD strA ConstSep - - String + strAPlusBTmp + Y -1 -1 - Y - - - - + String - strAPlusB ADD strAPlusBTmp strB - - String + strAPlusB + N -1 -1 - N - - - - + String - intCMinusB SUBTRACT intC intB - - Integer + intCMinusB + N -1 -1 - N - - - - + Integer - numCMinusB SUBTRACT numC numB - - Number + numCMinusB + N -1 -1 - N - - - - + Number - intATimesB MULTIPLY intA intB - - Integer + intATimesB + N -1 -1 - N - - - - + Integer - numATimesB MULTIPLY numA numB - - Number + numATimesB + N -1 -1 - N - - - - + Number - intCDivA DIVIDE intC intA - - Number + intCDivA + N -1 -1 - N - - - - + Number - numADivB DIVIDE numA numB - - Number + numADivB + N -1 -1 - N - - - - + Number - intA*A SQUARE intA intA - - Integer + intA*A + N -1 -1 - N - - - - + Integer - numA*A SQUARE numA numA - - Number + numA*A + N -1 -1 - N - - - - + Number - intSqrt(a) SQUARE_ROOT intA - - - Integer + intSqrt(a) + N -1 -1 - N - - - - + Integer - numSqrt(a) SQUARE_ROOT numA - - - Number + numSqrt(a) + N -1 -1 - N - - - - + Number - intPctAofB PERCENT_1 intA intB - - Integer + intPctAofB + N -1 -1 - N - - - - + Integer - numPctAofB PERCENT_1 numA numB - - Number + numPctAofB + N -1 -1 - N - - - - + Number diff --git a/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/CalculatorMeta.java b/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/CalculatorMeta.java index 25dbb13a9b..99a81babc2 100644 --- a/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/CalculatorMeta.java +++ b/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/CalculatorMeta.java @@ -56,11 +56,15 @@ public class CalculatorMeta extends BaseTransformMeta private static final Class PKG = CalculatorMeta.class; // For Translator /** The calculations to be performed */ - @HopMetadataProperty(key = "calculation") + @HopMetadataProperty( + key = "calculation", + injectionGroupKey = "Calculations", + injectionGroupDescription = "CalculatorMeta.Injection.Calculations") private List functions; /** Raise an error if file does not exist */ - @HopMetadataProperty private boolean failIfNoFile; + @HopMetadataProperty(injectionKeyDescription = "CalculatorMeta.Injection.FailIfNoFile") + private boolean failIfNoFile; public CalculatorMeta() { this.failIfNoFile = true; diff --git a/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/CalculatorMetaFunction.java b/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/CalculatorMetaFunction.java index 62cd6962d4..dc5a9dc3c8 100644 --- a/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/CalculatorMetaFunction.java +++ b/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/CalculatorMetaFunction.java @@ -416,43 +416,69 @@ public static CalculationType getTypeWithDescription(String description) { } } - @HopMetadataProperty(key = "field_name") + @HopMetadataProperty( + key = "field_name", + injectionKeyDescription = "CalculatorMeta.Injection.Calculation.FieldName") private String fieldName; - @HopMetadataProperty(key = "calc_type") + @HopMetadataProperty( + key = "calc_type", + injectionKeyDescription = "CalculatorMeta.Injection.Calculation.CalculationType") private CalculationType calcType; - @HopMetadataProperty(key = "field_a") + @HopMetadataProperty( + key = "field_a", + injectionKeyDescription = "CalculatorMeta.Injection.Calculation.FieldA") private String fieldA; - @HopMetadataProperty(key = "field_b") + @HopMetadataProperty( + key = "field_b", + injectionKeyDescription = "CalculatorMeta.Injection.Calculation.FieldB") private String fieldB; - @HopMetadataProperty(key = "field_c") + @HopMetadataProperty( + key = "field_c", + injectionKeyDescription = "CalculatorMeta.Injection.Calculation.FieldC") private String fieldC; - @HopMetadataProperty(key = "value_type") + @HopMetadataProperty( + key = "value_type", + injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValueType") private String valueType; - @HopMetadataProperty(key = "value_length") + @HopMetadataProperty( + key = "value_length", + injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValueLength") private int valueLength; - @HopMetadataProperty(key = "value_precision") + @HopMetadataProperty( + key = "value_precision", + injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValuePrecision") private int valuePrecision; - @HopMetadataProperty(key = "conversion_mask") + @HopMetadataProperty( + key = "conversion_mask", + injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValueFormat") private String conversionMask; - @HopMetadataProperty(key = "decimal_symbol") + @HopMetadataProperty( + key = "decimal_symbol", + injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValueDecimal") private String decimalSymbol; - @HopMetadataProperty(key = "grouping_symbol") + @HopMetadataProperty( + key = "grouping_symbol", + injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValueGroup") private String groupingSymbol; - @HopMetadataProperty(key = "currency_symbol") + @HopMetadataProperty( + key = "currency_symbol", + injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValueCurrency") private String currencySymbol; - @HopMetadataProperty(key = "remove") + @HopMetadataProperty( + key = "remove", + injectionKeyDescription = "CalculatorMeta.Injection.Calculation.Remove") private boolean removedFromResult; public CalculatorMetaFunction() { diff --git a/plugins/transforms/calculator/src/main/resources/org/apache/hop/pipeline/transforms/calculator/messages/messages_en_US.properties b/plugins/transforms/calculator/src/main/resources/org/apache/hop/pipeline/transforms/calculator/messages/messages_en_US.properties index 3a12329c90..c424d5c5d4 100644 --- a/plugins/transforms/calculator/src/main/resources/org/apache/hop/pipeline/transforms/calculator/messages/messages_en_US.properties +++ b/plugins/transforms/calculator/src/main/resources/org/apache/hop/pipeline/transforms/calculator/messages/messages_en_US.properties @@ -1,5 +1,4 @@ # -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -15,9 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# -# - BaseTransform.TypeLongDesc.Calculator=Calculator BaseTransform.TypeTooltipDesc.Calculator=Create new fields by performing simple calculations CalculatorDialog.LengthColumn.Column=Length @@ -129,3 +125,18 @@ CalculatorMetaFunction.CalcFunctions.Remainder=Remainder of A / B Calculator.Log.NoFile=File not found CalculatorDialog.FailIfNoFile=Throw an error on non existing files CalculatorDialog.FailIfNoFileTooltip=Check this option if you want the pipeline to throw an error\nif there are no files to process. +CalculatorMeta.Injection.Calculations=List of calculations +CalculatorMeta.Injection.FailIfNoFile=Fail if no file +CalculatorMeta.Injection.Calculation.FieldName=Output field name +CalculatorMeta.Injection.Calculation.CalculationType=Calculation type +CalculatorMeta.Injection.Calculation.FieldA=Name of field A +CalculatorMeta.Injection.Calculation.FieldB=Name of field B +CalculatorMeta.Injection.Calculation.FieldC=Name of field C +CalculatorMeta.Injection.Calculation.ValueType=Output value type +CalculatorMeta.Injection.Calculation.ValueLength=Length +CalculatorMeta.Injection.Calculation.ValuePrecision=Precision +CalculatorMeta.Injection.Calculation.ValueFormat=Format mask +CalculatorMeta.Injection.Calculation.ValueGroup=Grouping symbol +CalculatorMeta.Injection.Calculation.ValueDecimal=Decimal symbol +CalculatorMeta.Injection.Calculation.ValueCurrency=Currency symbol +CalculatorMeta.Injection.Calculation.Remove=Remove from result? diff --git a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java index e13cfdedda..eb374d4d25 100644 --- a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java +++ b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -58,167 +58,183 @@ * @author Matt * @since 2007-07-05 */ -public class MetaInject extends BaseTransform implements ITransform { +public class MetaInject extends BaseTransform + implements ITransform { private static final Class PKG = MetaInject.class; // For Translator private static final Lock repoSaveLock = new ReentrantLock(); public MetaInject( - TransformMeta transformMeta, MetaInjectMeta meta, MetaInjectData data, int copyNr, PipelineMeta pipelineMeta, Pipeline trans ) { - super( transformMeta, meta, data, copyNr, pipelineMeta, trans ); + TransformMeta transformMeta, + MetaInjectMeta meta, + MetaInjectData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline trans) { + super(transformMeta, meta, data, copyNr, pipelineMeta, trans); } - public boolean processRow( ) throws HopException { + public boolean processRow() throws HopException { // Read the data from all input transforms and keep it in memory... // Skip the transform from which we stream data. Keep that available for runtime action. // data.rowMap = new HashMap<>(); - for ( String prevTransformName : getPipelineMeta().getPrevTransformNames( getTransformMeta() ) ) { + for (String prevTransformName : getPipelineMeta().getPrevTransformNames(getTransformMeta())) { // Don't read from the streaming source transform // - if ( !data.streaming || !prevTransformName.equalsIgnoreCase( data.streamingSourceTransformName) ) { + if (!data.streaming + || !prevTransformName.equalsIgnoreCase(data.streamingSourceTransformName)) { List list = new ArrayList<>(); - IRowSet rowSet = findInputRowSet( prevTransformName ); - Object[] row = getRowFrom( rowSet ); - while ( row != null ) { + IRowSet rowSet = findInputRowSet(prevTransformName); + Object[] row = getRowFrom(rowSet); + while (row != null) { RowMetaAndData rd = new RowMetaAndData(); - rd.setRowMeta( rowSet.getRowMeta() ); - rd.setData( row ); - list.add( rd ); + rd.setRowMeta(rowSet.getRowMeta()); + rd.setData(row); + list.add(rd); - row = getRowFrom( rowSet ); + row = getRowFrom(rowSet); } - if ( !list.isEmpty() ) { - data.rowMap.put( prevTransformName, list ); + if (!list.isEmpty()) { + data.rowMap.put(prevTransformName, list); } } } List transforms = data.pipelineMeta.getTransforms(); - for ( Entry en : data.transformInjectionMetasMap.entrySet() ) { - newInjection( en.getKey(), en.getValue() ); + for (Entry en : data.transformInjectionMetasMap.entrySet()) { + newInjection(en.getKey(), en.getValue()); } /* * constants injection should be executed after transforms, because if constant should be inserted into target with array * in path, constants should be inserted into all arrays items */ - for ( Entry en : data.transformInjectionMetasMap.entrySet() ) { - newInjectionConstants( en.getKey(), en.getValue() ); + for (Entry en : data.transformInjectionMetasMap.entrySet()) { + newInjectionConstants(en.getKey(), en.getValue()); } - for ( Entry en : data.transformInjectionMetasMap.entrySet() ) { - en.getValue().searchInfoAndTargetTransforms( transforms ); + for (Entry en : data.transformInjectionMetasMap.entrySet()) { + en.getValue().searchInfoAndTargetTransforms(transforms); } - for ( String targetTransformName : data.transformInjectionMetasMap.keySet() ) { - if ( !data.transformInjectionMetasMap.containsKey( targetTransformName ) ) { - TransformMeta targetTransform = TransformMeta.findTransform( transforms, targetTransformName ); - if ( targetTransform != null ) { - targetTransform.getTransform().searchInfoAndTargetTransforms( transforms ); + for (String targetTransformName : data.transformInjectionMetasMap.keySet()) { + if (!data.transformInjectionMetasMap.containsKey(targetTransformName)) { + TransformMeta targetTransform = + TransformMeta.findTransform(transforms, targetTransformName); + if (targetTransform != null) { + targetTransform.getTransform().searchInfoAndTargetTransforms(transforms); } } } - if ( !meta.isNoExecution() ) { + if (!meta.isNoExecution()) { // Now we can execute this modified transformation metadata. // final Pipeline injectPipeline = createInjectPipeline(); - injectPipeline.setParentPipeline( getPipeline() ); - injectPipeline.setMetadataProvider( getMetadataProvider() ); - if ( getPipeline().getParentWorkflow() != null ) { - injectPipeline.setParentWorkflow( getPipeline().getParentWorkflow() ); + injectPipeline.setParentPipeline(getPipeline()); + injectPipeline.setMetadataProvider(getMetadataProvider()); + if (getPipeline().getParentWorkflow() != null) { + injectPipeline.setParentWorkflow(getPipeline().getParentWorkflow()); } // Copy all variables over... // - injectPipeline.copyFrom( this ); + injectPipeline.copyFrom(this); // Copy parameter definitions with empty values. // Then set those parameters to the values if have any. - injectPipeline.copyParametersFromDefinitions( data.pipelineMeta ); + injectPipeline.copyParametersFromDefinitions(data.pipelineMeta); for (String variableName : injectPipeline.getVariableNames()) { - String variableValue = getVariable( variableName ); - if ( StringUtils.isNotEmpty(variableValue)) { - injectPipeline.setParameterValue( variableName, variableValue ); + String variableValue = getVariable(variableName); + if (StringUtils.isNotEmpty(variableValue)) { + injectPipeline.setParameterValue(variableName, variableValue); } } getPipeline().addExecutionStoppedListener(e -> injectPipeline.stopAll()); - injectPipeline.setLogLevel( getLogLevel() ); + injectPipeline.setLogLevel(getLogLevel()); // Parameters get activated below so we need to make sure they have values // - injectPipeline.prepareExecution( ); + injectPipeline.prepareExecution(); // See if we need to stream some data over... // RowProducer rowProducer = null; - if ( data.streaming ) { - rowProducer = injectPipeline.addRowProducer( data.streamingTargetTransformName, 0 ); + if (data.streaming) { + rowProducer = injectPipeline.addRowProducer(data.streamingTargetTransformName, 0); } // Finally, add the mapping transformation to the active sub-transformations // map in the parent transformation // - getPipeline().addActiveSubPipeline( getTransformName(), injectPipeline ); - - if ( !Utils.isEmpty( meta.getSourceTransformName() ) ) { - ITransform transformInterface = injectPipeline.getTransformInterface( meta.getSourceTransformName(), 0 ); - if ( transformInterface == null ) { - throw new HopException( "Unable to find transform '" + meta.getSourceTransformName() + "' to read from." ); + getPipeline().addActiveSubPipeline(getTransformName(), injectPipeline); + + if (!Utils.isEmpty(meta.getSourceTransformName())) { + ITransform transformInterface = + injectPipeline.getTransformInterface(meta.getSourceTransformName(), 0); + if (transformInterface == null) { + throw new HopException( + "Unable to find transform '" + meta.getSourceTransformName() + "' to read from."); } - transformInterface.addRowListener( new RowAdapter() { - @Override - public void rowWrittenEvent(IRowMeta rowMeta, Object[] row ) throws HopTransformException { - // Just pass along the data as output of this transform... - // - MetaInject.this.putRow( rowMeta, row ); - } - } ); + transformInterface.addRowListener( + new RowAdapter() { + @Override + public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) + throws HopTransformException { + // Just pass along the data as output of this transform... + // + MetaInject.this.putRow(rowMeta, row); + } + }); } injectPipeline.startThreads(); - if ( data.streaming ) { + if (data.streaming) { // Deplete all the rows from the parent transformation into the modified transformation // - IRowSet rowSet = findInputRowSet( data.streamingSourceTransformName); - if ( rowSet == null ) { - throw new HopException( "Unable to find transform '" + data.streamingSourceTransformName + "' to stream data from" ); + IRowSet rowSet = findInputRowSet(data.streamingSourceTransformName); + if (rowSet == null) { + throw new HopException( + "Unable to find transform '" + + data.streamingSourceTransformName + + "' to stream data from"); } - Object[] row = getRowFrom( rowSet ); - while ( row != null && !isStopped() ) { - rowProducer.putRow( rowSet.getRowMeta(), row ); - row = getRowFrom( rowSet ); + Object[] row = getRowFrom(rowSet); + while (row != null && !isStopped()) { + rowProducer.putRow(rowSet.getRowMeta(), row); + row = getRowFrom(rowSet); } rowProducer.finished(); } // Wait until the child transformation finished processing... // - while ( !injectPipeline.isFinished() && !injectPipeline.isStopped() && !isStopped() ) { - copyResult( injectPipeline ); + while (!injectPipeline.isFinished() && !injectPipeline.isStopped() && !isStopped()) { + copyResult(injectPipeline); // Wait a little bit. try { - Thread.sleep( 50 ); - } catch ( Exception e ) { + Thread.sleep(50); + } catch (Exception e) { // Ignore errors } } - copyResult( injectPipeline ); - waitUntilFinished( injectPipeline ); + copyResult(injectPipeline); + waitUntilFinished(injectPipeline); } - // let the transformation complete it's execution to allow for any customizations to MDI to happen in the init methods of transforms - if ( log.isDetailed() ) { - logDetailed( "XML of transformation after injection: " + data.pipelineMeta.getXml() ); + // let the transformation complete it's execution to allow for any customizations to MDI to + // happen in the init methods of transforms + if (log.isDetailed()) { + logDetailed("XML of transformation after injection: " + data.pipelineMeta.getXml()); } - String targetFile = resolve( meta.getTargetFile() ); - if ( !Utils.isEmpty( targetFile ) ) { - writeInjectedHpl( targetFile ); + String targetFile = resolve(meta.getTargetFile()); + if (!Utils.isEmpty(targetFile)) { + writeInjectedHpl(targetFile); } // All done! @@ -228,178 +244,195 @@ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row ) throws HopTransform return false; } - void waitUntilFinished( Pipeline injectTrans ) { + void waitUntilFinished(Pipeline injectTrans) { injectTrans.waitUntilFinished(); } Pipeline createInjectPipeline() { - return new LocalPipelineEngine( data.pipelineMeta, this, this ); + return new LocalPipelineEngine(data.pipelineMeta, this, this); } - private void writeInjectedHpl(String targetFilPath ) throws HopException { + private void writeInjectedHpl(String targetFilPath) throws HopException { - writeInjectedHplToFs( targetFilPath ); + writeInjectedHplToFs(targetFilPath); } /** * Writes the generated meta injection transformation to the file system. + * * @param targetFilePath the filesystem path to which to save the generated injection hpl * @throws HopException */ - private void writeInjectedHplToFs(String targetFilePath ) throws HopException { + private void writeInjectedHplToFs(String targetFilePath) throws HopException { OutputStream os = null; try { - os = HopVfs.getOutputStream( targetFilePath, false ); - os.write( XmlHandler.getXmlHeader().getBytes( Const.XML_ENCODING ) ); - os.write( data.pipelineMeta.getXml().getBytes( Const.XML_ENCODING ) ); - } catch ( IOException e ) { - throw new HopException( "Unable to write target file (hpl after injection) to file '" - + targetFilePath + "'", e ); + os = HopVfs.getOutputStream(targetFilePath, false); + os.write(XmlHandler.getXmlHeader().getBytes(Const.XML_ENCODING)); + os.write(data.pipelineMeta.getXml().getBytes(Const.XML_ENCODING)); + } catch (IOException e) { + throw new HopException( + "Unable to write target file (hpl after injection) to file '" + targetFilePath + "'", e); } finally { - if ( os != null ) { + if (os != null) { try { os.close(); - } catch ( Exception e ) { - throw new HopException( e ); + } catch (Exception e) { + throw new HopException(e); } } } } - /** - * Inject values from transforms. - */ - private void newInjection( String targetTransform, ITransformMeta targetTransformMeta ) throws HopException { - if ( log.isDetailed() ) { - logDetailed( "Handing transform '" + targetTransform + "' injection!" ); + /** Inject values from transforms. */ + private void newInjection(String targetTransform, ITransformMeta targetTransformMeta) + throws HopException { + if (log.isDetailed()) { + logDetailed("Handing transform '" + targetTransform + "' injection!"); } - BeanInjectionInfo injectionInfo = new BeanInjectionInfo( targetTransformMeta.getClass() ); - BeanInjector injector = new BeanInjector( injectionInfo ); + BeanInjectionInfo injectionInfo = new BeanInjectionInfo(targetTransformMeta.getClass()); + BeanInjector injector = new BeanInjector(injectionInfo); // Collect all the metadata for this target transform... // Map targetMap = meta.getTargetSourceMapping(); boolean wasInjection = false; - for ( TargetTransformAttribute target : targetMap.keySet() ) { - SourceTransformField source = targetMap.get( target ); + for (TargetTransformAttribute target : targetMap.keySet()) { + SourceTransformField source = targetMap.get(target); - if ( target.getTransformName().equalsIgnoreCase( targetTransform ) ) { + if (target.getTransformName().equalsIgnoreCase(targetTransform)) { // This is the transform to collect data for... // We also know which transform to read the data from. (source) // - if ( source.getTransformName() != null ) { + if (source.getTransformName() != null) { // from specified transform - List rows = data.rowMap.get( source.getTransformName() ); - if ( rows != null && !rows.isEmpty() ) { - // Which metadata key is this referencing? Find the attribute key in the metadata entries... + List rows = data.rowMap.get(source.getTransformName()); + if (rows != null && !rows.isEmpty()) { + // Which metadata key is this referencing? Find the attribute key in the metadata + // entries... // - if ( injector.hasProperty( targetTransformMeta, target.getAttributeKey() ) ) { + if (injector.hasProperty(targetTransformMeta, target.getAttributeKey())) { // target transform has specified key boolean skip = false; - for ( RowMetaAndData r : rows ) { - if ( r.getRowMeta().indexOfValue( source.getField() ) < 0 ) { - logError( BaseMessages.getString( PKG, "MetaInject.SourceFieldIsNotDefined.Message", source - .getField(), getPipelineMeta().getName() ) ); + for (RowMetaAndData r : rows) { + if (r.getRowMeta().indexOfValue(source.getField()) < 0) { + logError( + BaseMessages.getString( + PKG, + "MetaInject.SourceFieldIsNotDefined.Message", + source.getField(), + getPipelineMeta().getName())); // source transform doesn't contain specified field skip = true; } } - if ( !skip ) { + if (!skip) { // specified field exist - need to inject - injector.setProperty( targetTransformMeta, target.getAttributeKey(), rows, source.getField() ); + injector.setProperty( + targetTransformMeta, target.getAttributeKey(), rows, source.getField()); wasInjection = true; } } else { - // target transform doesn't have specified key - just report but don't fail like in 6.0 (BACKLOG-6753) - logError( BaseMessages.getString( PKG, "MetaInject.TargetKeyIsNotDefined.Message", target - .getAttributeKey(), getPipelineMeta().getName() ) ); + // target transform doesn't have specified key - just report but don't fail like in + // 6.0 (BACKLOG-6753) + logError( + BaseMessages.getString( + PKG, + "MetaInject.TargetKeyIsNotDefined.Message", + target.getAttributeKey(), + getPipelineMeta().getName())); } } } } } - if ( wasInjection ) { - injector.runPostInjectionProcessing( targetTransformMeta ); + if (wasInjection) { + injector.runPostInjectionProcessing(targetTransformMeta); } } - /** - * Inject constant values. - */ - private void newInjectionConstants( String targetTransform, ITransformMeta targetTransformMeta ) throws HopException { - if ( log.isDetailed() ) { - logDetailed( "Handing transform '" + targetTransform + "' constants injection!" ); + /** Inject constant values. */ + private void newInjectionConstants(String targetTransform, ITransformMeta targetTransformMeta) + throws HopException { + if (log.isDetailed()) { + logDetailed("Handing transform '" + targetTransform + "' constants injection!"); } - BeanInjectionInfo injectionInfo = new BeanInjectionInfo( targetTransformMeta.getClass() ); - BeanInjector injector = new BeanInjector( injectionInfo ); + BeanInjectionInfo injectionInfo = new BeanInjectionInfo(targetTransformMeta.getClass()); + BeanInjector injector = new BeanInjector(injectionInfo); // Collect all the metadata for this target transform... // Map targetMap = meta.getTargetSourceMapping(); - for ( TargetTransformAttribute target : targetMap.keySet() ) { - SourceTransformField source = targetMap.get( target ); + for (TargetTransformAttribute target : targetMap.keySet()) { + SourceTransformField source = targetMap.get(target); - if ( target.getTransformName().equalsIgnoreCase( targetTransform ) ) { + if (target.getTransformName().equalsIgnoreCase(targetTransform)) { // This is the transform to collect data for... // We also know which transform to read the data from. (source) // - if ( source.getTransformName() == null ) { + if (source.getTransformName() == null) { // inject constant - if ( injector.hasProperty( targetTransformMeta, target.getAttributeKey() ) ) { + if (injector.hasProperty(targetTransformMeta, target.getAttributeKey())) { // target transform has specified key - injector.setProperty( targetTransformMeta, target.getAttributeKey(), null, source.getField() ); + injector.setProperty( + targetTransformMeta, target.getAttributeKey(), null, source.getField()); } else { - // target transform doesn't have specified key - just report but don't fail like in 6.0 (BACKLOG-6753) - logError( BaseMessages.getString( PKG, "MetaInject.TargetKeyIsNotDefined.Message", target.getAttributeKey(), - getPipelineMeta().getName() ) ); + // target transform doesn't have specified key - just report but don't fail like in 6.0 + // (BACKLOG-6753) + logError( + BaseMessages.getString( + PKG, + "MetaInject.TargetKeyIsNotDefined.Message", + target.getAttributeKey(), + getPipelineMeta().getName())); } } } } } - private void copyResult( Pipeline trans ) { + private void copyResult(Pipeline trans) { Result result = trans.getResult(); - setLinesInput( result.getNrLinesInput() ); - setLinesOutput( result.getNrLinesOutput() ); - setLinesRead( result.getNrLinesRead() ); - setLinesWritten( result.getNrLinesWritten() ); - setLinesUpdated( result.getNrLinesUpdated() ); - setLinesRejected( result.getNrLinesRejected() ); - setErrors( result.getNrErrors() ); + setLinesInput(result.getNrLinesInput()); + setLinesOutput(result.getNrLinesOutput()); + setLinesRead(result.getNrLinesRead()); + setLinesWritten(result.getNrLinesWritten()); + setLinesUpdated(result.getNrLinesUpdated()); + setLinesRejected(result.getNrLinesRejected()); + setErrors(result.getNrErrors()); } - public boolean init( ) { + public boolean init() { - if ( super.init( ) ) { + if (super.init()) { try { meta.actualizeMetaInjectMapping(); data.pipelineMeta = loadPipelineMeta(); - checkSoureTransformsAvailability(); + checkSourceTransformsAvailability(); checkTargetTransformsAvailability(); // Get a mapping between the transform name and the injection... // // Get new injection info data.transformInjectionMetasMap = new HashMap<>(); - for ( TransformMeta transformMeta : data.pipelineMeta.getUsedTransforms() ) { + for (TransformMeta transformMeta : data.pipelineMeta.getUsedTransforms()) { ITransformMeta meta = transformMeta.getTransform(); - if ( BeanInjectionInfo.isInjectionSupported( meta.getClass() ) ) { - data.transformInjectionMetasMap.put( transformMeta.getName(), meta ); + if (BeanInjectionInfo.isInjectionSupported(meta.getClass())) { + data.transformInjectionMetasMap.put(transformMeta.getName(), meta); } } // See if we need to stream data from a specific transform into the template // - if ( meta.getStreamSourceTransform() != null && !Utils.isEmpty( meta.getStreamTargetTransformName() ) ) { + if (meta.getStreamSourceTransform() != null + && !Utils.isEmpty(meta.getStreamTargetTransformName())) { data.streaming = true; data.streamingSourceTransformName = meta.getStreamSourceTransform().getName(); data.streamingTargetTransformName = meta.getStreamTargetTransformName(); } return true; - } catch ( Exception e ) { - logError( BaseMessages.getString( PKG, "MetaInject.BadEncoding.Message" ), e ); + } catch (Exception e) { + logError(BaseMessages.getString(PKG, "MetaInject.BadEncoding.Message"), e); return false; } } @@ -408,136 +441,154 @@ public boolean init( ) { } private void checkTargetTransformsAvailability() { - Set existedTransformNames = convertToUpperCaseSet( data.pipelineMeta.getTransformNames() ); + Set existedTransformNames = + convertToUpperCaseSet(data.pipelineMeta.getTransformNames()); Map targetMap = meta.getTargetSourceMapping(); - Set unavailableTargetTransforms = getUnavailableTargetTransforms( targetMap, data.pipelineMeta); + Set unavailableTargetTransforms = + getUnavailableTargetTransforms(targetMap, data.pipelineMeta); Set alreadyMarkedTransforms = new HashSet<>(); - for ( TargetTransformAttribute currentTarget : unavailableTargetTransforms ) { - if ( alreadyMarkedTransforms.contains( currentTarget.getTransformName() ) ) { + for (TargetTransformAttribute currentTarget : unavailableTargetTransforms) { + if (alreadyMarkedTransforms.contains(currentTarget.getTransformName())) { continue; } - alreadyMarkedTransforms.add( currentTarget.getTransformName() ); - if ( existedTransformNames.contains( currentTarget.getTransformName().toUpperCase() ) ) { - logError( BaseMessages.getString( PKG, "MetaInject.TargetTransformIsNotUsed.Message", currentTarget.getTransformName(), - data.pipelineMeta.getName() ) ); + alreadyMarkedTransforms.add(currentTarget.getTransformName()); + if (existedTransformNames.contains(currentTarget.getTransformName().toUpperCase())) { + logError( + BaseMessages.getString( + PKG, + "MetaInject.TargetTransformIsNotUsed.Message", + currentTarget.getTransformName(), + data.pipelineMeta.getName())); } else { - logError( BaseMessages.getString( PKG, "MetaInject.TargetTransformIsNotDefined.Message", currentTarget.getTransformName(), - data.pipelineMeta.getName() ) ); + logError( + BaseMessages.getString( + PKG, + "MetaInject.TargetTransformIsNotDefined.Message", + currentTarget.getTransformName(), + data.pipelineMeta.getName())); } } - // alreadyMarked contains wrong transforms. Hop-Gui can report error if it will not fail transformation [BACKLOG-6753] + // alreadyMarked contains wrong transforms. Hop-Gui can report error if it will not fail + // transformation [BACKLOG-6753] } - public static void removeUnavailableTransformsFromMapping(Map targetMap, - Set unavailableSourceTransforms, Set unavailableTargetTransforms ) { - Iterator> targetMapIterator = targetMap.entrySet().iterator(); - while ( targetMapIterator.hasNext() ) { + public static void removeUnavailableTransformsFromMapping( + Map targetMap, + Set unavailableSourceTransforms, + Set unavailableTargetTransforms) { + Iterator> targetMapIterator = + targetMap.entrySet().iterator(); + while (targetMapIterator.hasNext()) { Entry entry = targetMapIterator.next(); SourceTransformField currentSourceTransformField = entry.getValue(); TargetTransformAttribute currentTargetTransformAttribute = entry.getKey(); - if ( unavailableSourceTransforms.contains(currentSourceTransformField) || unavailableTargetTransforms.contains( - currentTargetTransformAttribute) ) { + if (unavailableSourceTransforms.contains(currentSourceTransformField) + || unavailableTargetTransforms.contains(currentTargetTransformAttribute)) { targetMapIterator.remove(); } } } - public static Set getUnavailableTargetTransforms(Map targetMap, - PipelineMeta injectedPipelineMeta ) { - Set usedTransformNames = getUsedTransformsForReferencendTransformation( injectedPipelineMeta ); + public static Set getUnavailableTargetTransforms( + Map targetMap, + PipelineMeta injectedPipelineMeta) { + Set usedTransformNames = getUsedTransformsForReferencedPipeline(injectedPipelineMeta); Set unavailableTargetTransforms = new HashSet<>(); - for ( TargetTransformAttribute currentTarget : targetMap.keySet() ) { - if ( !usedTransformNames.contains( currentTarget.getTransformName().toUpperCase() ) ) { - unavailableTargetTransforms.add( currentTarget ); + for (TargetTransformAttribute currentTarget : targetMap.keySet()) { + if (!usedTransformNames.contains(currentTarget.getTransformName().toUpperCase())) { + unavailableTargetTransforms.add(currentTarget); } } - return Collections.unmodifiableSet( unavailableTargetTransforms ); + return Collections.unmodifiableSet(unavailableTargetTransforms); } - public static Set getUnavailableTargetKeys(Map targetMap, - PipelineMeta injectedPipelineMeta, Set unavailableTargetTransforms ) { + public static Set getUnavailableTargetKeys( + Map targetMap, + PipelineMeta injectedPipelineMeta, + Set unavailableTargetTransforms) { Set missingKeys = new HashSet<>(); - Map beanInfos = getUsedTransformBeanInfos( injectedPipelineMeta ); - for ( TargetTransformAttribute key : targetMap.keySet() ) { - if ( !unavailableTargetTransforms.contains( key ) ) { - BeanInjectionInfo info = beanInfos.get( key.getTransformName().toUpperCase() ); - if ( info != null && !info.getProperties().containsKey( key.getAttributeKey() ) ) { - missingKeys.add( key ); + Map beanInfos = getUsedTransformBeanInfos(injectedPipelineMeta); + for (TargetTransformAttribute key : targetMap.keySet()) { + if (!unavailableTargetTransforms.contains(key)) { + BeanInjectionInfo info = beanInfos.get(key.getTransformName().toUpperCase()); + if (info != null && !info.getProperties().containsKey(key.getAttributeKey())) { + missingKeys.add(key); } } } return missingKeys; } - private static Map getUsedTransformBeanInfos(PipelineMeta pipelineMeta ) { + private static Map getUsedTransformBeanInfos( + PipelineMeta pipelineMeta) { Map res = new HashMap<>(); - for ( TransformMeta transformMeta : pipelineMeta.getUsedTransforms() ) { + for (TransformMeta transformMeta : pipelineMeta.getUsedTransforms()) { Class transformMetaClass = transformMeta.getTransform().getClass(); - if ( BeanInjectionInfo.isInjectionSupported( transformMetaClass ) ) { - res.put( transformMeta.getName().toUpperCase(), new BeanInjectionInfo( transformMetaClass ) ); + if (BeanInjectionInfo.isInjectionSupported(transformMetaClass)) { + res.put(transformMeta.getName().toUpperCase(), new BeanInjectionInfo(transformMetaClass)); } } return res; } - private static Set getUsedTransformsForReferencendTransformation(PipelineMeta pipelineMeta ) { + private static Set getUsedTransformsForReferencedPipeline(PipelineMeta pipelineMeta) { Set usedTransformNames = new HashSet<>(); - for ( TransformMeta currentTransform : pipelineMeta.getUsedTransforms() ) { - usedTransformNames.add( currentTransform.getName().toUpperCase() ); + for (TransformMeta currentTransform : pipelineMeta.getUsedTransforms()) { + usedTransformNames.add(currentTransform.getName().toUpperCase()); } return usedTransformNames; } - public static Set getUnavailableSourceTransforms(Map targetMap, - PipelineMeta sourcePipelineMeta, TransformMeta transformMeta ) { - String[] transformNamesArray = sourcePipelineMeta.getPrevTransformNames( transformMeta ); - Set existedTransformNames = convertToUpperCaseSet( transformNamesArray ); + public static Set getUnavailableSourceTransforms( + Map targetMap, + PipelineMeta sourcePipelineMeta, + TransformMeta transformMeta) { + String[] transformNamesArray = sourcePipelineMeta.getPrevTransformNames(transformMeta); + Set existedTransformNames = convertToUpperCaseSet(transformNamesArray); Set unavailableSourceTransforms = new HashSet<>(); - for ( SourceTransformField currentSource : targetMap.values() ) { - if ( currentSource.getTransformName() != null ) { - if ( !existedTransformNames.contains( currentSource.getTransformName().toUpperCase() ) ) { - unavailableSourceTransforms.add( currentSource ); + for (SourceTransformField currentSource : targetMap.values()) { + if (currentSource.getTransformName() != null) { + if (!existedTransformNames.contains(currentSource.getTransformName().toUpperCase())) { + unavailableSourceTransforms.add(currentSource); } } } - return Collections.unmodifiableSet( unavailableSourceTransforms ); + return Collections.unmodifiableSet(unavailableSourceTransforms); } - private void checkSoureTransformsAvailability() { + private void checkSourceTransformsAvailability() { Map targetMap = meta.getTargetSourceMapping(); Set unavailableSourceTransforms = - getUnavailableSourceTransforms( targetMap, getPipelineMeta(), getTransformMeta() ); + getUnavailableSourceTransforms(targetMap, getPipelineMeta(), getTransformMeta()); Set alreadyMarkedTransforms = new HashSet<>(); - for ( SourceTransformField currentSource : unavailableSourceTransforms ) { - if ( alreadyMarkedTransforms.contains( currentSource.getTransformName() ) ) { + for (SourceTransformField currentSource : unavailableSourceTransforms) { + if (alreadyMarkedTransforms.contains(currentSource.getTransformName())) { continue; } - alreadyMarkedTransforms.add( currentSource.getTransformName() ); - logError( BaseMessages.getString( PKG, "MetaInject.SourceTransformIsNotAvailable.Message", currentSource.getTransformName(), - getPipelineMeta().getName() ) ); + alreadyMarkedTransforms.add(currentSource.getTransformName()); + logError( + BaseMessages.getString( + PKG, + "MetaInject.SourceTransformIsNotAvailable.Message", + currentSource.getTransformName(), + getPipelineMeta().getName())); } - // alreadyMarked contains wrong transforms. Spoon can report error if it will not fail transformation [BACKLOG-6753] } - /** - * package-local visibility for testing purposes - */ - static Set convertToUpperCaseSet( String[] array ) { - if ( array == null ) { + /** package-local visibility for testing purposes */ + static Set convertToUpperCaseSet(String[] array) { + if (array == null) { return Collections.emptySet(); } Set strings = new HashSet<>(); - for ( String currentString : array ) { - strings.add( currentString.toUpperCase() ); + for (String currentString : array) { + strings.add(currentString.toUpperCase()); } return strings; } - /** - * package-local visibility for testing purposes - */ + /** package-local visibility for testing purposes */ PipelineMeta loadPipelineMeta() throws HopException { - return MetaInjectMeta.loadPipelineMeta( meta, getPipeline().getMetadataProvider(), this ); + return MetaInjectMeta.loadPipelineMeta(meta, getPipeline().getMetadataProvider(), this); } - } diff --git a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java index 8467bb094b..c6c98f934c 100644 --- a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java +++ b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -581,24 +581,20 @@ private void addInjectTab() { ToolBar treeTb = new ToolBar(wInjectComp, SWT.HORIZONTAL | SWT.FLAT); props.setLook(treeTb); - ToolItem wExpandAll = new ToolItem( treeTb, SWT.PUSH ); + ToolItem wExpandAll = new ToolItem(treeTb, SWT.PUSH); wExpandAll.setImage(GuiResource.getInstance().getImageExpandAll()); wExpandAll.setToolTipText( BaseMessages.getString(PKG, "MetaInjectDialog.InjectTab.FilterString.ExpandAll")); - wExpandAll.addListener( - SWT.Selection, - e -> setExpandedState(true) ); + wExpandAll.addListener(SWT.Selection, e -> setExpandedState(true)); - ToolItem wCollapseAll = new ToolItem( treeTb, SWT.PUSH ); + ToolItem wCollapseAll = new ToolItem(treeTb, SWT.PUSH); wCollapseAll.setImage(GuiResource.getInstance().getImageCollapseAll()); wCollapseAll.setToolTipText( BaseMessages.getString(PKG, "MetaInjectDialog.InjectTab.FilterString.CollapseAll")); - wCollapseAll.addListener( - SWT.Selection, - e -> setExpandedState(false) ); + wCollapseAll.addListener(SWT.Selection, e -> setExpandedState(false)); - ToolItem wFilter = new ToolItem( treeTb, SWT.SEPARATOR ); + ToolItem wFilter = new ToolItem(treeTb, SWT.SEPARATOR); wSearchText = new Text(treeTb, SWT.SEARCH | SWT.CANCEL); props.setLook(wSearchText); wSearchText.setToolTipText( @@ -607,7 +603,7 @@ private void addInjectTab() { wFilter.setWidth((int) (120 * props.getZoomFactor())); // The search bar - ToolItem wSearch = new ToolItem( treeTb, SWT.PUSH ); + ToolItem wSearch = new ToolItem(treeTb, SWT.PUSH); wSearch.setImage(GuiResource.getInstance().getImageSearch()); wSearch.setToolTipText( BaseMessages.getString(PKG, "MetaInjectDialog.InjectTab.FilterString.refresh.Label")); @@ -869,7 +865,7 @@ public void getData() { setActive(); refreshTree(); - setExpandedState( true ); + setExpandedState(true); wTabFolder.setSelection(0); @@ -887,7 +883,7 @@ protected void updateTransformationFilter() { filterString = wSearchText.getText().toUpperCase(); } refreshTree(); - setExpandedState( true ); + setExpandedState(true); } private void refreshTree() { @@ -916,7 +912,7 @@ private void refreshTree() { // ITransformMeta metaInterface = transformMeta.getTransform(); if (BeanInjectionInfo.isInjectionSupported(metaInterface.getClass())) { - processNewMDIDescription(transformMeta, transformItem, metaInterface); + processMDIDescription(transformMeta, transformItem, metaInterface); } } @@ -934,7 +930,7 @@ private void refreshTree() { } } - private void processNewMDIDescription( + private void processMDIDescription( TransformMeta transformMeta, TreeItem transformItem, ITransformMeta metaInterface) { BeanInjectionInfo transformInjectionInfo = new BeanInjectionInfo(metaInterface.getClass()); @@ -944,30 +940,30 @@ private void processNewMDIDescription( if (!gr.hasMatchingProperty(filterString)) { continue; } - boolean rootGroup = StringUtils.isEmpty(gr.getName()); + boolean rootGroup = StringUtils.isEmpty(gr.getKey()); TreeItem groupItem; if (!rootGroup) { groupItem = new TreeItem(transformItem, SWT.NONE); - groupItem.setText(gr.getName()); - groupItem.setText(1, gr.getDescription()); + groupItem.setText(gr.getKey()); + groupItem.setText(1, gr.getTranslatedDescription()); groupItem.setExpanded(true); } else { groupItem = null; } - List propertyList = gr.getGroupProperties(); + List propertyList = gr.getProperties(); for (BeanInjectionInfo.Property property : propertyList) { if (!property.hasMatch(filterString)) { continue; } TreeItem treeItem = new TreeItem(rootGroup ? transformItem : groupItem, SWT.NONE); - treeItem.setText(property.getName()); - treeItem.setText(1, property.getDescription()); + treeItem.setText(property.getKey()); + treeItem.setText(1, property.getTranslatedDescription()); TargetTransformAttribute target = - new TargetTransformAttribute(transformMeta.getName(), property.getName(), !rootGroup); + new TargetTransformAttribute(transformMeta.getName(), property.getKey(), !rootGroup); treeItemTargetMap.put(treeItem, target); SourceTransformField source = targetSourceMapping.get(target); @@ -1157,18 +1153,18 @@ private void enterMapping() { BeanInjectionInfo transformInjectionInfo = new BeanInjectionInfo(iTransformMeta.getClass()); List groupsList = transformInjectionInfo.getGroups(); for (BeanInjectionInfo.Group group : groupsList) { - boolean detail = StringUtils.isNotEmpty(group.getName()); - List propertyList = group.getGroupProperties(); + boolean detail = StringUtils.isNotEmpty(group.getKey()); + List propertyList = group.getProperties(); for (BeanInjectionInfo.Property property : propertyList) { - mappingTargets.add(new MappingTarget(transformMeta, property.getName(), detail)); - String groupName = group.getName(); + mappingTargets.add(new MappingTarget(transformMeta, property.getKey(), detail)); + String groupName = group.getKey(); String targetString = transformMeta.getName() + " | "; if (StringUtils.isNotEmpty(groupName)) { targetString += groupName + " - "; } - targetString += property.getName(); + targetString += property.getKey(); targetString += " : "; - targetString += property.getDescription(); + targetString += property.getTranslatedDescription(); targetStrings.add(targetString); } } From 7ddaf1ccbe157dec785de006f9edfdc8dd89cf2f Mon Sep 17 00:00:00 2001 From: Matt Casters Date: Mon, 10 May 2021 16:00:17 +0200 Subject: [PATCH 2/6] HOP-2858 : Allow metadata injection through metadata property annotations --- .../mdi/0001-calculator-child.hpl | 140 +++++++++ .../mdi/0001-calculator-parent.hpl | 185 +++++++++++ integration-tests/mdi/dev-env-config.json | 9 + integration-tests/mdi/hop-config.json | 290 ++++++++++++++++++ .../mdi/main-0001-calculator.hwf | 113 +++++++ .../pipeline-run-configuration/local.json | 17 + .../workflow-run-configuration/local.json | 9 + integration-tests/mdi/project-config.json | 9 + 8 files changed, 772 insertions(+) create mode 100644 integration-tests/mdi/0001-calculator-child.hpl create mode 100644 integration-tests/mdi/0001-calculator-parent.hpl create mode 100644 integration-tests/mdi/dev-env-config.json create mode 100644 integration-tests/mdi/hop-config.json create mode 100644 integration-tests/mdi/main-0001-calculator.hwf create mode 100644 integration-tests/mdi/metadata/pipeline-run-configuration/local.json create mode 100644 integration-tests/mdi/metadata/workflow-run-configuration/local.json create mode 100644 integration-tests/mdi/project-config.json diff --git a/integration-tests/mdi/0001-calculator-child.hpl b/integration-tests/mdi/0001-calculator-child.hpl new file mode 100644 index 0000000000..4b29c5049e --- /dev/null +++ b/integration-tests/mdi/0001-calculator-child.hpl @@ -0,0 +1,140 @@ + + + + 0001-calculator-child + Y + + + + Normal + + + N + 1000 + 100 + - + 2021/05/10 10:52:28.332 + - + 2021/05/10 10:52:28.332 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + Sample data + Calculator + Y + + + Calculator + Log results + Y + + + + Calculator + Calculator + + Y + + 1 + + none + + + Y + + + 384 + 160 + + + + Log results + WriteToLog + + Y + + 1 + + none + + + log_level_basic + Y + N + 0 + + + + A + + + B + + + C + + + D + + + + + 576 + 160 + + + + Sample data + DataGrid + + Y + + 1 + + none + + + + + A + Integer + + + + + -1 + -1 + N + + + B + Integer + + + + + -1 + -1 + N + + + + + 100 + 200 + + + + + 192 + 160 + + + + + + diff --git a/integration-tests/mdi/0001-calculator-parent.hpl b/integration-tests/mdi/0001-calculator-parent.hpl new file mode 100644 index 0000000000..32df9d7b03 --- /dev/null +++ b/integration-tests/mdi/0001-calculator-parent.hpl @@ -0,0 +1,185 @@ + + + + 0001-calculator-parent + Y + + + + Normal + + + N + 1000 + 100 + - + 2021/05/10 10:50:11.779 + - + 2021/05/10 10:50:11.779 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + Sample data + ETL metadata injection + Y + + + + ETL metadata injection + MetaInject + + Y + + 1 + + none + + + ${PROJECT_HOME}/0001-calculator-child.hpl + + + + N + + + + + Calculator + field_b + Y + Sample data + FieldB + + + Calculator + field_a + Y + Sample data + FieldA + + + Calculator + value_type + Y + Sample data + Type + + + Calculator + field_name + Y + Sample data + FieldName + + + Calculator + calc_type + Y + Sample data + Calculation + + + + + 352 + 128 + + + + Sample data + DataGrid + + Y + + 1 + + none + + + + + Calculation + String + + + + + -1 + -1 + N + + + FieldName + String + + + + + -1 + -1 + N + + + FieldA + String + + + + + -1 + -1 + N + + + FieldB + String + + + + + -1 + -1 + N + + + Type + String + + + + + -1 + -1 + N + + + + + ADD + C + A + B + Integer + + + SUBTRACT + D + B + A + Integer + + + + + 165 + 128 + + + + + + diff --git a/integration-tests/mdi/dev-env-config.json b/integration-tests/mdi/dev-env-config.json new file mode 100644 index 0000000000..a1ca51bc29 --- /dev/null +++ b/integration-tests/mdi/dev-env-config.json @@ -0,0 +1,9 @@ +{ + "variables": [ + { + "name": "SAMPLE", + "value": "sampleValue", + "description": "A sample variable value" + } + ] +} \ No newline at end of file diff --git a/integration-tests/mdi/hop-config.json b/integration-tests/mdi/hop-config.json new file mode 100644 index 0000000000..d9e1e6562e --- /dev/null +++ b/integration-tests/mdi/hop-config.json @@ -0,0 +1,290 @@ +{ + "variables": [ + { + "name": "HOP_LENIENT_STRING_TO_NUMBER_CONVERSION", + "value": "N", + "description": "System wide flag to allow lenient string to number conversion for backward compatibility. If this setting is set to \"Y\", an string starting with digits will be converted successfully into a number. (example: 192.168.1.1 will be converted into 192 or 192.168 or 192168 depending on the decimal and grouping symbol). The default (N) will be to throw an error if non-numeric symbols are found in the string." + }, + { + "name": "HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE", + "value": "N", + "description": "System wide flag to ignore timezone while writing date/timestamp value to the database." + }, + { + "name": "HOP_LOG_SIZE_LIMIT", + "value": "0", + "description": "The log size limit for all pipelines and workflows that don't have the \"log size limit\" property set in their respective properties." + }, + { + "name": "HOP_EMPTY_STRING_DIFFERS_FROM_NULL", + "value": "N", + "description": "NULL vs Empty String. If this setting is set to Y, an empty string and null are different. Otherwise they are not." + }, + { + "name": "HOP_MAX_LOG_SIZE_IN_LINES", + "value": "0", + "description": "The maximum number of log lines that are kept internally by Hop. Set to 0 to keep all rows (default)" + }, + { + "name": "HOP_MAX_LOG_TIMEOUT_IN_MINUTES", + "value": "1440", + "description": "The maximum age (in minutes) of a log line while being kept internally by Hop. Set to 0 to keep all rows indefinitely (default)" + }, + { + "name": "HOP_MAX_WORKFLOW_TRACKER_SIZE", + "value": "5000", + "description": "The maximum number of workflow trackers kept in memory" + }, + { + "name": "HOP_MAX_ACTIONS_LOGGED", + "value": "5000", + "description": "The maximum number of action results kept in memory for logging purposes." + }, + { + "name": "HOP_MAX_LOGGING_REGISTRY_SIZE", + "value": "10000", + "description": "The maximum number of logging registry entries kept in memory for logging purposes." + }, + { + "name": "HOP_LOG_TAB_REFRESH_DELAY", + "value": "1000", + "description": "The hop log tab refresh delay." + }, + { + "name": "HOP_LOG_TAB_REFRESH_PERIOD", + "value": "1000", + "description": "The hop log tab refresh period." + }, + { + "name": "HOP_PLUGIN_CLASSES", + "value": null, + "description": "A comma delimited list of classes to scan for plugin annotations" + }, + { + "name": "HOP_PLUGIN_PACKAGES", + "value": null, + "description": "A comma delimited list of packages to scan for plugin annotations (warning: slow!!)" + }, + { + "name": "HOP_TRANSFORM_PERFORMANCE_SNAPSHOT_LIMIT", + "value": "0", + "description": "The maximum number of transform performance snapshots to keep in memory. Set to 0 to keep all snapshots indefinitely (default)" + }, + { + "name": "HOP_ROWSET_GET_TIMEOUT", + "value": "50", + "description": "The name of the variable that optionally contains an alternative rowset get timeout (in ms). This only makes a difference for extremely short lived pipelines." + }, + { + "name": "HOP_ROWSET_PUT_TIMEOUT", + "value": "50", + "description": "The name of the variable that optionally contains an alternative rowset put timeout (in ms). This only makes a difference for extremely short lived pipelines." + }, + { + "name": "HOP_CORE_TRANSFORMS_FILE", + "value": null, + "description": "The name of the project variable that will contain the alternative location of the hop-transforms.xml file. You can use this to customize the list of available internal transforms outside of the codebase." + }, + { + "name": "HOP_CORE_WORKFLOW_ACTIONS_FILE", + "value": null, + "description": "The name of the project variable that will contain the alternative location of the hop-workflow-actions.xml file." + }, + { + "name": "HOP_SERVER_OBJECT_TIMEOUT_MINUTES", + "value": "1440", + "description": "This project variable will set a time-out after which waiting, completed or stopped pipelines and workflows will be automatically cleaned up. The default value is 1440 (one day)." + }, + { + "name": "HOP_PIPELINE_PAN_JVM_EXIT_CODE", + "value": null, + "description": "Set this variable to an integer that will be returned as the Pan JVM exit code." + }, + { + "name": "HOP_DISABLE_CONSOLE_LOGGING", + "value": "N", + "description": "Set this variable to Y to disable standard Hop logging to the console. (stdout)" + }, + { + "name": "HOP_REDIRECT_STDERR", + "value": "N", + "description": "Set this variable to Y to redirect stderr to Hop logging." + }, + { + "name": "HOP_REDIRECT_STDOUT", + "value": "N", + "description": "Set this variable to Y to redirect stdout to Hop logging." + }, + { + "name": "HOP_DEFAULT_NUMBER_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default number format" + }, + { + "name": "HOP_DEFAULT_BIGNUMBER_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default bignumber format" + }, + { + "name": "HOP_DEFAULT_INTEGER_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default integer format" + }, + { + "name": "HOP_DEFAULT_DATE_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default date format" + }, + { + "name": "HOP_DEFAULT_TIMESTAMP_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default timestamp format" + }, + { + "name": "HOP_DEFAULT_SERVLET_ENCODING", + "value": null, + "description": "Defines the default encoding for servlets, leave it empty to use Java default encoding" + }, + { + "name": "HOP_FAIL_ON_LOGGING_ERROR", + "value": "N", + "description": "Set this variable to Y when you want the workflow/pipeline fail with an error when the related logging process (e.g. to a database) fails." + }, + { + "name": "HOP_AGGREGATION_MIN_NULL_IS_VALUED", + "value": "N", + "description": "Set this variable to Y to set the minimum to NULL if NULL is within an aggregate. Otherwise by default NULL is ignored by the MIN aggregate and MIN is set to the minimum value that is not NULL. See also the variable HOP_AGGREGATION_ALL_NULLS_ARE_ZERO." + }, + { + "name": "HOP_AGGREGATION_ALL_NULLS_ARE_ZERO", + "value": "N", + "description": "Set this variable to Y to return 0 when all values within an aggregate are NULL. Otherwise by default a NULL is returned when all values are NULL." + }, + { + "name": "HOP_COMPATIBILITY_TEXT_FILE_OUTPUT_APPEND_NO_HEADER", + "value": "N", + "description": "Set this variable to Y for backward compatibility for the Text File Output transform. Setting this to Ywill add no header row at all when the append option is enabled, regardless if the file is existing or not." + }, + { + "name": "HOP_PASSWORD_ENCODER_PLUGIN", + "value": "Hop", + "description": "Specifies the password encoder plugin to use by ID (Hop is the default)." + }, + { + "name": "HOP_SYSTEM_HOSTNAME", + "value": null, + "description": "You can use this variable to speed up hostname lookup. Hostname lookup is performed by Hop so that it is capable of logging the server on which a workflow or pipeline is executed." + }, + { + "name": "HOP_SERVER_JETTY_ACCEPTORS", + "value": null, + "description": "A variable to configure jetty option: acceptors for Carte" + }, + { + "name": "HOP_SERVER_JETTY_ACCEPT_QUEUE_SIZE", + "value": null, + "description": "A variable to configure jetty option: acceptQueueSize for Carte" + }, + { + "name": "HOP_SERVER_JETTY_RES_MAX_IDLE_TIME", + "value": null, + "description": "A variable to configure jetty option: lowResourcesMaxIdleTime for Carte" + }, + { + "name": "HOP_COMPATIBILITY_MERGE_ROWS_USE_REFERENCE_STREAM_WHEN_IDENTICAL", + "value": "N", + "description": "Set this variable to Y for backward compatibility for the Merge Rows (diff) transform. Setting this to Y will use the data from the reference stream (instead of the comparison stream) in case the compared rows are identical." + }, + { + "name": "HOP_SPLIT_FIELDS_REMOVE_ENCLOSURE", + "value": "false", + "description": "Set this variable to false to preserve enclosure symbol after splitting the string in the Split fields transform. Changing it to true will remove first and last enclosure symbol from the resulting string chunks." + }, + { + "name": "HOP_ALLOW_EMPTY_FIELD_NAMES_AND_TYPES", + "value": "false", + "description": "Set this variable to TRUE to allow your pipeline to pass 'null' fields and/or empty types." + }, + { + "name": "HOP_GLOBAL_LOG_VARIABLES_CLEAR_ON_EXPORT", + "value": "false", + "description": "Set this variable to false to preserve global log variables defined in pipeline / workflow Properties -> Log panel. Changing it to true will clear it when export pipeline / workflow." + }, + { + "name": "HOP_FILE_OUTPUT_MAX_STREAM_COUNT", + "value": "1024", + "description": "This project variable is used by the Text File Output transform. It defines the max number of simultaneously open files within the transform. The transform will close/reopen files as necessary to insure the max is not exceeded" + }, + { + "name": "HOP_FILE_OUTPUT_MAX_STREAM_LIFE", + "value": "0", + "description": "This project variable is used by the Text File Output transform. It defines the max number of milliseconds between flushes of files opened by the transform." + }, + { + "name": "HOP_USE_NATIVE_FILE_DIALOG", + "value": "N", + "description": "Set this value to Y if you want to use the system file open/save dialog when browsing files" + }, + { + "name": "HOP_AUTO_CREATE_CONFIG", + "value": "Y", + "description": "Set this value to N if you don't want to automatically create a hop configuration file (hop-config.json) when it's missing" + } + ], + "LocaleDefault": "en_BE", + "guiProperties": { + "FontFixedSize": "13", + "MaxUndo": "100", + "DarkMode": "Y", + "FontNoteSize": "13", + "ShowOSLook": "Y", + "FontFixedStyle": "0", + "FontNoteName": ".AppleSystemUIFont", + "FontFixedName": "Monospaced", + "FontGraphStyle": "0", + "FontDefaultSize": "13", + "GraphColorR": "255", + "FontGraphSize": "13", + "IconSize": "32", + "BackgroundColorB": "255", + "FontNoteStyle": "0", + "FontGraphName": ".AppleSystemUIFont", + "FontDefaultName": ".AppleSystemUIFont", + "GraphColorG": "255", + "UseGlobalFileBookmarks": "Y", + "FontDefaultStyle": "0", + "GraphColorB": "255", + "BackgroundColorR": "255", + "BackgroundColorG": "255", + "WorkflowDialogStyle": "RESIZE,MAX,MIN", + "LineWidth": "1", + "ContextDialogShowCategories": "Y" + }, + "projectsConfig": { + "enabled": true, + "projectMandatory": true, + "environmentMandatory": false, + "defaultProject": "default", + "defaultEnvironment": null, + "standardParentProject": "default", + "standardProjectsFolder": null, + "projectConfigurations": [ + { + "projectName": "default", + "projectHome": "${HOP_CONFIG_FOLDER}", + "configFilename": "project-config.json" + } + ], + "lifecycleEnvironments": [ + { + "name": "dev", + "purpose": "Testing", + "projectName": "default", + "configurationFiles": [ + "${PROJECT_HOME}/dev-env-config.json" + ] + } + ], + "projectLifecycles": [] + } +} \ No newline at end of file diff --git a/integration-tests/mdi/main-0001-calculator.hwf b/integration-tests/mdi/main-0001-calculator.hwf new file mode 100644 index 0000000000..6831cca197 --- /dev/null +++ b/integration-tests/mdi/main-0001-calculator.hwf @@ -0,0 +1,113 @@ + + + main-0001-calculator + Y + + + + - + 2021/05/10 10:49:19.324 + - + 2021/05/10 10:49:19.324 + + + + + Start + + SPECIAL + + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + 112 + 96 + + + + Check pipeline log + + EVAL + + + N + 464 + 96 + + + + 0001-calculator-parent.hpl + + PIPELINE + + ${PROJECT_HOME}/0001-calculator-parent.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + local + + Y + + N + 288 + 96 + + + + + + Start + 0001-calculator-parent.hpl + Y + Y + Y + + + 0001-calculator-parent.hpl + Check pipeline log + Y + Y + N + + + + + + diff --git a/integration-tests/mdi/metadata/pipeline-run-configuration/local.json b/integration-tests/mdi/metadata/pipeline-run-configuration/local.json new file mode 100644 index 0000000000..63794efcaf --- /dev/null +++ b/integration-tests/mdi/metadata/pipeline-run-configuration/local.json @@ -0,0 +1,17 @@ +{ + "engineRunConfiguration": { + "Local": { + "feedback_size": "50000", + "sample_size": "100", + "sample_type_in_gui": "Last", + "rowset_size": "10000", + "safe_mode": false, + "show_feedback": false, + "topo_sort": false, + "gather_metrics": false + } + }, + "configurationVariables": [], + "name": "local", + "description": "Runs your pipelines locally with the standard local Hop pipeline engine" +} \ No newline at end of file diff --git a/integration-tests/mdi/metadata/workflow-run-configuration/local.json b/integration-tests/mdi/metadata/workflow-run-configuration/local.json new file mode 100644 index 0000000000..e37a93039a --- /dev/null +++ b/integration-tests/mdi/metadata/workflow-run-configuration/local.json @@ -0,0 +1,9 @@ +{ + "engineRunConfiguration": { + "Local": { + "safe_mode": false + } + }, + "name": "local", + "description": "Runs your workflows locally with the standard local Hop workflow engine" +} \ No newline at end of file diff --git a/integration-tests/mdi/project-config.json b/integration-tests/mdi/project-config.json new file mode 100644 index 0000000000..b249d738c2 --- /dev/null +++ b/integration-tests/mdi/project-config.json @@ -0,0 +1,9 @@ +{ + "metadataBaseFolder": "${PROJECT_HOME}/metadata", + "unitTestsBasePath": "${PROJECT_HOME}", + "dataSetsCsvFolder": "${PROJECT_HOME}/datasets", + "enforcingExecutionInHome": true, + "config": { + "variables": [] + } +} \ No newline at end of file From a1a3e9f41cce1ae5020b1b39b05c71a2ecd94715 Mon Sep 17 00:00:00 2001 From: Matt Casters Date: Mon, 10 May 2021 16:43:07 +0200 Subject: [PATCH 3/6] HOP-2858 : Allow metadata injection through metadata property annotations (test fix) --- .../apache/hop/core/injection/bean/BeanInjectionInfo.java | 4 ++-- .../org/apache/hop/core/injection/bean/BeanLevelInfo.java | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjectionInfo.java b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjectionInfo.java index cd5cc0cc6d..47b34b9e44 100644 --- a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjectionInfo.java +++ b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjectionInfo.java @@ -395,8 +395,8 @@ public String getDescription(String description) { private String calcPropertyName(Injection metaInj, BeanLevelInfo leaf) { String name = metaInj.name(); while (leaf != null) { - if (StringUtils.isNotBlank(leaf.nameKey)) { - name = leaf.nameKey; + if (StringUtils.isNotEmpty(leaf.nameKey)) { + name = leaf.nameKey + "." + name; } leaf = leaf.parent; } diff --git a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java index 5f52afa297..002bd88147 100644 --- a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java +++ b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java @@ -247,7 +247,11 @@ private String calculateNameKey(String prefix, String name) { if (StringUtils.isEmpty(prefix)) { return name; } else { - return prefix + "." + name; + if (StringUtils.isEmpty(name)) { + return prefix; + } else { + return prefix + "." + name; + } } } From 87db9c80956881123e378ca9cc3f8f263abad859 Mon Sep 17 00:00:00 2001 From: Matt Casters Date: Mon, 10 May 2021 18:38:38 +0200 Subject: [PATCH 4/6] HOP-2858 : Allow metadata injection through metadata property annotations (Testing, i18n) --- .../mdi/0001-calculator-parent.hpl | 34 +- integration-tests/mdi/0002-abort-child.hpl | 79 ++++ integration-tests/mdi/0002-abort-parent.hpl | 168 +++++++++ .../mdi/0003-add-sequence-child.hpl | 218 +++++++++++ .../mdi/0003-add-sequence-parent.hpl | 206 +++++++++++ integration-tests/mdi/main-0002-abort.hwf | 132 +++++++ .../mdi/main-0003-add-sequence.hwf | 75 ++++ .../transforms/0013-row-generator.hpl | 135 +++++++ .../datasets/golden-row-generator.csv | 16 + .../transforms/main-0013-row-generator.hwf | 62 ++++ .../dataset/golden-row-generator.json | 48 +++ .../unit-test/0013-row-generator UNIT.json | 44 +++ .../pipeline/transforms/abort/AbortMeta.java | 15 +- .../abort/messages/messages_en_US.properties | 10 +- .../addsequence/AddSequenceMeta.java | 13 +- .../rowgenerator/GeneratorField.java | 255 +++++++++++++ .../transforms/rowgenerator/RowGenerator.java | 33 +- .../rowgenerator/RowGeneratorDialog.java | 110 +++--- .../rowgenerator/RowGeneratorMeta.java | 348 ++++-------------- .../messages/messages_en_US.properties | 76 ++-- .../rowgenerator/RowGeneratorMetaTest.java | 159 +++++--- .../rowgenerator/RowGeneratorUnitTest.java | 38 +- 22 files changed, 1796 insertions(+), 478 deletions(-) create mode 100644 integration-tests/mdi/0002-abort-child.hpl create mode 100644 integration-tests/mdi/0002-abort-parent.hpl create mode 100644 integration-tests/mdi/0003-add-sequence-child.hpl create mode 100644 integration-tests/mdi/0003-add-sequence-parent.hpl create mode 100644 integration-tests/mdi/main-0002-abort.hwf create mode 100644 integration-tests/mdi/main-0003-add-sequence.hwf create mode 100644 integration-tests/transforms/0013-row-generator.hpl create mode 100644 integration-tests/transforms/datasets/golden-row-generator.csv create mode 100644 integration-tests/transforms/main-0013-row-generator.hwf create mode 100644 integration-tests/transforms/metadata/dataset/golden-row-generator.json create mode 100644 integration-tests/transforms/metadata/unit-test/0013-row-generator UNIT.json create mode 100644 plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/GeneratorField.java diff --git a/integration-tests/mdi/0001-calculator-parent.hpl b/integration-tests/mdi/0001-calculator-parent.hpl index 32df9d7b03..7a43c23981 100644 --- a/integration-tests/mdi/0001-calculator-parent.hpl +++ b/integration-tests/mdi/0001-calculator-parent.hpl @@ -23,13 +23,13 @@ - Sample data - ETL metadata injection + Calculator Metadata + 0001-calculator-child.hpl Y - ETL metadata injection + 0001-calculator-child.hpl MetaInject Y @@ -51,36 +51,36 @@ Calculator field_b Y - Sample data + Calculator Metadata FieldB Calculator - field_a + field_name Y - Sample data - FieldA + Calculator Metadata + FieldName Calculator - value_type + field_a Y - Sample data - Type + Calculator Metadata + FieldA Calculator - field_name + calc_type Y - Sample data - FieldName + Calculator Metadata + Calculation Calculator - calc_type + value_type Y - Sample data - Calculation + Calculator Metadata + Type @@ -90,7 +90,7 @@ - Sample data + Calculator Metadata DataGrid Y diff --git a/integration-tests/mdi/0002-abort-child.hpl b/integration-tests/mdi/0002-abort-child.hpl new file mode 100644 index 0000000000..ef4e24600d --- /dev/null +++ b/integration-tests/mdi/0002-abort-child.hpl @@ -0,0 +1,79 @@ + + + + 0002-abort-child + Y + + + + Normal + + + N + 1000 + 100 + - + 2021/05/10 10:52:28.332 + - + 2021/05/10 10:52:28.332 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + 10 rows + Abort after a few rows + Y + + + + 10 rows + RowGenerator + + Y + + 1 + + none + + + + + 10 + N + 5000 + now + FiveSecondsAgo + + + 250 + 160 + + + + Abort after a few rows + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + + 0 + + + 416 + 160 + + + + + + diff --git a/integration-tests/mdi/0002-abort-parent.hpl b/integration-tests/mdi/0002-abort-parent.hpl new file mode 100644 index 0000000000..505967f592 --- /dev/null +++ b/integration-tests/mdi/0002-abort-parent.hpl @@ -0,0 +1,168 @@ + + + + 0002-abort-parent + Y + + + + Normal + + + N + 1000 + 100 + - + 2021/05/10 10:50:11.779 + - + 2021/05/10 10:50:11.779 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + Abort metadata + 0002-abort-child.hpl + Y + + + Row Generator metadata + 0002-abort-child.hpl + Y + + + + Abort metadata + DataGrid + + Y + + 1 + + none + + + + + Treshold + String + + + + + -1 + -1 + N + + + Message + String + + + + + -1 + -1 + N + + + + + 5 + More than 5 rows received + + + + + 160 + 224 + + + + 0002-abort-child.hpl + MetaInject + + Y + + 1 + + none + + + ${PROJECT_HOME}/0002-abort-child.hpl + + + + N + + + + + 10 rows + limit + N + Row Generator metadata + nrRows + + + Abort after a few rows + message + N + Abort metadata + Message + + + Abort after a few rows + row_threshold + N + Abort metadata + Treshold + + + + + 336 + 176 + + + + Row Generator metadata + DataGrid + + Y + + 1 + + none + + + + + nrRows + String + + + + + -1 + -1 + N + + + + + 10 + + + + + 160 + 128 + + + + + + diff --git a/integration-tests/mdi/0003-add-sequence-child.hpl b/integration-tests/mdi/0003-add-sequence-child.hpl new file mode 100644 index 0000000000..ba976f2fe9 --- /dev/null +++ b/integration-tests/mdi/0003-add-sequence-child.hpl @@ -0,0 +1,218 @@ + + + + 0003-add-sequence-child + Y + + + + Normal + + + N + 1000 + 100 + - + 2021/05/10 10:52:28.332 + - + 2021/05/10 10:52:28.332 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + 10 rows + sequence + Y + + + sequence + minId, maxId + Y + + + minId, maxId + minId<>2 or maxId<>7 + Y + + + minId<>2 or maxId<>7 + Abort + Y + + + + 10 rows + RowGenerator + + Y + + 1 + + none + + + + + 5000 + FiveSecondsAgo + N + 10 + now + + + 250 + 160 + + + + sequence + Sequence + + Y + + 1 + + none + + + + Y + N + 1 + 9999999 + + SEQ_ + 1 + id + + + 416 + 160 + + + + minId, maxId + GroupBy + + Y + + 1 + + none + + + N + N + + ${java.io.tmpdir} + grp + N + + N + + + + + minId + id + MIN + + + + maxId + id + MAX + + + + + + 608 + 160 + + + + minId<>2 or maxId<>7 + FilterRows + + Y + + 1 + + none + + + + + + + N + + + N + minId + <> + + + constant + Integer + 2 + -1 + 0 + N + ####0;-####0 + + + + N + OR + maxId + <> + + + constant + Integer + 7 + -1 + 0 + N + ####0;-####0 + + + + + + + + 768 + 160 + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + Add sequence transform MDI didn't work as expected + 0 + + + 960 + 160 + + + + + + diff --git a/integration-tests/mdi/0003-add-sequence-parent.hpl b/integration-tests/mdi/0003-add-sequence-parent.hpl new file mode 100644 index 0000000000..4111ed9888 --- /dev/null +++ b/integration-tests/mdi/0003-add-sequence-parent.hpl @@ -0,0 +1,206 @@ + + + + 0003-add-sequence-parent + Y + + + + Normal + + + N + 1000 + 100 + - + 2021/05/10 10:50:11.779 + - + 2021/05/10 10:50:11.779 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + Add Sequence metadata + 0003-add-sequence-child.hpl + Y + + + Row Generator metadata + 0003-add-sequence-child.hpl + Y + + + + Add Sequence metadata + DataGrid + + Y + + 1 + + none + + + + + start + String + + + + + -1 + -1 + N + + + increment + String + + + + + -1 + -1 + N + + + max + String + + + + + -1 + -1 + N + + + fieldname + String + + + + + -1 + -1 + N + + + + + 2 + 1 + 7 + id + + + + + 160 + 160 + + + + 0003-add-sequence-child.hpl + MetaInject + + Y + + 1 + + none + + + ${PROJECT_HOME}/0003-add-sequence-child.hpl + + + + N + + + + + 10 rows + limit + N + Row Generator metadata + nrRows + + + sequence + valuename + N + Add Sequence metadata + fieldname + + + sequence + increment_by + N + Add Sequence metadata + increment + + + sequence + start_at + N + Add Sequence metadata + start + + + sequence + max_value + N + Add Sequence metadata + max + + + + + 352 + 112 + + + + Row Generator metadata + DataGrid + + Y + + 1 + + none + + + + + nrRows + String + + + + + -1 + -1 + N + + + + + 100 + + + + + 160 + 64 + + + + + + diff --git a/integration-tests/mdi/main-0002-abort.hwf b/integration-tests/mdi/main-0002-abort.hwf new file mode 100644 index 0000000000..aae30c39b0 --- /dev/null +++ b/integration-tests/mdi/main-0002-abort.hwf @@ -0,0 +1,132 @@ + + + main-0002-abort + Y + + + + - + 2021/05/10 10:49:19.324 + - + 2021/05/10 10:49:19.324 + + + + + Start + + SPECIAL + + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + 112 + 96 + + + + Check pipeline log + + EVAL + + + N + 544 + 96 + + + + 0002-abort-parent.hpl + + PIPELINE + + ${PROJECT_HOME}/0002-abort-parent.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + local + + Y + + N + 288 + 96 + + + + Error + + ABORT + + The Abort transform was not called! + N + 544 + 192 + + + + + + Start + 0002-abort-parent.hpl + Y + Y + Y + + + 0002-abort-parent.hpl + Check pipeline log + Y + N + N + + + 0002-abort-parent.hpl + Error + Y + Y + N + + + + + + diff --git a/integration-tests/mdi/main-0003-add-sequence.hwf b/integration-tests/mdi/main-0003-add-sequence.hwf new file mode 100644 index 0000000000..287651e4ad --- /dev/null +++ b/integration-tests/mdi/main-0003-add-sequence.hwf @@ -0,0 +1,75 @@ + + + main-0003-add-sequence + Y + + + + - + 2021/05/10 10:49:19.324 + - + 2021/05/10 10:49:19.324 + + + + + Start + + SPECIAL + + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + 112 + 96 + + + + 0003-add-sequence-parent.hpl + + PIPELINE + + ${PROJECT_HOME}/0003-add-sequence-parent.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + local + + Y + + N + 288 + 96 + + + + + + Start + 0003-add-sequence-parent.hpl + Y + Y + Y + + + + + + diff --git a/integration-tests/transforms/0013-row-generator.hpl b/integration-tests/transforms/0013-row-generator.hpl new file mode 100644 index 0000000000..3fbd54f1b8 --- /dev/null +++ b/integration-tests/transforms/0013-row-generator.hpl @@ -0,0 +1,135 @@ + + + + 0013-row-generator + Y + + + + Normal + + + N + 1000 + 100 + - + 2021/05/10 17:36:36.287 + - + 2021/05/10 17:36:36.287 + + N + + + + + + 15 rows + Verify + Y + + + + 15 rows + RowGenerator + + Y + + 1 + + none + + + + + A + Integer + # + + + + 9876 + -1 + -1 + N + + + B + String + + + + + A value + -1 + -1 + N + + + C + Number + #.00 + + + + 123.45 + -1 + -1 + N + + + D + Date + yyyy/MM/dd HH:mm:ss + + + + 2021/05/10 17:38:07 + -1 + -1 + N + + + E + String + + + + + + -1 + -1 + Y + + + 15 + N + 5000 + now + FiveSecondsAgo + + + 176 + 112 + + + + Verify + Dummy + + Y + + 1 + + none + + + + + 384 + 112 + + + + + + diff --git a/integration-tests/transforms/datasets/golden-row-generator.csv b/integration-tests/transforms/datasets/golden-row-generator.csv new file mode 100644 index 0000000000..6cce3b2e2b --- /dev/null +++ b/integration-tests/transforms/datasets/golden-row-generator.csv @@ -0,0 +1,16 @@ +A,B,C,D,E +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, +9876,A value,123.45,2021/05/10 17:38:07, diff --git a/integration-tests/transforms/main-0013-row-generator.hwf b/integration-tests/transforms/main-0013-row-generator.hwf new file mode 100644 index 0000000000..f57df23a14 --- /dev/null +++ b/integration-tests/transforms/main-0013-row-generator.hwf @@ -0,0 +1,62 @@ + + + main-0013-row-generator + Y + + + + 0 + - + 2020/12/16 12:24:26.983 + - + 2020/12/16 12:24:26.983 + + + + + Start + + SPECIAL + + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + 128 + 112 + + + + Run Pipeline Unit Tests + + RunPipelineTests + + + + 0013-row-generator UNIT + + + N + 320 + 112 + + + + + + Start + Run Pipeline Unit Tests + Y + Y + Y + + + + + + diff --git a/integration-tests/transforms/metadata/dataset/golden-row-generator.json b/integration-tests/transforms/metadata/dataset/golden-row-generator.json new file mode 100644 index 0000000000..fab54c6b21 --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/golden-row-generator.json @@ -0,0 +1,48 @@ +{ + "base_filename": "golden-row-generator.csv", + "name": "golden-row-generator", + "description": "", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 5, + "field_precision": 0, + "field_format": "#", + "field_name": "A" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_format": "", + "field_name": "B" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 1, + "field_precision": -1, + "field_format": "#.00", + "field_name": "C" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 3, + "field_precision": -1, + "field_format": "yyyy/MM/dd HH:mm:ss", + "field_name": "D" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_format": "", + "field_name": "E" + } + ], + "folder_name": "" +} \ No newline at end of file diff --git a/integration-tests/transforms/metadata/unit-test/0013-row-generator UNIT.json b/integration-tests/transforms/metadata/unit-test/0013-row-generator UNIT.json new file mode 100644 index 0000000000..55ebf16d54 --- /dev/null +++ b/integration-tests/transforms/metadata/unit-test/0013-row-generator UNIT.json @@ -0,0 +1,44 @@ +{ + "variableValues": [], + "database_replacements": [], + "autoOpening": true, + "basePath": "", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "A", + "data_set_field": "A" + }, + { + "transform_field": "B", + "data_set_field": "B" + }, + { + "transform_field": "C", + "data_set_field": "C" + }, + { + "transform_field": "D", + "data_set_field": "D" + }, + { + "transform_field": "E", + "data_set_field": "E" + } + ], + "field_order": [ + "A" + ], + "transform_name": "Verify", + "data_set_name": "golden-row-generator" + } + ], + "input_data_sets": [], + "name": "0013-row-generator UNIT", + "description": "", + "trans_test_tweaks": [], + "persist_filename": "", + "pipeline_filename": "./0013-row-generator.hpl", + "test_type": "UNIT_TEST" +} \ No newline at end of file diff --git a/plugins/transforms/abort/src/main/java/org/apache/hop/pipeline/transforms/abort/AbortMeta.java b/plugins/transforms/abort/src/main/java/org/apache/hop/pipeline/transforms/abort/AbortMeta.java index 16dbd6346a..fe9ad61ad4 100644 --- a/plugins/transforms/abort/src/main/java/org/apache/hop/pipeline/transforms/abort/AbortMeta.java +++ b/plugins/transforms/abort/src/main/java/org/apache/hop/pipeline/transforms/abort/AbortMeta.java @@ -56,17 +56,24 @@ public enum AbortOption { } /** Threshold to abort. */ - @HopMetadataProperty(key = "row_threshold") + @HopMetadataProperty( + key = "row_threshold", + injectionKeyDescription = "AbortDialog.Options.RowThreshold.Label") private String rowThreshold; /** Message to put in log when aborting. */ - @HopMetadataProperty private String message; + @HopMetadataProperty(injectionKeyDescription = "AbortDialog.Logging.AbortMessage.Tooltip") + private String message; /** Always log rows. */ - @HopMetadataProperty(key = "always_log_rows") + @HopMetadataProperty( + key = "always_log_rows", + injectionKeyDescription = "AbortDialog.Logging.AlwaysLogRows.Label") private boolean alwaysLogRows; - @HopMetadataProperty(key = "abort_option") + @HopMetadataProperty( + key = "abort_option", + injectionKeyDescription = "AbortMeta.Injection.AbortOption") private AbortOption abortOption; @Override diff --git a/plugins/transforms/abort/src/main/resources/org/apache/hop/pipeline/transforms/abort/messages/messages_en_US.properties b/plugins/transforms/abort/src/main/resources/org/apache/hop/pipeline/transforms/abort/messages/messages_en_US.properties index 576b93f182..d6dd3a7ede 100644 --- a/plugins/transforms/abort/src/main/resources/org/apache/hop/pipeline/transforms/abort/messages/messages_en_US.properties +++ b/plugins/transforms/abort/src/main/resources/org/apache/hop/pipeline/transforms/abort/messages/messages_en_US.properties @@ -1,5 +1,4 @@ # -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -15,9 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# -# - Abort.Name=Abort Abort.Description=Abort a pipeline AbortMeta.CheckResult.NoInputReceivedError=No input received from other transforms. @@ -38,4 +34,8 @@ AbortDialog.Logging.Group=Logging AbortDialog.Logging.AlwaysLogRows.Label=Always log rows AbortDialog.Logging.AlwaysLogRows.Tooltip=Always log rows even if below current log level AbortDialog.Logging.AbortMessage.Label=Abort message\: -AbortDialog.Logging.AbortMessage.Tooltip=Message to put in log upon aborting \ No newline at end of file +AbortDialog.Logging.AbortMessage.Tooltip=Message to put in log upon aborting +AbortMeta.Injection.AbortOption=Abort type (ABORT, ABORT_WITH_ERROR, SAFE_STOP) + + + diff --git a/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMeta.java b/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMeta.java index 83bad34505..f113a033c6 100644 --- a/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMeta.java +++ b/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMeta.java @@ -58,13 +58,20 @@ public class AddSequenceMeta extends BaseTransformMeta private static final Class PKG = AddSequenceMeta.class; // For Translator - @HopMetadataProperty(key = "valuename") + @HopMetadataProperty( + key = "valuename", + injectionKeyDescription = "AddSequenceDialog.Valuename.Label") private String valueName; - @HopMetadataProperty(key = "use_database") + @HopMetadataProperty( + key = "use_database", + injectionKeyDescription = "AddSequenceDialog.UseDatabase.Label") private boolean databaseUsed; - @HopMetadataProperty(key = "connection", storeWithName = true) + @HopMetadataProperty( + key = "connection", + storeWithName = true, + injectionKeyDescription = "AddSequenceMeta.Injection.Connection") private DatabaseMeta databaseMeta; @HopMetadataProperty(key = "schema") diff --git a/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/GeneratorField.java b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/GeneratorField.java new file mode 100644 index 0000000000..faa6d71374 --- /dev/null +++ b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/GeneratorField.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.rowgenerator; + +import org.apache.hop.metadata.api.HopMetadataProperty; + +import java.util.Objects; + +public class GeneratorField { + + @HopMetadataProperty(injectionKeyDescription = "RowGeneratorMeta.Injection.Field.Name") + private String name; + + @HopMetadataProperty(injectionKeyDescription = "RowGeneratorMeta.Injection.Field.Type") + private String type; + + @HopMetadataProperty(injectionKeyDescription = "RowGeneratorMeta.Injection.Field.Format") + private String format; + + @HopMetadataProperty(injectionKeyDescription = "RowGeneratorMeta.Injection.Field.Length") + private int length; + + @HopMetadataProperty(injectionKeyDescription = "RowGeneratorMeta.Injection.Field.Precision") + private int precision; + + @HopMetadataProperty(injectionKeyDescription = "RowGeneratorMeta.Injection.Field.Currency") + private String currency; + + @HopMetadataProperty(injectionKeyDescription = "RowGeneratorMeta.Injection.Field.Decimal") + private String decimal; + + @HopMetadataProperty(injectionKeyDescription = "RowGeneratorMeta.Injection.Field.Group") + private String group; + + // Yeah, it has a key of "nullif", keep it for backward compatibility + @HopMetadataProperty( + key = "nullif", + injectionKeyDescription = "RowGeneratorMeta.Injection.Field.Value") + private String value; + + @HopMetadataProperty( + key = "set_empty_string", + injectionKeyDescription = "RowGeneratorMeta.Injection.Field.SetEmptyString") + private boolean setEmptyString; + + public GeneratorField() {} + + public GeneratorField(GeneratorField f) { + this.name = f.name; + this.type = f.type; + this.format = f.format; + this.length = f.length; + this.precision = f.precision; + this.currency = f.currency; + this.decimal = f.decimal; + this.group = f.group; + this.value = f.value; + this.setEmptyString = f.setEmptyString; + } + + public GeneratorField( + String name, + String type, + String format, + int length, + int precision, + String currency, + String decimal, + String group, + String value, + boolean setEmptyString) { + this.name = name; + this.type = type; + this.format = format; + this.length = length; + this.precision = precision; + this.currency = currency; + this.decimal = decimal; + this.group = group; + this.value = value; + this.setEmptyString = setEmptyString; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + GeneratorField that = (GeneratorField) o; + return Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + /** + * Gets name + * + * @return value of name + */ + public String getName() { + return name; + } + + /** @param name The name to set */ + public void setName(String name) { + this.name = name; + } + + /** + * Gets type + * + * @return value of type + */ + public String getType() { + return type; + } + + /** @param type The type to set */ + public void setType(String type) { + this.type = type; + } + + /** + * Gets format + * + * @return value of format + */ + public String getFormat() { + return format; + } + + /** @param format The format to set */ + public void setFormat(String format) { + this.format = format; + } + + /** + * Gets length + * + * @return value of length + */ + public int getLength() { + return length; + } + + /** @param length The length to set */ + public void setLength(int length) { + this.length = length; + } + + /** + * Gets precision + * + * @return value of precision + */ + public int getPrecision() { + return precision; + } + + /** @param precision The precision to set */ + public void setPrecision(int precision) { + this.precision = precision; + } + + /** + * Gets currency + * + * @return value of currency + */ + public String getCurrency() { + return currency; + } + + /** @param currency The currency to set */ + public void setCurrency(String currency) { + this.currency = currency; + } + + /** + * Gets decimal + * + * @return value of decimal + */ + public String getDecimal() { + return decimal; + } + + /** @param decimal The decimal to set */ + public void setDecimal(String decimal) { + this.decimal = decimal; + } + + /** + * Gets group + * + * @return value of group + */ + public String getGroup() { + return group; + } + + /** @param group The group to set */ + public void setGroup(String group) { + this.group = group; + } + + /** + * Gets value + * + * @return value of value + */ + public String getValue() { + return value; + } + + /** @param value The value to set */ + public void setValue(String value) { + this.value = value; + } + + /** + * Gets setEmptyString + * + * @return value of setEmptyString + */ + public boolean isSetEmptyString() { + return setEmptyString; + } + + /** @param setEmptyString The setEmptyString to set */ + public void setSetEmptyString(boolean setEmptyString) { + this.setEmptyString = setEmptyString; + } +} diff --git a/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGenerator.java b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGenerator.java index 505d7824b8..0aa9f4f433 100644 --- a/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGenerator.java +++ b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGenerator.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,6 +17,7 @@ package org.apache.hop.pipeline.transforms.rowgenerator; +import org.apache.commons.lang.StringUtils; import org.apache.hop.core.CheckResult; import org.apache.hop.core.Const; import org.apache.hop.core.ICheckResult; @@ -67,7 +68,7 @@ public RowGenerator( public static final RowMetaAndData buildRow( RowGeneratorMeta meta, List remarks, String origin) throws HopPluginException { IRowMeta rowMeta = new RowMeta(); - Object[] rowData = RowDataUtil.allocateRowData(meta.getFieldName().length + 2); + Object[] rowData = RowDataUtil.allocateRowData(meta.getFields().size() + 2); int index = 0; if (meta.isNeverEnding()) { @@ -82,27 +83,27 @@ public static final RowMetaAndData buildRow( } } - for (int i = 0; i < meta.getFieldName().length; i++) { - int valtype = ValueMetaFactory.getIdForValueMeta(meta.getFieldType()[i]); - if (meta.getFieldName()[i] != null) { + for (GeneratorField field : meta.getFields()) { + int typeString = ValueMetaFactory.getIdForValueMeta(field.getType()); + if (StringUtils.isNotEmpty(field.getType())) { IValueMeta valueMeta = - ValueMetaFactory.createValueMeta(meta.getFieldName()[i], valtype); // build a + ValueMetaFactory.createValueMeta(field.getName(), typeString); // build a // value! - valueMeta.setLength(meta.getFieldLength()[i]); - valueMeta.setPrecision(meta.getFieldPrecision()[i]); - valueMeta.setConversionMask(meta.getFieldFormat()[i]); - valueMeta.setCurrencySymbol(meta.getCurrency()[i]); - valueMeta.setGroupingSymbol(meta.getGroup()[i]); - valueMeta.setDecimalSymbol(meta.getDecimal()[i]); + valueMeta.setLength(field.getLength()); + valueMeta.setPrecision(field.getPrecision()); + valueMeta.setConversionMask(field.getFormat()); + valueMeta.setCurrencySymbol(field.getCurrency()); + valueMeta.setGroupingSymbol(field.getGroup()); + valueMeta.setDecimalSymbol(field.getDecimal()); valueMeta.setOrigin(origin); IValueMeta stringMeta = ValueMetaFactory.cloneValueMeta(valueMeta, IValueMeta.TYPE_STRING); - if (meta.isSetEmptyString() != null && meta.isSetEmptyString()[i]) { + if (field.isSetEmptyString()) { // Set empty string rowData[index] = StringUtil.EMPTY_STRING; } else { - String stringValue = meta.getValue()[i]; + String stringValue = field.getValue(); // If the value is empty: consider it to be NULL. if (Utils.isEmpty(stringValue)) { @@ -273,9 +274,9 @@ public boolean init() { if (super.init()) { // Determine the number of rows to generate... - data.rowLimit = Const.toLong( resolve(meta.getRowLimit()), -1L); + data.rowLimit = Const.toLong(resolve(meta.getRowLimit()), -1L); data.rowsWritten = 0L; - data.delay = Const.toLong( resolve(meta.getIntervalInMs()), -1L); + data.delay = Const.toLong(resolve(meta.getIntervalInMs()), -1L); if (data.rowLimit < 0L) { // Unable to parse logError(BaseMessages.getString(PKG, "RowGenerator.Wrong.RowLimit.Number")); diff --git a/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorDialog.java b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorDialog.java index 80693a6102..a6d7380854 100644 --- a/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorDialog.java +++ b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorDialog.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -77,7 +77,7 @@ public class RowGeneratorDialog extends BaseTransformDialog implements ITransfor private final RowGeneratorMeta input; public RowGeneratorDialog( - Shell parent, IVariables variables, Object in, PipelineMeta pipelineMeta, String sname) { + Shell parent, IVariables variables, Object in, PipelineMeta pipelineMeta, String sname) { super(parent, variables, (BaseTransformMeta) in, pipelineMeta, sname); input = (RowGeneratorMeta) in; } @@ -241,7 +241,7 @@ public void widgetSelected(SelectionEvent e) { wlFields.setLayoutData(fdlFields); lastControl = wlFields; - final int FieldsRows = input.getFieldName().length; + final int nrFields = input.getFields().size(); ColumnInfo[] colinf = new ColumnInfo[] { @@ -296,7 +296,7 @@ public void widgetSelected(SelectionEvent e) { shell, SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI, colinf, - FieldsRows, + nrFields, lsMod, props); @@ -377,36 +377,35 @@ public void getData() { wRowTimeField.setText(Const.NVL(input.getRowTimeField(), "")); wLastTimeField.setText(Const.NVL(input.getLastTimeField(), "")); - for (int i = 0; i < input.getFieldName().length; i++) { - if (input.getFieldName()[i] != null) { - TableItem item = wFields.table.getItem(i); - int col = 1; - item.setText(col++, input.getFieldName()[i]); - - String type = input.getFieldType()[i]; - String format = input.getFieldFormat()[i]; - String length = input.getFieldLength()[i] < 0 ? "" : ("" + input.getFieldLength()[i]); - String prec = input.getFieldPrecision()[i] < 0 ? "" : ("" + input.getFieldPrecision()[i]); - - String curr = input.getCurrency()[i]; - String group = input.getGroup()[i]; - String decim = input.getDecimal()[i]; - String def = input.getValue()[i]; - - item.setText(col++, Const.NVL(type, "")); - item.setText(col++, Const.NVL(format, "")); - item.setText(col++, Const.NVL(length, "")); - item.setText(col++, Const.NVL(prec, "")); - item.setText(col++, Const.NVL(curr, "")); - item.setText(col++, Const.NVL(decim, "")); - item.setText(col++, Const.NVL(group, "")); - item.setText(col++, Const.NVL(def, "")); - item.setText( - col++, - input.isSetEmptyString()[i] - ? BaseMessages.getString(PKG, "System.Combo.Yes") - : BaseMessages.getString(PKG, "System.Combo.No")); - } + for (int i = 0; i < input.getFields().size(); i++) { + GeneratorField field = input.getFields().get(i); + TableItem item = wFields.table.getItem(i); + int col = 1; + item.setText(col++, Const.NVL(field.getName(), "")); + + String type = field.getType(); + String format = field.getFormat(); + String length = field.getLength() < 0 ? "" : ("" + field.getLength()); + String prec = field.getPrecision() < 0 ? "" : ("" + field.getPrecision()); + + String curr = field.getCurrency(); + String group = field.getGroup(); + String decim = field.getDecimal(); + String def = field.getValue(); + + item.setText(col++, Const.NVL(type, "")); + item.setText(col++, Const.NVL(format, "")); + item.setText(col++, Const.NVL(length, "")); + item.setText(col++, Const.NVL(prec, "")); + item.setText(col++, Const.NVL(curr, "")); + item.setText(col++, Const.NVL(decim, "")); + item.setText(col++, Const.NVL(group, "")); + item.setText(col++, Const.NVL(def, "")); + item.setText( + col++, + field.isSetEmptyString() + ? BaseMessages.getString(PKG, "System.Combo.Yes") + : BaseMessages.getString(PKG, "System.Combo.No")); } wFields.setRowNums(); @@ -450,29 +449,22 @@ private void getInfo(RowGeneratorMeta meta) throws HopException { meta.setRowTimeField(wRowTimeField.getText()); meta.setLastTimeField(wLastTimeField.getText()); - int nrFields = wFields.nrNonEmpty(); - - meta.allocate(nrFields); + meta.getFields().clear(); // CHECKSTYLE:Indentation:OFF - for (int i = 0; i < nrFields; i++) { - TableItem item = wFields.getNonEmpty(i); - - meta.getFieldName()[i] = item.getText(1); - - meta.getFieldFormat()[i] = item.getText(3); - String slength = item.getText(4); - String sprec = item.getText(5); - meta.getCurrency()[i] = item.getText(6); - meta.getDecimal()[i] = item.getText(7); - meta.getGroup()[i] = item.getText(8); - meta.isSetEmptyString()[i] = - BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(item.getText(10)); - - meta.getValue()[i] = meta.isSetEmptyString()[i] ? "" : item.getText(9); - meta.getFieldType()[i] = meta.isSetEmptyString()[i] ? "String" : item.getText(2); - meta.getFieldLength()[i] = Const.toInt(slength, -1); - meta.getFieldPrecision()[i] = Const.toInt(sprec, -1); + for (TableItem item : wFields.getNonEmptyItems()) { + GeneratorField field = new GeneratorField(); + field.setName(item.getText(1)); + field.setFormat(item.getText(3)); + field.setLength(Const.toInt(item.getText(4), -1)); + field.setPrecision(Const.toInt(item.getText(5), -1)); + field.setCurrency(item.getText(6)); + field.setDecimal(item.getText(7)); + field.setGroup(item.getText(8)); + field.setValue(field.isSetEmptyString() ? "" : item.getText(9)); + field.setSetEmptyString( + BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(item.getText(10))); + field.setType(field.isSetEmptyString() ? "String" : item.getText(2)); } // Performs checks... @@ -503,7 +495,7 @@ private void preview() { PipelineMeta previewMeta = PipelinePreviewFactory.generatePreviewPipeline( - pipelineMeta.getMetadataProvider(), oneMeta, wTransformName.getText()); + pipelineMeta.getMetadataProvider(), oneMeta, wTransformName.getText()); EnterNumberDialog numberDialog = new EnterNumberDialog( @@ -515,7 +507,11 @@ private void preview() { if (previewSize > 0) { PipelinePreviewProgressDialog progressDialog = new PipelinePreviewProgressDialog( - shell, variables, previewMeta, new String[] {wTransformName.getText()}, new int[] {previewSize}); + shell, + variables, + previewMeta, + new String[] {wTransformName.getText()}, + new int[] {previewSize}); progressDialog.open(); Pipeline pipeline = progressDialog.getPipeline(); diff --git a/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorMeta.java b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorMeta.java index a813709bb0..157f8821c6 100644 --- a/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorMeta.java +++ b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorMeta.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,13 +23,11 @@ import org.apache.hop.core.RowMetaAndData; import org.apache.hop.core.annotations.Transform; import org.apache.hop.core.exception.HopTransformException; -import org.apache.hop.core.exception.HopXmlException; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.row.IValueMeta; -import org.apache.hop.core.util.Utils; import org.apache.hop.core.variables.IVariables; -import org.apache.hop.core.xml.XmlHandler; import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; import org.apache.hop.metadata.api.IHopMetadataProvider; import org.apache.hop.pipeline.Pipeline; import org.apache.hop.pipeline.PipelineMeta; @@ -38,9 +36,7 @@ import org.apache.hop.pipeline.transform.ITransformMeta; import org.apache.hop.pipeline.transform.TransformIOMeta; import org.apache.hop.pipeline.transform.TransformMeta; -import org.w3c.dom.Node; -import java.text.DecimalFormat; import java.util.ArrayList; import java.util.List; @@ -58,141 +54,40 @@ public class RowGeneratorMeta extends BaseTransformMeta implements ITransformMeta { private static final Class PKG = RowGeneratorMeta.class; // For Translator + @HopMetadataProperty( + key = "never_ending", + injectionKeyDescription = "RowGeneratorMeta.Injection.NeverEnding") private boolean neverEnding; + @HopMetadataProperty( + key = "interval_in_ms", + injectionKeyDescription = "RowGeneratorMeta.Injection.IntervalInMs") private String intervalInMs; + @HopMetadataProperty( + key = "row_time_field", + injectionKeyDescription = "RowGeneratorMeta.Injection.RowTimeField") private String rowTimeField; + @HopMetadataProperty( + key = "last_time_field", + injectionKeyDescription = "RowGeneratorMeta.Injection.LastTimeField") private String lastTimeField; + @HopMetadataProperty( + key = "limit", + injectionKeyDescription = "RowGeneratorMeta.Injection.RowLimit") private String rowLimit; - private String[] currency; - - private String[] decimal; - - private String[] group; - - private String[] value; - - private String[] fieldName; - - private String[] fieldType; - - private String[] fieldFormat; - - private int[] fieldLength; - - private int[] fieldPrecision; - - /** Flag : set empty string */ - private boolean[] setEmptyString; + @HopMetadataProperty( + groupKey = "fields", + key = "field", + injectionGroupDescription = "RowGeneratorMeta.Injection.Fields", + injectionKeyDescription = "RowGeneratorMeta.Injection.Field") + private List fields; public RowGeneratorMeta() { - super(); // allocate BaseTransformMeta - } - - public void loadXml(Node transformNode, IHopMetadataProvider metadataProvider) - throws HopXmlException { - readData(transformNode); - } - - public void allocate(int nrFields) { - fieldName = new String[nrFields]; - fieldType = new String[nrFields]; - fieldFormat = new String[nrFields]; - fieldLength = new int[nrFields]; - fieldPrecision = new int[nrFields]; - currency = new String[nrFields]; - decimal = new String[nrFields]; - group = new String[nrFields]; - value = new String[nrFields]; - setEmptyString = new boolean[nrFields]; - } - - public Object clone() { - RowGeneratorMeta retval = (RowGeneratorMeta) super.clone(); - - int nrFields = fieldName.length; - - retval.allocate(nrFields); - System.arraycopy(fieldName, 0, retval.fieldName, 0, nrFields); - System.arraycopy(fieldType, 0, retval.fieldType, 0, nrFields); - System.arraycopy(fieldFormat, 0, retval.fieldFormat, 0, nrFields); - System.arraycopy(fieldLength, 0, retval.fieldLength, 0, nrFields); - System.arraycopy(fieldPrecision, 0, retval.fieldPrecision, 0, nrFields); - System.arraycopy(currency, 0, retval.currency, 0, nrFields); - System.arraycopy(decimal, 0, retval.decimal, 0, nrFields); - System.arraycopy(group, 0, retval.group, 0, nrFields); - System.arraycopy(value, 0, retval.value, 0, nrFields); - System.arraycopy(setEmptyString, 0, retval.setEmptyString, 0, nrFields); - - return retval; - } - - private void readData(Node transformNode) throws HopXmlException { - try { - Node fields = XmlHandler.getSubNode(transformNode, "fields"); - int nrFields = XmlHandler.countNodes(fields, "field"); - - allocate(nrFields); - - String slength, sprecision; - - for (int i = 0; i < nrFields; i++) { - Node fnode = XmlHandler.getSubNodeByNr(fields, "field", i); - - fieldName[i] = XmlHandler.getTagValue(fnode, "name"); - fieldType[i] = XmlHandler.getTagValue(fnode, "type"); - fieldFormat[i] = XmlHandler.getTagValue(fnode, "format"); - currency[i] = XmlHandler.getTagValue(fnode, "currency"); - decimal[i] = XmlHandler.getTagValue(fnode, "decimal"); - group[i] = XmlHandler.getTagValue(fnode, "group"); - value[i] = XmlHandler.getTagValue(fnode, "nullif"); - slength = XmlHandler.getTagValue(fnode, "length"); - sprecision = XmlHandler.getTagValue(fnode, "precision"); - - fieldLength[i] = Const.toInt(slength, -1); - fieldPrecision[i] = Const.toInt(sprecision, -1); - String emptyString = XmlHandler.getTagValue(fnode, "set_empty_string"); - setEmptyString[i] = !Utils.isEmpty(emptyString) && "Y".equalsIgnoreCase(emptyString); - } - - // Is there a limit on the number of rows we process? - rowLimit = XmlHandler.getTagValue(transformNode, "limit"); - - neverEnding = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "never_ending")); - intervalInMs = XmlHandler.getTagValue(transformNode, "interval_in_ms"); - rowTimeField = XmlHandler.getTagValue(transformNode, "row_time_field"); - lastTimeField = XmlHandler.getTagValue(transformNode, "last_time_field"); - - } catch (Exception e) { - throw new HopXmlException("Unable to load transform info from XML", e); - } - } - - public void setDefault() { - int i, nrFields = 0; - - allocate(nrFields); - - DecimalFormat decimalFormat = new DecimalFormat(); - - for (i = 0; i < nrFields; i++) { - fieldName[i] = "field" + i; - fieldType[i] = "Number"; - fieldFormat[i] = "\u00A40,000,000.00;\u00A4-0,000,000.00"; - fieldLength[i] = 9; - fieldPrecision[i] = 2; - currency[i] = decimalFormat.getDecimalFormatSymbols().getCurrencySymbol(); - decimal[i] = - new String(new char[] {decimalFormat.getDecimalFormatSymbols().getDecimalSeparator()}); - group[i] = - new String(new char[] {decimalFormat.getDecimalFormatSymbols().getGroupingSeparator()}); - value[i] = "-"; - setEmptyString[i] = false; - } + fields = new ArrayList<>(); rowLimit = "10"; neverEnding = false; @@ -201,6 +96,23 @@ public void setDefault() { lastTimeField = "FiveSecondsAgo"; } + public RowGeneratorMeta(RowGeneratorMeta m) { + this.neverEnding = m.neverEnding; + this.intervalInMs = m.intervalInMs; + this.rowTimeField = m.rowTimeField; + this.lastTimeField = m.lastTimeField; + this.rowLimit = m.rowLimit; + this.fields = new ArrayList<>(); + for (GeneratorField field : m.fields) { + this.fields.add(new GeneratorField(field)); + } + } + + public RowGeneratorMeta clone() { + return new RowGeneratorMeta(this); + } + + @Override public void getFields( IRowMeta row, String origin, @@ -230,38 +142,6 @@ public void getFields( } } - public String getXml() { - StringBuilder retval = new StringBuilder(300); - - retval.append(" ").append(Const.CR); - for (int i = 0; i < fieldName.length; i++) { - if (fieldName[i] != null && fieldName[i].length() != 0) { - retval.append(" ").append(Const.CR); - retval.append(" ").append(XmlHandler.addTagValue("name", fieldName[i])); - retval.append(" ").append(XmlHandler.addTagValue("type", fieldType[i])); - retval.append(" ").append(XmlHandler.addTagValue("format", fieldFormat[i])); - retval.append(" ").append(XmlHandler.addTagValue("currency", currency[i])); - retval.append(" ").append(XmlHandler.addTagValue("decimal", decimal[i])); - retval.append(" ").append(XmlHandler.addTagValue("group", group[i])); - retval.append(" ").append(XmlHandler.addTagValue("nullif", value[i])); - retval.append(" ").append(XmlHandler.addTagValue("length", fieldLength[i])); - retval.append(" ").append(XmlHandler.addTagValue("precision", fieldPrecision[i])); - retval - .append(" ") - .append(XmlHandler.addTagValue("set_empty_string", setEmptyString[i])); - retval.append(" ").append(Const.CR); - } - } - retval.append(" ").append(Const.CR); - retval.append(" ").append(XmlHandler.addTagValue("limit", rowLimit)); - retval.append(" ").append(XmlHandler.addTagValue("never_ending", neverEnding ? "Y" : "N")); - retval.append(" ").append(XmlHandler.addTagValue("interval_in_ms", intervalInMs)); - retval.append(" ").append(XmlHandler.addTagValue("row_time_field", rowTimeField)); - retval.append(" ").append(XmlHandler.addTagValue("last_time_field", lastTimeField)); - - return retval.toString(); - } - public void check( List remarks, PipelineMeta pipelineMeta, @@ -346,153 +226,87 @@ public ITransformIOMeta getTransformIOMeta() { return new TransformIOMeta(false, true, false, false, false, false); } + /** + * Gets neverEnding + * + * @return value of neverEnding + */ public boolean isNeverEnding() { return neverEnding; } + /** @param neverEnding The neverEnding to set */ public void setNeverEnding(boolean neverEnding) { this.neverEnding = neverEnding; } + /** + * Gets intervalInMs + * + * @return value of intervalInMs + */ public String getIntervalInMs() { return intervalInMs; } + /** @param intervalInMs The intervalInMs to set */ public void setIntervalInMs(String intervalInMs) { this.intervalInMs = intervalInMs; } + /** + * Gets rowTimeField + * + * @return value of rowTimeField + */ public String getRowTimeField() { return rowTimeField; } + /** @param rowTimeField The rowTimeField to set */ public void setRowTimeField(String rowTimeField) { this.rowTimeField = rowTimeField; } + /** + * Gets lastTimeField + * + * @return value of lastTimeField + */ public String getLastTimeField() { return lastTimeField; } + /** @param lastTimeField The lastTimeField to set */ public void setLastTimeField(String lastTimeField) { this.lastTimeField = lastTimeField; } - public boolean[] getSetEmptyString() { - return setEmptyString; - } - - public void setSetEmptyString(boolean[] setEmptyString) { - this.setEmptyString = setEmptyString; - } - - /** @return Returns the currency. */ - public String[] getCurrency() { - return currency; - } - - /** @param currency The currency to set. */ - public void setCurrency(String[] currency) { - this.currency = currency; - } - - /** @return Returns the decimal. */ - public String[] getDecimal() { - return decimal; - } - - /** @param decimal The decimal to set. */ - public void setDecimal(String[] decimal) { - this.decimal = decimal; - } - - /** @return Returns the fieldFormat. */ - public String[] getFieldFormat() { - return fieldFormat; - } - - /** @param fieldFormat The fieldFormat to set. */ - public void setFieldFormat(String[] fieldFormat) { - this.fieldFormat = fieldFormat; - } - - /** @return Returns the fieldLength. */ - public int[] getFieldLength() { - return fieldLength; - } - - /** @param fieldLength The fieldLength to set. */ - public void setFieldLength(int[] fieldLength) { - this.fieldLength = fieldLength; - } - - /** @return Returns the fieldName. */ - public String[] getFieldName() { - return fieldName; - } - - /** @param fieldName The fieldName to set. */ - public void setFieldName(String[] fieldName) { - this.fieldName = fieldName; - } - - /** @return Returns the fieldPrecision. */ - public int[] getFieldPrecision() { - return fieldPrecision; - } - - /** @param fieldPrecision The fieldPrecision to set. */ - public void setFieldPrecision(int[] fieldPrecision) { - this.fieldPrecision = fieldPrecision; - } - - /** @return Returns the fieldType. */ - public String[] getFieldType() { - return fieldType; - } - - /** @param fieldType The fieldType to set. */ - public void setFieldType(String[] fieldType) { - this.fieldType = fieldType; - } - - /** @return Returns the group. */ - public String[] getGroup() { - return group; - } - - /** @param group The group to set. */ - public void setGroup(String[] group) { - this.group = group; - } - - /** @return the setEmptyString */ - public boolean[] isSetEmptyString() { - return setEmptyString; - } - - /** @param setEmptyString the setEmptyString to set */ - public void setEmptyString(boolean[] setEmptyString) { - this.setEmptyString = setEmptyString; - } - - /** @return Returns the rowLimit. */ + /** + * Gets rowLimit + * + * @return value of rowLimit + */ public String getRowLimit() { return rowLimit; } - /** @param rowLimit The rowLimit to set. */ + /** @param rowLimit The rowLimit to set */ public void setRowLimit(String rowLimit) { this.rowLimit = rowLimit; } - /** @return Returns the value. */ - public String[] getValue() { - return value; + /** + * Gets fields + * + * @return value of fields + */ + public List getFields() { + return fields; } - /** @param value The value to set. */ - public void setValue(String[] value) { - this.value = value; + /** @param fields The fields to set */ + public void setFields(List fields) { + this.fields = fields; } } diff --git a/plugins/transforms/rowgenerator/src/main/resources/org/apache/hop/pipeline/transforms/rowgenerator/messages/messages_en_US.properties b/plugins/transforms/rowgenerator/src/main/resources/org/apache/hop/pipeline/transforms/rowgenerator/messages/messages_en_US.properties index 5a7014e9ce..0c03974eea 100644 --- a/plugins/transforms/rowgenerator/src/main/resources/org/apache/hop/pipeline/transforms/rowgenerator/messages/messages_en_US.properties +++ b/plugins/transforms/rowgenerator/src/main/resources/org/apache/hop/pipeline/transforms/rowgenerator/messages/messages_en_US.properties @@ -1,5 +1,4 @@ # -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -15,34 +14,51 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# -# - BaseTransform.TypeLongDesc.GenerateRows=Generate rows BaseTransform.TypeTooltipDesc.GenerateRows=Generate a number of empty or equal rows. -RowGeneratorDialog.DialogTitle = Generate rows -RowGeneratorDialog.Limit.Label = Limit -RowGeneratorDialog.Fields.Label = Fields : -RowGeneratorDialog.Illegal.Dialog.Settings.Title = Error -RowGeneratorDialog.Illegal.Dialog.Settings.Message = There is a problem with one or more settings in this dialog: -RowGeneratorDialog.Wrong.RowLimit.Number = Unable to parse the number of rows to generate (or set to negative number) -RowGeneratorDialog.NeverEnding.Label = Never stop generating rows -RowGeneratorDialog.Interval.Label = Interval in ms (delay) -RowGeneratorDialog.RowTimeField.Label = Current row time field name -RowGeneratorDialog.LastTimeField.Label = Previous row time field name -RowGeneratorMeta.CheckResult.NoInputStreamsError = This transform type can''t read from the input stream(s) -RowGeneratorMeta.CheckResult.NoInputStreamOk = transform doesn''t read from the input stream(s) -RowGeneratorMeta.CheckResult.WarnNoRows = transform will not return any rows. -RowGeneratorMeta.CheckResult.WillReturnRows = transform will return {0} rows -RowGeneratorMeta.CheckResult.NoInputError = This transform is not expecting nor reading any input -RowGeneratorMeta.CheckResult.NoInputOk = Not receiving any input from other transforms. -RowGenerator.CheckResult.SpecifyTypeError = Please specify the value type of field [{0}] with value [{1}] -RowGenerator.Wrong.RowLimit.Number = Unable to parse the number of rows to generate (or set to negative number) -RowGenerator.Log.Wrote.Row = Wrote row {0} : {1} -RowGenerator.Log.LineNr = LineNr : {0} -RowGenerator.BuildRow.Error.Parsing.Number = Couldn''t parse Number field [{0}] with value [{1}] --> {2} -RowGenerator.BuildRow.Error.Parsing.Date = Couldn''t parse Date field [{0}] with value [{1}] --> {2} -RowGenerator.BuildRow.Error.Parsing.Integer = Couldn''t parse Integer field [{0}] with value [{1}] --> {2} -RowGenerator.BuildRow.Error.Parsing.BigNumber= Couldn''t parse BigNumber field [{0}] with value [{1}] --> {2} -RowGenerator.BuildRow.Error.Parsing.Timestamp = Couldn''t parse Timestamp field [{0}] with value [{1}] --> {2} -System.Column.SetEmptyString=Set empty string? \ No newline at end of file +RowGeneratorDialog.DialogTitle=Generate rows +RowGeneratorDialog.Limit.Label=Limit +RowGeneratorDialog.Fields.Label=Fields : +RowGeneratorDialog.Illegal.Dialog.Settings.Title=Error +RowGeneratorDialog.Illegal.Dialog.Settings.Message=There is a problem with one or more settings in this dialog: +RowGeneratorDialog.Wrong.RowLimit.Number=Unable to parse the number of rows to generate (or set to negative number) +RowGeneratorDialog.NeverEnding.Label=Never stop generating rows +RowGeneratorDialog.Interval.Label=Interval in ms (delay) +RowGeneratorDialog.RowTimeField.Label=Current row time field name +RowGeneratorDialog.LastTimeField.Label=Previous row time field name +RowGeneratorMeta.CheckResult.NoInputStreamsError=This transform type can''t read from the input stream(s) +RowGeneratorMeta.CheckResult.NoInputStreamOk=transform doesn''t read from the input stream(s) +RowGeneratorMeta.CheckResult.WarnNoRows=transform will not return any rows. +RowGeneratorMeta.CheckResult.WillReturnRows=transform will return {0} rows +RowGeneratorMeta.CheckResult.NoInputError=This transform is not expecting nor reading any input +RowGeneratorMeta.CheckResult.NoInputOk=Not receiving any input from other transforms. +RowGenerator.CheckResult.SpecifyTypeError=Please specify the value type of field [{0}] with value [{1}] +RowGenerator.Wrong.RowLimit.Number=Unable to parse the number of rows to generate (or set to negative number) +RowGenerator.Log.Wrote.Row=Wrote row {0} : {1} +RowGenerator.Log.LineNr=LineNr : {0} +RowGenerator.BuildRow.Error.Parsing.Number=Couldn''t parse Number field [{0}] with value [{1}] --> {2} +RowGenerator.BuildRow.Error.Parsing.Date=Couldn''t parse Date field [{0}] with value [{1}] --> {2} +RowGenerator.BuildRow.Error.Parsing.Integer=Couldn''t parse Integer field [{0}] with value [{1}] --> {2} +RowGenerator.BuildRow.Error.Parsing.BigNumber=Couldn''t parse BigNumber field [{0}] with value [{1}] --> {2} +RowGenerator.BuildRow.Error.Parsing.Timestamp=Couldn''t parse Timestamp field [{0}] with value [{1}] --> {2} +System.Column.SetEmptyString=Set empty string? +RowGeneratorMeta.Injection.NeverEnding=Never ending? +RowGeneratorMeta.Injection.IntervalInMs=Interval in ms +RowGeneratorMeta.Injection.RowTimeField=Row time field +RowGeneratorMeta.Injection.LastTimeField=Last time field +RowGeneratorMeta.Injection.RowLimit=Row limit +RowGeneratorMeta.Injection.Fields=Fields to generate +RowGeneratorMeta.Injection.Field=Field to generate +RowGeneratorMeta.Injection.Field.Name=Name +RowGeneratorMeta.Injection.Field.Type=Type +RowGeneratorMeta.Injection.Field.Length=Length +RowGeneratorMeta.Injection.Field.Precision=Precision +RowGeneratorMeta.Injection.Field.Format=Format +RowGeneratorMeta.Injection.Field.Decimal=Decimal symbol +RowGeneratorMeta.Injection.Field.Group=Grouping symbol +RowGeneratorMeta.Injection.Field.Currency=Currency symbol +RowGeneratorMeta.Injection.Field.Value=Value +RowGeneratorMeta.Injection.Field.SetEmptyString=Set empty string? + + + diff --git a/plugins/transforms/rowgenerator/src/test/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorMetaTest.java b/plugins/transforms/rowgenerator/src/test/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorMetaTest.java index eff7db0a31..a2bb273846 100644 --- a/plugins/transforms/rowgenerator/src/test/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorMetaTest.java +++ b/plugins/transforms/rowgenerator/src/test/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorMetaTest.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,9 +17,11 @@ package org.apache.hop.pipeline.transforms.rowgenerator; +import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.hop.core.HopEnvironment; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.row.value.ValueMetaFactory; import org.apache.hop.junit.rules.RestoreHopEngineEnvironment; import org.apache.hop.pipeline.transform.ITransformMeta; import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester; @@ -27,6 +29,7 @@ import org.apache.hop.pipeline.transforms.loadsave.validator.ArrayLoadSaveValidator; import org.apache.hop.pipeline.transforms.loadsave.validator.IFieldLoadSaveValidator; import org.apache.hop.pipeline.transforms.loadsave.validator.IntLoadSaveValidator; +import org.apache.hop.pipeline.transforms.loadsave.validator.ListLoadSaveValidator; import org.apache.hop.pipeline.transforms.loadsave.validator.PrimitiveIntArrayLoadSaveValidator; import org.apache.hop.pipeline.transforms.loadsave.validator.StringLoadSaveValidator; import org.junit.Before; @@ -38,6 +41,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; +import java.util.UUID; public class RowGeneratorMetaTest implements IInitializer { @ClassRule public static RestoreHopEngineEnvironment env = new RestoreHopEngineEnvironment(); @@ -49,79 +54,69 @@ public class RowGeneratorMetaTest implements IInitializer { private Class testMetaClass = RowGeneratorMeta.class; @Before - public void setUp() throws HopException { - } + public void setUp() throws HopException {} @Before public void setUpLoadSave() throws Exception { HopEnvironment.init(); - PluginRegistry.init( false ); + PluginRegistry.init(false); List attributes = - Arrays.asList( "neverEnding", "intervalInMs", "rowTimeField", "lastTimeField", "rowLimit", "currency", "decimal", "group", - "value", "fieldName", "fieldType", "fieldFormat", "fieldLength", "fieldPrecision" ); - - Map getterMap = new HashMap() { - { - put( "neverEnding", "isNeverEnding" ); - put( "intervalInMs", "getIntervalInMs" ); - put( "rowTimeField", "getRowTimeField" ); - put( "lastTimeField", "getLastTimeField" ); - put( "rowLimit", "getRowLimit" ); - put( "currency", "getCurrency" ); - put( "decimal", "getDecimal" ); - put( "group", "getGroup" ); - put( "value", "getValue" ); - put( "fieldName", "getFieldName" ); - put( "fieldType", "getFieldType" ); - put( "fieldFormat", "getFieldFormat" ); - put( "fieldLength", "getFieldLength" ); - put( "fieldPrecision", "getFieldPrecision" ); - } - }; - Map setterMap = new HashMap() { - { - put( "neverEnding", "setNeverEnding" ); - put( "intervalInMs", "setIntervalInMs" ); - put( "rowTimeField", "setRowTimeField" ); - put( "lastTimeField", "setLastTimeField" ); - put( "rowLimit", "setRowLimit" ); - put( "currency", "setCurrency" ); - put( "decimal", "setDecimal" ); - put( "group", "setGroup" ); - put( "value", "setValue" ); - put( "fieldName", "setFieldName" ); - put( "fieldType", "setFieldType" ); - put( "fieldFormat", "setFieldFormat" ); - put( "fieldLength", "setFieldLength" ); - put( "fieldPrecision", "setFieldPrecision" ); - } - }; - IFieldLoadSaveValidator stringArrayLoadSaveValidator = - new ArrayLoadSaveValidator<>( new StringLoadSaveValidator(), 5 ); + Arrays.asList( + "neverEnding", "intervalInMs", "rowTimeField", "lastTimeField", "rowLimit", "fields"); - Map> attrValidatorMap = new HashMap<>(); - attrValidatorMap.put( "currency", stringArrayLoadSaveValidator ); - attrValidatorMap.put( "decimal", stringArrayLoadSaveValidator ); - attrValidatorMap.put( "group", stringArrayLoadSaveValidator ); - attrValidatorMap.put( "value", stringArrayLoadSaveValidator ); - attrValidatorMap.put( "fieldName", stringArrayLoadSaveValidator ); - attrValidatorMap.put( "fieldType", stringArrayLoadSaveValidator ); - attrValidatorMap.put( "fieldFormat", stringArrayLoadSaveValidator ); - attrValidatorMap.put( "fieldLength", new PrimitiveIntArrayLoadSaveValidator( new IntLoadSaveValidator( 100 ), 5 ) ); - attrValidatorMap.put( "fieldPrecision", new PrimitiveIntArrayLoadSaveValidator( new IntLoadSaveValidator( 9 ), 5 ) ); + Map getterMap = + new HashMap() { + { + put("neverEnding", "isNeverEnding"); + put("intervalInMs", "getIntervalInMs"); + put("rowTimeField", "getRowTimeField"); + put("lastTimeField", "getLastTimeField"); + put("rowLimit", "getRowLimit"); + } + }; + Map setterMap = + new HashMap() { + { + put("neverEnding", "setNeverEnding"); + put("intervalInMs", "setIntervalInMs"); + put("rowTimeField", "setRowTimeField"); + put("lastTimeField", "setLastTimeField"); + put("rowLimit", "setRowLimit"); + } + }; + Map> attrValidatorMap = new HashMap<>(); + attrValidatorMap.put( + "fields", new ListLoadSaveValidator<>(new GeneratorFieldInputFieldLoadSaveValidator(), 5)); Map> typeValidatorMap = new HashMap<>(); loadSaveTester = - new LoadSaveTester( testMetaClass, attributes, new ArrayList<>(), - getterMap, setterMap, attrValidatorMap, typeValidatorMap, this ); + new LoadSaveTester( + testMetaClass, + attributes, + new ArrayList<>(), + getterMap, + setterMap, + attrValidatorMap, + typeValidatorMap, + this); } // Call the allocate method on the LoadSaveTester meta class - public void modify( ITransformMeta someMeta ) { - if ( someMeta instanceof RowGeneratorMeta ) { - ( (RowGeneratorMeta) someMeta ).allocate( 5 ); + public void modify(ITransformMeta someMeta) { + if (someMeta instanceof RowGeneratorMeta) { + ((RowGeneratorMeta) someMeta).getFields().clear(); + ((RowGeneratorMeta) someMeta) + .getFields() + .addAll( + Arrays.asList( + new GeneratorField("a", "String", null, 50, -1, null, null, null, "AAAA", false), + new GeneratorField("b", "String", null, 50, -1, null, null, null, "BBBB", false), + new GeneratorField("c", "String", null, 50, -1, null, null, null, "CCCC", false), + new GeneratorField("d", "String", null, 50, -1, null, null, null, "DDDD", false), + new GeneratorField( + "e", "String", null, 50, -1, null, null, null, "EEEE", false))); } } @@ -129,4 +124,48 @@ public void modify( ITransformMeta someMeta ) { public void testSerialization() throws HopException { loadSaveTester.testSerialization(); } + + public class GeneratorFieldInputFieldLoadSaveValidator + implements IFieldLoadSaveValidator { + final Random rand = new Random(); + + @Override + public GeneratorField getTestObject() { + String[] types = ValueMetaFactory.getAllValueMetaNames(); + + GeneratorField field = + new GeneratorField( + UUID.randomUUID().toString(), + types[Math.abs(rand.nextInt(types.length))], + UUID.randomUUID().toString(), + rand.nextInt(20), + rand.nextInt(20), + UUID.randomUUID().toString(), + UUID.randomUUID().toString(), + UUID.randomUUID().toString(), + UUID.randomUUID().toString(), + rand.nextInt(20) < 0); + + return field; + } + + @Override + public boolean validateTestObject(GeneratorField testObject, Object actual) { + if (!(actual instanceof GeneratorField)) { + return false; + } + GeneratorField another = (GeneratorField) actual; + return new EqualsBuilder() + .append(testObject.getName(), another.getName()) + .append(testObject.getType(), another.getType()) + .append(testObject.getFormat(), another.getFormat()) + .append(testObject.getLength(), another.getLength()) + .append(testObject.getPrecision(), another.getPrecision()) + .append(testObject.getDecimal(), another.getDecimal()) + .append(testObject.getGroup(), another.getGroup()) + .append(testObject.getValue(), another.getValue()) + .append(testObject.isSetEmptyString(), another.isSetEmptyString()) + .isEquals(); + } + } } diff --git a/plugins/transforms/rowgenerator/src/test/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorUnitTest.java b/plugins/transforms/rowgenerator/src/test/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorUnitTest.java index a826e35073..ee4b7f0c2c 100644 --- a/plugins/transforms/rowgenerator/src/test/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorUnitTest.java +++ b/plugins/transforms/rowgenerator/src/test/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorUnitTest.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -30,7 +30,9 @@ import org.junit.ClassRule; import org.junit.Test; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -51,40 +53,38 @@ public static void initEnvironment() throws Exception { @Before public void setUp() throws HopException { // add variable to row generator transform - ITransformMeta transformMetaInterface = spy( new RowGeneratorMeta() ); + ITransformMeta transformMetaInterface = spy(new RowGeneratorMeta()); RowGeneratorMeta meta = (RowGeneratorMeta) transformMetaInterface; - meta.setRowLimit( "${ROW_LIMIT}" ); - String[] strings = {}; - when( meta.getFieldName() ).thenReturn( strings ); + meta.setRowLimit("${ROW_LIMIT}"); + List fields = new ArrayList<>(); + when(meta.getFields()).thenReturn(fields); TransformMeta transformMeta = new TransformMeta(); - transformMeta.setTransform( transformMetaInterface ); - transformMeta.setName( "ROW_TRANSFORM_META" ); + transformMeta.setTransform(transformMetaInterface); + transformMeta.setName("ROW_TRANSFORM_META"); RowGeneratorData data = (RowGeneratorData) transformMeta.getTransform().getTransformData(); + PipelineMeta pipelineMeta = spy(new PipelineMeta()); + when(pipelineMeta.findTransform(anyString())).thenReturn(transformMeta); - PipelineMeta pipelineMeta = spy( new PipelineMeta() ); - when( pipelineMeta.findTransform( anyString() ) ).thenReturn( transformMeta ); - - Pipeline pipeline = spy( new LocalPipelineEngine( pipelineMeta ) ); + Pipeline pipeline = spy(new LocalPipelineEngine(pipelineMeta)); // add variable to pipeline variable variables Map map = new HashMap<>(); - map.put( "ROW_LIMIT", "1440" ); - pipeline.setVariables( map ); + map.put("ROW_LIMIT", "1440"); + pipeline.setVariables(map); - when( pipeline.getLogChannelId() ).thenReturn( "ROW_LIMIT" ); + when(pipeline.getLogChannelId()).thenReturn("ROW_LIMIT"); - //prepare row generator, substitutes variable by value from pipeline variable variables - rowGenerator = spy( new RowGenerator( transformMeta, meta, data, 0, pipelineMeta, pipeline ) ); - rowGenerator.initializeFrom( pipeline ); + // prepare row generator, substitutes variable by value from pipeline variable variables + rowGenerator = spy(new RowGenerator(transformMeta, meta, data, 0, pipelineMeta, pipeline)); + rowGenerator.initializeFrom(pipeline); rowGenerator.init(); } @Test public void testReadRowLimitAsPipelineVar() throws HopException { long rowLimit = rowGenerator.getData().rowLimit; - assertEquals( rowLimit, 1440 ); + assertEquals(rowLimit, 1440); } - } From 1b6cc66942075940199ffa15dd9eda6ccfeb52e8 Mon Sep 17 00:00:00 2001 From: Matt Casters Date: Mon, 10 May 2021 20:53:38 +0200 Subject: [PATCH 5/6] HOP-2858 : Allow metadata injection through metadata property annotations (Test fixes) --- .../UserDefinedJavaClassDialog.java | 38 +- .../javascript/ScriptValuesMetaModDialog.java | 2079 +++++++++-------- .../transforms/jsonoutput/JsonOutputTest.java | 332 +-- 3 files changed, 1313 insertions(+), 1136 deletions(-) diff --git a/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/userdefinedjavaclass/UserDefinedJavaClassDialog.java b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/userdefinedjavaclass/UserDefinedJavaClassDialog.java index bc8d44460b..3e35235db6 100644 --- a/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/userdefinedjavaclass/UserDefinedJavaClassDialog.java +++ b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/userdefinedjavaclass/UserDefinedJavaClassDialog.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -35,6 +35,7 @@ import org.apache.hop.pipeline.transform.BaseTransformMeta; import org.apache.hop.pipeline.transform.ITransformDialog; import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.pipeline.transforms.rowgenerator.GeneratorField; import org.apache.hop.pipeline.transforms.rowgenerator.RowGeneratorMeta; import org.apache.hop.pipeline.transforms.userdefinedjavaclass.UserDefinedJavaClassCodeSnippits.Category; import org.apache.hop.pipeline.transforms.userdefinedjavaclass.UserDefinedJavaClassCodeSnippits.Snippit; @@ -1386,44 +1387,44 @@ private boolean test() { if (genMeta == null) { genMeta = new RowGeneratorMeta(); genMeta.setRowLimit("10"); - genMeta.allocate(rowMeta.size()); // CHECKSTYLE:Indentation:OFF for (int i = 0; i < rowMeta.size(); i++) { IValueMeta valueMeta = rowMeta.getValueMeta(i); if (valueMeta.isStorageBinaryString()) { valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL); } - genMeta.getFieldName()[i] = valueMeta.getName(); - genMeta.getFieldType()[i] = valueMeta.getTypeDesc(); - genMeta.getFieldLength()[i] = valueMeta.getLength(); - genMeta.getFieldPrecision()[i] = valueMeta.getPrecision(); - genMeta.getCurrency()[i] = valueMeta.getCurrencySymbol(); - genMeta.getDecimal()[i] = valueMeta.getDecimalSymbol(); - genMeta.getGroup()[i] = valueMeta.getGroupingSymbol(); + GeneratorField field = new GeneratorField(); + field.setName(valueMeta.getName()); + field.setType(valueMeta.getTypeDesc()); + field.setLength(valueMeta.getLength()); + field.setPrecision(valueMeta.getPrecision()); + field.setCurrency(valueMeta.getCurrencySymbol()); + field.setDecimal(valueMeta.getDecimalSymbol()); + field.setGroup(valueMeta.getGroupingSymbol()); String string = null; switch (valueMeta.getType()) { case IValueMeta.TYPE_DATE: - genMeta.getFieldFormat()[i] = "yyyy/MM/dd HH:mm:ss"; - valueMeta.setConversionMask(genMeta.getFieldFormat()[i]); + field.setFormat("yyyy/MM/dd HH:mm:ss"); + valueMeta.setConversionMask(field.getFormat()); string = valueMeta.getString(new Date()); break; case IValueMeta.TYPE_STRING: string = "test value test value"; break; case IValueMeta.TYPE_INTEGER: - genMeta.getFieldFormat()[i] = "#"; - valueMeta.setConversionMask(genMeta.getFieldFormat()[i]); + field.setFormat("#"); + valueMeta.setConversionMask(field.getFormat()); string = valueMeta.getString(Long.valueOf(0L)); break; case IValueMeta.TYPE_NUMBER: - genMeta.getFieldFormat()[i] = "#.#"; - valueMeta.setConversionMask(genMeta.getFieldFormat()[i]); + field.setFormat("#.#"); + valueMeta.setConversionMask(field.getFormat()); string = valueMeta.getString(Double.valueOf(0.0D)); break; case IValueMeta.TYPE_BIGNUMBER: - genMeta.getFieldFormat()[i] = "#.#"; - valueMeta.setConversionMask(genMeta.getFieldFormat()[i]); + field.setFormat("#.#"); + valueMeta.setConversionMask(field.getFormat()); string = valueMeta.getString(BigDecimal.ZERO); break; case IValueMeta.TYPE_BOOLEAN: @@ -1440,7 +1441,8 @@ private boolean test() { break; } - genMeta.getValue()[i] = string; + field.setValue(string); + genMeta.getFields().add(field); } } TransformMeta genTransform = diff --git a/plugins/transforms/javascript/src/main/java/org/apache/hop/pipeline/transforms/javascript/ScriptValuesMetaModDialog.java b/plugins/transforms/javascript/src/main/java/org/apache/hop/pipeline/transforms/javascript/ScriptValuesMetaModDialog.java index 8046328607..7b9fa82344 100644 --- a/plugins/transforms/javascript/src/main/java/org/apache/hop/pipeline/transforms/javascript/ScriptValuesMetaModDialog.java +++ b/plugins/transforms/javascript/src/main/java/org/apache/hop/pipeline/transforms/javascript/ScriptValuesMetaModDialog.java @@ -1,4 +1,3 @@ -//CHECKSTYLE:FileLength:OFF /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -7,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,9 +15,9 @@ * limitations under the License. */ +// CHECKSTYLE:FileLength:OFF package org.apache.hop.pipeline.transforms.javascript; - import org.apache.hop.core.Const; import org.apache.hop.core.Props; import org.apache.hop.core.exception.HopException; @@ -36,6 +35,7 @@ import org.apache.hop.pipeline.transform.BaseTransformMeta; import org.apache.hop.pipeline.transform.ITransformDialog; import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.pipeline.transforms.rowgenerator.GeneratorField; import org.apache.hop.pipeline.transforms.rowgenerator.RowGeneratorMeta; import org.apache.hop.ui.core.dialog.EnterTextDialog; import org.apache.hop.ui.core.dialog.ErrorDialog; @@ -72,8 +72,11 @@ public class ScriptValuesMetaModDialog extends BaseTransformDialog implements ITransformDialog { private static final Class PKG = ScriptValuesMetaMod.class; // For Translator - private static final String[] YES_NO_COMBO = new String[] { - BaseMessages.getString( PKG, "System.Combo.No" ), BaseMessages.getString( PKG, "System.Combo.Yes" ) }; + private static final String[] YES_NO_COMBO = + new String[] { + BaseMessages.getString(PKG, "System.Combo.No"), + BaseMessages.getString(PKG, "System.Combo.Yes") + }; private ModifyListener lsMod; @@ -134,9 +137,10 @@ public class ScriptValuesMetaModDialog extends BaseTransformDialog implements IT private RowGeneratorMeta genMeta; - public ScriptValuesMetaModDialog( Shell parent, IVariables variables, Object in, PipelineMeta pipelineMeta, String sname ) { + public ScriptValuesMetaModDialog( + Shell parent, IVariables variables, Object in, PipelineMeta pipelineMeta, String sname) { - super( parent, variables, (BaseTransformMeta) in, pipelineMeta, sname ); + super(parent, variables, (BaseTransformMeta) in, pipelineMeta, sname); input = (ScriptValuesMetaMod) in; genMeta = null; try { @@ -144,7 +148,7 @@ public ScriptValuesMetaModDialog( Shell parent, IVariables variables, Object in, imageInactiveScript = guiresource.getImage("ui/images/script-inactive.svg"); imageActiveStartScript = guiresource.getImage("ui/images/script-start.svg"); imageActiveEndScript = guiresource.getImage("ui/images/script-end.svg"); - } catch ( Exception e ) { + } catch (Exception e) { imageActiveScript = guiresource.getImageEmpty(); imageInactiveScript = guiresource.getImageEmpty(); imageActiveStartScript = guiresource.getImageEmpty(); @@ -152,12 +156,15 @@ public ScriptValuesMetaModDialog( Shell parent, IVariables variables, Object in, } try { - scVHelp = new ScriptValuesHelp("org/apache/hop/pipeline/transforms/javascript/jsFunctionHelp.xml"); - } catch ( Exception e ) { + scVHelp = + new ScriptValuesHelp("org/apache/hop/pipeline/transforms/javascript/jsFunctionHelp.xml"); + } catch (Exception e) { new ErrorDialog( - shell, "Unexpected error", "There was an unexpected error reading the javascript functions help", e ); + shell, + "Unexpected error", + "There was an unexpected error reading the javascript functions help", + e); } - } public String open() { @@ -165,9 +172,9 @@ public String open() { Shell parent = getParent(); Display display = parent.getDisplay(); - shell = new Shell( parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN ); - props.setLook( shell ); - setShellImage( shell, input ); + shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN); + props.setLook(shell); + setShellImage(shell, input); lsMod = e -> input.setChanged(); changed = input.hasChanged(); @@ -176,30 +183,31 @@ public String open() { formLayout.marginWidth = Const.FORM_MARGIN; formLayout.marginHeight = Const.FORM_MARGIN; - shell.setLayout( formLayout ); - shell.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.Shell.Title" ) ); + shell.setLayout(formLayout); + shell.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.Shell.Title")); int middle = props.getMiddlePct(); int margin = props.getMargin(); // Filename line - wlTransformName = new Label( shell, SWT.RIGHT ); - wlTransformName.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.TransformName.Label" ) ); - props.setLook( wlTransformName ); + wlTransformName = new Label(shell, SWT.RIGHT); + wlTransformName.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.TransformName.Label")); + props.setLook(wlTransformName); fdlTransformName = new FormData(); - fdlTransformName.left = new FormAttachment( 0, 0 ); - fdlTransformName.right = new FormAttachment( middle, -margin ); - fdlTransformName.top = new FormAttachment( 0, margin ); - wlTransformName.setLayoutData( fdlTransformName ); - wTransformName = new Text( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); - wTransformName.setText( transformName ); - props.setLook( wTransformName ); - wTransformName.addModifyListener( lsMod ); + fdlTransformName.left = new FormAttachment(0, 0); + fdlTransformName.right = new FormAttachment(middle, -margin); + fdlTransformName.top = new FormAttachment(0, margin); + wlTransformName.setLayoutData(fdlTransformName); + wTransformName = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wTransformName.setText(transformName); + props.setLook(wTransformName); + wTransformName.addModifyListener(lsMod); fdTransformName = new FormData(); - fdTransformName.left = new FormAttachment( middle, 0 ); - fdTransformName.top = new FormAttachment( 0, margin ); - fdTransformName.right = new FormAttachment( 100, 0 ); - wTransformName.setLayoutData( fdTransformName ); + fdTransformName.left = new FormAttachment(middle, 0); + fdTransformName.top = new FormAttachment(0, margin); + fdTransformName.right = new FormAttachment(100, 0); + wTransformName.setLayoutData(fdTransformName); SashForm wSash = new SashForm(shell, SWT.VERTICAL); @@ -211,87 +219,88 @@ public String open() { FormLayout topLayout = new FormLayout(); topLayout.marginWidth = Const.FORM_MARGIN; topLayout.marginHeight = Const.FORM_MARGIN; - wTop.setLayout( topLayout ); + wTop.setLayout(topLayout); // Script line Label wlScriptFunctions = new Label(wTop, SWT.NONE); - wlScriptFunctions.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.JavascriptFunctions.Label" ) ); + wlScriptFunctions.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.JavascriptFunctions.Label")); props.setLook(wlScriptFunctions); FormData fdlScriptFunctions = new FormData(); - fdlScriptFunctions.left = new FormAttachment( 0, 0 ); - fdlScriptFunctions.top = new FormAttachment( 0, 0 ); + fdlScriptFunctions.left = new FormAttachment(0, 0); + fdlScriptFunctions.top = new FormAttachment(0, 0); wlScriptFunctions.setLayoutData(fdlScriptFunctions); // Tree View Test - wTree = new Tree(wTop, SWT.BORDER | SWT.H_SCROLL | SWT.V_SCROLL ); - props.setLook( wTree ); + wTree = new Tree(wTop, SWT.BORDER | SWT.H_SCROLL | SWT.V_SCROLL); + props.setLook(wTree); FormData fdlTree = new FormData(); - fdlTree.left = new FormAttachment( 0, 0 ); - fdlTree.top = new FormAttachment(wlScriptFunctions, margin ); - fdlTree.right = new FormAttachment( 20, 0 ); - fdlTree.bottom = new FormAttachment( 100, -margin ); + fdlTree.left = new FormAttachment(0, 0); + fdlTree.top = new FormAttachment(wlScriptFunctions, margin); + fdlTree.right = new FormAttachment(20, 0); + fdlTree.bottom = new FormAttachment(100, -margin); wTree.setLayoutData(fdlTree); // Script line at the very top // Label wlScript = new Label(wTop, SWT.NONE); - wlScript.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.Javascript.Label" ) ); + wlScript.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.Javascript.Label")); props.setLook(wlScript); FormData fdlScript = new FormData(); - fdlScript.left = new FormAttachment( wTree, margin ); - fdlScript.top = new FormAttachment( 0, 0 ); + fdlScript.left = new FormAttachment(wTree, margin); + fdlScript.top = new FormAttachment(0, 0); wlScript.setLayoutData(fdlScript); // some information at the bottom... // // The optimisation level... // - Label wlOptimizationLevel = new Label(wTop, SWT.NONE ); - wlOptimizationLevel.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.OptimizationLevel.Label" ) ); - props.setLook( wlOptimizationLevel ); + Label wlOptimizationLevel = new Label(wTop, SWT.NONE); + wlOptimizationLevel.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.OptimizationLevel.Label")); + props.setLook(wlOptimizationLevel); FormData fdlOptimizationLevel = new FormData(); - fdlOptimizationLevel.left = new FormAttachment( wTree, margin * 2 ); - fdlOptimizationLevel.bottom = new FormAttachment( 100, -2*margin ); - wlOptimizationLevel.setLayoutData( fdlOptimizationLevel ); - - wOptimizationLevel = new TextVar( variables, wTop, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); - wOptimizationLevel.setToolTipText( BaseMessages.getString( - PKG, "ScriptValuesDialogMod.OptimizationLevel.Tooltip" ) ); - props.setLook( wOptimizationLevel ); + fdlOptimizationLevel.left = new FormAttachment(wTree, margin * 2); + fdlOptimizationLevel.bottom = new FormAttachment(100, -2 * margin); + wlOptimizationLevel.setLayoutData(fdlOptimizationLevel); + + wOptimizationLevel = new TextVar(variables, wTop, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wOptimizationLevel.setToolTipText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.OptimizationLevel.Tooltip")); + props.setLook(wOptimizationLevel); FormData fdOptimizationLevel = new FormData(); - fdOptimizationLevel.left = new FormAttachment( wlOptimizationLevel, margin ); - fdOptimizationLevel.top = new FormAttachment( wlOptimizationLevel, 0, SWT.CENTER ); - fdOptimizationLevel.right = new FormAttachment( 100, margin ); - wOptimizationLevel.setLayoutData( fdOptimizationLevel ); - wOptimizationLevel.addModifyListener( lsMod ); + fdOptimizationLevel.left = new FormAttachment(wlOptimizationLevel, margin); + fdOptimizationLevel.top = new FormAttachment(wlOptimizationLevel, 0, SWT.CENTER); + fdOptimizationLevel.right = new FormAttachment(100, margin); + wOptimizationLevel.setLayoutData(fdOptimizationLevel); + wOptimizationLevel.addModifyListener(lsMod); // The position just above that and below the script... // - wlPosition = new Label(wTop, SWT.LEFT ); - wlPosition.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.Position.Label" ) ); - props.setLook( wlPosition ); + wlPosition = new Label(wTop, SWT.LEFT); + wlPosition.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.Position.Label")); + props.setLook(wlPosition); FormData fdlPosition = new FormData(); - fdlPosition.left = new FormAttachment( wTree, 2*margin ); - fdlPosition.right = new FormAttachment( 100, 0 ); - fdlPosition.bottom = new FormAttachment( wOptimizationLevel, -2*margin ); + fdlPosition.left = new FormAttachment(wTree, 2 * margin); + fdlPosition.right = new FormAttachment(100, 0); + fdlPosition.bottom = new FormAttachment(wOptimizationLevel, -2 * margin); wlPosition.setLayoutData(fdlPosition); - - folder = new CTabFolder(wTop, SWT.BORDER | SWT.RESIZE ); - folder.setUnselectedImageVisible( true ); - folder.setUnselectedCloseVisible( true ); + folder = new CTabFolder(wTop, SWT.BORDER | SWT.RESIZE); + folder.setUnselectedImageVisible(true); + folder.setUnselectedCloseVisible(true); FormData fdScript = new FormData(); - fdScript.left = new FormAttachment( wTree, margin ); - fdScript.top = new FormAttachment(wlScript, margin ); - fdScript.right = new FormAttachment( 100, -5 ); - fdScript.bottom = new FormAttachment( wlPosition, -2*margin ); + fdScript.left = new FormAttachment(wTree, margin); + fdScript.top = new FormAttachment(wlScript, margin); + fdScript.right = new FormAttachment(100, -5); + fdScript.bottom = new FormAttachment(wlPosition, -2 * margin); folder.setLayoutData(fdScript); FormData fdTop = new FormData(); - fdTop.left = new FormAttachment( 0, 0 ); - fdTop.top = new FormAttachment( 0, 0 ); - fdTop.right = new FormAttachment( 100, 0 ); - fdTop.bottom = new FormAttachment( 100, 0 ); + fdTop.left = new FormAttachment(0, 0); + fdTop.top = new FormAttachment(0, 0); + fdTop.right = new FormAttachment(100, 0); + fdTop.bottom = new FormAttachment(100, 0); wTop.setLayoutData(fdTop); Composite wBottom = new Composite(wSash, SWT.NONE); @@ -300,83 +309,96 @@ public String open() { FormLayout bottomLayout = new FormLayout(); bottomLayout.marginWidth = Const.FORM_MARGIN; bottomLayout.marginHeight = Const.FORM_MARGIN; - wBottom.setLayout( bottomLayout ); + wBottom.setLayout(bottomLayout); Label wSeparator = new Label(wBottom, SWT.SEPARATOR | SWT.HORIZONTAL); FormData fdSeparator = new FormData(); - fdSeparator.left = new FormAttachment( 0, 0 ); - fdSeparator.right = new FormAttachment( 100, 0 ); - fdSeparator.top = new FormAttachment( 0, -margin + 2 ); + fdSeparator.left = new FormAttachment(0, 0); + fdSeparator.right = new FormAttachment(100, 0); + fdSeparator.top = new FormAttachment(0, -margin + 2); wSeparator.setLayoutData(fdSeparator); Label wlFields = new Label(wBottom, SWT.NONE); - wlFields.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.Fields.Label" ) ); + wlFields.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.Fields.Label")); props.setLook(wlFields); FormData fdlFields = new FormData(); - fdlFields.left = new FormAttachment( 0, 0 ); - fdlFields.top = new FormAttachment(wSeparator, 0 ); + fdlFields.left = new FormAttachment(0, 0); + fdlFields.top = new FormAttachment(wSeparator, 0); wlFields.setLayoutData(fdlFields); final int FieldsRows = input.getFieldname().length; ColumnInfo[] colinf = - new ColumnInfo[] { - new ColumnInfo( - BaseMessages.getString( PKG, "ScriptValuesDialogMod.ColumnInfo.Filename" ), - ColumnInfo.COLUMN_TYPE_TEXT, false ), - new ColumnInfo( - BaseMessages.getString( PKG, "ScriptValuesDialogMod.ColumnInfo.RenameTo" ), - ColumnInfo.COLUMN_TYPE_TEXT, false ), - new ColumnInfo( - BaseMessages.getString( PKG, "ScriptValuesDialogMod.ColumnInfo.Type" ), - ColumnInfo.COLUMN_TYPE_CCOMBO, ValueMetaFactory.getValueMetaNames() ), - new ColumnInfo( - BaseMessages.getString( PKG, "ScriptValuesDialogMod.ColumnInfo.Length" ), - ColumnInfo.COLUMN_TYPE_TEXT, false ), - new ColumnInfo( - BaseMessages.getString( PKG, "ScriptValuesDialogMod.ColumnInfo.Precision" ), - ColumnInfo.COLUMN_TYPE_TEXT, false ), - new ColumnInfo( - BaseMessages.getString( PKG, "ScriptValuesDialogMod.ColumnInfo.Replace" ), - ColumnInfo.COLUMN_TYPE_CCOMBO, YES_NO_COMBO ), }; + new ColumnInfo[] { + new ColumnInfo( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.ColumnInfo.Filename"), + ColumnInfo.COLUMN_TYPE_TEXT, + false), + new ColumnInfo( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.ColumnInfo.RenameTo"), + ColumnInfo.COLUMN_TYPE_TEXT, + false), + new ColumnInfo( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.ColumnInfo.Type"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + ValueMetaFactory.getValueMetaNames()), + new ColumnInfo( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.ColumnInfo.Length"), + ColumnInfo.COLUMN_TYPE_TEXT, + false), + new ColumnInfo( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.ColumnInfo.Precision"), + ColumnInfo.COLUMN_TYPE_TEXT, + false), + new ColumnInfo( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.ColumnInfo.Replace"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + YES_NO_COMBO), + }; wFields = - new TableView( - variables, wBottom, SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI, colinf, FieldsRows, lsMod, props ); + new TableView( + variables, + wBottom, + SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI, + colinf, + FieldsRows, + lsMod, + props); FormData fdFields = new FormData(); - fdFields.left = new FormAttachment( 0, 0 ); - fdFields.top = new FormAttachment(wlFields, margin ); - fdFields.right = new FormAttachment( 100, 0 ); - fdFields.bottom = new FormAttachment( 100, 0 ); + fdFields.left = new FormAttachment(0, 0); + fdFields.top = new FormAttachment(wlFields, margin); + fdFields.right = new FormAttachment(100, 0); + fdFields.bottom = new FormAttachment(100, 0); wFields.setLayoutData(fdFields); FormData fdBottom = new FormData(); - fdBottom.left = new FormAttachment( 0, 0 ); - fdBottom.top = new FormAttachment( 0, 0 ); - fdBottom.right = new FormAttachment( 100, 0 ); - fdBottom.bottom = new FormAttachment( 100, 0 ); + fdBottom.left = new FormAttachment(0, 0); + fdBottom.top = new FormAttachment(0, 0); + fdBottom.right = new FormAttachment(100, 0); + fdBottom.bottom = new FormAttachment(100, 0); wBottom.setLayoutData(fdBottom); FormData fdSash = new FormData(); - fdSash.left = new FormAttachment( 0, 0 ); - fdSash.top = new FormAttachment( wTransformName, 0 ); - fdSash.right = new FormAttachment( 100, 0 ); - fdSash.bottom = new FormAttachment( 100, -50 ); + fdSash.left = new FormAttachment(0, 0); + fdSash.top = new FormAttachment(wTransformName, 0); + fdSash.right = new FormAttachment(100, 0); + fdSash.bottom = new FormAttachment(100, -50); wSash.setLayoutData(fdSash); - wSash.setWeights( new int[] { 75, 25 } ); + wSash.setWeights(new int[] {75, 25}); - wOk = new Button( shell, SWT.PUSH ); - wOk.setText( BaseMessages.getString( PKG, "System.Button.OK" ) ); + wOk = new Button(shell, SWT.PUSH); + wOk.setText(BaseMessages.getString(PKG, "System.Button.OK")); Button wVars = new Button(shell, SWT.PUSH); - wVars.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.GetVariables.Button" ) ); + wVars.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.GetVariables.Button")); Button wTest = new Button(shell, SWT.PUSH); - wTest.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.TestScript.Button" ) ); - wCancel = new Button( shell, SWT.PUSH ); - wCancel.setText( BaseMessages.getString( PKG, "System.Button.Cancel" ) ); + wTest.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.TestScript.Button")); + wCancel = new Button(shell, SWT.PUSH); + wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel")); - setButtonPositions( new Button[] { wOk, wCancel, wVars, wTest}, margin, null ); + setButtonPositions(new Button[] {wOk, wCancel, wVars, wTest}, margin, null); // Add listeners lsCancel = e -> cancel(); @@ -387,59 +409,65 @@ public String open() { Listener lsTree = e -> treeDblClick(e); // lsHelp = new Listener(){public void handleEvent(Event e){ wlHelpLabel.setVisible(true); }}; - wCancel.addListener( SWT.Selection, lsCancel ); + wCancel.addListener(SWT.Selection, lsCancel); // wGet.addListener (SWT.Selection, lsGet ); - wTest.addListener( SWT.Selection, lsTest); - wVars.addListener( SWT.Selection, lsVars); - wOk.addListener( SWT.Selection, lsOk ); - wTree.addListener( SWT.MouseDoubleClick, lsTree); - - lsDef = new SelectionAdapter() { - public void widgetDefaultSelected( SelectionEvent e ) { - ok(); - } - }; - wTransformName.addSelectionListener( lsDef ); + wTest.addListener(SWT.Selection, lsTest); + wVars.addListener(SWT.Selection, lsVars); + wOk.addListener(SWT.Selection, lsOk); + wTree.addListener(SWT.MouseDoubleClick, lsTree); + + lsDef = + new SelectionAdapter() { + public void widgetDefaultSelected(SelectionEvent e) { + ok(); + } + }; + wTransformName.addSelectionListener(lsDef); // Detect X or ALT-F4 or something that kills this window... - shell.addShellListener( new ShellAdapter() { - public void shellClosed( ShellEvent e ) { - if ( !cancel() ) { - e.doit = false; - } - } - } ); - - folder.addCTabFolder2Listener( new CTabFolder2Adapter() { - public void close( CTabFolderEvent event ) { - CTabItem cItem = (CTabItem) event.item; - event.doit = false; - if ( cItem != null && folder.getItemCount() > 1 ) { - MessageBox messageBox = new MessageBox( shell, SWT.ICON_QUESTION | SWT.NO | SWT.YES ); - messageBox.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.DeleteItem.Label" ) ); - messageBox.setMessage( BaseMessages.getString( - PKG, "ScriptValuesDialogMod.ConfirmDeleteItem.Label", cItem.getText() ) ); - switch ( messageBox.open() ) { - case SWT.YES: - modifyScriptTree( cItem, DELETE_ITEM ); - event.doit = true; - break; - default: - break; + shell.addShellListener( + new ShellAdapter() { + public void shellClosed(ShellEvent e) { + if (!cancel()) { + e.doit = false; + } } - } - } - } ); + }); - cMenu = new Menu( shell, SWT.POP_UP ); + folder.addCTabFolder2Listener( + new CTabFolder2Adapter() { + public void close(CTabFolderEvent event) { + CTabItem cItem = (CTabItem) event.item; + event.doit = false; + if (cItem != null && folder.getItemCount() > 1) { + MessageBox messageBox = new MessageBox(shell, SWT.ICON_QUESTION | SWT.NO | SWT.YES); + messageBox.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.DeleteItem.Label")); + messageBox.setMessage( + BaseMessages.getString( + PKG, "ScriptValuesDialogMod.ConfirmDeleteItem.Label", cItem.getText())); + switch (messageBox.open()) { + case SWT.YES: + modifyScriptTree(cItem, DELETE_ITEM); + event.doit = true; + break; + default: + break; + } + } + } + }); + + cMenu = new Menu(shell, SWT.POP_UP); buildingFolderMenu(); - tMenu = new Menu( shell, SWT.POP_UP ); + tMenu = new Menu(shell, SWT.POP_UP); buildingTreeMenu(); // Adding the Default Transform Scripts Item to the Tree - wTreeScriptsItem = new TreeItem( wTree, SWT.NULL ); - wTreeScriptsItem.setImage( guiresource.getImageFolder() ); - wTreeScriptsItem.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.TransformScript.Label" ) ); + wTreeScriptsItem = new TreeItem(wTree, SWT.NULL); + wTreeScriptsItem.setImage(guiresource.getImageFolder()); + wTreeScriptsItem.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.TransformScript.Label")); // Set the shell size, based upon previous time... setSize(); @@ -449,182 +477,192 @@ public void close( CTabFolderEvent event ) { buildSpecialFunctionsTree(); // Input Fields - iteminput = new TreeItem( wTree, SWT.NULL ); - iteminput.setImage( guiresource.getImageInput() ); - iteminput.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.InputFields.Label" ) ); + iteminput = new TreeItem(wTree, SWT.NULL); + iteminput.setImage(guiresource.getImageInput()); + iteminput.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.InputFields.Label")); // Output Fields - itemoutput = new TreeItem( wTree, SWT.NULL ); - itemoutput.setImage( guiresource.getImageOutput() ); - itemoutput.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.OutputFields.Label" ) ); + itemoutput = new TreeItem(wTree, SWT.NULL); + itemoutput.setImage(guiresource.getImageOutput()); + itemoutput.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.OutputFields.Label")); // Display waiting message for input TreeItem itemWaitFieldsIn = new TreeItem(iteminput, SWT.NULL); - itemWaitFieldsIn.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.GettingFields.Label" ) ); - itemWaitFieldsIn.setForeground( guiresource.getColorDirectory() ); - iteminput.setExpanded( true ); + itemWaitFieldsIn.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.GettingFields.Label")); + itemWaitFieldsIn.setForeground(guiresource.getColorDirectory()); + iteminput.setExpanded(true); // Display waiting message for output TreeItem itemWaitFieldsOut = new TreeItem(itemoutput, SWT.NULL); - itemWaitFieldsOut.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.GettingFields.Label" ) ); - itemWaitFieldsOut.setForeground( guiresource.getColorDirectory() ); - itemoutput.setExpanded( true ); + itemWaitFieldsOut.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.GettingFields.Label")); + itemWaitFieldsOut.setForeground(guiresource.getColorDirectory()); + itemoutput.setExpanded(true); // // Search the fields in the background // - final Runnable runnable = () -> { - TransformMeta transformMeta = pipelineMeta.findTransform( transformName ); - if ( transformMeta != null ) { - try { - rowPrevTransformFields = pipelineMeta.getPrevTransformFields( variables, transformMeta ); - if ( rowPrevTransformFields != null ) { - setInputOutputFields(); - } else { - // Can not get fields...end of wait message - iteminput.removeAll(); - itemoutput.removeAll(); + final Runnable runnable = + () -> { + TransformMeta transformMeta = pipelineMeta.findTransform(transformName); + if (transformMeta != null) { + try { + rowPrevTransformFields = + pipelineMeta.getPrevTransformFields(variables, transformMeta); + if (rowPrevTransformFields != null) { + setInputOutputFields(); + } else { + // Can not get fields...end of wait message + iteminput.removeAll(); + itemoutput.removeAll(); + } + } catch (HopException e) { + logError(BaseMessages.getString(PKG, "System.Dialog.GetFieldsFailed.Message")); + } } - } catch ( HopException e ) { - logError( BaseMessages.getString( PKG, "System.Dialog.GetFieldsFailed.Message" ) ); - } - } - }; - new Thread( runnable ).start(); + }; + new Thread(runnable).start(); // rebuildInputFieldsTree(); // buildOutputFieldsTree(); buildAddClassesListTree(); addRenameTowTreeScriptItems(); - input.setChanged( changed ); + input.setChanged(changed); // Create the drag source on the tree - DragSource ds = new DragSource( wTree, DND.DROP_MOVE ); - ds.setTransfer( new Transfer[] { TextTransfer.getInstance() } ); - ds.addDragListener( new DragSourceAdapter() { - - public void dragStart( DragSourceEvent event ) { - TreeItem item = wTree.getSelection()[ 0 ]; - - // Qualifikation where the Drag Request Comes from - if ( item != null && item.getParentItem() != null ) { - if ( item.getParentItem().equals( wTreeScriptsItem ) ) { - event.doit = false; - } else if ( !item.getData().equals( "Function" ) ) { - String strInsert = (String) item.getData(); - if ( strInsert.equals( "jsFunction" ) ) { - event.doit = true; + DragSource ds = new DragSource(wTree, DND.DROP_MOVE); + ds.setTransfer(new Transfer[] {TextTransfer.getInstance()}); + ds.addDragListener( + new DragSourceAdapter() { + + public void dragStart(DragSourceEvent event) { + TreeItem item = wTree.getSelection()[0]; + + // Qualifikation where the Drag Request Comes from + if (item != null && item.getParentItem() != null) { + if (item.getParentItem().equals(wTreeScriptsItem)) { + event.doit = false; + } else if (!item.getData().equals("Function")) { + String strInsert = (String) item.getData(); + if (strInsert.equals("jsFunction")) { + event.doit = true; + } else { + event.doit = false; + } + } else { + event.doit = false; + } } else { event.doit = false; } - } else { - event.doit = false; } - } else { - event.doit = false; - } - - } - public void dragSetData( DragSourceEvent event ) { - // Set the data to be the first selected item's text - event.data = wTree.getSelection()[ 0 ].getText(); - } - } ); + public void dragSetData(DragSourceEvent event) { + // Set the data to be the first selected item's text + event.data = wTree.getSelection()[0].getText(); + } + }); shell.open(); - while ( !shell.isDisposed() ) { - if ( !display.readAndDispatch() ) { + while (!shell.isDisposed()) { + if (!display.readAndDispatch()) { display.sleep(); } } return transformName; } - private void setActiveCtab( String strName ) { - if ( strName.length() == 0 ) { - folder.setSelection( 0 ); + private void setActiveCtab(String strName) { + if (strName.length() == 0) { + folder.setSelection(0); } else { - folder.setSelection( getCTabPosition( strName ) ); + folder.setSelection(getCTabPosition(strName)); } } - private void addCtab( String cScriptName, String strScript, int iType ) { - CTabItem item = new CTabItem( folder, SWT.CLOSE ); + private void addCtab(String cScriptName, String strScript, int iType) { + CTabItem item = new CTabItem(folder, SWT.CLOSE); - switch ( iType ) { + switch (iType) { case ADD_DEFAULT: - item.setText( cScriptName ); + item.setText(cScriptName); break; default: - item.setText( getNextName( cScriptName ) ); + item.setText(getNextName(cScriptName)); break; } StyledTextComp wScript = - new StyledTextComp( variables, item.getParent(), SWT.MULTI | SWT.LEFT | SWT.H_SCROLL | SWT.V_SCROLL, false ); - if ( ( strScript != null ) && strScript.length() > 0 ) { - wScript.setText( strScript ); + new StyledTextComp( + variables, item.getParent(), SWT.MULTI | SWT.LEFT | SWT.H_SCROLL | SWT.V_SCROLL, false); + if ((strScript != null) && strScript.length() > 0) { + wScript.setText(strScript); } else { - wScript.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.ScriptHere.Label" ) - + Const.CR + Const.CR ); + wScript.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.ScriptHere.Label") + + Const.CR + + Const.CR); } - item.setImage( imageInactiveScript ); - props.setLook( wScript, Props.WIDGET_STYLE_FIXED ); + item.setImage(imageInactiveScript); + props.setLook(wScript, Props.WIDGET_STYLE_FIXED); - wScript.addKeyListener( new KeyAdapter() { - public void keyPressed( KeyEvent e ) { - setPosition(); - } + wScript.addKeyListener( + new KeyAdapter() { + public void keyPressed(KeyEvent e) { + setPosition(); + } - public void keyReleased( KeyEvent e ) { - setPosition(); - } - } ); - wScript.addFocusListener( new FocusAdapter() { - public void focusGained( FocusEvent e ) { - setPosition(); - } + public void keyReleased(KeyEvent e) { + setPosition(); + } + }); + wScript.addFocusListener( + new FocusAdapter() { + public void focusGained(FocusEvent e) { + setPosition(); + } - public void focusLost( FocusEvent e ) { - setPosition(); - } - } ); - wScript.addMouseListener( new MouseAdapter() { - public void mouseDoubleClick( MouseEvent e ) { - setPosition(); - } + public void focusLost(FocusEvent e) { + setPosition(); + } + }); + wScript.addMouseListener( + new MouseAdapter() { + public void mouseDoubleClick(MouseEvent e) { + setPosition(); + } - public void mouseDown( MouseEvent e ) { - setPosition(); - } + public void mouseDown(MouseEvent e) { + setPosition(); + } - public void mouseUp( MouseEvent e ) { - setPosition(); - } - } ); + public void mouseUp(MouseEvent e) { + setPosition(); + } + }); - wScript.addModifyListener( lsMod ); + wScript.addModifyListener(lsMod); - item.setControl( wScript ); + item.setControl(wScript); // Adding new Item to Tree - modifyScriptTree( item, ADD_ITEM ); + modifyScriptTree(item, ADD_ITEM); } - private void modifyScriptTree( CTabItem ctabitem, int iModType ) { + private void modifyScriptTree(CTabItem ctabitem, int iModType) { - switch ( iModType ) { + switch (iModType) { case DELETE_ITEM: - TreeItem dItem = getTreeItemByName( ctabitem.getText() ); - if ( dItem != null ) { + TreeItem dItem = getTreeItemByName(ctabitem.getText()); + if (dItem != null) { dItem.dispose(); input.setChanged(); } break; case ADD_ITEM: - TreeItem item = new TreeItem( wTreeScriptsItem, SWT.NULL ); - item.setImage( imageActiveScript ); - item.setText( ctabitem.getText() ); + TreeItem item = new TreeItem(wTreeScriptsItem, SWT.NULL); + item.setImage(imageActiveScript); + item.setText(ctabitem.getText()); input.setChanged(); break; @@ -639,7 +677,7 @@ private void modifyScriptTree( CTabItem ctabitem, int iModType ) { } } - private TreeItem getTreeItemByName( String strTabName ) { + private TreeItem getTreeItemByName(String strTabName) { TreeItem[] tItems = wTreeScriptsItem.getItems(); for (TreeItem tItem : tItems) { if (tItem.getText().equals(strTabName)) { @@ -649,17 +687,17 @@ private TreeItem getTreeItemByName( String strTabName ) { return null; } - private int getCTabPosition( String strTabName ) { + private int getCTabPosition(String strTabName) { CTabItem[] cItems = folder.getItems(); - for ( int i = 0; i < cItems.length; i++ ) { - if ( cItems[ i ].getText().equals( strTabName ) ) { + for (int i = 0; i < cItems.length; i++) { + if (cItems[i].getText().equals(strTabName)) { return i; } } return -1; } - private CTabItem getCTabItemByName( String strTabName ) { + private CTabItem getCTabItemByName(String strTabName) { CTabItem[] cItems = folder.getItems(); for (CTabItem cItem : cItems) { if (cItem.getText().equals(strTabName)) { @@ -669,41 +707,41 @@ private CTabItem getCTabItemByName( String strTabName ) { return null; } - private void modifyCTabItem( TreeItem tItem, int iModType, String strOption ) { + private void modifyCTabItem(TreeItem tItem, int iModType, String strOption) { - switch ( iModType ) { + switch (iModType) { case DELETE_ITEM: - CTabItem dItem = folder.getItem( getCTabPosition( tItem.getText() ) ); - if ( dItem != null ) { + CTabItem dItem = folder.getItem(getCTabPosition(tItem.getText())); + if (dItem != null) { dItem.dispose(); input.setChanged(); } break; case RENAME_ITEM: - CTabItem rItem = folder.getItem( getCTabPosition( tItem.getText() ) ); - if ( rItem != null ) { - rItem.setText( strOption ); + CTabItem rItem = folder.getItem(getCTabPosition(tItem.getText())); + if (rItem != null) { + rItem.setText(strOption); input.setChanged(); - if ( rItem.getImage().equals( imageActiveScript ) ) { + if (rItem.getImage().equals(imageActiveScript)) { strActiveScript = strOption; - } else if ( rItem.getImage().equals( imageActiveStartScript ) ) { + } else if (rItem.getImage().equals(imageActiveStartScript)) { strActiveStartScript = strOption; - } else if ( rItem.getImage().equals( imageActiveEndScript ) ) { + } else if (rItem.getImage().equals(imageActiveEndScript)) { strActiveEndScript = strOption; } } break; case SET_ACTIVE_ITEM: - CTabItem aItem = folder.getItem( getCTabPosition( tItem.getText() ) ); - if ( aItem != null ) { + CTabItem aItem = folder.getItem(getCTabPosition(tItem.getText())); + if (aItem != null) { input.setChanged(); strActiveScript = tItem.getText(); - for ( int i = 0; i < folder.getItemCount(); i++ ) { - if ( folder.getItem( i ).equals( aItem ) ) { - aItem.setImage( imageActiveScript ); + for (int i = 0; i < folder.getItemCount(); i++) { + if (folder.getItem(i).equals(aItem)) { + aItem.setImage(imageActiveScript); } else { - folder.getItem( i ).setImage( imageInactiveScript ); + folder.getItem(i).setImage(imageInactiveScript); } } } @@ -711,19 +749,18 @@ private void modifyCTabItem( TreeItem tItem, int iModType, String strOption ) { default: break; } - } private StyledTextComp getStyledTextComp() { CTabItem item = folder.getSelection(); - if ( item.getControl().isDisposed() ) { + if (item.getControl().isDisposed()) { return null; } else { return (StyledTextComp) item.getControl(); } } - private StyledTextComp getStyledTextComp( CTabItem item ) { + private StyledTextComp getStyledTextComp(CTabItem item) { return (StyledTextComp) item.getControl(); } @@ -735,15 +772,15 @@ private StyledTextComp getStyledTextComp( CTabItem item ) { * ((StyledTextComp)item.getControl()).setText(strText); } */ - private String getNextName( String strActualName ) { + private String getNextName(String strActualName) { String strRC = ""; - if ( strActualName.length() == 0 ) { + if (strActualName.length() == 0) { strActualName = "Item"; } int i = 0; strRC = strActualName + "_" + i; - while ( getCTabItemByName( strRC ) != null ) { + while (getCTabItemByName(strRC) != null) { i++; strRC = strActualName + "_" + i; } @@ -754,40 +791,40 @@ public void setPosition() { StyledTextComp wScript = getStyledTextComp(); int lineNumber = wScript.getLineNumber(); int columnNumber = wScript.getColumnNumber(); - wlPosition.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.Position.Label2", "" + lineNumber, "" + columnNumber ) ); + wlPosition.setText( + BaseMessages.getString( + PKG, "ScriptValuesDialogMod.Position.Label2", "" + lineNumber, "" + columnNumber)); } - /** - * Copy information from the meta-data input to the dialog fields. - */ + /** Copy information from the meta-data input to the dialog fields. */ public void getData() { -// wCompatible.setSelection( input.isCompatible() ); - if ( !Utils.isEmpty( Const.trim( input.getOptimizationLevel() ) ) ) { - wOptimizationLevel.setText( input.getOptimizationLevel().trim() ); + // wCompatible.setSelection( input.isCompatible() ); + if (!Utils.isEmpty(Const.trim(input.getOptimizationLevel()))) { + wOptimizationLevel.setText(input.getOptimizationLevel().trim()); } else { - wOptimizationLevel.setText( ScriptValuesMetaMod.OPTIMIZATION_LEVEL_DEFAULT ); + wOptimizationLevel.setText(ScriptValuesMetaMod.OPTIMIZATION_LEVEL_DEFAULT); } - for ( int i = 0; i < input.getFieldname().length; i++ ) { - if ( input.getFieldname()[ i ] != null && input.getFieldname()[ i ].length() > 0 ) { - TableItem item = wFields.table.getItem( i ); - item.setText( 1, input.getFieldname()[ i ] ); - if ( input.getRename()[ i ] != null && !input.getFieldname()[ i ].equals( input.getRename()[ i ] ) ) { - item.setText( 2, input.getRename()[ i ] ); + for (int i = 0; i < input.getFieldname().length; i++) { + if (input.getFieldname()[i] != null && input.getFieldname()[i].length() > 0) { + TableItem item = wFields.table.getItem(i); + item.setText(1, input.getFieldname()[i]); + if (input.getRename()[i] != null && !input.getFieldname()[i].equals(input.getRename()[i])) { + item.setText(2, input.getRename()[i]); } - item.setText( 3, ValueMetaFactory.getValueMetaName( input.getType()[ i ] ) ); - if ( input.getLength()[ i ] >= 0 ) { - item.setText( 4, "" + input.getLength()[ i ] ); + item.setText(3, ValueMetaFactory.getValueMetaName(input.getType()[i])); + if (input.getLength()[i] >= 0) { + item.setText(4, "" + input.getLength()[i]); } - if ( input.getPrecision()[ i ] >= 0 ) { - item.setText( 5, "" + input.getPrecision()[ i ] ); + if (input.getPrecision()[i] >= 0) { + item.setText(5, "" + input.getPrecision()[i]); } - item.setText( 6, input.getReplace()[ i ] ? YES_NO_COMBO[ 1 ] : YES_NO_COMBO[ 0 ] ); + item.setText(6, input.getReplace()[i] ? YES_NO_COMBO[1] : YES_NO_COMBO[0]); } } ScriptValuesScript[] jsScripts = input.getJSScripts(); - if ( jsScripts.length > 0 ) { + if (jsScripts.length > 0) { for (ScriptValuesScript jsScript : jsScripts) { if (jsScript.isTransformScript()) { strActiveScript = jsScript.getScriptName(); @@ -799,13 +836,13 @@ public void getData() { addCtab(jsScript.getScriptName(), jsScript.getScript(), ADD_DEFAULT); } } else { - addCtab( "", "", ADD_DEFAULT ); + addCtab("", "", ADD_DEFAULT); } - setActiveCtab( strActiveScript ); + setActiveCtab(strActiveScript); refresh(); wFields.setRowNums(); - wFields.optWidth( true ); + wFields.optWidth(true); wTransformName.selectAll(); wTransformName.setFocus(); @@ -814,16 +851,16 @@ public void getData() { // Setting default active Script private void refresh() { // CTabItem item = getCTabItemByName(strActiveScript); - for ( int i = 0; i < folder.getItemCount(); i++ ) { - CTabItem item = folder.getItem( i ); - if ( item.getText().equals( strActiveScript ) ) { - item.setImage( imageActiveScript ); - } else if ( item.getText().equals( strActiveStartScript ) ) { - item.setImage( imageActiveStartScript ); - } else if ( item.getText().equals( strActiveEndScript ) ) { - item.setImage( imageActiveEndScript ); + for (int i = 0; i < folder.getItemCount(); i++) { + CTabItem item = folder.getItem(i); + if (item.getText().equals(strActiveScript)) { + item.setImage(imageActiveScript); + } else if (item.getText().equals(strActiveStartScript)) { + item.setImage(imageActiveStartScript); + } else if (item.getText().equals(strActiveEndScript)) { + item.setImage(imageActiveEndScript); } else { - item.setImage( imageInactiveScript ); + item.setImage(imageInactiveScript); } } // modifyScriptTree(null, SET_ACTIVE_ITEM); @@ -841,69 +878,72 @@ private void refreshScripts() { } private boolean cancel() { - if ( input.hasChanged() ) { - MessageBox box = new MessageBox( shell, SWT.YES | SWT.NO | SWT.APPLICATION_MODAL | SWT.SHEET ); - box.setText( BaseMessages.getString( PKG, "ScriptValuesModDialog.WarningDialogChanged.Title" ) ); - box - .setMessage( BaseMessages - .getString( PKG, "ScriptValuesModDialog.WarningDialogChanged.Message", Const.CR ) ); + if (input.hasChanged()) { + MessageBox box = new MessageBox(shell, SWT.YES | SWT.NO | SWT.APPLICATION_MODAL | SWT.SHEET); + box.setText(BaseMessages.getString(PKG, "ScriptValuesModDialog.WarningDialogChanged.Title")); + box.setMessage( + BaseMessages.getString( + PKG, "ScriptValuesModDialog.WarningDialogChanged.Message", Const.CR)); int answer = box.open(); - if ( answer == SWT.NO ) { + if (answer == SWT.NO) { return false; } } transformName = null; - input.setChanged( changed ); + input.setChanged(changed); dispose(); return true; } - private void getInfo( ScriptValuesMetaMod meta ) { -// meta.setCompatible( wCompatible.getSelection() ); - meta.setOptimizationLevel( wOptimizationLevel.getText() ); + private void getInfo(ScriptValuesMetaMod meta) { + // meta.setCompatible( wCompatible.getSelection() ); + meta.setOptimizationLevel(wOptimizationLevel.getText()); int nrFields = wFields.nrNonEmpty(); - meta.allocate( nrFields ); - //CHECKSTYLE:Indentation:OFF - for ( int i = 0; i < nrFields; i++ ) { - TableItem item = wFields.getNonEmpty( i ); - meta.getFieldname()[ i ] = item.getText( 1 ); - meta.getRename()[ i ] = item.getText( 2 ); - if ( meta.getRename()[ i ] == null - || meta.getRename()[ i ].length() == 0 || meta.getRename()[ i ].equalsIgnoreCase( meta.getFieldname()[ i ] ) ) { - meta.getRename()[ i ] = meta.getFieldname()[ i ]; + meta.allocate(nrFields); + // CHECKSTYLE:Indentation:OFF + for (int i = 0; i < nrFields; i++) { + TableItem item = wFields.getNonEmpty(i); + meta.getFieldname()[i] = item.getText(1); + meta.getRename()[i] = item.getText(2); + if (meta.getRename()[i] == null + || meta.getRename()[i].length() == 0 + || meta.getRename()[i].equalsIgnoreCase(meta.getFieldname()[i])) { + meta.getRename()[i] = meta.getFieldname()[i]; } - meta.getType()[ i ] = ValueMetaFactory.getIdForValueMeta( item.getText( 3 ) ); - String slen = item.getText( 4 ); - String sprc = item.getText( 5 ); - meta.getLength()[ i ] = Const.toInt( slen, -1 ); - meta.getPrecision()[ i ] = Const.toInt( sprc, -1 ); - meta.getReplace()[ i ] = YES_NO_COMBO[ 1 ].equalsIgnoreCase( item.getText( 6 ) ); + meta.getType()[i] = ValueMetaFactory.getIdForValueMeta(item.getText(3)); + String slen = item.getText(4); + String sprc = item.getText(5); + meta.getLength()[i] = Const.toInt(slen, -1); + meta.getPrecision()[i] = Const.toInt(sprc, -1); + meta.getReplace()[i] = YES_NO_COMBO[1].equalsIgnoreCase(item.getText(6)); } // input.setActiveJSScript(strActiveScript); CTabItem[] cTabs = folder.getItems(); - if ( cTabs.length > 0 ) { - ScriptValuesScript[] jsScripts = new ScriptValuesScript[ cTabs.length ]; - for ( int i = 0; i < cTabs.length; i++ ) { + if (cTabs.length > 0) { + ScriptValuesScript[] jsScripts = new ScriptValuesScript[cTabs.length]; + for (int i = 0; i < cTabs.length; i++) { ScriptValuesScript jsScript = - new ScriptValuesScript( ScriptValuesScript.NORMAL_SCRIPT, cTabs[ i ].getText(), getStyledTextComp( - cTabs[ i ] ).getText() ); - if ( cTabs[ i ].getImage().equals( imageActiveScript ) ) { - jsScript.setScriptType( ScriptValuesScript.TRANSFORM_SCRIPT ); - } else if ( cTabs[ i ].getImage().equals( imageActiveStartScript ) ) { - jsScript.setScriptType( ScriptValuesScript.START_SCRIPT ); - } else if ( cTabs[ i ].getImage().equals( imageActiveEndScript ) ) { - jsScript.setScriptType( ScriptValuesScript.END_SCRIPT ); + new ScriptValuesScript( + ScriptValuesScript.NORMAL_SCRIPT, + cTabs[i].getText(), + getStyledTextComp(cTabs[i]).getText()); + if (cTabs[i].getImage().equals(imageActiveScript)) { + jsScript.setScriptType(ScriptValuesScript.TRANSFORM_SCRIPT); + } else if (cTabs[i].getImage().equals(imageActiveStartScript)) { + jsScript.setScriptType(ScriptValuesScript.START_SCRIPT); + } else if (cTabs[i].getImage().equals(imageActiveEndScript)) { + jsScript.setScriptType(ScriptValuesScript.END_SCRIPT); } - jsScripts[ i ] = jsScript; + jsScripts[i] = jsScript; } - meta.setJSScripts( jsScripts ); + meta.setJSScripts(jsScripts); } } private void ok() { - if ( Utils.isEmpty( wTransformName.getText() ) ) { + if (Utils.isEmpty(wTransformName.getText())) { return; } @@ -912,13 +952,13 @@ private void ok() { boolean bInputOK = false; // Check if Active Script has set, otherwise Ask - if ( getCTabItemByName( strActiveScript ) == null ) { - MessageBox mb = new MessageBox( shell, SWT.OK | SWT.CANCEL | SWT.ICON_ERROR ); - mb.setMessage( BaseMessages.getString( PKG, "ScriptValuesDialogMod.NoActiveScriptSet" ) ); - mb.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.ERROR.Label" ) ); - switch ( mb.open() ) { + if (getCTabItemByName(strActiveScript) == null) { + MessageBox mb = new MessageBox(shell, SWT.OK | SWT.CANCEL | SWT.ICON_ERROR); + mb.setMessage(BaseMessages.getString(PKG, "ScriptValuesDialogMod.NoActiveScriptSet")); + mb.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.ERROR.Label")); + switch (mb.open()) { case SWT.OK: - strActiveScript = folder.getItem( 0 ).getText(); + strActiveScript = folder.getItem(0).getText(); refresh(); bInputOK = true; break; @@ -932,35 +972,35 @@ private void ok() { bInputOK = true; } -/* - if ( bInputOK && wCompatible.getSelection() ) { - // If in compatibility mode the "replace" column must not be "Yes" - int nrFields = wFields.nrNonEmpty(); - for ( int i = 0; i < nrFields; i++ ) { - TableItem item = wFields.getNonEmpty( i ); - if ( YES_NO_COMBO[ 1 ].equalsIgnoreCase( item.getText( 6 ) ) ) { - bInputOK = false; + /* + if ( bInputOK && wCompatible.getSelection() ) { + // If in compatibility mode the "replace" column must not be "Yes" + int nrFields = wFields.nrNonEmpty(); + for ( int i = 0; i < nrFields; i++ ) { + TableItem item = wFields.getNonEmpty( i ); + if ( YES_NO_COMBO[ 1 ].equalsIgnoreCase( item.getText( 6 ) ) ) { + bInputOK = false; + } + } + if ( !bInputOK ) { + MessageBox mb = new MessageBox( shell, SWT.OK | SWT.CANCEL | SWT.ICON_ERROR ); + mb + .setMessage( BaseMessages + .getString( PKG, "ScriptValuesDialogMod.ReplaceNotAllowedInCompatibilityMode" ) ); + mb.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.ERROR.Label" ) ); + mb.open(); + } } - } - if ( !bInputOK ) { - MessageBox mb = new MessageBox( shell, SWT.OK | SWT.CANCEL | SWT.ICON_ERROR ); - mb - .setMessage( BaseMessages - .getString( PKG, "ScriptValuesDialogMod.ReplaceNotAllowedInCompatibilityMode" ) ); - mb.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.ERROR.Label" ) ); - mb.open(); - } - } -*/ + */ - if ( bInputOK ) { - getInfo( input ); + if (bInputOK) { + getInfo(input); dispose(); } } public boolean test() { - return test( false, false ); + return test(false, false); } private boolean newTest() { @@ -971,145 +1011,182 @@ private boolean newTest() { try { // What fields are coming into the transform? // - IRowMeta rowMeta = pipelineMeta.getPrevTransformFields( variables, transformName ).clone(); - if ( rowMeta != null ) { + IRowMeta rowMeta = pipelineMeta.getPrevTransformFields(variables, transformName).clone(); + if (rowMeta != null) { // Create a new RowGenerator transform to generate rows for the test data... // Only create a new instance the first time to help the user. // Otherwise he/she has to key in the same test data all the time // - if ( genMeta == null ) { + if (genMeta == null) { genMeta = new RowGeneratorMeta(); - genMeta.setRowLimit( "10" ); - genMeta.allocate( rowMeta.size() ); - //CHECKSTYLE:Indentation:OFF - for ( int i = 0; i < rowMeta.size(); i++ ) { - IValueMeta valueMeta = rowMeta.getValueMeta( i ); - if ( valueMeta.isStorageBinaryString() ) { - valueMeta.setStorageType( IValueMeta.STORAGE_TYPE_NORMAL ); + genMeta.setRowLimit("10"); + + // CHECKSTYLE:Indentation:OFF + for (int i = 0; i < rowMeta.size(); i++) { + IValueMeta valueMeta = rowMeta.getValueMeta(i); + if (valueMeta.isStorageBinaryString()) { + valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL); } - genMeta.getFieldName()[ i ] = valueMeta.getName(); - genMeta.getFieldType()[ i ] = valueMeta.getTypeDesc(); - genMeta.getFieldLength()[ i ] = valueMeta.getLength(); - genMeta.getFieldPrecision()[ i ] = valueMeta.getPrecision(); - genMeta.getCurrency()[ i ] = valueMeta.getCurrencySymbol(); - genMeta.getDecimal()[ i ] = valueMeta.getDecimalSymbol(); - genMeta.getGroup()[ i ] = valueMeta.getGroupingSymbol(); String string = null; - switch ( valueMeta.getType() ) { + String format = valueMeta.getFormatMask(); + boolean setValueMetaFormat = false; + switch (valueMeta.getType()) { case IValueMeta.TYPE_DATE: - genMeta.getFieldFormat()[ i ] = "yyyy/MM/dd HH:mm:ss"; - valueMeta.setConversionMask( genMeta.getFieldFormat()[ i ] ); - string = valueMeta.getString( new Date() ); + format = "yyyy/MM/dd HH:mm:ss"; + setValueMetaFormat = true; + string = valueMeta.getString(new Date()); break; case IValueMeta.TYPE_STRING: string = "test value test value"; break; case IValueMeta.TYPE_INTEGER: - genMeta.getFieldFormat()[ i ] = "#"; - valueMeta.setConversionMask( genMeta.getFieldFormat()[ i ] ); + format = "#"; + setValueMetaFormat = true; string = valueMeta.getString(0L); break; case IValueMeta.TYPE_NUMBER: - genMeta.getFieldFormat()[ i ] = "#.#"; - valueMeta.setConversionMask( genMeta.getFieldFormat()[ i ] ); + format = "#.#"; + setValueMetaFormat = true; string = valueMeta.getString(0.0D); break; case IValueMeta.TYPE_BIGNUMBER: - genMeta.getFieldFormat()[ i ] = "#.#"; - valueMeta.setConversionMask( genMeta.getFieldFormat()[ i ] ); - string = valueMeta.getString( BigDecimal.ZERO ); + format = "#.#"; + setValueMetaFormat = true; + string = valueMeta.getString(BigDecimal.ZERO); break; case IValueMeta.TYPE_BOOLEAN: - string = valueMeta.getString( Boolean.TRUE ); + string = valueMeta.getString(Boolean.TRUE); break; case IValueMeta.TYPE_BINARY: - string = valueMeta.getString( new byte[] { 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, } ); + string = + valueMeta.getString( + new byte[] { + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + }); break; default: break; } + if (setValueMetaFormat) { + valueMeta.setConversionMask(format); + } - genMeta.getValue()[ i ] = string; + GeneratorField field = + new GeneratorField( + valueMeta.getName(), + valueMeta.getTypeDesc(), + format, + valueMeta.getLength(), + valueMeta.getPrecision(), + valueMeta.getCurrencySymbol(), + valueMeta.getDecimalSymbol(), + valueMeta.getGroupingSymbol(), + string, + false); + + genMeta.getFields().add(field); } } TransformMeta genTransform = - new TransformMeta( registry.getPluginId( TransformPluginType.class, genMeta ), "## TEST DATA ##", genMeta ); - genTransform.setLocation( 50, 50 ); + new TransformMeta( + registry.getPluginId(TransformPluginType.class, genMeta), + "## TEST DATA ##", + genMeta); + genTransform.setLocation(50, 50); // Now create a JavaScript transform with the information in this dialog // ScriptValuesMetaMod scriptMeta = new ScriptValuesMetaMod(); - getInfo( scriptMeta ); + getInfo(scriptMeta); TransformMeta scriptTransform = - new TransformMeta( registry.getPluginId( TransformPluginType.class, scriptMeta ), Const.NVL( - scriptTransformName, "## SCRIPT ##" ), scriptMeta ); + new TransformMeta( + registry.getPluginId(TransformPluginType.class, scriptMeta), + Const.NVL(scriptTransformName, "## SCRIPT ##"), + scriptMeta); scriptTransformName = scriptTransform.getName(); - scriptTransform.setLocation( 150, 50 ); + scriptTransform.setLocation(150, 50); // Create a hop between both transforms... // - PipelineHopMeta hop = new PipelineHopMeta( genTransform, scriptTransform ); + PipelineHopMeta hop = new PipelineHopMeta(genTransform, scriptTransform); // Generate a new test pipeline... // PipelineMeta pipelineMeta = new PipelineMeta(); - pipelineMeta.setName( wTransformName.getText() + " - PREVIEW" ); - pipelineMeta.addTransform( genTransform ); - pipelineMeta.addTransform( scriptTransform ); - pipelineMeta.addPipelineHop( hop ); + pipelineMeta.setName(wTransformName.getText() + " - PREVIEW"); + pipelineMeta.addTransform(genTransform); + pipelineMeta.addTransform(scriptTransform); + pipelineMeta.addPipelineHop(hop); // OK, now we ask the user to edit this dialog... // -// if ( HopGui.getInstance().editTransform( pipelineMeta, genTransform ) != null ) { - // Now run this pipeline and grab the results... - // - PipelinePreviewProgressDialog progressDialog = + // if ( HopGui.getInstance().editTransform( pipelineMeta, genTransform ) != null ) { + // Now run this pipeline and grab the results... + // + PipelinePreviewProgressDialog progressDialog = new PipelinePreviewProgressDialog( - shell, variables, pipelineMeta, new String[] { scriptTransformName, }, new int[] { Const.toInt( genMeta - .getRowLimit(), 10 ), } ); - progressDialog.open(); - - Pipeline pipeline = progressDialog.getPipeline(); - String loggingText = progressDialog.getLoggingText(); - - if ( !progressDialog.isCancelled() ) { - if ( pipeline.getResult() != null && pipeline.getResult().getNrErrors() > 0 ) { - EnterTextDialog etd = + shell, + variables, + pipelineMeta, + new String[] { + scriptTransformName, + }, + new int[] { + Const.toInt(genMeta.getRowLimit(), 10), + }); + progressDialog.open(); + + Pipeline pipeline = progressDialog.getPipeline(); + String loggingText = progressDialog.getLoggingText(); + + if (!progressDialog.isCancelled()) { + if (pipeline.getResult() != null && pipeline.getResult().getNrErrors() > 0) { + EnterTextDialog etd = new EnterTextDialog( - shell, BaseMessages.getString( PKG, "System.Dialog.PreviewError.Title" ), BaseMessages - .getString( PKG, "System.Dialog.PreviewError.Message" ), loggingText, true ); - etd.setReadOnly(); - etd.open(); - } + shell, + BaseMessages.getString(PKG, "System.Dialog.PreviewError.Title"), + BaseMessages.getString(PKG, "System.Dialog.PreviewError.Message"), + loggingText, + true); + etd.setReadOnly(); + etd.open(); } + } - IRowMeta previewRowsMeta = progressDialog.getPreviewRowsMeta( wTransformName.getText() ); - List previewRows = progressDialog.getPreviewRows( wTransformName.getText() ); + IRowMeta previewRowsMeta = progressDialog.getPreviewRowsMeta(wTransformName.getText()); + List previewRows = progressDialog.getPreviewRows(wTransformName.getText()); - if ( previewRowsMeta != null && previewRows != null && previewRows.size() > 0 ) { - PreviewRowsDialog prd = + if (previewRowsMeta != null && previewRows != null && previewRows.size() > 0) { + PreviewRowsDialog prd = new PreviewRowsDialog( - shell, variables, SWT.NONE, wTransformName.getText(), previewRowsMeta, previewRows, loggingText ); - prd.open(); - } + shell, + variables, + SWT.NONE, + wTransformName.getText(), + previewRowsMeta, + previewRows, + loggingText); + prd.open(); } + } - return true; -// } else { -// throw new HopException( BaseMessages.getString( -// PKG, "ScriptValuesDialogMod.Exception.CouldNotGetFields" ) ); -// } - } catch ( Exception e ) { + return true; + // } else { + // throw new HopException( BaseMessages.getString( + // PKG, "ScriptValuesDialogMod.Exception.CouldNotGetFields" ) ); + // } + } catch (Exception e) { new ErrorDialog( - shell, BaseMessages.getString( PKG, "ScriptValuesDialogMod.TestFailed.DialogTitle" ), BaseMessages - .getString( PKG, "ScriptValuesDialogMod.TestFailed.DialogMessage" ), e ); + shell, + BaseMessages.getString(PKG, "ScriptValuesDialogMod.TestFailed.DialogTitle"), + BaseMessages.getString(PKG, "ScriptValuesDialogMod.TestFailed.DialogMessage"), + e); return false; } - } - private boolean test( boolean getvars, boolean popup ) { + private boolean test(boolean getvars, boolean popup) { boolean retval = true; StyledTextComp wScript = getStyledTextComp(); String scr = wScript.getText(); @@ -1123,108 +1200,118 @@ private boolean test( boolean getvars, boolean popup ) { refreshScripts(); jscx = ContextFactory.getGlobal().enterContext(); - jscx.setOptimizationLevel( -1 ); - jsscope = jscx.initStandardObjects( null, false ); + jscx.setOptimizationLevel(-1); + jsscope = jscx.initStandardObjects(null, false); // Adding the existing Scripts to the Context - for ( int i = 0; i < folder.getItemCount(); i++ ) { - StyledTextComp sItem = getStyledTextComp( folder.getItem( i ) ); - Scriptable jsR = Context.toObject( sItem.getText(), jsscope ); - jsscope.put( folder.getItem( i ).getText(), jsscope, jsR ); + for (int i = 0; i < folder.getItemCount(); i++) { + StyledTextComp sItem = getStyledTextComp(folder.getItem(i)); + Scriptable jsR = Context.toObject(sItem.getText(), jsscope); + jsscope.put(folder.getItem(i).getText(), jsscope, jsR); } // Adding the Name of the Pipeline to the Context - jsscope.put( "_PipelineName_", jsscope, this.transformName ); + jsscope.put("_PipelineName_", jsscope, this.transformName); try { - IRowMeta rowMeta = pipelineMeta.getPrevTransformFields( variables, transformName ); - if ( rowMeta != null ) { + IRowMeta rowMeta = pipelineMeta.getPrevTransformFields(variables, transformName); + if (rowMeta != null) { - ScriptValuesModDummy dummyTransform = new ScriptValuesModDummy( rowMeta, pipelineMeta.getTransformFields(variables, transformName ) ); - Scriptable jsvalue = Context.toObject( dummyTransform, jsscope ); - jsscope.put( "_transform_", jsscope, jsvalue ); + ScriptValuesModDummy dummyTransform = + new ScriptValuesModDummy( + rowMeta, pipelineMeta.getTransformFields(variables, transformName)); + Scriptable jsvalue = Context.toObject(dummyTransform, jsscope); + jsscope.put("_transform_", jsscope, jsvalue); // Modification for Additional Script parsing try { - if ( input.getAddClasses() != null ) { - for ( int i = 0; i < input.getAddClasses().length; i++ ) { - Object jsOut = Context.javaToJS( input.getAddClasses()[ i ].getAddObject(), jsscope ); - ScriptableObject.putProperty( jsscope, input.getAddClasses()[ i ].getJSName(), jsOut ); + if (input.getAddClasses() != null) { + for (int i = 0; i < input.getAddClasses().length; i++) { + Object jsOut = Context.javaToJS(input.getAddClasses()[i].getAddObject(), jsscope); + ScriptableObject.putProperty(jsscope, input.getAddClasses()[i].getJSName(), jsOut); } } - } catch ( Exception e ) { + } catch (Exception e) { testException = - new HopException( BaseMessages.getString( PKG, "ScriptValuesDialogMod.CouldNotAddToContext", e - .toString() ) ); + new HopException( + BaseMessages.getString( + PKG, "ScriptValuesDialogMod.CouldNotAddToContext", e.toString())); retval = false; } // Adding some default JavaScriptFunctions to the System try { - Context.javaToJS( ScriptValuesAddedFunctions.class, jsscope ); - ( (ScriptableObject) jsscope ).defineFunctionProperties( - jsFunctionList, ScriptValuesAddedFunctions.class, ScriptableObject.DONTENUM ); - } catch ( Exception ex ) { + Context.javaToJS(ScriptValuesAddedFunctions.class, jsscope); + ((ScriptableObject) jsscope) + .defineFunctionProperties( + jsFunctionList, ScriptValuesAddedFunctions.class, ScriptableObject.DONTENUM); + } catch (Exception ex) { testException = - new HopException( BaseMessages.getString( - PKG, "ScriptValuesDialogMod.CouldNotAddDefaultFunctions", ex.toString() ) ); + new HopException( + BaseMessages.getString( + PKG, "ScriptValuesDialogMod.CouldNotAddDefaultFunctions", ex.toString())); retval = false; } // Adding some Constants to the JavaScript try { - jsscope.put( "SKIP_PIPELINE", jsscope, SKIP_PIPELINE); - jsscope.put( "ABORT_PIPELINE", jsscope, ABORT_PIPELINE); - jsscope.put( "ERROR_PIPELINE", jsscope, ERROR_PIPELINE); - jsscope.put( "CONTINUE_PIPELINE", jsscope, CONTINUE_PIPELINE); - } catch ( Exception ex ) { + jsscope.put("SKIP_PIPELINE", jsscope, SKIP_PIPELINE); + jsscope.put("ABORT_PIPELINE", jsscope, ABORT_PIPELINE); + jsscope.put("ERROR_PIPELINE", jsscope, ERROR_PIPELINE); + jsscope.put("CONTINUE_PIPELINE", jsscope, CONTINUE_PIPELINE); + } catch (Exception ex) { testException = - new HopException( BaseMessages.getString( - PKG, "ScriptValuesDialogMod.CouldNotAddPipelineConstants", ex.toString() ) ); + new HopException( + BaseMessages.getString( + PKG, "ScriptValuesDialogMod.CouldNotAddPipelineConstants", ex.toString())); retval = false; } try { - Object[] row = new Object[ rowMeta.size() ]; - Scriptable jsRowMeta = Context.toObject( rowMeta, jsscope ); - jsscope.put( "rowMeta", jsscope, jsRowMeta ); - for ( int i = 0; i < rowMeta.size(); i++ ) { - IValueMeta valueMeta = rowMeta.getValueMeta( i ); + Object[] row = new Object[rowMeta.size()]; + Scriptable jsRowMeta = Context.toObject(rowMeta, jsscope); + jsscope.put("rowMeta", jsscope, jsRowMeta); + for (int i = 0; i < rowMeta.size(); i++) { + IValueMeta valueMeta = rowMeta.getValueMeta(i); Object valueData = null; // Set date and string values to something to simulate real thing // - if ( valueMeta.isDate() ) { + if (valueMeta.isDate()) { valueData = new Date(); } - if ( valueMeta.isString() ) { - valueData = "test value test value test value test value test value " - + "test value test value test value test value test value"; + if (valueMeta.isString()) { + valueData = + "test value test value test value test value test value " + + "test value test value test value test value test value"; } - if ( valueMeta.isInteger() ) { + if (valueMeta.isInteger()) { valueData = 0L; } - if ( valueMeta.isNumber() ) { + if (valueMeta.isNumber()) { valueData = 0.0; } - if ( valueMeta.isBigNumber() ) { + if (valueMeta.isBigNumber()) { valueData = BigDecimal.ZERO; } - if ( valueMeta.isBoolean() ) { + if (valueMeta.isBoolean()) { valueData = Boolean.TRUE; } - if ( valueMeta.isBinary() ) { - valueData = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, }; + if (valueMeta.isBinary()) { + valueData = + new byte[] { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + }; } - if ( valueMeta.isStorageBinaryString() ) { - valueMeta.setStorageType( IValueMeta.STORAGE_TYPE_NORMAL ); + if (valueMeta.isStorageBinaryString()) { + valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL); } - row[ i ] = valueData; + row[i] = valueData; - Scriptable jsarg = Context.toObject( valueData, jsscope ); - jsscope.put( valueMeta.getName(), jsscope, jsarg ); + Scriptable jsarg = Context.toObject(valueData, jsscope); + jsscope.put(valueMeta.getName(), jsscope, jsarg); } // OK, for these input values, we're going to allow the user to edit the default values... @@ -1232,141 +1319,156 @@ private boolean test( boolean getvars, boolean popup ) { // 2) // Add support for Value class (new Value()) -// Scriptable jsval = Context.toObject( Value.class, jsscope ); -// jsscope.put( "Value", jsscope, jsval ); + // Scriptable jsval = Context.toObject( Value.class, jsscope ); + // jsscope.put( "Value", jsscope, jsval ); - Scriptable jsRow = Context.toObject( row, jsscope ); - jsscope.put( "row", jsscope, jsRow ); + Scriptable jsRow = Context.toObject(row, jsscope); + jsscope.put("row", jsscope, jsRow); - } catch ( Exception ev ) { + } catch (Exception ev) { testException = - new HopException( BaseMessages.getString( PKG, "ScriptValuesDialogMod.CouldNotAddInputFields", ev - .toString() ) ); + new HopException( + BaseMessages.getString( + PKG, "ScriptValuesDialogMod.CouldNotAddInputFields", ev.toString())); retval = false; } try { // Checking for StartScript - if ( strActiveStartScript != null - && !folder.getSelection().getText().equals( strActiveStartScript ) - && strActiveStartScript.length() > 0 ) { + if (strActiveStartScript != null + && !folder.getSelection().getText().equals(strActiveStartScript) + && strActiveStartScript.length() > 0) { String strStartScript = - getStyledTextComp( folder.getItem( getCTabPosition( strActiveStartScript ) ) ).getText(); - /* Object startScript = */ - jscx.evaluateString( jsscope, strStartScript, "pipeline_Start", 1, null ); + getStyledTextComp(folder.getItem(getCTabPosition(strActiveStartScript))).getText(); + /* Object startScript = */ jscx.evaluateString( + jsscope, strStartScript, "pipeline_Start", 1, null); } - } catch ( Exception e ) { + } catch (Exception e) { testException = - new HopException( BaseMessages.getString( PKG, "ScriptValuesDialogMod.CouldProcessStartScript", e - .toString() ) ); + new HopException( + BaseMessages.getString( + PKG, "ScriptValuesDialogMod.CouldProcessStartScript", e.toString())); retval = false; } try { - Script evalScript = jscx.compileString( scr, "script", 1, null ); - evalScript.exec( jscx, jsscope ); + Script evalScript = jscx.compileString(scr, "script", 1, null); + evalScript.exec(jscx, jsscope); - if ( getvars ) { - ScriptNode tree = parseVariables( jscx, jsscope, scr, "script", 1, null ); - for ( int i = 0; i < tree.getParamAndVarCount(); i++ ) { - String varname = tree.getParamOrVarName( i ); - if ( !varname.equalsIgnoreCase( "row" ) && !varname.equalsIgnoreCase( "pipeline_Status" ) ) { + if (getvars) { + ScriptNode tree = parseVariables(jscx, jsscope, scr, "script", 1, null); + for (int i = 0; i < tree.getParamAndVarCount(); i++) { + String varname = tree.getParamOrVarName(i); + if (!varname.equalsIgnoreCase("row") + && !varname.equalsIgnoreCase("pipeline_Status")) { int type = IValueMeta.TYPE_STRING; int length = -1, precision = -1; - Object result = jsscope.get( varname, jsscope ); - if ( result != null ) { + Object result = jsscope.get(varname, jsscope); + if (result != null) { String classname = result.getClass().getName(); - if ( classname.equalsIgnoreCase( "java.lang.Byte" ) ) { + if (classname.equalsIgnoreCase("java.lang.Byte")) { // MAX = 127 type = IValueMeta.TYPE_INTEGER; length = 3; precision = 0; - } else if ( classname.equalsIgnoreCase( "java.lang.Integer" ) ) { + } else if (classname.equalsIgnoreCase("java.lang.Integer")) { // MAX = 2147483647 type = IValueMeta.TYPE_INTEGER; length = 9; precision = 0; - } else if ( classname.equalsIgnoreCase( "java.lang.Long" ) ) { + } else if (classname.equalsIgnoreCase("java.lang.Long")) { // MAX = 9223372036854775807 type = IValueMeta.TYPE_INTEGER; length = 18; precision = 0; - } else if ( classname.equalsIgnoreCase( "java.lang.Double" ) ) { + } else if (classname.equalsIgnoreCase("java.lang.Double")) { type = IValueMeta.TYPE_NUMBER; length = 16; precision = 2; - } else if ( classname.equalsIgnoreCase( "org.mozilla.javascript.NativeDate" ) - || classname.equalsIgnoreCase( "java.util.Date" ) ) { + } else if (classname.equalsIgnoreCase("org.mozilla.javascript.NativeDate") + || classname.equalsIgnoreCase("java.util.Date")) { type = IValueMeta.TYPE_DATE; - } else if ( classname.equalsIgnoreCase( "java.lang.Boolean" ) ) { + } else if (classname.equalsIgnoreCase("java.lang.Boolean")) { type = IValueMeta.TYPE_BOOLEAN; } } - TableItem ti = new TableItem( wFields.table, SWT.NONE ); - ti.setText( 1, varname ); - ti.setText( 2, "" ); - ti.setText( 3, ValueMetaFactory.getValueMetaName( type ) ); - ti.setText( 4, length >= 0 ? ( "" + length ) : "" ); - ti.setText( 5, precision >= 0 ? ( "" + precision ) : "" ); + TableItem ti = new TableItem(wFields.table, SWT.NONE); + ti.setText(1, varname); + ti.setText(2, ""); + ti.setText(3, ValueMetaFactory.getValueMetaName(type)); + ti.setText(4, length >= 0 ? ("" + length) : ""); + ti.setText(5, precision >= 0 ? ("" + precision) : ""); // If the variable name exists in the input, suggest to replace the value // - ti.setText( 6, ( rowMeta.indexOfValue( varname ) >= 0 ) ? YES_NO_COMBO[ 1 ] : YES_NO_COMBO[ 0 ] ); + ti.setText( + 6, (rowMeta.indexOfValue(varname) >= 0) ? YES_NO_COMBO[1] : YES_NO_COMBO[0]); } } wFields.removeEmptyRows(); wFields.setRowNums(); - wFields.optWidth( true ); + wFields.optWidth(true); } // End Script! - } catch ( EvaluatorException e ) { + } catch (EvaluatorException e) { String position = "(" + e.lineNumber() + ":" + e.columnNumber() + ")"; String message = - BaseMessages.getString( PKG, "ScriptValuesDialogMod.Exception.CouldNotExecuteScript", position ); - testException = new HopException( message, e ); + BaseMessages.getString( + PKG, "ScriptValuesDialogMod.Exception.CouldNotExecuteScript", position); + testException = new HopException(message, e); retval = false; - } catch ( JavaScriptException e ) { + } catch (JavaScriptException e) { String position = "(" + e.lineNumber() + ":" + e.columnNumber() + ")"; String message = - BaseMessages.getString( PKG, "ScriptValuesDialogMod.Exception.CouldNotExecuteScript", position ); - testException = new HopException( message, e ); + BaseMessages.getString( + PKG, "ScriptValuesDialogMod.Exception.CouldNotExecuteScript", position); + testException = new HopException(message, e); retval = false; - } catch ( Exception e ) { + } catch (Exception e) { testException = - new HopException( BaseMessages.getString( - PKG, "ScriptValuesDialogMod.Exception.CouldNotExecuteScript2" ), e ); + new HopException( + BaseMessages.getString( + PKG, "ScriptValuesDialogMod.Exception.CouldNotExecuteScript2"), + e); retval = false; } } else { testException = - new HopException( BaseMessages.getString( PKG, "ScriptValuesDialogMod.Exception.CouldNotGetFields" ) ); + new HopException( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.Exception.CouldNotGetFields")); retval = false; } - if ( popup ) { - if ( retval ) { - if ( !getvars ) { - MessageBox mb = new MessageBox( shell, SWT.OK | SWT.ICON_INFORMATION ); - mb.setMessage( BaseMessages.getString( PKG, "ScriptValuesDialogMod.ScriptCompilationOK" ) + Const.CR ); - mb.setText( "OK" ); + if (popup) { + if (retval) { + if (!getvars) { + MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); + mb.setMessage( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.ScriptCompilationOK") + + Const.CR); + mb.setText("OK"); mb.open(); } } else { new ErrorDialog( - shell, BaseMessages.getString( PKG, "ScriptValuesDialogMod.TestFailed.DialogTitle" ), BaseMessages - .getString( PKG, "ScriptValuesDialogMod.TestFailed.DialogMessage" ), testException ); + shell, + BaseMessages.getString(PKG, "ScriptValuesDialogMod.TestFailed.DialogTitle"), + BaseMessages.getString(PKG, "ScriptValuesDialogMod.TestFailed.DialogMessage"), + testException); } } - } catch ( HopException ke ) { + } catch (HopException ke) { retval = false; new ErrorDialog( - shell, BaseMessages.getString( PKG, "ScriptValuesDialogMod.TestFailed.DialogTitle" ), BaseMessages - .getString( PKG, "ScriptValuesDialogMod.TestFailed.DialogMessage" ), ke ); + shell, + BaseMessages.getString(PKG, "ScriptValuesDialogMod.TestFailed.DialogTitle"), + BaseMessages.getString(PKG, "ScriptValuesDialogMod.TestFailed.DialogMessage"), + ke); } finally { - if ( jscx != null ) { + if (jscx != null) { Context.exit(); } } @@ -1375,58 +1477,61 @@ private boolean test( boolean getvars, boolean popup ) { private void buildSpecialFunctionsTree() { - TreeItem item = new TreeItem( wTree, SWT.NULL ); - item.setImage( guiresource.getImageFolder() ); - item.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.TansformConstant.Label" ) ); - TreeItem itemT = new TreeItem( item, SWT.NULL ); - itemT.setImage( GuiResource.getInstance().getImageLabel() ); - itemT.setText( "SKIP_PIPELINE" ); - itemT.setData( "SKIP_PIPELINE" ); + TreeItem item = new TreeItem(wTree, SWT.NULL); + item.setImage(guiresource.getImageFolder()); + item.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.TansformConstant.Label")); + TreeItem itemT = new TreeItem(item, SWT.NULL); + itemT.setImage(GuiResource.getInstance().getImageLabel()); + itemT.setText("SKIP_PIPELINE"); + itemT.setData("SKIP_PIPELINE"); // itemT = new TreeItem(item, SWT.NULL); // itemT.setText("ABORT_PIPELINE"); // itemT.setData("ABORT_PIPELINE"); - itemT = new TreeItem( item, SWT.NULL ); - itemT.setImage( GuiResource.getInstance().getImageLabel() ); - itemT.setText( "ERROR_PIPELINE" ); - itemT.setData( "ERROR_PIPELINE" ); - itemT = new TreeItem( item, SWT.NULL ); - itemT.setImage( GuiResource.getInstance().getImageLabel() ); - itemT.setText( "CONTINUE_PIPELINE" ); - itemT.setData( "CONTINUE_PIPELINE" ); - - item = new TreeItem( wTree, SWT.NULL ); - item.setImage( guiresource.getImageFolder() ); - item.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.TransformFunctions.Label" ) ); + itemT = new TreeItem(item, SWT.NULL); + itemT.setImage(GuiResource.getInstance().getImageLabel()); + itemT.setText("ERROR_PIPELINE"); + itemT.setData("ERROR_PIPELINE"); + itemT = new TreeItem(item, SWT.NULL); + itemT.setImage(GuiResource.getInstance().getImageLabel()); + itemT.setText("CONTINUE_PIPELINE"); + itemT.setData("CONTINUE_PIPELINE"); + + item = new TreeItem(wTree, SWT.NULL); + item.setImage(guiresource.getImageFolder()); + item.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.TransformFunctions.Label")); String strData = ""; // Adding the Grouping Items to the Tree - TreeItem itemStringFunctionsGroup = new TreeItem( item, SWT.NULL ); - itemStringFunctionsGroup.setImage( GuiResource.getInstance().getImageFolder() ); - itemStringFunctionsGroup - .setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.StringFunctions.Label" ) ); - itemStringFunctionsGroup.setData( "Function" ); - TreeItem itemNumericFunctionsGroup = new TreeItem( item, SWT.NULL ); - itemNumericFunctionsGroup.setImage( GuiResource.getInstance().getImageFolder() ); - itemNumericFunctionsGroup.setText( BaseMessages - .getString( PKG, "ScriptValuesDialogMod.NumericFunctions.Label" ) ); - itemNumericFunctionsGroup.setData( "Function" ); - TreeItem itemDateFunctionsGroup = new TreeItem( item, SWT.NULL ); - itemDateFunctionsGroup.setImage( GuiResource.getInstance().getImageFolder() ); - itemDateFunctionsGroup.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.DateFunctions.Label" ) ); - itemDateFunctionsGroup.setData( "Function" ); - TreeItem itemLogicFunctionsGroup = new TreeItem( item, SWT.NULL ); - itemLogicFunctionsGroup.setImage( GuiResource.getInstance().getImageFolder() ); - itemLogicFunctionsGroup.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.LogicFunctions.Label" ) ); - itemLogicFunctionsGroup.setData( "Function" ); - TreeItem itemSpecialFunctionsGroup = new TreeItem( item, SWT.NULL ); - itemSpecialFunctionsGroup.setImage( GuiResource.getInstance().getImageFolder() ); - itemSpecialFunctionsGroup.setText( BaseMessages - .getString( PKG, "ScriptValuesDialogMod.SpecialFunctions.Label" ) ); - itemSpecialFunctionsGroup.setData( "Function" ); - TreeItem itemFileFunctionsGroup = new TreeItem( item, SWT.NULL ); - itemFileFunctionsGroup.setImage( GuiResource.getInstance().getImageFolder() ); - itemFileFunctionsGroup.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.FileFunctions.Label" ) ); - itemFileFunctionsGroup.setData( "Function" ); + TreeItem itemStringFunctionsGroup = new TreeItem(item, SWT.NULL); + itemStringFunctionsGroup.setImage(GuiResource.getInstance().getImageFolder()); + itemStringFunctionsGroup.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.StringFunctions.Label")); + itemStringFunctionsGroup.setData("Function"); + TreeItem itemNumericFunctionsGroup = new TreeItem(item, SWT.NULL); + itemNumericFunctionsGroup.setImage(GuiResource.getInstance().getImageFolder()); + itemNumericFunctionsGroup.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.NumericFunctions.Label")); + itemNumericFunctionsGroup.setData("Function"); + TreeItem itemDateFunctionsGroup = new TreeItem(item, SWT.NULL); + itemDateFunctionsGroup.setImage(GuiResource.getInstance().getImageFolder()); + itemDateFunctionsGroup.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.DateFunctions.Label")); + itemDateFunctionsGroup.setData("Function"); + TreeItem itemLogicFunctionsGroup = new TreeItem(item, SWT.NULL); + itemLogicFunctionsGroup.setImage(GuiResource.getInstance().getImageFolder()); + itemLogicFunctionsGroup.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.LogicFunctions.Label")); + itemLogicFunctionsGroup.setData("Function"); + TreeItem itemSpecialFunctionsGroup = new TreeItem(item, SWT.NULL); + itemSpecialFunctionsGroup.setImage(GuiResource.getInstance().getImageFolder()); + itemSpecialFunctionsGroup.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.SpecialFunctions.Label")); + itemSpecialFunctionsGroup.setData("Function"); + TreeItem itemFileFunctionsGroup = new TreeItem(item, SWT.NULL); + itemFileFunctionsGroup.setImage(GuiResource.getInstance().getImageFolder()); + itemFileFunctionsGroup.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.FileFunctions.Label")); + itemFileFunctionsGroup.setData("Function"); // Loading the Default delivered JScript Functions // Method[] methods = ScriptValuesAddedFunctions.class.getMethods(); @@ -1435,47 +1540,47 @@ private void buildSpecialFunctionsTree() { Hashtable hatFunctions = scVHelp.getFunctionList(); Vector v = new Vector<>(hatFunctions.keySet()); - Collections.sort( v ); + Collections.sort(v); - for ( String strFunction : v ) { - String strFunctionType = hatFunctions.get( strFunction ); + for (String strFunction : v) { + String strFunctionType = hatFunctions.get(strFunction); int iFunctionType = Integer.valueOf(strFunctionType); TreeItem itemFunction = null; - switch ( iFunctionType ) { + switch (iFunctionType) { case ScriptValuesAddedFunctions.STRING_FUNCTION: - itemFunction = new TreeItem( itemStringFunctionsGroup, SWT.NULL ); + itemFunction = new TreeItem(itemStringFunctionsGroup, SWT.NULL); break; case ScriptValuesAddedFunctions.NUMERIC_FUNCTION: - itemFunction = new TreeItem( itemNumericFunctionsGroup, SWT.NULL ); + itemFunction = new TreeItem(itemNumericFunctionsGroup, SWT.NULL); break; case ScriptValuesAddedFunctions.DATE_FUNCTION: - itemFunction = new TreeItem( itemDateFunctionsGroup, SWT.NULL ); + itemFunction = new TreeItem(itemDateFunctionsGroup, SWT.NULL); break; case ScriptValuesAddedFunctions.LOGIC_FUNCTION: - itemFunction = new TreeItem( itemLogicFunctionsGroup, SWT.NULL ); + itemFunction = new TreeItem(itemLogicFunctionsGroup, SWT.NULL); break; case ScriptValuesAddedFunctions.SPECIAL_FUNCTION: - itemFunction = new TreeItem( itemSpecialFunctionsGroup, SWT.NULL ); + itemFunction = new TreeItem(itemSpecialFunctionsGroup, SWT.NULL); break; case ScriptValuesAddedFunctions.FILE_FUNCTION: - itemFunction = new TreeItem( itemFileFunctionsGroup, SWT.NULL ); + itemFunction = new TreeItem(itemFileFunctionsGroup, SWT.NULL); break; default: break; } - if ( itemFunction != null ) { - itemFunction.setText( strFunction ); - itemFunction.setImage( GuiResource.getInstance().getImageFunction() ); + if (itemFunction != null) { + itemFunction.setText(strFunction); + itemFunction.setImage(GuiResource.getInstance().getImageFunction()); strData = "jsFunction"; - itemFunction.setData( strData ); + itemFunction.setData(strData); } } } - public boolean TreeItemExist( TreeItem itemToCheck, String strItemName ) { + public boolean TreeItemExist(TreeItem itemToCheck, String strItemName) { boolean bRC = false; - if ( itemToCheck.getItemCount() > 0 ) { + if (itemToCheck.getItemCount() > 0) { TreeItem[] items = itemToCheck.getItems(); for (TreeItem item : items) { if (item.getText().equals(strItemName)) { @@ -1487,61 +1592,69 @@ public boolean TreeItemExist( TreeItem itemToCheck, String strItemName ) { } private void setInputOutputFields() { - shell.getDisplay().syncExec( () -> { - // fields are got...end of wait message - iteminput.removeAll(); - itemoutput.removeAll(); - - String strItemInToAdd = ""; - String strItemToAddOut = ""; - - // try{ - - // IRowMeta r = pipelineMeta.getPrevTransformFields(transformName); - if ( rowPrevTransformFields != null ) { - // TreeItem item = new TreeItem(wTree, SWT.NULL); - // item.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.OutputFields.Label")); - // String strItemToAdd=""; - - for ( int i = 0; i < rowPrevTransformFields.size(); i++ ) { - IValueMeta valueMeta = rowPrevTransformFields.getValueMeta( i ); - strItemToAddOut = valueMeta.getName() + ".setValue(var)"; - strItemInToAdd = valueMeta.getName(); - TreeItem itemFields = new TreeItem( iteminput, SWT.NULL ); - itemFields.setImage( GuiResource.getInstance().getImage(valueMeta) ); - itemFields.setText( strItemInToAdd ); - itemFields.setData( strItemInToAdd ); - } - TreeItem itemFields = new TreeItem( itemoutput, SWT.NULL ); - itemFields.setData( "" ); - itemFields.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.OutputFiels.CompatibilityOff" ) ); - } - } ); + shell + .getDisplay() + .syncExec( + () -> { + // fields are got...end of wait message + iteminput.removeAll(); + itemoutput.removeAll(); + + String strItemInToAdd = ""; + String strItemToAddOut = ""; + + // try{ + + // IRowMeta r = pipelineMeta.getPrevTransformFields(transformName); + if (rowPrevTransformFields != null) { + // TreeItem item = new TreeItem(wTree, SWT.NULL); + // item.setText(BaseMessages.getString(PKG, + // "ScriptValuesDialogMod.OutputFields.Label")); + // String strItemToAdd=""; + + for (int i = 0; i < rowPrevTransformFields.size(); i++) { + IValueMeta valueMeta = rowPrevTransformFields.getValueMeta(i); + strItemToAddOut = valueMeta.getName() + ".setValue(var)"; + strItemInToAdd = valueMeta.getName(); + TreeItem itemFields = new TreeItem(iteminput, SWT.NULL); + itemFields.setImage(GuiResource.getInstance().getImage(valueMeta)); + itemFields.setText(strItemInToAdd); + itemFields.setData(strItemInToAdd); + } + TreeItem itemFields = new TreeItem(itemoutput, SWT.NULL); + itemFields.setData(""); + itemFields.setText( + BaseMessages.getString( + PKG, "ScriptValuesDialogMod.OutputFiels.CompatibilityOff")); + } + }); } // Adds the Current item to the current Position - private void treeDblClick( Event event ) { + private void treeDblClick(Event event) { StyledTextComp wScript = getStyledTextComp(); - Point point = new Point( event.x, event.y ); - TreeItem item = wTree.getItem( point ); + Point point = new Point(event.x, event.y); + TreeItem item = wTree.getItem(point); // Qualification where the Click comes from - if ( item != null && item.getParentItem() != null ) { - if ( item.getParentItem().equals( wTreeScriptsItem ) ) { - setActiveCtab( item.getText() ); - } else if ( !item.getData().equals( "Function" ) ) { + if (item != null && item.getParentItem() != null) { + if (item.getParentItem().equals(wTreeScriptsItem)) { + setActiveCtab(item.getText()); + } else if (!item.getData().equals("Function")) { int iStart = wScript.getTextWidget().getCaretPosition(); - int selCount = wScript.getSelectionCount(); // this selection will be replaced by wScript.insert - iStart = iStart - selCount; // when a selection is already there we need to subtract the position - if ( iStart < 0 ) { + int selCount = + wScript.getSelectionCount(); // this selection will be replaced by wScript.insert + iStart = + iStart - selCount; // when a selection is already there we need to subtract the position + if (iStart < 0) { iStart = 0; // just safety } String strInsert = (String) item.getData(); - if ( strInsert.equals( "jsFunction" ) ) { + if (strInsert.equals("jsFunction")) { strInsert = item.getText(); } - wScript.insert( strInsert ); - wScript.setSelection( iStart, iStart + strInsert.length() ); + wScript.insert(strInsert); + wScript.setSelection(iStart, iStart + strInsert.length()); } } /* @@ -1553,17 +1666,17 @@ private void treeDblClick( Event event ) { // Building the Tree for Additional Classes private void buildAddClassesListTree() { - if ( wTreeClassesitem != null ) { + if (wTreeClassesitem != null) { wTreeClassesitem.dispose(); } - if ( input.getAddClasses() != null ) { - for ( int i = 0; i < input.getAddClasses().length; i++ ) { + if (input.getAddClasses() != null) { + for (int i = 0; i < input.getAddClasses().length; i++) { try { - Method[] methods = input.getAddClasses()[ i ].getAddClass().getMethods(); - String strClassType = input.getAddClasses()[ i ].getAddClass().toString(); + Method[] methods = input.getAddClasses()[i].getAddClass().getMethods(); + String strClassType = input.getAddClasses()[i].getAddClass().toString(); String strParams; - wTreeClassesitem = new TreeItem( wTree, SWT.NULL ); - wTreeClassesitem.setText( input.getAddClasses()[ i ].getJSName() ); + wTreeClassesitem = new TreeItem(wTree, SWT.NULL); + wTreeClassesitem.setText(input.getAddClasses()[i].getJSName()); for (Method method : methods) { String strDeclaringClass = method.getDeclaringClass().toString(); if (strClassType.equals(strDeclaringClass)) { @@ -1571,18 +1684,23 @@ private void buildAddClassesListTree() { strParams = buildAddClassFunctionName(method); item2.setText(method.getName() + "(" + strParams + ")"); String strData = - input.getAddClasses()[i].getJSName() + "." + method.getName() + "(" + strParams + ")"; + input.getAddClasses()[i].getJSName() + + "." + + method.getName() + + "(" + + strParams + + ")"; item2.setData(strData); } } - } catch ( Exception e ) { + } catch (Exception e) { // Ignore errors } } } } - private String buildAddClassFunctionName( Method metForParams ) { + private String buildAddClassFunctionName(Method metForParams) { StringBuilder sbRC = new StringBuilder(); String strRC = ""; Class[] clsParamType = metForParams.getParameterTypes(); @@ -1602,307 +1720,334 @@ private String buildAddClassFunctionName( Method metForParams ) { sbRC.append(", "); } } - } strRC = sbRC.toString(); - if ( strRC.length() > 0 ) { - strRC = strRC.substring( 0, sbRC.length() - 2 ); + if (strRC.length() > 0) { + strRC = strRC.substring(0, sbRC.length() - 2); } return strRC; } private void buildingFolderMenu() { // styledTextPopupmenu = new Menu(, SWT.POP_UP); - MenuItem addNewItem = new MenuItem( cMenu, SWT.PUSH ); - addNewItem.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.AddNewTab" ) ); - addNewItem.setImage( guiresource.getImageAdd() ); - addNewItem.addListener( SWT.Selection, e -> addCtab( "", "", ADD_BLANK ) ); - - MenuItem copyItem = new MenuItem( cMenu, SWT.PUSH ); - copyItem.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.AddCopy" ) ); - copyItem.setImage( guiresource.getImageDuplicate() ); - copyItem.addListener( SWT.Selection, e -> { - CTabItem item = folder.getSelection(); - StyledTextComp st = (StyledTextComp) item.getControl(); - addCtab( item.getText(), st.getText(), ADD_COPY ); - } ); - new MenuItem( cMenu, SWT.SEPARATOR ); - - MenuItem setActiveScriptItem = new MenuItem( cMenu, SWT.PUSH ); - setActiveScriptItem.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.SetTransformScript" ) ); - setActiveScriptItem.setImage( imageActiveScript ); - setActiveScriptItem.addListener( SWT.Selection, e -> { - CTabItem item = folder.getSelection(); - for ( int i = 0; i < folder.getItemCount(); i++ ) { - if ( folder.getItem( i ).equals( item ) ) { - if ( item.getImage().equals( imageActiveScript ) ) { - strActiveScript = ""; - } else if ( item.getImage().equals( imageActiveStartScript ) ) { - strActiveStartScript = ""; - } else if ( item.getImage().equals( imageActiveEndScript ) ) { - strActiveEndScript = ""; + MenuItem addNewItem = new MenuItem(cMenu, SWT.PUSH); + addNewItem.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.AddNewTab")); + addNewItem.setImage(guiresource.getImageAdd()); + addNewItem.addListener(SWT.Selection, e -> addCtab("", "", ADD_BLANK)); + + MenuItem copyItem = new MenuItem(cMenu, SWT.PUSH); + copyItem.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.AddCopy")); + copyItem.setImage(guiresource.getImageDuplicate()); + copyItem.addListener( + SWT.Selection, + e -> { + CTabItem item = folder.getSelection(); + StyledTextComp st = (StyledTextComp) item.getControl(); + addCtab(item.getText(), st.getText(), ADD_COPY); + }); + new MenuItem(cMenu, SWT.SEPARATOR); + + MenuItem setActiveScriptItem = new MenuItem(cMenu, SWT.PUSH); + setActiveScriptItem.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.SetTransformScript")); + setActiveScriptItem.setImage(imageActiveScript); + setActiveScriptItem.addListener( + SWT.Selection, + e -> { + CTabItem item = folder.getSelection(); + for (int i = 0; i < folder.getItemCount(); i++) { + if (folder.getItem(i).equals(item)) { + if (item.getImage().equals(imageActiveScript)) { + strActiveScript = ""; + } else if (item.getImage().equals(imageActiveStartScript)) { + strActiveStartScript = ""; + } else if (item.getImage().equals(imageActiveEndScript)) { + strActiveEndScript = ""; + } + item.setImage(imageActiveScript); + strActiveScript = item.getText(); + } else if (folder.getItem(i).getImage().equals(imageActiveScript)) { + folder.getItem(i).setImage(imageInactiveScript); + } } - item.setImage( imageActiveScript ); - strActiveScript = item.getText(); - } else if ( folder.getItem( i ).getImage().equals( imageActiveScript ) ) { - folder.getItem( i ).setImage( imageInactiveScript ); - } - } - modifyScriptTree( item, SET_ACTIVE_ITEM ); - } ); - - MenuItem setStartScriptItem = new MenuItem( cMenu, SWT.PUSH ); - setStartScriptItem.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.SetStartScript" ) ); - setStartScriptItem.setImage( imageActiveStartScript ); - setStartScriptItem.addListener( SWT.Selection, e -> { - CTabItem item = folder.getSelection(); - for ( int i = 0; i < folder.getItemCount(); i++ ) { - if ( folder.getItem( i ).equals( item ) ) { - if ( item.getImage().equals( imageActiveScript ) ) { - strActiveScript = ""; - } else if ( item.getImage().equals( imageActiveStartScript ) ) { - strActiveStartScript = ""; - } else if ( item.getImage().equals( imageActiveEndScript ) ) { - strActiveEndScript = ""; + modifyScriptTree(item, SET_ACTIVE_ITEM); + }); + + MenuItem setStartScriptItem = new MenuItem(cMenu, SWT.PUSH); + setStartScriptItem.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.SetStartScript")); + setStartScriptItem.setImage(imageActiveStartScript); + setStartScriptItem.addListener( + SWT.Selection, + e -> { + CTabItem item = folder.getSelection(); + for (int i = 0; i < folder.getItemCount(); i++) { + if (folder.getItem(i).equals(item)) { + if (item.getImage().equals(imageActiveScript)) { + strActiveScript = ""; + } else if (item.getImage().equals(imageActiveStartScript)) { + strActiveStartScript = ""; + } else if (item.getImage().equals(imageActiveEndScript)) { + strActiveEndScript = ""; + } + item.setImage(imageActiveStartScript); + strActiveStartScript = item.getText(); + } else if (folder.getItem(i).getImage().equals(imageActiveStartScript)) { + folder.getItem(i).setImage(imageInactiveScript); + } } - item.setImage( imageActiveStartScript ); - strActiveStartScript = item.getText(); - } else if ( folder.getItem( i ).getImage().equals( imageActiveStartScript ) ) { - folder.getItem( i ).setImage( imageInactiveScript ); - } - } - modifyScriptTree( item, SET_ACTIVE_ITEM ); - } ); - - MenuItem setEndScriptItem = new MenuItem( cMenu, SWT.PUSH ); - setEndScriptItem.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.SetEndScript" ) ); - setEndScriptItem.setImage( imageActiveEndScript ); - setEndScriptItem.addListener( SWT.Selection, e -> { - CTabItem item = folder.getSelection(); - for ( int i = 0; i < folder.getItemCount(); i++ ) { - if ( folder.getItem( i ).equals( item ) ) { - if ( item.getImage().equals( imageActiveScript ) ) { + modifyScriptTree(item, SET_ACTIVE_ITEM); + }); + + MenuItem setEndScriptItem = new MenuItem(cMenu, SWT.PUSH); + setEndScriptItem.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.SetEndScript")); + setEndScriptItem.setImage(imageActiveEndScript); + setEndScriptItem.addListener( + SWT.Selection, + e -> { + CTabItem item = folder.getSelection(); + for (int i = 0; i < folder.getItemCount(); i++) { + if (folder.getItem(i).equals(item)) { + if (item.getImage().equals(imageActiveScript)) { + strActiveScript = ""; + } else if (item.getImage().equals(imageActiveStartScript)) { + strActiveStartScript = ""; + } else if (item.getImage().equals(imageActiveEndScript)) { + strActiveEndScript = ""; + } + item.setImage(imageActiveEndScript); + strActiveEndScript = item.getText(); + } else if (folder.getItem(i).getImage().equals(imageActiveEndScript)) { + folder.getItem(i).setImage(imageInactiveScript); + } + } + modifyScriptTree(item, SET_ACTIVE_ITEM); + }); + new MenuItem(cMenu, SWT.SEPARATOR); + MenuItem setRemoveScriptItem = new MenuItem(cMenu, SWT.PUSH); + setRemoveScriptItem.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.RemoveScriptType")); + setRemoveScriptItem.setImage(imageInactiveScript); + setRemoveScriptItem.addListener( + SWT.Selection, + e -> { + CTabItem item = folder.getSelection(); + input.setChanged(true); + if (item.getImage().equals(imageActiveScript)) { strActiveScript = ""; - } else if ( item.getImage().equals( imageActiveStartScript ) ) { + } else if (item.getImage().equals(imageActiveStartScript)) { strActiveStartScript = ""; - } else if ( item.getImage().equals( imageActiveEndScript ) ) { + } else if (item.getImage().equals(imageActiveEndScript)) { strActiveEndScript = ""; } - item.setImage( imageActiveEndScript ); - strActiveEndScript = item.getText(); - } else if ( folder.getItem( i ).getImage().equals( imageActiveEndScript ) ) { - folder.getItem( i ).setImage( imageInactiveScript ); - } - } - modifyScriptTree( item, SET_ACTIVE_ITEM ); - } ); - new MenuItem( cMenu, SWT.SEPARATOR ); - MenuItem setRemoveScriptItem = new MenuItem( cMenu, SWT.PUSH ); - setRemoveScriptItem.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.RemoveScriptType" ) ); - setRemoveScriptItem.setImage( imageInactiveScript ); - setRemoveScriptItem.addListener( SWT.Selection, e -> { - CTabItem item = folder.getSelection(); - input.setChanged( true ); - if ( item.getImage().equals( imageActiveScript ) ) { - strActiveScript = ""; - } else if ( item.getImage().equals( imageActiveStartScript ) ) { - strActiveStartScript = ""; - } else if ( item.getImage().equals( imageActiveEndScript ) ) { - strActiveEndScript = ""; - } - item.setImage( imageInactiveScript ); - } ); + item.setImage(imageInactiveScript); + }); - folder.setMenu( cMenu ); + folder.setMenu(cMenu); } private void buildingTreeMenu() { // styledTextPopupmenu = new Menu(, SWT.POP_UP); - MenuItem addDeleteItem = new MenuItem( tMenu, SWT.PUSH ); - addDeleteItem.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.Delete.Label" ) ); - addDeleteItem.setImage( guiresource.getImageDelete() ); - addDeleteItem.addListener( SWT.Selection, e -> { - if ( wTree.getSelectionCount() <= 0 ) { - return; - } + MenuItem addDeleteItem = new MenuItem(tMenu, SWT.PUSH); + addDeleteItem.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.Delete.Label")); + addDeleteItem.setImage(guiresource.getImageDelete()); + addDeleteItem.addListener( + SWT.Selection, + e -> { + if (wTree.getSelectionCount() <= 0) { + return; + } - TreeItem tItem = wTree.getSelection()[ 0 ]; - if ( tItem != null ) { - MessageBox messageBox = new MessageBox( shell, SWT.ICON_QUESTION | SWT.NO | SWT.YES ); - messageBox.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.DeleteItem.Label" ) ); - messageBox.setMessage( BaseMessages.getString( - PKG, "ScriptValuesDialogMod.ConfirmDeleteItem.Label", tItem.getText() ) ); - switch ( messageBox.open() ) { - case SWT.YES: - modifyCTabItem( tItem, DELETE_ITEM, "" ); - tItem.dispose(); - input.setChanged(); - break; - default: - break; - } - } - } ); - - MenuItem renItem = new MenuItem( tMenu, SWT.PUSH ); - renItem.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.Rename.Label" ) ); - renItem.addListener( SWT.Selection, e -> renameFunction( wTree.getSelection()[ 0 ] ) ); - - new MenuItem( tMenu, SWT.SEPARATOR ); - MenuItem helpItem = new MenuItem( tMenu, SWT.PUSH ); - helpItem.setText( BaseMessages.getString( PKG, "ScriptValuesDialogMod.Sample.Label" ) ); - helpItem.addListener( SWT.Selection, e -> { - String strFunctionName = wTree.getSelection()[ 0 ].getText(); - String strFunctionNameWithArgs = strFunctionName; - strFunctionName = strFunctionName.substring( 0, strFunctionName.indexOf( '(' ) ); - String strHelpTabName = strFunctionName + "_Sample"; - - if ( getCTabPosition( strHelpTabName ) == -1 ) { - addCtab( strHelpTabName, scVHelp.getSample( strFunctionName, strFunctionNameWithArgs ), 0 ); - } + TreeItem tItem = wTree.getSelection()[0]; + if (tItem != null) { + MessageBox messageBox = new MessageBox(shell, SWT.ICON_QUESTION | SWT.NO | SWT.YES); + messageBox.setText( + BaseMessages.getString(PKG, "ScriptValuesDialogMod.DeleteItem.Label")); + messageBox.setMessage( + BaseMessages.getString( + PKG, "ScriptValuesDialogMod.ConfirmDeleteItem.Label", tItem.getText())); + switch (messageBox.open()) { + case SWT.YES: + modifyCTabItem(tItem, DELETE_ITEM, ""); + tItem.dispose(); + input.setChanged(); + break; + default: + break; + } + } + }); + + MenuItem renItem = new MenuItem(tMenu, SWT.PUSH); + renItem.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.Rename.Label")); + renItem.addListener(SWT.Selection, e -> renameFunction(wTree.getSelection()[0])); + + new MenuItem(tMenu, SWT.SEPARATOR); + MenuItem helpItem = new MenuItem(tMenu, SWT.PUSH); + helpItem.setText(BaseMessages.getString(PKG, "ScriptValuesDialogMod.Sample.Label")); + helpItem.addListener( + SWT.Selection, + e -> { + String strFunctionName = wTree.getSelection()[0].getText(); + String strFunctionNameWithArgs = strFunctionName; + strFunctionName = strFunctionName.substring(0, strFunctionName.indexOf('(')); + String strHelpTabName = strFunctionName + "_Sample"; + + if (getCTabPosition(strHelpTabName) == -1) { + addCtab(strHelpTabName, scVHelp.getSample(strFunctionName, strFunctionNameWithArgs), 0); + } - if ( getCTabPosition( strHelpTabName ) != -1 ) { - setActiveCtab( strHelpTabName ); - } - } ); + if (getCTabPosition(strHelpTabName) != -1) { + setActiveCtab(strHelpTabName); + } + }); - wTree.addListener( SWT.MouseDown, e -> { - if ( wTree.getSelectionCount() <= 0 ) { - return; - } + wTree.addListener( + SWT.MouseDown, + e -> { + if (wTree.getSelectionCount() <= 0) { + return; + } - TreeItem tItem = wTree.getSelection()[ 0 ]; - if ( tItem != null ) { - TreeItem pItem = tItem.getParentItem(); + TreeItem tItem = wTree.getSelection()[0]; + if (tItem != null) { + TreeItem pItem = tItem.getParentItem(); - if ( pItem != null && pItem.equals( wTreeScriptsItem ) ) { - if ( folder.getItemCount() > 1 ) { - tMenu.getItem( 0 ).setEnabled( true ); - } else { - tMenu.getItem( 0 ).setEnabled( false ); + if (pItem != null && pItem.equals(wTreeScriptsItem)) { + if (folder.getItemCount() > 1) { + tMenu.getItem(0).setEnabled(true); + } else { + tMenu.getItem(0).setEnabled(false); + } + tMenu.getItem(1).setEnabled(true); + tMenu.getItem(3).setEnabled(false); + } else if (tItem.equals(wTreeClassesitem)) { + tMenu.getItem(0).setEnabled(false); + tMenu.getItem(1).setEnabled(false); + tMenu.getItem(3).setEnabled(false); + } else if (tItem.getData() != null && tItem.getData().equals("jsFunction")) { + tMenu.getItem(0).setEnabled(false); + tMenu.getItem(1).setEnabled(false); + tMenu.getItem(3).setEnabled(true); + } else { + tMenu.getItem(0).setEnabled(false); + tMenu.getItem(1).setEnabled(false); + tMenu.getItem(3).setEnabled(false); + } } - tMenu.getItem( 1 ).setEnabled( true ); - tMenu.getItem( 3 ).setEnabled( false ); - } else if ( tItem.equals( wTreeClassesitem ) ) { - tMenu.getItem( 0 ).setEnabled( false ); - tMenu.getItem( 1 ).setEnabled( false ); - tMenu.getItem( 3 ).setEnabled( false ); - } else if ( tItem.getData() != null && tItem.getData().equals( "jsFunction" ) ) { - tMenu.getItem( 0 ).setEnabled( false ); - tMenu.getItem( 1 ).setEnabled( false ); - tMenu.getItem( 3 ).setEnabled( true ); - } else { - tMenu.getItem( 0 ).setEnabled( false ); - tMenu.getItem( 1 ).setEnabled( false ); - tMenu.getItem( 3 ).setEnabled( false ); - } - } - } ); - wTree.setMenu( tMenu ); + }); + wTree.setMenu(tMenu); } private void addRenameTowTreeScriptItems() { - lastItem = new TreeItem[ 1 ]; - editor = new TreeEditor( wTree ); - wTree.addListener( SWT.Selection, event -> { - final TreeItem item = (TreeItem) event.item; - renameFunction( item ); - } ); + lastItem = new TreeItem[1]; + editor = new TreeEditor(wTree); + wTree.addListener( + SWT.Selection, + event -> { + final TreeItem item = (TreeItem) event.item; + renameFunction(item); + }); } // This function is for a Windows Like renaming inside the tree - private void renameFunction( TreeItem tItem ) { + private void renameFunction(TreeItem tItem) { final TreeItem item = tItem; - if ( item.getParentItem() != null && item.getParentItem().equals( wTreeScriptsItem ) ) { - if ( item != null && item == lastItem[ 0 ] ) { - boolean isCarbon = SWT.getPlatform().equals( "carbon" ); - final Composite composite = new Composite( wTree, SWT.NONE ); - if ( !isCarbon ) { - composite.setBackground( shell.getDisplay().getSystemColor( SWT.COLOR_BLACK ) ); + if (item.getParentItem() != null && item.getParentItem().equals(wTreeScriptsItem)) { + if (item != null && item == lastItem[0]) { + boolean isCarbon = SWT.getPlatform().equals("carbon"); + final Composite composite = new Composite(wTree, SWT.NONE); + if (!isCarbon) { + composite.setBackground(shell.getDisplay().getSystemColor(SWT.COLOR_BLACK)); } - final Text text = new Text( composite, SWT.NONE ); + final Text text = new Text(composite, SWT.NONE); final int inset = isCarbon ? 0 : 1; - composite.addListener( SWT.Resize, e -> { - Rectangle rect = composite.getClientArea(); - text.setBounds( rect.x + inset, rect.y + inset, rect.width - inset * 2, rect.height - inset * 2 ); - } ); - Listener textListener = e -> { - switch ( e.type ) { - case SWT.FocusOut: - if ( text.getText().length() > 0 ) { - // Check if the name Exists - if ( getCTabItemByName( text.getText() ) == null ) { - modifyCTabItem( item, RENAME_ITEM, text.getText() ); - item.setText( text.getText() ); - } - } - composite.dispose(); - break; - case SWT.Verify: - String newText = text.getText(); - String leftText = newText.substring( 0, e.start ); - String rightText = newText.substring( e.end, newText.length() ); - Point size = TextSizeUtilFacade.textExtent( leftText + e.text + rightText ); - size = text.computeSize( size.x, SWT.DEFAULT ); - editor.horizontalAlignment = SWT.LEFT; - Rectangle itemRect = item.getBounds(), - rect = wTree.getClientArea(); - editor.minimumWidth = Math.max( size.x, itemRect.width ) + inset * 2; - int left = itemRect.x, - right = rect.x + rect.width; - editor.minimumWidth = Math.min( editor.minimumWidth, right - left ); - editor.minimumHeight = size.y + inset * 2; - editor.layout(); - break; - case SWT.Traverse: - switch ( e.detail ) { - case SWT.TRAVERSE_RETURN: - if ( text.getText().length() > 0 ) { + composite.addListener( + SWT.Resize, + e -> { + Rectangle rect = composite.getClientArea(); + text.setBounds( + rect.x + inset, rect.y + inset, rect.width - inset * 2, rect.height - inset * 2); + }); + Listener textListener = + e -> { + switch (e.type) { + case SWT.FocusOut: + if (text.getText().length() > 0) { // Check if the name Exists - if ( getCTabItemByName( text.getText() ) == null ) { - modifyCTabItem( item, RENAME_ITEM, text.getText() ); - item.setText( text.getText() ); + if (getCTabItemByName(text.getText()) == null) { + modifyCTabItem(item, RENAME_ITEM, text.getText()); + item.setText(text.getText()); } } - break; - case SWT.TRAVERSE_ESCAPE: composite.dispose(); - e.doit = false; + break; + case SWT.Verify: + String newText = text.getText(); + String leftText = newText.substring(0, e.start); + String rightText = newText.substring(e.end, newText.length()); + Point size = TextSizeUtilFacade.textExtent(leftText + e.text + rightText); + size = text.computeSize(size.x, SWT.DEFAULT); + editor.horizontalAlignment = SWT.LEFT; + Rectangle itemRect = item.getBounds(), rect = wTree.getClientArea(); + editor.minimumWidth = Math.max(size.x, itemRect.width) + inset * 2; + int left = itemRect.x, right = rect.x + rect.width; + editor.minimumWidth = Math.min(editor.minimumWidth, right - left); + editor.minimumHeight = size.y + inset * 2; + editor.layout(); + break; + case SWT.Traverse: + switch (e.detail) { + case SWT.TRAVERSE_RETURN: + if (text.getText().length() > 0) { + // Check if the name Exists + if (getCTabItemByName(text.getText()) == null) { + modifyCTabItem(item, RENAME_ITEM, text.getText()); + item.setText(text.getText()); + } + } + break; + case SWT.TRAVERSE_ESCAPE: + composite.dispose(); + e.doit = false; + break; + default: + break; + } break; default: break; } - break; - default: - break; - } - }; - text.addListener( SWT.FocusOut, textListener ); - text.addListener( SWT.Traverse, textListener ); - text.addListener( SWT.Verify, textListener ); - editor.setEditor( composite, item ); - text.setText( item.getText() ); + }; + text.addListener(SWT.FocusOut, textListener); + text.addListener(SWT.Traverse, textListener); + text.addListener(SWT.Verify, textListener); + editor.setEditor(composite, item); + text.setText(item.getText()); text.selectAll(); text.setFocus(); - } } - lastItem[ 0 ] = item; + lastItem[0] = item; } // This could be useful for further improvements - public static ScriptNode parseVariables( Context cx, Scriptable scope, String source, String sourceName, - int lineno, Object securityDomain ) { + public static ScriptNode parseVariables( + Context cx, + Scriptable scope, + String source, + String sourceName, + int lineno, + Object securityDomain) { // Interpreter compiler = new Interpreter(); CompilerEnvirons evn = new CompilerEnvirons(); // evn.setLanguageVersion(Context.VERSION_1_5); - evn.setOptimizationLevel( -1 ); - evn.setGeneratingSource( true ); - evn.setGenerateDebugInfo( true ); - ErrorReporter errorReporter = new ToolErrorReporter( false ); - Parser p = new Parser( evn, errorReporter ); - ScriptNode tree = p.parse( source, "", 0 ); // IOException - new NodeTransformer().transform( tree, evn ); + evn.setOptimizationLevel(-1); + evn.setGeneratingSource(true); + evn.setGenerateDebugInfo(true); + ErrorReporter errorReporter = new ToolErrorReporter(false); + Parser p = new Parser(evn, errorReporter); + ScriptNode tree = p.parse(source, "", 0); // IOException + new NodeTransformer().transform(tree, evn); // Script result = (Script)compiler.compile(scope, evn, tree, p.getEncodedSource(),false, null); return tree; } diff --git a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutput/JsonOutputTest.java b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutput/JsonOutputTest.java index 9a6a465ca3..5b63e4416f 100644 --- a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutput/JsonOutputTest.java +++ b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutput/JsonOutputTest.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -41,6 +41,7 @@ import org.apache.hop.pipeline.transform.TransformMeta; import org.apache.hop.pipeline.transforms.dummy.DummyMeta; import org.apache.hop.pipeline.transforms.mock.TransformMockHelper; +import org.apache.hop.pipeline.transforms.rowgenerator.GeneratorField; import org.apache.hop.pipeline.transforms.rowgenerator.RowGeneratorMeta; import org.apache.hop.pipeline.transforms.TransformRowsCollector; import org.json.simple.JSONObject; @@ -49,6 +50,7 @@ import java.io.File; import java.io.Writer; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import static org.mockito.Matchers.any; @@ -63,9 +65,10 @@ /** * This class was a "copy and modification" of Hop's JsonOutputTests. - * - * @author Hendy Irawan Modified by Sean Flatley, removing dependency on external text file to hold - * expected results and modifying code to handle "Compatibility Mode". + * + * @author Hendy Irawan Modified by Sean Flatley, removing dependency on + * external text file to hold expected results and modifying code to handle "Compatibility + * Mode". */ public class JsonOutputTest extends TestCase { @@ -92,32 +95,31 @@ public class JsonOutputTest extends TestCase { /** * Creates a row generator transform for this class.. - * + * * @param name * @param registry * @return */ - private TransformMeta createRowGeneratorTransform( String name, PluginRegistry registry ) { + private TransformMeta createRowGeneratorTransform(String name, PluginRegistry registry) { // Default the name if it is empty - String testFileOutputName = ( Utils.isEmpty( name ) ? "generate rows" : name ); + String testFileOutputName = (Utils.isEmpty(name) ? "generate rows" : name); // create the RowGenerator and Transform Meta RowGeneratorMeta rowGeneratorMeta = new RowGeneratorMeta(); - String rowGeneratorPid = registry.getPluginId( TransformPluginType.class, rowGeneratorMeta ); - TransformMeta generateRowsTransform = new TransformMeta( rowGeneratorPid, testFileOutputName, rowGeneratorMeta ); + String rowGeneratorPid = registry.getPluginId(TransformPluginType.class, rowGeneratorMeta); + TransformMeta generateRowsTransform = + new TransformMeta(rowGeneratorPid, testFileOutputName, rowGeneratorMeta); // Set the field names, types and values - rowGeneratorMeta.setFieldName( new String[] { "Id", "State", "City" } ); - rowGeneratorMeta.setFieldType( new String[] { "Integer", "String", "String" } ); - rowGeneratorMeta.setValue( new String[] { "1", "Florida", "Orlando" } ); - rowGeneratorMeta.setFieldLength( new int[] { -1, -1, -1 } ); - rowGeneratorMeta.setFieldPrecision( new int[] { -1, -1, -1 } ); - rowGeneratorMeta.setGroup( new String[] { "", "", "" } ); - rowGeneratorMeta.setDecimal( new String[] { "", "", "" } ); - rowGeneratorMeta.setCurrency( new String[] { "", "", "" } ); - rowGeneratorMeta.setFieldFormat( new String[] { "", "", "" } ); - rowGeneratorMeta.setRowLimit( "10" ); + rowGeneratorMeta + .getFields() + .addAll( + Arrays.asList( + new GeneratorField("Id", "Integer", "", -1, -1, "", "", "", "1", false), + new GeneratorField("State", "String", "", -1, -1, "", "", "", "Florida", false), + new GeneratorField("City", "String", "", -1, -1, "", "", "", "Orlando", false))); + rowGeneratorMeta.setRowLimit("10"); // return the transform meta return generateRowsTransform; @@ -125,25 +127,25 @@ private TransformMeta createRowGeneratorTransform( String name, PluginRegistry r /** * Create a dummy transform for this class. - * + * * @param name * @param registry * @return */ - private TransformMeta createDummyTransform( String name, PluginRegistry registry ) { + private TransformMeta createDummyTransform(String name, PluginRegistry registry) { // Create a dummy transform 1 and add it to the tranMeta String dummyTransformName = "dummy transform"; DummyMeta dm1 = new DummyMeta(); - String dummyPid1 = registry.getPluginId( TransformPluginType.class, dm1 ); - TransformMeta dummyTransform = new TransformMeta( dummyPid1, dummyTransformName, dm1 ); + String dummyPid1 = registry.getPluginId(TransformPluginType.class, dm1); + TransformMeta dummyTransform = new TransformMeta(dummyPid1, dummyTransformName, dm1); return dummyTransform; } /** - * Create result data for test case 1. Each Object array in element in list should mirror the data written by the row - * generator created by the createRowGenerator method. - * + * Create result data for test case 1. Each Object array in element in list should mirror the data + * written by the row generator created by the createRowGenerator method. + * * @return list of metadata/data couples of how the result should look like. */ public List createResultData1() { @@ -151,41 +153,43 @@ public List createResultData1() { IRowMeta rowMetaInterface = createResultRowMeta(); - Object[] r1 = new Object[] { new Long( 1L ), "Orlando", "Florida" }; - Object[] r2 = new Object[] { new Long( 1L ), "Orlando", "Florida" }; - Object[] r3 = new Object[] { new Long( 1L ), "Orlando", "Florida" }; - Object[] r4 = new Object[] { new Long( 1L ), "Orlando", "Florida" }; - Object[] r5 = new Object[] { new Long( 1L ), "Orlando", "Florida" }; - Object[] r6 = new Object[] { new Long( 1L ), "Orlando", "Florida" }; - Object[] r7 = new Object[] { new Long( 1L ), "Orlando", "Florida" }; - Object[] r8 = new Object[] { new Long( 1L ), "Orlando", "Florida" }; - Object[] r9 = new Object[] { new Long( 1L ), "Orlando", "Florida" }; - Object[] r10 = new Object[] { new Long( 1L ), "Orlando", "Florida" }; - - list.add( new RowMetaAndData( rowMetaInterface, r1 ) ); - list.add( new RowMetaAndData( rowMetaInterface, r2 ) ); - list.add( new RowMetaAndData( rowMetaInterface, r3 ) ); - list.add( new RowMetaAndData( rowMetaInterface, r4 ) ); - list.add( new RowMetaAndData( rowMetaInterface, r5 ) ); - list.add( new RowMetaAndData( rowMetaInterface, r6 ) ); - list.add( new RowMetaAndData( rowMetaInterface, r7 ) ); - list.add( new RowMetaAndData( rowMetaInterface, r8 ) ); - list.add( new RowMetaAndData( rowMetaInterface, r9 ) ); - list.add( new RowMetaAndData( rowMetaInterface, r10 ) ); + Object[] r1 = new Object[] {new Long(1L), "Orlando", "Florida"}; + Object[] r2 = new Object[] {new Long(1L), "Orlando", "Florida"}; + Object[] r3 = new Object[] {new Long(1L), "Orlando", "Florida"}; + Object[] r4 = new Object[] {new Long(1L), "Orlando", "Florida"}; + Object[] r5 = new Object[] {new Long(1L), "Orlando", "Florida"}; + Object[] r6 = new Object[] {new Long(1L), "Orlando", "Florida"}; + Object[] r7 = new Object[] {new Long(1L), "Orlando", "Florida"}; + Object[] r8 = new Object[] {new Long(1L), "Orlando", "Florida"}; + Object[] r9 = new Object[] {new Long(1L), "Orlando", "Florida"}; + Object[] r10 = new Object[] {new Long(1L), "Orlando", "Florida"}; + + list.add(new RowMetaAndData(rowMetaInterface, r1)); + list.add(new RowMetaAndData(rowMetaInterface, r2)); + list.add(new RowMetaAndData(rowMetaInterface, r3)); + list.add(new RowMetaAndData(rowMetaInterface, r4)); + list.add(new RowMetaAndData(rowMetaInterface, r5)); + list.add(new RowMetaAndData(rowMetaInterface, r6)); + list.add(new RowMetaAndData(rowMetaInterface, r7)); + list.add(new RowMetaAndData(rowMetaInterface, r8)); + list.add(new RowMetaAndData(rowMetaInterface, r9)); + list.add(new RowMetaAndData(rowMetaInterface, r10)); return list; } /** * Creates a IRowMeta with a IValueMeta with the name "filename". - * + * * @return */ public IRowMeta createIRowMeta() { IRowMeta rowMetaInterface = new RowMeta(); - IValueMeta[] valuesMeta = { new ValueMetaString( "filename" ), }; - for ( int i = 0; i < valuesMeta.length; i++ ) { - rowMetaInterface.addValueMeta( valuesMeta[i] ); + IValueMeta[] valuesMeta = { + new ValueMetaString("filename"), + }; + for (int i = 0; i < valuesMeta.length; i++) { + rowMetaInterface.addValueMeta(valuesMeta[i]); } return rowMetaInterface; @@ -193,47 +197,50 @@ public IRowMeta createIRowMeta() { /** * Creates data... Will add more as I figure what the data is. - * + * * @return */ public List createData() { List list = new ArrayList<>(); IRowMeta rowMetaInterface = createIRowMeta(); Object[] r1 = new Object[] {}; - list.add( new RowMetaAndData( rowMetaInterface, r1 ) ); + list.add(new RowMetaAndData(rowMetaInterface, r1)); return list; } /** - * Creates a row meta interface for the fields that are defined by performing a getFields and by checking "Result - * filenames - Add filenames to result from "Text File Input" dialog. - * + * Creates a row meta interface for the fields that are defined by performing a getFields and by + * checking "Result filenames - Add filenames to result from "Text File Input" dialog. + * * @return */ public IRowMeta createResultRowMeta() { IRowMeta rowMetaInterface = new RowMeta(); - IValueMeta[] valuesMeta = - { new ValueMetaInteger( "Id" ), new ValueMetaString( "State" ), new ValueMetaString( "City" ) }; + IValueMeta[] valuesMeta = { + new ValueMetaInteger("Id"), new ValueMetaString("State"), new ValueMetaString("City") + }; - for ( int i = 0; i < valuesMeta.length; i++ ) { - rowMetaInterface.addValueMeta( valuesMeta[i] ); + for (int i = 0; i < valuesMeta.length; i++) { + rowMetaInterface.addValueMeta(valuesMeta[i]); } return rowMetaInterface; } - private TransformMeta createJsonOutputTransform( String name, String jsonFileName, PluginRegistry registry ) { + private TransformMeta createJsonOutputTransform( + String name, String jsonFileName, PluginRegistry registry) { // Create a Text File Output transform String testFileOutputName = name; JsonOutputMeta jsonOutputMeta = new JsonOutputMeta(); - String textFileInputPid = registry.getPluginId( TransformPluginType.class, jsonOutputMeta ); - TransformMeta jsonOutputTransform = new TransformMeta( textFileInputPid, testFileOutputName, jsonOutputMeta ); + String textFileInputPid = registry.getPluginId(TransformPluginType.class, jsonOutputMeta); + TransformMeta jsonOutputTransform = + new TransformMeta(textFileInputPid, testFileOutputName, jsonOutputMeta); // initialize the fields JsonOutputField[] fields = new JsonOutputField[3]; - for ( int idx = 0; idx < fields.length; idx++ ) { + for (int idx = 0; idx < fields.length; idx++) { fields[idx] = new JsonOutputField(); } @@ -241,18 +248,18 @@ private TransformMeta createJsonOutputTransform( String name, String jsonFileNam // it is important that the setPosition(int) // is invoked with the correct position as // we are testing the reading of a delimited file. - fields[0].setFieldName( "id" ); - fields[0].setElementName( "id" ); + fields[0].setFieldName("id"); + fields[0].setElementName("id"); - fields[1].setFieldName( "state" ); - fields[1].setElementName( "state" ); + fields[1].setFieldName("state"); + fields[1].setElementName("state"); - fields[2].setFieldName( "city" ); - fields[2].setElementName( "city" ); + fields[2].setFieldName("city"); + fields[2].setElementName("city"); // call this to allocate the number of fields - jsonOutputMeta.allocate( fields.length ); - jsonOutputMeta.setOutputFields( fields ); + jsonOutputMeta.allocate(fields.length); + jsonOutputMeta.setOutputFields(fields); // set meta properties- these were determined by running Spoon // and setting up the transformation we are setting up here. @@ -260,74 +267,80 @@ private TransformMeta createJsonOutputTransform( String name, String jsonFileNam // NPEs during the transformation. // We need a file name so we will generate a temp file - jsonOutputMeta.setOperationType( JsonOutputMeta.OPERATION_TYPE_WRITE_TO_FILE ); - jsonOutputMeta.setOutputValue( "data" ); - jsonOutputMeta.setFileName( jsonFileName ); - jsonOutputMeta.setExtension( "js" ); - jsonOutputMeta.setNrRowsInBloc( "0" ); // a single "data" contains an array of all records - jsonOutputMeta.setJsonBloc( "data" ); + jsonOutputMeta.setOperationType(JsonOutputMeta.OPERATION_TYPE_WRITE_TO_FILE); + jsonOutputMeta.setOutputValue("data"); + jsonOutputMeta.setFileName(jsonFileName); + jsonOutputMeta.setExtension("js"); + jsonOutputMeta.setNrRowsInBloc("0"); // a single "data" contains an array of all records + jsonOutputMeta.setJsonBloc("data"); return jsonOutputTransform; } - public String test( boolean compatibilityMode ) throws Exception { + public String test(boolean compatibilityMode) throws Exception { HopEnvironment.init(); // Create a new transformation... // PipelineMeta pipelineMeta = new PipelineMeta(); - pipelineMeta.setName( "testJsonOutput" ); + pipelineMeta.setName("testJsonOutput"); PluginRegistry registry = PluginRegistry.getInstance(); // create an injector transform String injectorTransformName = "injector transform"; - TransformMeta injectorTransform = TestUtilities.createInjectorTransform( injectorTransformName, registry ); - pipelineMeta.addTransform( injectorTransform ); + TransformMeta injectorTransform = + TestUtilities.createInjectorTransform(injectorTransformName, registry); + pipelineMeta.addTransform(injectorTransform); // create a row generator transform - TransformMeta rowGeneratorTransform = createRowGeneratorTransform( "Create rows for testJsonOutput1", registry ); - pipelineMeta.addTransform( rowGeneratorTransform ); + TransformMeta rowGeneratorTransform = + createRowGeneratorTransform("Create rows for testJsonOutput1", registry); + pipelineMeta.addTransform(rowGeneratorTransform); // create a PipelineHopMeta for injector and add it to the pipelineMeta - PipelineHopMeta hop_injectoryRowGenerator = new PipelineHopMeta( injectorTransform, rowGeneratorTransform ); - pipelineMeta.addPipelineHop( hop_injectoryRowGenerator ); + PipelineHopMeta hop_injectoryRowGenerator = + new PipelineHopMeta(injectorTransform, rowGeneratorTransform); + pipelineMeta.addPipelineHop(hop_injectoryRowGenerator); // create the json output transform // but first lets get a filename - String jsonFileName = TestUtilities.createEmptyTempFile( "testJsonOutput1_" ); - TransformMeta jsonOutputTransform = createJsonOutputTransform( "json output transform", jsonFileName, registry ); - ( (JsonOutputMeta) jsonOutputTransform.getTransform() ).setCompatibilityMode( compatibilityMode ); - pipelineMeta.addTransform( jsonOutputTransform ); + String jsonFileName = TestUtilities.createEmptyTempFile("testJsonOutput1_"); + TransformMeta jsonOutputTransform = + createJsonOutputTransform("json output transform", jsonFileName, registry); + ((JsonOutputMeta) jsonOutputTransform.getTransform()).setCompatibilityMode(compatibilityMode); + pipelineMeta.addTransform(jsonOutputTransform); // create a PipelineHopMeta for jsonOutputTransform and add it to the pipelineMeta - PipelineHopMeta hop_RowGeneratorOutputTextFile = new PipelineHopMeta( rowGeneratorTransform, jsonOutputTransform ); - pipelineMeta.addPipelineHop( hop_RowGeneratorOutputTextFile ); + PipelineHopMeta hop_RowGeneratorOutputTextFile = + new PipelineHopMeta(rowGeneratorTransform, jsonOutputTransform); + pipelineMeta.addPipelineHop(hop_RowGeneratorOutputTextFile); // Create a dummy transform and add it to the tranMeta String dummyTransformName = "dummy transform"; - TransformMeta dummyTransform = createDummyTransform( dummyTransformName, registry ); - pipelineMeta.addTransform( dummyTransform ); + TransformMeta dummyTransform = createDummyTransform(dummyTransformName, registry); + pipelineMeta.addTransform(dummyTransform); // create a PipelineHopMeta for the - PipelineHopMeta hopOutputJson_dummyTransform = new PipelineHopMeta( jsonOutputTransform, dummyTransform ); - pipelineMeta.addPipelineHop( hopOutputJson_dummyTransform ); + PipelineHopMeta hopOutputJson_dummyTransform = + new PipelineHopMeta(jsonOutputTransform, dummyTransform); + pipelineMeta.addPipelineHop(hopOutputJson_dummyTransform); // Now execute the transformation... - Pipeline pipeline = new LocalPipelineEngine( pipelineMeta ); + Pipeline pipeline = new LocalPipelineEngine(pipelineMeta); pipeline.prepareExecution(); // Create a row collector and add it to the dummy transform interface - IEngineComponent dummyITransform = pipeline.findComponent( dummyTransformName, 0 ); + IEngineComponent dummyITransform = pipeline.findComponent(dummyTransformName, 0); TransformRowsCollector dummyRowCollector = new TransformRowsCollector(); - dummyITransform.addRowListener( dummyRowCollector ); + dummyITransform.addRowListener(dummyRowCollector); // RowProducer rowProducer = pipeline.addRowProducer(injectorTransformName, 0); pipeline.startThreads(); pipeline.waitUntilFinished(); // get the results and return it - File outputFile = new File( jsonFileName + ".js" ); - String jsonStructure = FileUtils.readFileToString( outputFile ); + File outputFile = new File(jsonFileName + ".js"); + String jsonStructure = FileUtils.readFileToString(outputFile); return jsonStructure; } @@ -335,91 +348,108 @@ public String test( boolean compatibilityMode ) throws Exception { // The actual tests public void testNonCompatibilityMode() throws Exception { - String jsonStructure = test( false ); - Assert.assertTrue( jsonEquals( EXPECTED_NON_COMPATIBILITY_JSON, jsonStructure ) ); + String jsonStructure = test(false); + Assert.assertTrue(jsonEquals(EXPECTED_NON_COMPATIBILITY_JSON, jsonStructure)); } public void testCompatibilityMode() throws Exception { - String jsonStructure = test( true ); - Assert.assertEquals( EXPECTED_COMPATIBILITY_MODE_JSON, jsonStructure ); + String jsonStructure = test(true); + Assert.assertEquals(EXPECTED_COMPATIBILITY_MODE_JSON, jsonStructure); } public void testNpeIsNotThrownOnNullInput() throws Exception { TransformMockHelper mockHelper = - new TransformMockHelper<>( "jsonOutput", JsonOutputMeta.class, JsonOutputData.class ); - when( mockHelper.logChannelFactory.create( any(), any( ILoggingObject.class ) ) ).thenReturn( - mockHelper.iLogChannel ); - when( mockHelper.pipeline.isRunning() ).thenReturn( true ); - when( mockHelper.transformMeta.getTransform() ).thenReturn( new JsonOutputMeta() ); + new TransformMockHelper<>("jsonOutput", JsonOutputMeta.class, JsonOutputData.class); + when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class))) + .thenReturn(mockHelper.iLogChannel); + when(mockHelper.pipeline.isRunning()).thenReturn(true); + when(mockHelper.transformMeta.getTransform()).thenReturn(new JsonOutputMeta()); JsonOutput transform = - new JsonOutput( mockHelper.transformMeta, mockHelper.iTransformMeta, mockHelper.iTransformData, 0, mockHelper.pipelineMeta, mockHelper.pipeline ); - transform = spy( transform ); + new JsonOutput( + mockHelper.transformMeta, + mockHelper.iTransformMeta, + mockHelper.iTransformData, + 0, + mockHelper.pipelineMeta, + mockHelper.pipeline); + transform = spy(transform); - doReturn( null ).when( transform ).getRow(); + doReturn(null).when(transform).getRow(); transform.processRow(); } public void testEmptyDoesntWriteToFile() throws Exception { TransformMockHelper mockHelper = - new TransformMockHelper<>( "jsonOutput", JsonOutputMeta.class, JsonOutputData.class ); - when( mockHelper.logChannelFactory.create( any(), any( ILoggingObject.class ) ) ).thenReturn( - mockHelper.iLogChannel ); - when( mockHelper.pipeline.isRunning() ).thenReturn( true ); - when( mockHelper.transformMeta.getTransform() ).thenReturn( new JsonOutputMeta() ); + new TransformMockHelper<>("jsonOutput", JsonOutputMeta.class, JsonOutputData.class); + when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class))) + .thenReturn(mockHelper.iLogChannel); + when(mockHelper.pipeline.isRunning()).thenReturn(true); + when(mockHelper.transformMeta.getTransform()).thenReturn(new JsonOutputMeta()); JsonOutputData transformData = new JsonOutputData(); transformData.writeToFile = true; JsonOutput transform = - new JsonOutput( mockHelper.transformMeta, mockHelper.iTransformMeta, transformData, 0, mockHelper.pipelineMeta, mockHelper.pipeline ); - transform = spy( transform ); - - doReturn( null ).when( transform ).getRow(); - doReturn( true ).when( transform ).openNewFile(); - doReturn( true ).when( transform ).closeFile(); + new JsonOutput( + mockHelper.transformMeta, + mockHelper.iTransformMeta, + transformData, + 0, + mockHelper.pipelineMeta, + mockHelper.pipeline); + transform = spy(transform); + + doReturn(null).when(transform).getRow(); + doReturn(true).when(transform).openNewFile(); + doReturn(true).when(transform).closeFile(); transform.processRow(); - verify( transform, times( 0 ) ).openNewFile(); - verify( transform, times( 0 ) ).closeFile(); + verify(transform, times(0)).openNewFile(); + verify(transform, times(0)).closeFile(); } - @SuppressWarnings( "unchecked" ) + @SuppressWarnings("unchecked") public void testWriteToFile() throws Exception { TransformMockHelper mockHelper = - new TransformMockHelper<>( "jsonOutput", JsonOutputMeta.class, JsonOutputData.class ); - when( mockHelper.logChannelFactory.create( any(), any( ILoggingObject.class ) ) ).thenReturn( - mockHelper.iLogChannel ); - when( mockHelper.pipeline.isRunning() ).thenReturn( true ); - when( mockHelper.transformMeta.getTransform() ).thenReturn( new JsonOutputMeta() ); + new TransformMockHelper<>("jsonOutput", JsonOutputMeta.class, JsonOutputData.class); + when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class))) + .thenReturn(mockHelper.iLogChannel); + when(mockHelper.pipeline.isRunning()).thenReturn(true); + when(mockHelper.transformMeta.getTransform()).thenReturn(new JsonOutputMeta()); JsonOutputData transformData = new JsonOutputData(); transformData.writeToFile = true; JSONObject jsonObject = new JSONObject(); - jsonObject.put( "key", "value" ); - transformData.ja.add( jsonObject ); - transformData.writer = mock( Writer.class ); - - JsonOutput transform = new JsonOutput( mockHelper.transformMeta, mockHelper.iTransformMeta, transformData, 0, mockHelper.pipelineMeta, mockHelper.pipeline ); - transform = spy( transform ); + jsonObject.put("key", "value"); + transformData.ja.add(jsonObject); + transformData.writer = mock(Writer.class); - doReturn( null ).when( transform ).getRow(); - doReturn( true ).when( transform ).openNewFile(); - doReturn( true ).when( transform ).closeFile(); - doNothing().when( transformData.writer ).write( anyString() ); + JsonOutput transform = + new JsonOutput( + mockHelper.transformMeta, + mockHelper.iTransformMeta, + transformData, + 0, + mockHelper.pipelineMeta, + mockHelper.pipeline); + transform = spy(transform); + + doReturn(null).when(transform).getRow(); + doReturn(true).when(transform).openNewFile(); + doReturn(true).when(transform).closeFile(); + doNothing().when(transformData.writer).write(anyString()); transform.processRow(); - verify( transform ).openNewFile(); - verify( transform ).closeFile(); + verify(transform).openNewFile(); + verify(transform).closeFile(); } - /** - * compare json (deep equals ignoring order) - */ - protected boolean jsonEquals( String json1, String json2 ) throws Exception { + /** compare json (deep equals ignoring order) */ + protected boolean jsonEquals(String json1, String json2) throws Exception { ObjectMapper om = new ObjectMapper(); - JsonNode parsedJson1 = om.readTree( json1 ); - JsonNode parsedJson2 = om.readTree( json2 ); - return parsedJson1.equals( parsedJson2 ); + JsonNode parsedJson1 = om.readTree(json1); + JsonNode parsedJson2 = om.readTree(json2); + return parsedJson1.equals(parsedJson2); } } From 64b76a3abe066bc8c156b21e271d8b8873b3511f Mon Sep 17 00:00:00 2001 From: Matt Casters Date: Mon, 10 May 2021 21:18:07 +0200 Subject: [PATCH 6/6] HOP-2858 : Allow metadata injection through metadata property annotations (Extra IT MDI test for CSV Input) --- .../mdi/0004-csv-input-child.hpl | 83 +++ .../mdi/0004-csv-input-parent.hpl | 555 ++++++++++++++++++ .../mdi/datasets/golden-csv-input-parent.csv | 101 ++++ integration-tests/mdi/files/customers-100.txt | 101 ++++ integration-tests/mdi/main-0004-csv-input.hwf | 61 ++ .../dataset/golden-csv-input-parent.json | 88 +++ .../unit-test/0004-csv-input-parent UNIT.json | 64 ++ 7 files changed, 1053 insertions(+) create mode 100644 integration-tests/mdi/0004-csv-input-child.hpl create mode 100644 integration-tests/mdi/0004-csv-input-parent.hpl create mode 100644 integration-tests/mdi/datasets/golden-csv-input-parent.csv create mode 100644 integration-tests/mdi/files/customers-100.txt create mode 100644 integration-tests/mdi/main-0004-csv-input.hwf create mode 100644 integration-tests/mdi/metadata/dataset/golden-csv-input-parent.json create mode 100644 integration-tests/mdi/metadata/unit-test/0004-csv-input-parent UNIT.json diff --git a/integration-tests/mdi/0004-csv-input-child.hpl b/integration-tests/mdi/0004-csv-input-child.hpl new file mode 100644 index 0000000000..92ba7c0cbc --- /dev/null +++ b/integration-tests/mdi/0004-csv-input-child.hpl @@ -0,0 +1,83 @@ + + + + 0004-csv-input-child + Y + + + + Normal + + + N + 1000 + 100 + - + 2021/05/10 10:52:28.332 + - + 2021/05/10 10:52:28.332 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + CSV file input + Output + Y + + + + CSV file input + CSVInput + + Y + + 1 + + none + + + + + + N + + +
N
+ 50000 + N + N + N + N + + + + + + 160 + 128 + + + + Output + Dummy + + Y + + 1 + + none + + + + + 336 + 128 + + + + + + diff --git a/integration-tests/mdi/0004-csv-input-parent.hpl b/integration-tests/mdi/0004-csv-input-parent.hpl new file mode 100644 index 0000000000..c0524f8f8c --- /dev/null +++ b/integration-tests/mdi/0004-csv-input-parent.hpl @@ -0,0 +1,555 @@ + + + + 0004-csv-input-parent + Y + + + + Normal + + + N + 1000 + 100 + - + 2021/05/10 10:50:11.779 + - + 2021/05/10 10:50:11.779 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + CSV Input fields + 0004-csv-input-child.hpl + Y + + + 0004-csv-input-child.hpl + Verify + Y + + + CSV Input metadata + 0004-csv-input-child.hpl + Y + + + + 0004-csv-input-child.hpl + MetaInject + + Y + + 1 + + none + + + ${PROJECT_HOME}/0004-csv-input-child.hpl + Output + + + id + Integer + 15 + 0 + + + name + String + 50 + -1 + + + firstname + String + 50 + -1 + + + zip + String + 40 + 0 + + + city + String + 50 + -1 + + + birthdate + Date + -1 + -1 + + + street + String + 11 + -1 + + + housenr + String + 20 + 0 + + + stateCode + String + 10 + -1 + + + state + String + 50 + -1 + + + + N + + + + + CSV file input + INPUT_CURRENCY + Y + CSV Input fields + currency + + + CSV file input + INPUT_DECIMAL + Y + CSV Input fields + decimal + + + CSV file input + DELIMITER + N + CSV Input metadata + separator + + + CSV file input + HEADER_PRESENT + N + CSV Input metadata + header + + + CSV file input + INPUT_LENGTH + Y + CSV Input fields + length + + + CSV file input + ENCLOSURE + N + CSV Input metadata + enclosure + + + CSV file input + INPUT_PRECISION + Y + CSV Input fields + precision + + + CSV file input + INPUT_FORMAT + Y + CSV Input fields + format + + + CSV file input + LAZY_CONVERSION + N + CSV Input metadata + lazy + + + CSV file input + INPUT_GROUP + Y + CSV Input fields + group + + + CSV file input + FIELD_TRIM_TYPE + Y + CSV Input fields + trimType + + + CSV file input + FILENAME + N + CSV Input metadata + filename + + + CSV file input + FIELD_TYPE + Y + CSV Input fields + type + + + CSV file input + INPUT_NAME + Y + CSV Input fields + name + + + + + 304 + 96 + + + + CSV Input fields + DataGrid + + Y + + 1 + + none + + + + + name + String + + + + + -1 + -1 + N + + + type + + + + + + -1 + -1 + N + + + format + String + + + + + -1 + -1 + N + + + length + String + + + + + -1 + -1 + N + + + precision + String + + + + + -1 + -1 + N + + + currency + String + + + + + -1 + -1 + N + + + decimal + String + + + + + -1 + -1 + N + + + group + String + + + + + -1 + -1 + N + + + trimType + String + + + + + -1 + -1 + N + + + + + id + Integer + # + 15 + 0 + $ + . + , + none + + + name + String + + 50 + + $ + . + , + none + + + firstname + String + + 50 + + $ + . + , + none + + + zip + String + + 40 + 0 + $ + . + , + none + + + city + String + + 50 + + $ + . + , + none + + + birthdate + Date + yyyy/MM/dd + + + $ + . + , + none + + + street + String + + 11 + + $ + . + , + none + + + housenr + String + + 20 + 0 + $ + . + , + none + + + stateCode + String + + 10 + + $ + . + , + none + + + state + String + + 50 + + $ + . + , + none + + + + + 128 + 96 + + + + Verify + Dummy + + Y + + 1 + + none + + + + + 464 + 96 + + + + CSV Input metadata + GetVariable + + Y + + 1 + + none + + + + + filename + ${PROJECT_HOME}/files/customers-100.txt + String + + + + + -1 + -1 + none + + + separator + ; + String + + + + + -1 + -1 + none + + + enclosure + " + String + + + + + -1 + -1 + none + + + lazy + false + String + + + + + -1 + -1 + none + + + header + true + String + + + + + -1 + -1 + none + + + + + 128 + 192 + + + + + + diff --git a/integration-tests/mdi/datasets/golden-csv-input-parent.csv b/integration-tests/mdi/datasets/golden-csv-input-parent.csv new file mode 100644 index 0000000000..220a5951b2 --- /dev/null +++ b/integration-tests/mdi/datasets/golden-csv-input-parent.csv @@ -0,0 +1,101 @@ +id,name,firstname,zip,city,birthdate,street,housenr,stateCode,state +" 1",jwcdf-name,fsj-firstname," 13520",oem-city,1954/02/07,amrb-street," 145",AK,ALASKA +" 2",flhxu-name,tum-firstname," 17520",buo-city,1966/04/24,wfyz-street," 96",GA,GEORGIA +" 3",xthfg-name,gfe-firstname," 12560",vtz-city,1990/01/11,doxx-street," 46",NJ,NEW JERSEY +" 4",ulzrz-name,bnl-firstname," 11620",prz-city,1966/08/02,bxqn-street," 104",NY,NEW YORK +" 5",oxhyr-name,onx-firstname," 15180",bpn-city,1970/11/14,pksn-street," 133",IN,INDIANA +" 6",fiqjz-name,sce-firstname," 16020",fnn-city,1954/09/24,wbhg-street," 35",MD,MARYLAND +" 7",tkiat-name,xti-firstname," 12720",stt-city,1966/08/11,tvnf-street," 21",PA,PENNSYLVANIA +" 8",kljcz-name,uqd-firstname," 13340",ntt-city,1987/01/15,jyje-street," 10",PW,PALAU +" 9",pgunz-name,hcm-firstname," 16680",gxh-city,1970/11/08,shbe-street," 184",NC,NORTH CAROLINA +" 10",oyjha-name,uhj-firstname," 18880",uyg-city,1966/04/10,bjgw-street," 176",AR,ARKANSAS +" 11",igxbd-name,uph-firstname," 13480",ndh-city,1962/12/03,jdcd-street," 151",NH,NEW HAMPSHIRE +" 12",vnaov-name,wha-firstname," 13120",egm-city,1954/03/28,hpep-street," 20",CA,CALIFORNIA +" 13",dauuz-name,hwg-firstname," 13740",khn-city,1958/05/15,etqx-street," 5",OK,OKLAHOMA +" 14",gkuuo-name,kkb-firstname," 13560",xdt-city,1962/04/07,sdoj-street," 35",MT,MONTANA +" 15",wdhze-name,jjk-firstname," 16900",due-city,1970/07/17,pmmu-street," 174",AS,AMERICAN SAMOA +" 16",ncayz-name,ynb-firstname," 15720",lxj-city,1974/04/27,mdtb-street," 109",MA,MASSACHUSETTS +" 17",rdjin-name,hhu-firstname," 14480",lpc-city,1958/11/16,wxik-street," 145",KY,KENTUCKY +" 18",nxzij-name,bdl-firstname," 10740",avx-city,1958/02/20,nybz-street," 138",WI,WISCONSIN +" 19",xgrzc-name,dxw-firstname," 18900",vpq-city,1990/11/16,wzjh-street," 58",ME,MAINE +" 20",ehgrn-name,vbe-firstname," 17500",cik-city,1978/05/21,ucnw-street," 135",MD,MARYLAND +" 21",gctjx-name,upx-firstname," 11960",yqr-city,1958/03/03,rlko-street," 141",TN,TENNESSEE +" 22",ptzmg-name,hva-firstname," 15740",gux-city,1978/05/04,pugy-street," 122",VI,VIRGIN ISLANDS +" 23",eyeti-name,gnw-firstname," 17420",eko-city,1962/10/26,ylph-street," 61",NC,NORTH CAROLINA +" 24",wccwo-name,zpj-firstname," 16600",uim-city,1962/09/29,ygih-street," 26",WA,WASHINGTON +" 25",bwkoe-name,ayl-firstname," 18660",rtw-city,1978/07/16,mzww-street," 179",CA,CALIFORNIA +" 26",rezku-name,zio-firstname," 19080",nvt-city,1982/07/14,wwkd-street," 91",CA,CALIFORNIA +" 27",mjlsk-name,ecx-firstname," 10800",yxu-city,1950/12/11,vttb-street," 195",MO,MISSOURI +" 28",wdjsi-name,aoq-firstname," 13660",smo-city,1954/02/01,kako-street," 7",NV,NEVADA +" 29",mwfnd-name,nyb-firstname," 19760",bbu-city,1986/09/23,apdi-street," 91",MS,MISSISSIPPI +" 30",vtuoz-name,jhh-firstname," 17620",vad-city,1982/05/05,kzup-street," 79",GA,GEORGIA +" 31",rhhxk-name,ndr-firstname," 16760",fub-city,1978/11/12,regd-street," 55",OK,OKLAHOMA +" 32",lpstk-name,mqz-firstname," 18940",tnr-city,1982/09/16,cdhf-street," 4",SD,SOUTH DAKOTA +" 33",ldhyr-name,yts-firstname," 12000",auk-city,1986/11/14,abph-street," 147",IN,INDIANA +" 34",cjdml-name,iti-firstname," 16900",wkq-city,1970/06/05,npow-street," 96",NH,NEW HAMPSHIRE +" 35",cpenz-name,sbi-firstname," 16380",ssl-city,1962/08/19,kilz-street," 44",MS,MISSISSIPPI +" 36",rxtbg-name,anr-firstname," 14720",bqc-city,1958/08/10,pudg-street," 140",NV,NEVADA +" 37",udblf-name,raa-firstname," 11500",wli-city,1978/12/13,xomd-street," 41",PW,PALAU +" 38",vvyce-name,gep-firstname," 13740",gtd-city,1982/05/23,kwbv-street," 123",undefined,undefined +" 39",kwfnz-name,ucu-firstname," 10580",sns-city,1978/08/18,nnun-street," 20",OK,OKLAHOMA +" 40",zxydx-name,tml-firstname," 14680",jda-city,1974/05/29,wfjn-street," 157",DC,DISTRICT OF COLUMBIA +" 41",bfscx-name,jnl-firstname," 16920",yyg-city,1970/11/30,cgfh-street," 178",CO,COLORADO +" 42",qitur-name,yra-firstname," 15560",ijp-city,1978/01/30,fonc-street," 155",AK,ALASKA +" 43",msixi-name,ynb-firstname," 12720",ksl-city,1958/07/17,zpjw-street," 46",VI,VIRGIN ISLANDS +" 44",wzkjq-name,rgh-firstname," 19000",hkm-city,1974/08/12,yixf-street," 134",CA,CALIFORNIA +" 45",dqfmf-name,yxr-firstname," 13840",vie-city,1962/10/23,stvx-street," 39",TX,TEXAS +" 46",biluz-name,uqe-firstname," 17760",wkq-city,1962/07/27,embn-street," 183",PW,PALAU +" 47",wahfx-name,zwd-firstname," 13240",vic-city,1974/03/27,axpw-street," 131",UT,UTAH +" 48",denwt-name,bta-firstname," 17300",hhj-city,1986/12/20,orwy-street," 11",WV,WEST VIRGINIA +" 49",akdmy-name,ybz-firstname," 14560",wtx-city,1962/11/08,nwba-street," 123",MP,NORTHERN MARIANA ISLANDS +" 50",hqafg-name,nht-firstname," 16080",gfu-city,1951/01/12,spsq-street," 45",LA,LOUISIANA +" 51",zhmbl-name,lnw-firstname," 17460",hse-city,1986/12/21,scis-street," 97",GA,GEORGIA +" 52",snwnj-name,jyy-firstname," 16400",hsz-city,1966/02/15,imhl-street," 42",NC,NORTH CAROLINA +" 53",fuyla-name,mmp-firstname," 11840",hgu-city,1986/08/16,ixiz-street," 145",NC,NORTH CAROLINA +" 54",yvfqz-name,prz-firstname," 11260",wjl-city,1982/05/06,fbzd-street," 97",MO,MISSOURI +" 55",usbgq-name,vhd-firstname," 14080",dsb-city,1958/04/01,ggoc-street," 54",KS,KANSAS +" 56",yaeni-name,zpy-firstname," 19100",sen-city,1954/12/10,sbsw-street," 158",HI,HAWAII +" 57",fgxvr-name,vzi-firstname," 17520",lcf-city,1958/11/01,nbdv-street," 10",GU,GUAM +" 58",tqpbq-name,rwr-firstname," 19140",zpd-city,1978/08/23,npvb-street," 190",DC,DISTRICT OF COLUMBIA +" 59",ieigg-name,ayq-firstname," 12960",ljc-city,1962/07/05,dnjz-street," 163",FL,FLORIDA +" 60",rfvzu-name,edm-firstname," 13340",kvz-city,1954/12/08,eijd-street," 4",RI,RHODE ISLAND +" 61",pduwm-name,gqb-firstname," 14240",cyr-city,1954/07/03,ndux-street," 13",SD,SOUTH DAKOTA +" 62",yyixf-name,yzt-firstname," 18020",lwx-city,1974/01/29,iede-street," 120",NV,NEVADA +" 63",dkszq-name,ytd-firstname," 14700",zwh-city,1979/01/11,nbjz-street," 65",AS,AMERICAN SAMOA +" 64",slkzv-name,zbg-firstname," 19880",oee-city,1978/11/01,sphg-street," 119",OK,OKLAHOMA +" 65",nvxim-name,phc-firstname," 19220",vgg-city,1991/01/24,juok-street," 106",FM,FEDERATED STATES OF MICRONESIA +" 66",piyfg-name,xtn-firstname," 13760",nde-city,1954/07/22,vfrv-street," 11",NY,NEW YORK +" 67",jnusz-name,mjw-firstname," 12640",nwb-city,1986/08/23,kcsa-street," 138",VA,VIRGINIA +" 68",jnypj-name,ioq-firstname," 17000",zqy-city,1986/01/09,croe-street," 119",PW,PALAU +" 69",uohts-name,btx-firstname," 13480",dal-city,1990/10/22,llyw-street," 150",WA,WASHINGTON +" 70",aavpj-name,pvw-firstname," 13780",lai-city,1954/09/23,nygu-street," 171",FL,FLORIDA +" 71",nbjcj-name,rsf-firstname," 12000",kjl-city,1986/06/30,ijsb-street," 123",ID,IDAHO +" 72",syjxh-name,gkq-firstname," 19960",rmd-city,1978/10/26,qmyp-street," 161",MN,MINNESOTA +" 73",vkojz-name,ryo-firstname," 14300",bmz-city,1954/09/11,gcpj-street," 71",ND,NORTH DAKOTA +" 74",pqzfw-name,kld-firstname," 16400",qvq-city,1962/09/09,dhbv-street," 92",ND,NORTH DAKOTA +" 75",owvjk-name,fez-firstname," 19740",ldb-city,1978/06/14,kabf-street," 87",VA,VIRGINIA +" 76",qsfih-name,ixe-firstname," 16860",qvr-city,1987/01/07,qean-street," 159",CO,COLORADO +" 77",slixq-name,gmb-firstname," 19980",ftt-city,1982/06/22,xinx-street," 111",VT,VERMONT +" 78",eegsa-name,xlc-firstname," 12680",byk-city,1954/04/23,beul-street," 56",MD,MARYLAND +" 79",phevp-name,ihs-firstname," 16120",adc-city,1978/04/25,voig-street," 98",NM,NEW MEXICO +" 80",njfoe-name,tag-firstname," 16580",tnr-city,1966/12/04,dhky-street," 108",LA,LOUISIANA +" 81",bdncx-name,hcd-firstname," 11260",xcl-city,1970/07/02,jvlp-street," 49",GA,GEORGIA +" 82",ikedo-name,tks-firstname," 17460",odl-city,1958/08/25,iaaq-street," 8",GU,GUAM +" 83",iafxy-name,vur-firstname," 11480",hgt-city,1962/08/03,hmec-street," 164",TX,TEXAS +" 84",lafhf-name,ssz-firstname," 19560",wwp-city,1951/01/25,mxmq-street," 96",IN,INDIANA +" 85",okyny-name,hbu-firstname," 16800",yok-city,1978/03/28,ipjz-street," 135",NV,NEVADA +" 86",hznby-name,fwy-firstname," 13680",wbi-city,1970/07/25,mxui-street," 170",CT,CONNECTICUT +" 87",ztpoa-name,rzk-firstname," 18500",qum-city,1970/07/26,blqr-street," 152",ME,MAINE +" 88",gitxz-name,axt-firstname," 11800",fck-city,1974/01/12,tmjw-street," 189",SD,SOUTH DAKOTA +" 89",ziomm-name,mcv-firstname," 12940",iwq-city,1950/10/22,hqgj-street," 140",DC,DISTRICT OF COLUMBIA +" 90",otncg-name,tuy-firstname," 16540",ulk-city,1971/01/24,yuia-street," 166",TX,TEXAS +" 91",cnabb-name,hoq-firstname," 16300",tuw-city,1962/06/17,ujvv-street," 61",ME,MAINE +" 92",ucogf-name,ggc-firstname," 14500",fsj-city,1978/02/08,asfi-street," 53",WV,WEST VIRGINIA +" 93",lbpmf-name,sdt-firstname," 10780",ewj-city,1978/03/08,hxsp-street," 102",NV,NEVADA +" 94",tieqq-name,uyu-firstname," 17740",wea-city,1966/10/31,abpl-street," 187",MO,MISSOURI +" 95",fsgwf-name,vjd-firstname," 12460",ads-city,1970/11/29,yeou-street," 10",MA,MASSACHUSETTS +" 96",reeba-name,kzs-firstname," 13100",zhc-city,1966/07/08,abmv-street," 88",FL,FLORIDA +" 97",shybc-name,gcp-firstname," 10660",ahg-city,1950/12/15,hrqy-street," 174",KS,KANSAS +" 98",phszr-name,sst-firstname," 13080",ydd-city,1954/09/23,quqn-street," 2",RI,RHODE ISLAND +" 99",jteco-name,fxc-firstname," 19760",agr-city,1986/05/06,dzxc-street," 108",MD,MARYLAND +" 100",qvaar-name,icx-firstname," 16120",boc-city,1978/08/04,bfzf-street," 12",NM,NEW MEXICO diff --git a/integration-tests/mdi/files/customers-100.txt b/integration-tests/mdi/files/customers-100.txt new file mode 100644 index 0000000000..8b8bd04955 --- /dev/null +++ b/integration-tests/mdi/files/customers-100.txt @@ -0,0 +1,101 @@ +id;name;firstname;zip;city;birthdate;street;housenr;stateCode;state + 1;jwcdf-name;fsj-firstname; 13520;oem-city;1954/02/07;amrb-street; 145;AK;ALASKA + 2;flhxu-name;tum-firstname; 17520;buo-city;1966/04/24;wfyz-street; 96;GA;GEORGIA + 3;xthfg-name;gfe-firstname; 12560;vtz-city;1990/01/11;doxx-street; 46;NJ;NEW JERSEY + 4;ulzrz-name;bnl-firstname; 11620;prz-city;1966/08/02;bxqn-street; 104;NY;NEW YORK + 5;oxhyr-name;onx-firstname; 15180;bpn-city;1970/11/14;pksn-street; 133;IN;INDIANA + 6;fiqjz-name;sce-firstname; 16020;fnn-city;1954/09/24;wbhg-street; 35;MD;MARYLAND + 7;tkiat-name;xti-firstname; 12720;stt-city;1966/08/11;tvnf-street; 21;PA;PENNSYLVANIA + 8;kljcz-name;uqd-firstname; 13340;ntt-city;1987/01/15;jyje-street; 10;PW;PALAU + 9;pgunz-name;hcm-firstname; 16680;gxh-city;1970/11/08;shbe-street; 184;NC;NORTH CAROLINA + 10;oyjha-name;uhj-firstname; 18880;uyg-city;1966/04/10;bjgw-street; 176;AR;ARKANSAS + 11;igxbd-name;uph-firstname; 13480;ndh-city;1962/12/03;jdcd-street; 151;NH;NEW HAMPSHIRE + 12;vnaov-name;wha-firstname; 13120;egm-city;1954/03/28;hpep-street; 20;CA;CALIFORNIA + 13;dauuz-name;hwg-firstname; 13740;khn-city;1958/05/15;etqx-street; 5;OK;OKLAHOMA + 14;gkuuo-name;kkb-firstname; 13560;xdt-city;1962/04/07;sdoj-street; 35;MT;MONTANA + 15;wdhze-name;jjk-firstname; 16900;due-city;1970/07/17;pmmu-street; 174;AS;AMERICAN SAMOA + 16;ncayz-name;ynb-firstname; 15720;lxj-city;1974/04/27;mdtb-street; 109;MA;MASSACHUSETTS + 17;rdjin-name;hhu-firstname; 14480;lpc-city;1958/11/16;wxik-street; 145;KY;KENTUCKY + 18;nxzij-name;bdl-firstname; 10740;avx-city;1958/02/20;nybz-street; 138;WI;WISCONSIN + 19;xgrzc-name;dxw-firstname; 18900;vpq-city;1990/11/16;wzjh-street; 58;ME;MAINE + 20;ehgrn-name;vbe-firstname; 17500;cik-city;1978/05/21;ucnw-street; 135;MD;MARYLAND + 21;gctjx-name;upx-firstname; 11960;yqr-city;1958/03/03;rlko-street; 141;TN;TENNESSEE + 22;ptzmg-name;hva-firstname; 15740;gux-city;1978/05/04;pugy-street; 122;VI;VIRGIN ISLANDS + 23;eyeti-name;gnw-firstname; 17420;eko-city;1962/10/26;ylph-street; 61;NC;NORTH CAROLINA + 24;wccwo-name;zpj-firstname; 16600;uim-city;1962/09/29;ygih-street; 26;WA;WASHINGTON + 25;bwkoe-name;ayl-firstname; 18660;rtw-city;1978/07/16;mzww-street; 179;CA;CALIFORNIA + 26;rezku-name;zio-firstname; 19080;nvt-city;1982/07/14;wwkd-street; 91;CA;CALIFORNIA + 27;mjlsk-name;ecx-firstname; 10800;yxu-city;1950/12/11;vttb-street; 195;MO;MISSOURI + 28;wdjsi-name;aoq-firstname; 13660;smo-city;1954/02/01;kako-street; 7;NV;NEVADA + 29;mwfnd-name;nyb-firstname; 19760;bbu-city;1986/09/23;apdi-street; 91;MS;MISSISSIPPI + 30;vtuoz-name;jhh-firstname; 17620;vad-city;1982/05/05;kzup-street; 79;GA;GEORGIA + 31;rhhxk-name;ndr-firstname; 16760;fub-city;1978/11/12;regd-street; 55;OK;OKLAHOMA + 32;lpstk-name;mqz-firstname; 18940;tnr-city;1982/09/16;cdhf-street; 4;SD;SOUTH DAKOTA + 33;ldhyr-name;yts-firstname; 12000;auk-city;1986/11/14;abph-street; 147;IN;INDIANA + 34;cjdml-name;iti-firstname; 16900;wkq-city;1970/06/05;npow-street; 96;NH;NEW HAMPSHIRE + 35;cpenz-name;sbi-firstname; 16380;ssl-city;1962/08/19;kilz-street; 44;MS;MISSISSIPPI + 36;rxtbg-name;anr-firstname; 14720;bqc-city;1958/08/10;pudg-street; 140;NV;NEVADA + 37;udblf-name;raa-firstname; 11500;wli-city;1978/12/13;xomd-street; 41;PW;PALAU + 38;vvyce-name;gep-firstname; 13740;gtd-city;1982/05/23;kwbv-street; 123;undefined;undefined + 39;kwfnz-name;ucu-firstname; 10580;sns-city;1978/08/18;nnun-street; 20;OK;OKLAHOMA + 40;zxydx-name;tml-firstname; 14680;jda-city;1974/05/29;wfjn-street; 157;DC;DISTRICT OF COLUMBIA + 41;bfscx-name;jnl-firstname; 16920;yyg-city;1970/11/30;cgfh-street; 178;CO;COLORADO + 42;qitur-name;yra-firstname; 15560;ijp-city;1978/01/30;fonc-street; 155;AK;ALASKA + 43;msixi-name;ynb-firstname; 12720;ksl-city;1958/07/17;zpjw-street; 46;VI;VIRGIN ISLANDS + 44;wzkjq-name;rgh-firstname; 19000;hkm-city;1974/08/12;yixf-street; 134;CA;CALIFORNIA + 45;dqfmf-name;yxr-firstname; 13840;vie-city;1962/10/23;stvx-street; 39;TX;TEXAS + 46;biluz-name;uqe-firstname; 17760;wkq-city;1962/07/27;embn-street; 183;PW;PALAU + 47;wahfx-name;zwd-firstname; 13240;vic-city;1974/03/27;axpw-street; 131;UT;UTAH + 48;denwt-name;bta-firstname; 17300;hhj-city;1986/12/20;orwy-street; 11;WV;WEST VIRGINIA + 49;akdmy-name;ybz-firstname; 14560;wtx-city;1962/11/08;nwba-street; 123;MP;NORTHERN MARIANA ISLANDS + 50;hqafg-name;nht-firstname; 16080;gfu-city;1951/01/12;spsq-street; 45;LA;LOUISIANA + 51;zhmbl-name;lnw-firstname; 17460;hse-city;1986/12/21;scis-street; 97;GA;GEORGIA + 52;snwnj-name;jyy-firstname; 16400;hsz-city;1966/02/15;imhl-street; 42;NC;NORTH CAROLINA + 53;fuyla-name;mmp-firstname; 11840;hgu-city;1986/08/16;ixiz-street; 145;NC;NORTH CAROLINA + 54;yvfqz-name;prz-firstname; 11260;wjl-city;1982/05/06;fbzd-street; 97;MO;MISSOURI + 55;usbgq-name;vhd-firstname; 14080;dsb-city;1958/04/01;ggoc-street; 54;KS;KANSAS + 56;yaeni-name;zpy-firstname; 19100;sen-city;1954/12/10;sbsw-street; 158;HI;HAWAII + 57;fgxvr-name;vzi-firstname; 17520;lcf-city;1958/11/01;nbdv-street; 10;GU;GUAM + 58;tqpbq-name;rwr-firstname; 19140;zpd-city;1978/08/23;npvb-street; 190;DC;DISTRICT OF COLUMBIA + 59;ieigg-name;ayq-firstname; 12960;ljc-city;1962/07/05;dnjz-street; 163;FL;FLORIDA + 60;rfvzu-name;edm-firstname; 13340;kvz-city;1954/12/08;eijd-street; 4;RI;RHODE ISLAND + 61;pduwm-name;gqb-firstname; 14240;cyr-city;1954/07/03;ndux-street; 13;SD;SOUTH DAKOTA + 62;yyixf-name;yzt-firstname; 18020;lwx-city;1974/01/29;iede-street; 120;NV;NEVADA + 63;dkszq-name;ytd-firstname; 14700;zwh-city;1979/01/11;nbjz-street; 65;AS;AMERICAN SAMOA + 64;slkzv-name;zbg-firstname; 19880;oee-city;1978/11/01;sphg-street; 119;OK;OKLAHOMA + 65;nvxim-name;phc-firstname; 19220;vgg-city;1991/01/24;juok-street; 106;FM;FEDERATED STATES OF MICRONESIA + 66;piyfg-name;xtn-firstname; 13760;nde-city;1954/07/22;vfrv-street; 11;NY;NEW YORK + 67;jnusz-name;mjw-firstname; 12640;nwb-city;1986/08/23;kcsa-street; 138;VA;VIRGINIA + 68;jnypj-name;ioq-firstname; 17000;zqy-city;1986/01/09;croe-street; 119;PW;PALAU + 69;uohts-name;btx-firstname; 13480;dal-city;1990/10/22;llyw-street; 150;WA;WASHINGTON + 70;aavpj-name;pvw-firstname; 13780;lai-city;1954/09/23;nygu-street; 171;FL;FLORIDA + 71;nbjcj-name;rsf-firstname; 12000;kjl-city;1986/06/30;ijsb-street; 123;ID;IDAHO + 72;syjxh-name;gkq-firstname; 19960;rmd-city;1978/10/26;qmyp-street; 161;MN;MINNESOTA + 73;vkojz-name;ryo-firstname; 14300;bmz-city;1954/09/11;gcpj-street; 71;ND;NORTH DAKOTA + 74;pqzfw-name;kld-firstname; 16400;qvq-city;1962/09/09;dhbv-street; 92;ND;NORTH DAKOTA + 75;owvjk-name;fez-firstname; 19740;ldb-city;1978/06/14;kabf-street; 87;VA;VIRGINIA + 76;qsfih-name;ixe-firstname; 16860;qvr-city;1987/01/07;qean-street; 159;CO;COLORADO + 77;slixq-name;gmb-firstname; 19980;ftt-city;1982/06/22;xinx-street; 111;VT;VERMONT + 78;eegsa-name;xlc-firstname; 12680;byk-city;1954/04/23;beul-street; 56;MD;MARYLAND + 79;phevp-name;ihs-firstname; 16120;adc-city;1978/04/25;voig-street; 98;NM;NEW MEXICO + 80;njfoe-name;tag-firstname; 16580;tnr-city;1966/12/04;dhky-street; 108;LA;LOUISIANA + 81;bdncx-name;hcd-firstname; 11260;xcl-city;1970/07/02;jvlp-street; 49;GA;GEORGIA + 82;ikedo-name;tks-firstname; 17460;odl-city;1958/08/25;iaaq-street; 8;GU;GUAM + 83;iafxy-name;vur-firstname; 11480;hgt-city;1962/08/03;hmec-street; 164;TX;TEXAS + 84;lafhf-name;ssz-firstname; 19560;wwp-city;1951/01/25;mxmq-street; 96;IN;INDIANA + 85;okyny-name;hbu-firstname; 16800;yok-city;1978/03/28;ipjz-street; 135;NV;NEVADA + 86;hznby-name;fwy-firstname; 13680;wbi-city;1970/07/25;mxui-street; 170;CT;CONNECTICUT + 87;ztpoa-name;rzk-firstname; 18500;qum-city;1970/07/26;blqr-street; 152;ME;MAINE + 88;gitxz-name;axt-firstname; 11800;fck-city;1974/01/12;tmjw-street; 189;SD;SOUTH DAKOTA + 89;ziomm-name;mcv-firstname; 12940;iwq-city;1950/10/22;hqgj-street; 140;DC;DISTRICT OF COLUMBIA + 90;otncg-name;tuy-firstname; 16540;ulk-city;1971/01/24;yuia-street; 166;TX;TEXAS + 91;cnabb-name;hoq-firstname; 16300;tuw-city;1962/06/17;ujvv-street; 61;ME;MAINE + 92;ucogf-name;ggc-firstname; 14500;fsj-city;1978/02/08;asfi-street; 53;WV;WEST VIRGINIA + 93;lbpmf-name;sdt-firstname; 10780;ewj-city;1978/03/08;hxsp-street; 102;NV;NEVADA + 94;tieqq-name;uyu-firstname; 17740;wea-city;1966/10/31;abpl-street; 187;MO;MISSOURI + 95;fsgwf-name;vjd-firstname; 12460;ads-city;1970/11/29;yeou-street; 10;MA;MASSACHUSETTS + 96;reeba-name;kzs-firstname; 13100;zhc-city;1966/07/08;abmv-street; 88;FL;FLORIDA + 97;shybc-name;gcp-firstname; 10660;ahg-city;1950/12/15;hrqy-street; 174;KS;KANSAS + 98;phszr-name;sst-firstname; 13080;ydd-city;1954/09/23;quqn-street; 2;RI;RHODE ISLAND + 99;jteco-name;fxc-firstname; 19760;agr-city;1986/05/06;dzxc-street; 108;MD;MARYLAND + 100;qvaar-name;icx-firstname; 16120;boc-city;1978/08/04;bfzf-street; 12;NM;NEW MEXICO diff --git a/integration-tests/mdi/main-0004-csv-input.hwf b/integration-tests/mdi/main-0004-csv-input.hwf new file mode 100644 index 0000000000..a612fb96ef --- /dev/null +++ b/integration-tests/mdi/main-0004-csv-input.hwf @@ -0,0 +1,61 @@ + + + main-0004-csv-input + Y + + + + - + 2021/05/10 10:49:19.324 + - + 2021/05/10 10:49:19.324 + + + + + Start + + SPECIAL + + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + 112 + 96 + + + + Run Pipeline Unit Tests + + RunPipelineTests + + + + 0004-csv-input-parent UNIT + + + N + 320 + 96 + + + + + + Start + Run Pipeline Unit Tests + Y + Y + Y + + + + + + diff --git a/integration-tests/mdi/metadata/dataset/golden-csv-input-parent.json b/integration-tests/mdi/metadata/dataset/golden-csv-input-parent.json new file mode 100644 index 0000000000..2e2f71d7e8 --- /dev/null +++ b/integration-tests/mdi/metadata/dataset/golden-csv-input-parent.json @@ -0,0 +1,88 @@ +{ + "base_filename": "golden-csv-input-parent.csv", + "name": "golden-csv-input-parent", + "description": "", + "dataset_fields": [ + { + "field_comment": "", + "field_length": 15, + "field_type": 5, + "field_precision": 0, + "field_format": " #", + "field_name": "id" + }, + { + "field_comment": "", + "field_length": 50, + "field_type": 2, + "field_precision": -1, + "field_format": "", + "field_name": "name" + }, + { + "field_comment": "", + "field_length": 50, + "field_type": 2, + "field_precision": -1, + "field_format": "", + "field_name": "firstname" + }, + { + "field_comment": "", + "field_length": 40, + "field_type": 2, + "field_precision": -1, + "field_format": "", + "field_name": "zip" + }, + { + "field_comment": "", + "field_length": 50, + "field_type": 2, + "field_precision": -1, + "field_format": "", + "field_name": "city" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 3, + "field_precision": -1, + "field_format": "yyyy/MM/dd", + "field_name": "birthdate" + }, + { + "field_comment": "", + "field_length": 11, + "field_type": 2, + "field_precision": -1, + "field_format": "", + "field_name": "street" + }, + { + "field_comment": "", + "field_length": 20, + "field_type": 2, + "field_precision": -1, + "field_format": "", + "field_name": "housenr" + }, + { + "field_comment": "", + "field_length": 10, + "field_type": 2, + "field_precision": -1, + "field_format": "", + "field_name": "stateCode" + }, + { + "field_comment": "", + "field_length": 50, + "field_type": 2, + "field_precision": -1, + "field_format": "", + "field_name": "state" + } + ], + "folder_name": "" +} \ No newline at end of file diff --git a/integration-tests/mdi/metadata/unit-test/0004-csv-input-parent UNIT.json b/integration-tests/mdi/metadata/unit-test/0004-csv-input-parent UNIT.json new file mode 100644 index 0000000000..fe2a220c3b --- /dev/null +++ b/integration-tests/mdi/metadata/unit-test/0004-csv-input-parent UNIT.json @@ -0,0 +1,64 @@ +{ + "variableValues": [], + "database_replacements": [], + "autoOpening": true, + "basePath": "", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "birthdate", + "data_set_field": "birthdate" + }, + { + "transform_field": "city", + "data_set_field": "city" + }, + { + "transform_field": "firstname", + "data_set_field": "firstname" + }, + { + "transform_field": "housenr", + "data_set_field": "housenr" + }, + { + "transform_field": "id", + "data_set_field": "id" + }, + { + "transform_field": "name", + "data_set_field": "name" + }, + { + "transform_field": "state", + "data_set_field": "state" + }, + { + "transform_field": "stateCode", + "data_set_field": "stateCode" + }, + { + "transform_field": "street", + "data_set_field": "street" + }, + { + "transform_field": "zip", + "data_set_field": "zip" + } + ], + "field_order": [ + "id" + ], + "transform_name": "Verify", + "data_set_name": "golden-csv-input-parent" + } + ], + "input_data_sets": [], + "name": "0004-csv-input-parent UNIT", + "description": "", + "trans_test_tweaks": [], + "persist_filename": "", + "pipeline_filename": "./0004-csv-input-parent.hpl", + "test_type": "UNIT_TEST" +} \ No newline at end of file