Skip to content

Commit

Permalink
Merge pull request #4266 from tmcsantos/BACKLOG-17632
Browse files Browse the repository at this point in the history
[Backlog-17632] Decouple of append step from Kettle engine
  • Loading branch information
Steve Maring committed Aug 10, 2017
2 parents ae8f6bd + f87077d commit bd281ca
Show file tree
Hide file tree
Showing 30 changed files with 202 additions and 27 deletions.
7 changes: 7 additions & 0 deletions assemblies/pdi-ce/pom.xml
Expand Up @@ -82,6 +82,7 @@
<!-- plugins -->
<pdi-abort-plugin.version>${project.version}</pdi-abort-plugin.version>
<pdi-autodoc-plugin.version>${project.version}</pdi-autodoc-plugin.version>
<pdi-append-plugin.version>${project.version}</pdi-append-plugin.version>
<kettle-version-checker.version>${project.version}</kettle-version-checker.version>
<kettle-dummy-plugin.version>${project.version}</kettle-dummy-plugin.version>
<kettle-s3csvinput-plugin.version>${project.version}</kettle-s3csvinput-plugin.version>
Expand Down Expand Up @@ -785,6 +786,12 @@
<version>${pdi-autodoc-plugin.version}</version>
<type>zip</type>
</artifactItem>
<artifactItem>
<groupId>org.pentaho.di.plugins</groupId>
<artifactId>pdi-append-plugin</artifactId>
<version>${pdi-append-plugin.version}</version>
<type>zip</type>
</artifactItem>
<artifactItem>
<groupId>org.pentaho.di.plugins</groupId>
<artifactId>kettle-version-checker</artifactId>
Expand Down
1 change: 0 additions & 1 deletion engine/src/main/resources/kettle-steps.xml
@@ -1,7 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<steps>
<step id="Dummy"> <description>i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.Dummy</description> <classname>org.pentaho.di.trans.steps.dummytrans.DummyTransMeta</classname> <category>i18n:org.pentaho.di.trans.step:BaseStep.Category.Flow</category> <tooltip>i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.Dummy</tooltip> <iconfile>ui/images/DUM.svg</iconfile> <documentation_url>http://wiki.pentaho.com/display/EAI/Dummy+%28do+nothing%29</documentation_url> <cases_url/> <forum_url/> </step>
<step id="Append"> <description>i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.Append</description> <classname>org.pentaho.di.trans.steps.append.AppendMeta</classname> <category>i18n:org.pentaho.di.trans.step:BaseStep.Category.Flow</category> <tooltip>i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.Append</tooltip> <iconfile>ui/images/APP.svg</iconfile> <documentation_url>http://wiki.pentaho.com/display/EAI/Append+streams</documentation_url> <cases_url/> <forum_url/> </step>
<step id="TableInput"> <description>i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.TableInput</description> <classname>org.pentaho.di.trans.steps.tableinput.TableInputMeta</classname> <category>i18n:org.pentaho.di.trans.step:BaseStep.Category.Input</category> <tooltip>i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.TableInput</tooltip> <iconfile>ui/images/TIP.svg</iconfile> <documentation_url>http://wiki.pentaho.com/display/EAI/Table+Input</documentation_url> <cases_url>http://jira.pentaho.com/browse/PDI</cases_url> <forum_url>http://forums.pentaho.com/forumdisplay.php?135-Pentaho-Data-Integration-Kettle</forum_url> </step>
<step id="SocketReader"> <description>i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.SocketReader</description> <classname>org.pentaho.di.trans.steps.socketreader.SocketReaderMeta</classname> <category>i18n:org.pentaho.di.trans.step:BaseStep.Category.Inline</category> <tooltip>i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.SocketReader</tooltip> <iconfile>ui/images/SKR.svg</iconfile> <documentation_url>http://wiki.pentaho.com/display/EAI/Socket+reader</documentation_url> <cases_url/> <forum_url/> </step>
<step id="SocketWriter"> <description>i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.SocketWriter</description> <classname>org.pentaho.di.trans.steps.socketwriter.SocketWriterMeta</classname> <category>i18n:org.pentaho.di.trans.step:BaseStep.Category.Inline</category> <tooltip>i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.SocketWriter</tooltip> <iconfile>ui/images/SKW.svg</iconfile> <documentation_url>http://wiki.pentaho.com/display/EAI/Socket+writer</documentation_url> <cases_url/> <forum_url/> </step>
Expand Down
Expand Up @@ -155,7 +155,6 @@ BaseStep.Log.MaxPercentageRejectedReached=The maximum percentage of rejected row
BaseStep.TypeTooltipDesc.MergeRows=Merge two streams of rows, sorted on a certain key. The two streams are compared and the equals, changed, deleted and new rows are flagged.
BaseStep.TypeTooltipDesc.StringCut=Strings cut (substring).
BaseStep.TypeTooltipDesc.Groupby=Builds aggregates in a group by fashion.\nThis works only on a sorted input.\nIf the input is not sorted, only double consecutive rows are handled correctly.
BaseStep.TypeTooltipDesc.Append=Append 2 streams in an ordered way
BaseStep.TypeTooltipDesc.Validator=Validates passing data based on a set of rules
BaseStepDialog.FailedToGetFieldsPrevious.DialogTitle=Error fields
BaseStep.Log.InputRowInfo=input rel is {0}\:{1}
Expand Down Expand Up @@ -269,7 +268,6 @@ BaseStep.SafeMode.Exception.DoubleFieldnames=The field with name [{0}] occurs tw
BaseStep.TypeLongDesc.PGBulkLoader=PostgreSQL Bulk Loader
BaseStep.TypeLongDesc.PropertyInput=Property Input
StepMeta.Exception.UnableToLoadStepInfo=Unable to load step info from XML step node
BaseStep.TypeLongDesc.Append=Append streams
BaseStep.TypeTooltipDesc.XSDValidator=Validate XML source (files or streams) against XML Schema Definition.
BaseStep.TypeLongDesc.ValueMapper=Value Mapper
BaseStep.TypeTooltipDesc.MergeJoin=Joins two streams on a given key and outputs a joined set. The input streams must be sorted on the join key
Expand Down
Expand Up @@ -202,7 +202,6 @@ BaseStep.Log.MaxPercentageRejectedReached=Le nombre maximal {0} % de lignes reje
BaseStep.TypeTooltipDesc.MergeRows=Comparer deux flux, tri\u00E9s suivant une certaine cl\u00E9.\n Les deux flux sont compar\u00E9s suivant certaines cl\u00E9s.\n Les nouvelles lignes,les lignes modifi\u00E9es et les lignes supprim\u00E9s sont marqu\u00E9es.
BaseStep.TypeTooltipDesc.StringCut=Extraction depuis des cha\u00EEnes de caract\u00E8res.
BaseStep.TypeTooltipDesc.Groupby=R\u00E9aliser des groupements.Fonctionne uniquement sur des champs pr\u00E9alablement tri\u00E9s.\nSi les lignes ne sont pas tri\u00E9es, les lignes identiques et cons\u00E9cutives seront trait\u00E9es correctement.
BaseStep.TypeTooltipDesc.Append=Permet de transmettre en priorit\u00E9 un flux.\n Les lignes du second flux seront transmises apr\u00E8s que la derni\u00E8re ligne\n du flux prioritaire le soit.
BaseStep.TypeTooltipDesc.Validator=Validation des champs de flux entrants gr\u00E2ce \u00E0 la d\u00E9finition de r\u00E8gles.
BaseStep.TypeTooltipDesc.LDAPOutput=Insertion, mise \u00E0 jour, ajout ou suppression d''\u00E9l\u00E9ments bas\u00E9s sur leur DN (Distinguished Name).
BaseStepDialog.FailedToGetFieldsPrevious.DialogTitle=Erreur r\u00E9cup\u00E9ration champs
Expand Down Expand Up @@ -346,7 +345,6 @@ BaseStep.TypeLongDesc.PGBulkLoader=Chargement en bloc vers PostgreSQL
BaseStep.TypeLongDesc.PropertyInput=Extraction depuis fichier propri\u00E9t\u00E9s
BaseStep.TypeLongDesc.RandomCCNumberGenerator=G\u00E9n\u00E9ration de num\u00E9ros de carte
StepMeta.Exception.UnableToLoadStepInfo=Impossible de charger les informations de l''\u00E9tape depuis la node XML
BaseStep.TypeLongDesc.Append=Prioritisation de deux flux
BaseStep.TypeTooltipDesc.GetTableNames=R\u00E9cup\u00E9ration de la liste des tables, vues\n, proc\u00E9dures stoqu\u00E9es et synonymes depuis une base de donn\u00E9es.
BaseStep.TypeTooltipDesc.XSDValidator=Valider une source XML (fichiers ou flux) par une d\u00E9finition de sch\u00E9ma XML.
BaseStep.TypeLongDesc.JobExecutor=Ex\u00E9cution de t\u00E2che
Expand Down
Expand Up @@ -165,7 +165,6 @@ BaseStep.Log.MaxPercentageRejectedReached={0}\u306e\u62d2\u5426\u3055\u308c\u305
BaseStep.TypeTooltipDesc.MergeRows=Merge Rows (diff)\n\u30ad\u30fc\u3067\u30bd\u30fc\u30c8\u3055\u308c\u305f\u4e8c\u3064\u306e\u30b9\u30c8\u30ea\u30fc\u30e0\u306e\u884c\u3092\u30bd\u30fc\u30c8\u3057\u307e\u3059\u3002\u305d\u308c\u3089\u306e\u30b9\u30c8\u30ea\u30fc\u30e0\u306f\u6bd4\u8f03\u3001\u5909\u66f4\u3001\u524a\u9664\u3055\u308c\u3001\u65b0\u305f\u306a\u884c\u304c\u4f5c\u6210\u3055\u308c\u307e\u3059\u3002
BaseStep.TypeTooltipDesc.StringCut=Strings cut\n\u6587\u5b57\u5217\u3092\u30ab\u30c3\u30c8\u3057\u307e\u3059\u3002\uff08\u30b5\u30d6\u30b9\u30c8\u30ea\u30f3\u30b0\uff09
BaseStep.TypeTooltipDesc.Groupby=Group by\ngroup by\u306e\u5f62\u3067\u96c6\u5408\u3092\u5f62\u6210\u3057\u307e\u3059\u3002\n\u30bd\u30fc\u30c8\u3055\u308c\u305f\u5165\u529b\u306e\u307f\u4f7f\u7528\u3067\u304d\u307e\u3059\u3002\n\u5165\u529b\u304c\u30bd\u30fc\u30c8\u3055\u308c\u3066\u3044\u306a\u3044\u5834\u5408\u306f, \u9023\u7d9a\u3057\u305f\u884c\u306e\u307f\u6b63\u3057\u304f\u51e6\u7406\u3055\u308c\u307e\u3059\u3002
BaseStep.TypeTooltipDesc.Append=Append streams\n2\u3064\u306e\u30b9\u30c8\u30ea\u30fc\u30e0\u3092\u7d50\u5408\u3057\u307e\u3059\u3002
BaseStep.TypeTooltipDesc.Validator=Data Validator\n\u30eb\u30fc\u30eb\u306b\u57fa\u3065\u304d\u3001\u53d7\u3051\u6e21\u3059\u30c7\u30fc\u30bf\u3092\u6709\u52b9\u5316\u3057\u307e\u3059\u3002
BaseStepDialog.FailedToGetFieldsPrevious.DialogTitle=\u30a8\u30e9\u30fc\u30d5\u30a3\u30fc\u30eb\u30c9
BaseStep.Log.InputRowInfo=\u5165\u529brel\u306f{0}\:{1}\u3067\u3059\u3002
Expand Down Expand Up @@ -287,7 +286,6 @@ BaseStep.TypeTooltipDesc.XMLInputSax=Streaming XML Input\n\u30b9\u30c8\u30ea\u30
BaseStep.TypeLongDesc.PGBulkLoader=PostgreSQL\u30d0\u30eb\u30af\u30ed\u30fc\u30c9
BaseStep.TypeLongDesc.PropertyInput=\u30d7\u30ed\u30d1\u30c6\u30a3\u30fc\u30d5\u30a1\u30a4\u30eb\u5165\u529b
StepMeta.Exception.UnableToLoadStepInfo=\u30b9\u30c6\u30c3\u30d7\u304c\u8aad\u307f\u53d6\u308a\u3067\u304d\u307e\u305b\u3093\u3002
BaseStep.TypeLongDesc.Append=\u30b9\u30c8\u30ea\u30fc\u30e0\u7d50\u5408
BaseStep.TypeTooltipDesc.XSDValidator=XSD Validator\nXML\u30b9\u30ad\u30fc\u30de\u5b9a\u7fa9\u306b\u5bfe\u3057\u3066XML\u30bd\u30fc\u30b9\uff08\u30d5\u30a1\u30a4\u30eb\u3001\u307e\u305f\u306f\u30b9\u30c8\u30ea\u30fc\u30e0\uff09\u3092\u6709\u52b9\u5316\u3057\u307e\u3059\u3002
BaseStep.TypeLongDesc.ValueMapper=\u5024\u30de\u30c3\u30d4\u30f3\u30b0
BaseStep.TypeTooltipDesc.MergeJoin=Merge Join\n\u4e0e\u3048\u3089\u308c\u305f\u30ad\u30fc\u3067\u4e8c\u3064\u306e\u30b9\u30c8\u30ea\u30fc\u30e0\u3092\u7d50\u5408\u3057\u307e\u3059\u3002 \u5165\u529b\u30b9\u30c8\u30ea\u30fc\u30e0\u306f\u30bd\u30fc\u30c8\u3055\u308c\u3066\u3044\u308b\u5fc5\u8981\u304c\u3042\u308a\u307e\u3059\u3002
Expand Down
Expand Up @@ -199,7 +199,6 @@ BaseStep.TypeLongDesc.AccessInput=Access \u8f93\u5165
BaseStep.TypeLongDesc.AccessOutput=Access \u8f93\u51fa
BaseStep.TypeLongDesc.AddXML=\u589e\u52a0XML\u5217
BaseStep.TypeLongDesc.AnalyticQuery=\u5206\u6790\u67e5\u8be2
BaseStep.TypeLongDesc.Append=\u8ffd\u52a0\u6d41
BaseStep.TypeLongDesc.BeanShell=BeanShell \u8f93\u5165
BaseStep.TypeLongDesc.BlockUntilStepsFinish=\u963b\u585e\u6570\u636e\u76f4\u5230\u6b65\u9aa4\u90fd\u5b8c\u6210
BaseStep.TypeLongDesc.BlockingStep=\u963b\u585e\u6570\u636e
Expand Down
24 changes: 24 additions & 0 deletions plugins/core/append/assemblies/plugin/pom.xml
@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>pdi-append-assemblies</artifactId>
<groupId>org.pentaho.di.plugins</groupId>
<version>8.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>pdi-append-plugin</artifactId>
<name>PDI Append Step Plugin Distribution</name>
<packaging>pom</packaging>

<dependencies>
<dependency>
<groupId>org.pentaho.di.plugins</groupId>
<artifactId>pdi-append-plugin-impl</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>

</project>
26 changes: 26 additions & 0 deletions plugins/core/append/assemblies/plugin/src/assembly/assembly.xml
@@ -0,0 +1,26 @@
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
<id>plugin</id>
<formats>
<format>zip</format>
</formats>
<baseDirectory>pdi-append-plugin</baseDirectory>

<files>
<file>
<source>${project.basedir}/src/main/resources/version.xml</source>
<outputDirectory>.</outputDirectory>
<filtered>true</filtered>
</file>
</files>

<dependencySets>
<dependencySet>
<outputDirectory>.</outputDirectory>
<includes>
<include>org.pentaho.di.plugins:pdi-append-plugin-impl:jar</include>
</includes>
<useProjectArtifact>false</useProjectArtifact>
</dependencySet>
</dependencySets>
</assembly>
@@ -0,0 +1,2 @@
<?xml version="1.0" encoding="UTF-8"?>
<version branch='TRUNK'>${project.version}</version>
20 changes: 20 additions & 0 deletions plugins/core/append/assemblies/pom.xml
@@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>append</artifactId>
<groupId>org.pentaho.di.plugins</groupId>
<version>8.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>pdi-append-assemblies</artifactId>
<name>PDI Append Step Plugin Assemblies</name>
<packaging>pom</packaging>

<modules>
<module>plugin</module>
</modules>

</project>
66 changes: 66 additions & 0 deletions plugins/core/append/impl/pom.xml
@@ -0,0 +1,66 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>append</artifactId>
<groupId>org.pentaho.di.plugins</groupId>
<version>8.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>pdi-append-plugin-impl</artifactId>
<name>PDI Append Step Plugin Impl</name>

<properties>
<kettle.version>8.0-SNAPSHOT</kettle.version>
</properties>

<dependencies>
<dependency>
<groupId>pentaho-kettle</groupId>
<artifactId>kettle-core</artifactId>
<version>${kettle.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>pentaho-kettle</groupId>
<artifactId>kettle-engine</artifactId>
<version>${kettle.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>pentaho-kettle</groupId>
<artifactId>kettle-ui-swt</artifactId>
<version>${kettle.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>pentaho</groupId>
<artifactId>metastore</artifactId>
<scope>provided</scope>
</dependency>

<!-- SWT it required to compile any version of any architecture will work -->
<dependency>
<groupId>org.eclipse.swt</groupId>
<artifactId>org.eclipse.swt.gtk.linux.x86_64</artifactId>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>pentaho-kettle</groupId>
<artifactId>kettle-engine</artifactId>
<version>${kettle.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
</dependencies>

</project>
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com
* Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
*
*******************************************************************************
*
Expand Down Expand Up @@ -151,9 +151,9 @@ public boolean init( StepMetaInterface smi, StepDataInterface sdi ) {
/**
* Checks whether 2 template rows are compatible for the mergestep.
*
* @param referenceRow
* @param referenceRowMeta
* Reference row
* @param compareRow
* @param compareRowMeta
* Row to compare to
*
* @return true when templates are compatible.
Expand Down
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com
* Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
*
*******************************************************************************
*
Expand All @@ -26,6 +26,7 @@

import org.pentaho.di.core.CheckResult;
import org.pentaho.di.core.CheckResultInterface;
import org.pentaho.di.core.annotations.Step;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleStepException;
Expand Down Expand Up @@ -59,6 +60,10 @@
* @author Sven Boden
* @since 3-june-2007
*/
@Step( id = "Append", image = "APP.svg", i18nPackageName = "org.pentaho.di.trans.steps.append",
name = "Append.Name", description = "Append.Description",
documentationUrl = "http://wiki.pentaho.com/display/EAI/Append+streams",
categoryDescription = "i18n:org.pentaho.di.trans.step:BaseStep.Category.Flow" )
@InjectionSupported( localizationPrefix = "AppendMeta.Injection." )
public class AppendMeta extends BaseStepMeta implements StepMetaInterface {
private static Class<?> PKG = Append.class; // for i18n purposes, needed by Translator2!!
Expand Down
@@ -0,0 +1,4 @@
#${project.name} build information

version=${project.version}
builddate=${maven.build.timestamp}
File renamed without changes
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Expand Up @@ -13,6 +13,8 @@ AppendDialog.TailHop.Label=Tail hop:
## AppendRows
##
#####################################################################
Append.Name=Append streams
Append.Description=Append 2 streams in an ordered way
AppendRows.LineNumber=linenr
AppendRows.Log.BothHopsAreNeeded=Both head and tail step need to be supplied.
Append.Exception.InvalidLayoutDetected=Invalid layout detected in input streams, rows to append have to be of identical structure
Expand Down

0 comments on commit bd281ca

Please sign in to comment.