Permalink
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
executable file 1465 lines (1464 sloc) 34.8 KB
<?xml version="1.0" encoding="UTF-8"?>
<transformation>
<info>
<name>csv api import</name>
<description />
<extended_description />
<trans_version />
<trans_type>Normal</trans_type>
<trans_status>0</trans_status>
<directory>/</directory>
<parameters>
<parameter>
<name>entity</name>
<default_value />
<description>CiviCRM API entity</description>
</parameter>
<parameter>
<name>filename</name>
<default_value />
<description>filename to import</description>
</parameter>
</parameters>
<log>
<trans-log-table>
<connection />
<schema />
<table />
<size_limit_lines />
<interval />
<timeout_days />
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>CHANNEL_ID</id>
<enabled>Y</enabled>
<name>CHANNEL_ID</name>
</field>
<field>
<id>TRANSNAME</id>
<enabled>Y</enabled>
<name>TRANSNAME</name>
</field>
<field>
<id>STATUS</id>
<enabled>Y</enabled>
<name>STATUS</name>
</field>
<field>
<id>LINES_READ</id>
<enabled>Y</enabled>
<name>LINES_READ</name>
<subject />
</field>
<field>
<id>LINES_WRITTEN</id>
<enabled>Y</enabled>
<name>LINES_WRITTEN</name>
<subject />
</field>
<field>
<id>LINES_UPDATED</id>
<enabled>Y</enabled>
<name>LINES_UPDATED</name>
<subject />
</field>
<field>
<id>LINES_INPUT</id>
<enabled>Y</enabled>
<name>LINES_INPUT</name>
<subject />
</field>
<field>
<id>LINES_OUTPUT</id>
<enabled>Y</enabled>
<name>LINES_OUTPUT</name>
<subject />
</field>
<field>
<id>LINES_REJECTED</id>
<enabled>Y</enabled>
<name>LINES_REJECTED</name>
<subject />
</field>
<field>
<id>ERRORS</id>
<enabled>Y</enabled>
<name>ERRORS</name>
</field>
<field>
<id>STARTDATE</id>
<enabled>Y</enabled>
<name>STARTDATE</name>
</field>
<field>
<id>ENDDATE</id>
<enabled>Y</enabled>
<name>ENDDATE</name>
</field>
<field>
<id>LOGDATE</id>
<enabled>Y</enabled>
<name>LOGDATE</name>
</field>
<field>
<id>DEPDATE</id>
<enabled>Y</enabled>
<name>DEPDATE</name>
</field>
<field>
<id>REPLAYDATE</id>
<enabled>Y</enabled>
<name>REPLAYDATE</name>
</field>
<field>
<id>LOG_FIELD</id>
<enabled>Y</enabled>
<name>LOG_FIELD</name>
</field>
<field>
<id>EXECUTING_SERVER</id>
<enabled>N</enabled>
<name>EXECUTING_SERVER</name>
</field>
<field>
<id>EXECUTING_USER</id>
<enabled>N</enabled>
<name>EXECUTING_USER</name>
</field>
<field>
<id>CLIENT</id>
<enabled>N</enabled>
<name>CLIENT</name>
</field>
</trans-log-table>
<perf-log-table>
<connection />
<schema />
<table />
<interval />
<timeout_days />
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>SEQ_NR</id>
<enabled>Y</enabled>
<name>SEQ_NR</name>
</field>
<field>
<id>LOGDATE</id>
<enabled>Y</enabled>
<name>LOGDATE</name>
</field>
<field>
<id>TRANSNAME</id>
<enabled>Y</enabled>
<name>TRANSNAME</name>
</field>
<field>
<id>STEPNAME</id>
<enabled>Y</enabled>
<name>STEPNAME</name>
</field>
<field>
<id>STEP_COPY</id>
<enabled>Y</enabled>
<name>STEP_COPY</name>
</field>
<field>
<id>LINES_READ</id>
<enabled>Y</enabled>
<name>LINES_READ</name>
</field>
<field>
<id>LINES_WRITTEN</id>
<enabled>Y</enabled>
<name>LINES_WRITTEN</name>
</field>
<field>
<id>LINES_UPDATED</id>
<enabled>Y</enabled>
<name>LINES_UPDATED</name>
</field>
<field>
<id>LINES_INPUT</id>
<enabled>Y</enabled>
<name>LINES_INPUT</name>
</field>
<field>
<id>LINES_OUTPUT</id>
<enabled>Y</enabled>
<name>LINES_OUTPUT</name>
</field>
<field>
<id>LINES_REJECTED</id>
<enabled>Y</enabled>
<name>LINES_REJECTED</name>
</field>
<field>
<id>ERRORS</id>
<enabled>Y</enabled>
<name>ERRORS</name>
</field>
<field>
<id>INPUT_BUFFER_ROWS</id>
<enabled>Y</enabled>
<name>INPUT_BUFFER_ROWS</name>
</field>
<field>
<id>OUTPUT_BUFFER_ROWS</id>
<enabled>Y</enabled>
<name>OUTPUT_BUFFER_ROWS</name>
</field>
</perf-log-table>
<channel-log-table>
<connection />
<schema />
<table />
<timeout_days />
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>CHANNEL_ID</id>
<enabled>Y</enabled>
<name>CHANNEL_ID</name>
</field>
<field>
<id>LOG_DATE</id>
<enabled>Y</enabled>
<name>LOG_DATE</name>
</field>
<field>
<id>LOGGING_OBJECT_TYPE</id>
<enabled>Y</enabled>
<name>LOGGING_OBJECT_TYPE</name>
</field>
<field>
<id>OBJECT_NAME</id>
<enabled>Y</enabled>
<name>OBJECT_NAME</name>
</field>
<field>
<id>OBJECT_COPY</id>
<enabled>Y</enabled>
<name>OBJECT_COPY</name>
</field>
<field>
<id>REPOSITORY_DIRECTORY</id>
<enabled>Y</enabled>
<name>REPOSITORY_DIRECTORY</name>
</field>
<field>
<id>FILENAME</id>
<enabled>Y</enabled>
<name>FILENAME</name>
</field>
<field>
<id>OBJECT_ID</id>
<enabled>Y</enabled>
<name>OBJECT_ID</name>
</field>
<field>
<id>OBJECT_REVISION</id>
<enabled>Y</enabled>
<name>OBJECT_REVISION</name>
</field>
<field>
<id>PARENT_CHANNEL_ID</id>
<enabled>Y</enabled>
<name>PARENT_CHANNEL_ID</name>
</field>
<field>
<id>ROOT_CHANNEL_ID</id>
<enabled>Y</enabled>
<name>ROOT_CHANNEL_ID</name>
</field>
</channel-log-table>
<step-log-table>
<connection />
<schema />
<table />
<timeout_days />
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>CHANNEL_ID</id>
<enabled>Y</enabled>
<name>CHANNEL_ID</name>
</field>
<field>
<id>LOG_DATE</id>
<enabled>Y</enabled>
<name>LOG_DATE</name>
</field>
<field>
<id>TRANSNAME</id>
<enabled>Y</enabled>
<name>TRANSNAME</name>
</field>
<field>
<id>STEPNAME</id>
<enabled>Y</enabled>
<name>STEPNAME</name>
</field>
<field>
<id>STEP_COPY</id>
<enabled>Y</enabled>
<name>STEP_COPY</name>
</field>
<field>
<id>LINES_READ</id>
<enabled>Y</enabled>
<name>LINES_READ</name>
</field>
<field>
<id>LINES_WRITTEN</id>
<enabled>Y</enabled>
<name>LINES_WRITTEN</name>
</field>
<field>
<id>LINES_UPDATED</id>
<enabled>Y</enabled>
<name>LINES_UPDATED</name>
</field>
<field>
<id>LINES_INPUT</id>
<enabled>Y</enabled>
<name>LINES_INPUT</name>
</field>
<field>
<id>LINES_OUTPUT</id>
<enabled>Y</enabled>
<name>LINES_OUTPUT</name>
</field>
<field>
<id>LINES_REJECTED</id>
<enabled>Y</enabled>
<name>LINES_REJECTED</name>
</field>
<field>
<id>ERRORS</id>
<enabled>Y</enabled>
<name>ERRORS</name>
</field>
<field>
<id>LOG_FIELD</id>
<enabled>N</enabled>
<name>LOG_FIELD</name>
</field>
</step-log-table>
<metrics-log-table>
<connection />
<schema />
<table />
<timeout_days />
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>CHANNEL_ID</id>
<enabled>Y</enabled>
<name>CHANNEL_ID</name>
</field>
<field>
<id>LOG_DATE</id>
<enabled>Y</enabled>
<name>LOG_DATE</name>
</field>
<field>
<id>METRICS_DATE</id>
<enabled>Y</enabled>
<name>METRICS_DATE</name>
</field>
<field>
<id>METRICS_CODE</id>
<enabled>Y</enabled>
<name>METRICS_CODE</name>
</field>
<field>
<id>METRICS_DESCRIPTION</id>
<enabled>Y</enabled>
<name>METRICS_DESCRIPTION</name>
</field>
<field>
<id>METRICS_SUBJECT</id>
<enabled>Y</enabled>
<name>METRICS_SUBJECT</name>
</field>
<field>
<id>METRICS_TYPE</id>
<enabled>Y</enabled>
<name>METRICS_TYPE</name>
</field>
<field>
<id>METRICS_VALUE</id>
<enabled>Y</enabled>
<name>METRICS_VALUE</name>
</field>
</metrics-log-table>
</log>
<maxdate>
<connection />
<table />
<field />
<offset>0.0</offset>
<maxdiff>0.0</maxdiff>
</maxdate>
<size_rowset>10000</size_rowset>
<sleep_time_empty>50</sleep_time_empty>
<sleep_time_full>50</sleep_time_full>
<unique_connections>N</unique_connections>
<feedback_shown>Y</feedback_shown>
<feedback_size>50000</feedback_size>
<using_thread_priorities>Y</using_thread_priorities>
<shared_objects_file />
<capture_step_performance>N</capture_step_performance>
<step_performance_capturing_delay>1000</step_performance_capturing_delay>
<step_performance_capturing_size_limit>100</step_performance_capturing_size_limit>
<dependencies>
</dependencies>
<partitionschemas>
</partitionschemas>
<slaveservers>
</slaveservers>
<clusterschemas>
</clusterschemas>
<created_user>-</created_user>
<created_date>2016/01/18 16:34:51.228</created_date>
<modified_user>-</modified_user>
<modified_date>2016/01/18 16:34:51.228</modified_date>
<key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
<is_key_private>N</is_key_private>
</info>
<notepads>
</notepads>
<order>
<hop>
<from>Run SSH commands</from>
<to>Text file output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>errors only</from>
<to>Get line number</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>import file</from>
<to>Add a line number</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Add a line number</from>
<to>Merge Join</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Add a line number</from>
<to>line 1 only</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Merge Join</from>
<to>Drop StdOut and line_number_1</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Drop StdOut and line_number_1</from>
<to>Prioritize streams</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>line 1 only</from>
<to>Add constants</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Add constants</from>
<to>Reorder</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Reorder</from>
<to>Prioritize streams</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Text file output</from>
<to>rename stdOut to original_result</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>rename stdOut to original_result</from>
<to>Split field to rows on newline</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>trim stdOut</from>
<to>errors only</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Split field to rows on newline</from>
<to>Drop original_result</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Drop original_result</from>
<to>trim stdOut</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Get line number</from>
<to>Quotes around error_description</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Quotes around error_description</from>
<to>Merge Join</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Prioritize streams</from>
<to>Count the failed import rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Count the failed import rows</from>
<to>Delete header row if no failed import rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Delete header row if no failed import rows</from>
<to>Remove count</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Remove count</from>
<to>noimport_log</to>
<enabled>Y</enabled>
</hop>
</order>
<step>
<name>Add a line number</name>
<type>Sequence</type>
<description />
<distribute>N</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<valuename>line_number</valuename>
<use_database>N</use_database>
<connection />
<schema />
<seqname>SEQ_</seqname>
<use_counter>Y</use_counter>
<counter_name />
<start_at>1</start_at>
<increment_by>1</increment_by>
<max_value>999999999</max_value>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>272</xloc>
<yloc>400</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Add constants</name>
<type>Constant</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<fields>
<field>
<name>error_description</name>
<type>String</type>
<format />
<currency />
<decimal />
<group />
<nullif>error_description</nullif>
<length>-1</length>
<precision>-1</precision>
<set_empty_string>N</set_empty_string>
</field>
</fields>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>272</xloc>
<yloc>560</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Count the failed import rows</name>
<type>GroupBy</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<all_rows>Y</all_rows>
<ignore_aggregate>N</ignore_aggregate>
<field_ignore />
<directory>%%java.io.tmpdir%%</directory>
<prefix>grp</prefix>
<add_linenr>N</add_linenr>
<linenr_fieldname />
<give_back_row>N</give_back_row>
<group>
</group>
<fields>
<field>
<aggregate>count</aggregate>
<subject>line_number</subject>
<type>COUNT_ALL</type>
<valuefield />
</field>
</fields>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>144</xloc>
<yloc>832</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Delete header row if no failed import rows</name>
<type>FilterRows</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<send_true_to />
<send_false_to />
<compare>
<condition>
<negated>N</negated>
<leftvalue>count</leftvalue>
<function>&gt;</function>
<rightvalue />
<value>
<name>constant</name>
<type>Integer</type>
<text>1</text>
<length>-1</length>
<precision>0</precision>
<isnull>N</isnull>
<mask>####0;-####0</mask>
</value>
</condition>
</compare>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>144</xloc>
<yloc>912</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Drop StdOut and line_number_1</name>
<type>SelectValues</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<fields>
<select_unspecified>N</select_unspecified>
<remove>
<name>stdOut</name>
</remove>
<remove>
<name>line_number_1</name>
</remove>
</fields>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>144</xloc>
<yloc>656</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Drop original_result</name>
<type>SelectValues</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<fields>
<select_unspecified>N</select_unspecified>
<remove>
<name>original_result</name>
</remove>
</fields>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>64</xloc>
<yloc>256</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Get line number</name>
<type>RegexEval</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<script>^ERROR line (\d+): (.*)</script>
<matcher>stdOut</matcher>
<resultfieldname />
<usevar>N</usevar>
<allowcapturegroups>Y</allowcapturegroups>
<replacefields>Y</replacefields>
<canoneq>N</canoneq>
<caseinsensitive>N</caseinsensitive>
<comment>N</comment>
<dotall>N</dotall>
<multiline>N</multiline>
<unicode>N</unicode>
<unix>N</unix>
<fields>
<field>
<name>line_number</name>
<type>Integer</type>
<format />
<group />
<decimal />
<length>-1</length>
<precision>-1</precision>
<nullif />
<ifnull />
<trimtype>both</trimtype>
<currency />
</field>
<field>
<name>error_description</name>
<type>String</type>
<format />
<group />
<decimal />
<length>-1</length>
<precision>-1</precision>
<nullif />
<ifnull />
<trimtype>both</trimtype>
<currency />
</field>
</fields>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>64</xloc>
<yloc>448</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Merge Join</name>
<type>MergeJoin</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<join_type>INNER</join_type>
<step1>Quotes around error_description</step1>
<step2>Add a line number</step2>
<keys_1>
<key>line_number</key>
</keys_1>
<keys_2>
<key>line_number</key>
</keys_2>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>144</xloc>
<yloc>576</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Prioritize streams</name>
<type>PrioritizeStreams</type>
<description />
<distribute>N</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<steps>
<step>
<name>Reorder</name>
</step>
<step>
<name>Drop StdOut and line_number_1</name>
</step>
</steps>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>144</xloc>
<yloc>752</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Quotes around error_description</name>
<type>ScriptValueMod</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<compatible>N</compatible>
<optimizationLevel>9</optimizationLevel>
<jsScripts>
<jsScript>
<jsScript_type>0</jsScript_type>
<jsScript_name>Script 1</jsScript_name>
<jsScript_script>var error_description = '"' + error_description + '"'
</jsScript_script>
</jsScript>
</jsScripts>
<fields>
<field>
<name>error_description</name>
<rename>error_description</rename>
<type>String</type>
<length>-1</length>
<precision>-1</precision>
<replace>Y</replace>
</field>
</fields>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>64</xloc>
<yloc>512</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Remove count</name>
<type>SelectValues</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<fields>
<select_unspecified>N</select_unspecified>
<remove>
<name>count</name>
</remove>
</fields>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>144</xloc>
<yloc>992</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Reorder</name>
<type>SelectValues</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<fields>
<field>
<name>line_number</name>
</field>
<field>
<name>error_description</name>
</field>
<field>
<name>line</name>
</field>
<select_unspecified>N</select_unspecified>
</fields>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>272</xloc>
<yloc>672</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Run SSH commands</name>
<type>SSH</type>
<description />
<distribute>N</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<dynamicCommandField>N</dynamicCommandField>
<command>#this sed command adds commas to the end of each line, which fixes single-field imports
#sed -i 's/$/,/' ${SQLDUMP}/csvs/${filename}
php ${IMPORTSCRIPT} -e ${entity} --file=${SQLDUMP}/csvs/${filename}
</command>
<commandfieldname />
<port>22</port>
<servername>${DEVSERVER}</servername>
<userName>jon</userName>
<password>Encrypted </password>
<usePrivateKey>Y</usePrivateKey>
<keyFileName>${KEYFILE}</keyFileName>
<passPhrase>Encrypted </passPhrase>
<stdOutFieldName>stdOut</stdOutFieldName>
<stdErrFieldName>stdErr</stdErrFieldName>
<timeOut>0</timeOut>
<proxyHost />
<proxyPort />
<proxyUsername />
<proxyPassword>Encrypted </proxyPassword>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>64</xloc>
<yloc>16</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Split field to rows on newline</name>
<type>SplitFieldToRows3</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<splitfield>original_result</splitfield>
<delimiter>$[0A]</delimiter>
<newfield>stdOut</newfield>
<rownum>N</rownum>
<rownum_field />
<resetrownumber>Y</resetrownumber>
<delimiter_is_regex>N</delimiter_is_regex>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>64</xloc>
<yloc>192</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Text file output</name>
<type>TextFileOutput</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<separator>;</separator>
<enclosure>"</enclosure>
<enclosure_forced>N</enclosure_forced>
<enclosure_fix_disabled>N</enclosure_fix_disabled>
<header>Y</header>
<footer>N</footer>
<format>DOS</format>
<compression>None</compression>
<encoding>UTF-8</encoding>
<endedLine />
<fileNameInField>N</fileNameInField>
<fileNameField />
<create_parent_folder>N</create_parent_folder>
<file>
<name>${LOCALFILES}\importlogs\raw_log_${filename}</name>
<is_command>N</is_command>
<servlet_output>N</servlet_output>
<do_not_open_new_file_init>N</do_not_open_new_file_init>
<extention>txt</extention>
<append>N</append>
<split>N</split>
<haspartno>N</haspartno>
<add_date>N</add_date>
<add_time>N</add_time>
<SpecifyFormat>N</SpecifyFormat>
<date_time_format />
<add_to_result_filenames>Y</add_to_result_filenames>
<pad>N</pad>
<fast_dump>N</fast_dump>
<splitevery>0</splitevery>
</file>
<fields>
<field>
<name>stdOut</name>
<type>String</type>
<format />
<currency />
<decimal />
<group />
<nullif />
<trim_type>none</trim_type>
<length>-1</length>
<precision>-1</precision>
</field>
</fields>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>64</xloc>
<yloc>80</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>errors only</name>
<type>FilterRows</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<send_true_to />
<send_false_to />
<compare>
<condition>
<negated>N</negated>
<leftvalue>stdOut</leftvalue>
<function>STARTS WITH</function>
<rightvalue />
<value>
<name>constant</name>
<type>String</type>
<text>ERROR line</text>
<length>-1</length>
<precision>-1</precision>
<isnull>N</isnull>
<mask />
</value>
</condition>
</compare>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>64</xloc>
<yloc>384</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>import file</name>
<type>CsvInput</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<filename>${LOCALFILES}\data\${filename}</filename>
<filename_field />
<rownum_field />
<include_filename>N</include_filename>
<separator>~~~~~~</separator>
<enclosure />
<header>N</header>
<buffer_size>50000</buffer_size>
<lazy_conversion>Y</lazy_conversion>
<add_filename_result>N</add_filename_result>
<parallel>N</parallel>
<newline_possible>N</newline_possible>
<encoding />
<fields>
<field>
<name>line</name>
<type>String</type>
<format />
<currency />
<decimal />
<group />
<length>-1</length>
<precision>-1</precision>
<trim_type>none</trim_type>
</field>
</fields>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>272</xloc>
<yloc>320</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>line 1 only</name>
<type>FilterRows</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<send_true_to />
<send_false_to />
<compare>
<condition>
<negated>N</negated>
<leftvalue>line_number</leftvalue>
<function>=</function>
<rightvalue />
<value>
<name>constant</name>
<type>Integer</type>
<text>1</text>
<length>-1</length>
<precision>0</precision>
<isnull>N</isnull>
<mask>####0;-####0</mask>
</value>
</condition>
</compare>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>272</xloc>
<yloc>480</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>noimport_log</name>
<type>TextFileOutput</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<separator>,</separator>
<enclosure />
<enclosure_forced>Y</enclosure_forced>
<enclosure_fix_disabled>N</enclosure_fix_disabled>
<header>N</header>
<footer>N</footer>
<format>DOS</format>
<compression>None</compression>
<encoding>UTF-8</encoding>
<endedLine />
<fileNameInField>N</fileNameInField>
<fileNameField />
<create_parent_folder>N</create_parent_folder>
<file>
<name>${LOCALFILES}/errors/noimport_${filename}</name>
<is_command>N</is_command>
<servlet_output>N</servlet_output>
<do_not_open_new_file_init>N</do_not_open_new_file_init>
<extention />
<append>N</append>
<split>N</split>
<haspartno>N</haspartno>
<add_date>N</add_date>
<add_time>N</add_time>
<SpecifyFormat>N</SpecifyFormat>
<date_time_format />
<add_to_result_filenames>Y</add_to_result_filenames>
<pad>N</pad>
<fast_dump>N</fast_dump>
<splitevery>0</splitevery>
</file>
<fields>
<field>
<name>line_number</name>
<type>Integer</type>
<format>0</format>
<currency />
<decimal />
<group />
<nullif />
<trim_type>both</trim_type>
<length>-1</length>
<precision>-1</precision>
</field>
<field>
<name>error_description</name>
<type>String</type>
<format />
<currency />
<decimal />
<group />
<nullif />
<trim_type>both</trim_type>
<length>-1</length>
<precision>-1</precision>
</field>
<field>
<name>line</name>
<type>String</type>
<format />
<currency />
<decimal />
<group />
<nullif />
<trim_type>both</trim_type>
<length>-1</length>
<precision>-1</precision>
</field>
</fields>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>144</xloc>
<yloc>1072</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>rename stdOut to original_result</name>
<type>SelectValues</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<fields>
<field>
<name>stdOut</name>
<rename>original_result</rename>
</field>
<select_unspecified>N</select_unspecified>
</fields>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>64</xloc>
<yloc>144</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>trim stdOut</name>
<type>StringOperations</type>
<description />
<distribute>Y</distribute>
<custom_distribution />
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name />
</partitioning>
<fields>
<field>
<in_stream_name>stdOut</in_stream_name>
<out_stream_name />
<trim_type>both</trim_type>
<lower_upper>none</lower_upper>
<padding_type>none</padding_type>
<pad_char />
<pad_len />
<init_cap>no</init_cap>
<mask_xml>none</mask_xml>
<digits>none</digits>
<remove_special_characters>cr</remove_special_characters>
</field>
</fields>
<cluster_schema />
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>64</xloc>
<yloc>320</yloc>
<draw>Y</draw>
</GUI>
</step>
<step_error_handling>
</step_error_handling>
<slave-step-copy-partition-distribution>
</slave-step-copy-partition-distribution>
<slave_transformation>N</slave_transformation>
</transformation>