th step of the data set
--T c -- generate cutomers ONLY
--T l -- generate nation/region ONLY
--T L -- generate lineitem ONLY
--T n -- generate nation ONLY
--T o -- generate orders/lineitem ONLY
--T O -- generate orders ONLY
--T p -- generate parts/partsupp ONLY
--T P -- generate parts ONLY
--T r -- generate region ONLY
--T s -- generate suppliers ONLY
--T S -- generate partsupp ONLY
--U -- generate update sets
--v -- enable VERBOSE mode
--x -- do hashed pre-partitioning, distributing
- flat file records across
partitions
--y -- actually write hashed flat file records
- ONLY for (0-relative) partition number
--z -- generate no more than approximately
- records, regardless of the scale factor
-
-NOTE: The -y and -z arguments work ONLY in conjunction
- with the -x argument. When generating 'paired'
- tables (e.g. Order & Lineitem together), the
- value applies to the sum of the rows
- generated for the two tables.
-
-To generate the SF=1 (1GB) database population , use:
- dbgen -vfF -s 1
-
-To generate the qualification database population (100 MB), use:
-
- dbgen -vfF -s 0.1
-
-To generate updates for a SF=1 (1GB), use:
- dbgen -v -O s -s 1
- dbgen -v -U 1 -s 1
diff --git a/core/sql/regress/seabase/EXPECTED021 b/core/sql/regress/seabase/EXPECTED021
index 590ab6b39b..01cb0d5c15 100644
--- a/core/sql/regress/seabase/EXPECTED021
+++ b/core/sql/regress/seabase/EXPECTED021
@@ -1,7 +1,7 @@
>>
>>sh sqlci -i"TEST021(s0)" >> LOG021;
-Hewlett-Packard NonStop(TM) SQL/MX Conversational Interface 0.7.0
-(c) Copyright 2003-2010 Hewlett-Packard Development Company, LP.
+Apache Trafodion Conversational Interface 2.0.0
+Copyright (c) 2015 Apache Software Foundation
>>?section s0
>>initialize trafodion, drop;
@@ -15,8 +15,8 @@ End of MXCI Session
>>
>>sh sqlci -i"TEST021(s1)" >> LOG021;
-Hewlett-Packard NonStop(TM) SQL/MX Conversational Interface 0.7.0
-(c) Copyright 2003-2010 Hewlett-Packard Development Company, LP.
+Apache Trafodion Conversational Interface 2.0.0
+Copyright (c) 2015 Apache Software Foundation
>>?section s1
>>initialize trafodion;
@@ -28,8 +28,8 @@ End of MXCI Session
>>
>>sh sqlci -i"TEST021(s11)" >> LOG021;
-Hewlett-Packard NonStop(TM) SQL/MX Conversational Interface 0.7.0
-(c) Copyright 2003-2010 Hewlett-Packard Development Company, LP.
+Apache Trafodion Conversational Interface 2.0.0
+Copyright (c) 2015 Apache Software Foundation
>>?section s11
>>create table t021 (a int not null);
@@ -73,8 +73,8 @@ End of MXCI Session
>>
>>sh sqlci -i"TEST021(s2)" >> LOG021;
-Hewlett-Packard NonStop(TM) SQL/MX Conversational Interface 0.7.0
-(c) Copyright 2003-2010 Hewlett-Packard Development Company, LP.
+Apache Trafodion Conversational Interface 2.0.0
+Copyright (c) 2015 Apache Software Foundation
>>?section s2
>>set parserflags 131072;
@@ -124,8 +124,8 @@ End of MXCI Session
>>
>>sh sqlci -i"TEST021(s3)" >> LOG021;
-Hewlett-Packard NonStop(TM) SQL/MX Conversational Interface 0.7.0
-(c) Copyright 2003-2010 Hewlett-Packard Development Company, LP.
+Apache Trafodion Conversational Interface 2.0.0
+Copyright (c) 2015 Apache Software Foundation
>>?section s3
>>initialize trafodion, upgrade;
Metadata Upgrade: started
@@ -174,10 +174,10 @@ End of MXCI Session
>>
>>sh sqlci -i"TEST021(s4)" >> LOG021;
-Hewlett-Packard NonStop(TM) SQL/MX Conversational Interface 0.7.0
-(c) Copyright 2003-2010 Hewlett-Packard Development Company, LP.
+Apache Trafodion Conversational Interface 2.0.0
+Copyright (c) 2015 Apache Software Foundation
>>?section s4
->>select * from trafodion."_MD_".indexes;
+select * from trafodion."_MD_".indexes;
BASE_TABLE_UID KEYTAG IS_UNIQUE KEY_COLCOUNT NONKEY_COLCOUNT IS_EXPLICIT INDEX_UID
-------------------- ----------- ----------- ------------ --------------- ----------- --------------------
diff --git a/core/sql/regress/tools/regress-filter b/core/sql/regress/tools/regress-filter
index a5a8ddced3..060260d08d 100755
--- a/core/sql/regress/tools/regress-filter
+++ b/core/sql/regress/tools/regress-filter
@@ -254,10 +254,6 @@ s/^\([A-Za-z0-9]\+\.m[df]?\)[ ]\+([A-Za-z]:\\\\.\+\\\\\(.\+\..\+\..\+\))/\1 (\2)
/^[A-Za-z0-9]*\.mdf[ ]*([A-Za-z0-9]*\.[A-Za-z0-9]*\.[A-Za-z0-9]*)/s/[ ]*([A-Za-z0-9]*\.[A-Za-z0-9]*/ (#CAT.#SCH/
s/^NonStop SQL\/MX \([A-Za-z\/ +]*\) [0-9][0-9]*\.[0-9][0-9]*$/\1 #n.#n/
/^Copyright (c).*All Rights Reserved. *$/s/[0-9]/#/g
-/^Copyright (c).*All Rights Reserved. *$/s/Tandem[^.]*\./#CPQ#TDM#./
-/^Copyright (c).*All Rights Reserved. *$/s/Compaq[^.]*\./#CPQ#TDM#./
-/^(c) Copyright [0-9][0-9][0-9][0-9] Hewlett-Packard Development Company.*$/s/^.*$/Copyright (c) ####-#### #CPQ#TDM#\. All Rights Reserved./
-/^(c) Copyright [0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9] Hewlett-Packard Development Company.*$/s/^.*$/Copyright (c) ####-#### #CPQ#TDM#\. All Rights Reserved./
/^[Hh]istory file .* Access is denied/d
s/^\(MESSAGEFILE [ ]*\).*/\1#f/
/^MESSAGEFILE VRSN[ ]*{/s/{.*/#v/
@@ -296,7 +292,6 @@ s/^\([ ][ ]*[0-9][0-9]*[ ][ ]*[0-9][0-9]*[ ][ ]*CAT.SCH\)[A-Z][0-9A-Z][0-9A-Z]\(
s/^--- SQL operation completed with errors./--- SQL operation failed with errors./
# Filter out Microsoft compiler and linker information
s/Microsoft (R) .*/Microsoft (R) xxxxxx/g
-s/Copyright (C) Microsoft Corp.*/Copyright (C) Microsoft Corp xxxxxx/g
# ... NonStop(TM) SQL/MX Compiler 2.0
# ^^^
diff --git a/core/sql/regress/tools/regress-filter-linux b/core/sql/regress/tools/regress-filter-linux
index 003d39273f..f345fd2503 100755
--- a/core/sql/regress/tools/regress-filter-linux
+++ b/core/sql/regress/tools/regress-filter-linux
@@ -320,10 +320,6 @@ s/^\([A-Za-z0-9]\+\.m[df]?\)[ ]\+([A-Za-z]:\\\\.\+\\\\\(.\+\..\+\..\+\))/\1 (\2)
/^[A-Za-z0-9]*\.mdf[ ]*([A-Za-z0-9]*\.[A-Za-z0-9]*\.[A-Za-z0-9]*)/s/[ ]*([A-Za-z0-9]*\.[A-Za-z0-9]*/ (#CAT.#SCH/
s/^NonStop SQL\/MX \([A-Za-z\/ +]*\) [0-9][0-9]*\.[0-9][0-9]*$/\1 #n.#n/
/^Copyright (c).*All Rights Reserved. *$/s/[0-9]/#/g
-/^Copyright (c).*All Rights Reserved. *$/s/Tandem[^.]*\./#CPQ#TDM#./
-/^Copyright (c).*All Rights Reserved. *$/s/Compaq[^.]*\./#CPQ#TDM#./
-/^(c) Copyright [0-9][0-9][0-9][0-9] Hewlett-Packard Development Company.*$/s/^.*$/Copyright (c) ####-#### #CPQ#TDM#\. All Rights Reserved./
-/^(c) Copyright [0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9] Hewlett-Packard Development Company.*$/s/^.*$/Copyright (c) ####-#### #CPQ#TDM#\. All Rights Reserved./
s/^Copyright (c) .*/Copyright (c) ######/
/^[Hh]istory file .* Access is denied/d
s/^\(MESSAGEFILE [ ]*\).*/\1#f/
@@ -366,7 +362,6 @@ s/^\([ ][ ]*[0-9][0-9]*[ ][ ]*[0-9][0-9]*[ ][ ]*CAT.SCH\)[A-Z][0-9A-Z][0-9A-Z]\(
s/^--- SQL operation completed with errors./--- SQL operation failed with errors./
# Filter out Microsoft compiler and linker information
s/Microsoft (R) .*/Microsoft (R) xxxxxx/g
-s/Copyright (C) Microsoft Corp.*/Copyright (C) Microsoft Corp xxxxxx/g
#Added for nist test nist586
s/nist586\.mdf.*/nist586.mdf/g
diff --git a/core/sql/regress/udr/EXPECTED100.SB b/core/sql/regress/udr/EXPECTED100.SB
index f7ac5143ce..8ceca903f4 100644
--- a/core/sql/regress/udr/EXPECTED100.SB
+++ b/core/sql/regress/udr/EXPECTED100.SB
@@ -1706,8 +1706,8 @@ OUT2
>>
>>obey TEST100(udrtrace);
>>log;
-Conversational Interface @version@
-(c) Copyright 2014 Hewlett-Packard Development Company, LP.
+Apache Trafodion Conversational Interface 2.0.0
+Copyright (c) 2015 Apache Software Foundation
>>?section udrtrace2
>>--
>>-- This section will be executed by the mxci fired off while executing
diff --git a/core/updateCopyrightCheck.py b/core/updateCopyrightCheck.py
index 6ccdd726dc..73b531d8f2 100644
--- a/core/updateCopyrightCheck.py
+++ b/core/updateCopyrightCheck.py
@@ -202,11 +202,12 @@ def __init__(self):
if parseResult:
# The copyright line of interest looks like:
#
- # // (x) Copyright 1998-2014 Hewlett-Packard Development Company, L.P.
+ # // (x) 1998-2014 Hewlett-Packard Development Company, L.P.
#
- # (It is actually (C), not (x); I obfuscated the comment so this
+ # (It is actually (C), not (x) and Copyright not .
+ # I obfuscated the comment so this
# script wouldn't mistakenly report itself as having multiple
- # copyrights :-)
+ # copyrights and to pass copyright checks from Apache :-)
#
# Of course the comment delimiter will vary with the file language.
cmdGrep = ['grep',
diff --git a/dcs/LICENSE.txt b/dcs/LICENSE.txt
deleted file mode 100644
index d645695673..0000000000
--- a/dcs/LICENSE.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/dcs/NOTICE.txt b/dcs/NOTICE.txt
deleted file mode 100644
index 48780e03ff..0000000000
--- a/dcs/NOTICE.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).
-
-In addition, this product includes software developed by:
-
-JUnit (http://www.junit.org/) included under the Common Public License v1.0. See
-the full text here: http://junit.sourceforge.net/cpl-v10.html
\ No newline at end of file
diff --git a/dcs/src/main/java/org/trafodion/dcs/util/ByteBufferUtils.java b/dcs/src/main/java/org/trafodion/dcs/util/ByteBufferUtils.java
index 699463aad5..49325d3148 100644
--- a/dcs/src/main/java/org/trafodion/dcs/util/ByteBufferUtils.java
+++ b/dcs/src/main/java/org/trafodion/dcs/util/ByteBufferUtils.java
@@ -1,6 +1,4 @@
/*
- * Copyright 2015 the original author or authors.
- *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
diff --git a/dcs/src/main/java/org/trafodion/dcs/util/SqlUtils.java b/dcs/src/main/java/org/trafodion/dcs/util/SqlUtils.java
index e90a301add..2bb7f80994 100644
--- a/dcs/src/main/java/org/trafodion/dcs/util/SqlUtils.java
+++ b/dcs/src/main/java/org/trafodion/dcs/util/SqlUtils.java
@@ -1,6 +1,4 @@
/*
- * Copyright 2015 the original author or authors.
- *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
diff --git a/dcs/src/main/resources/dcs-webapps/master/css/stylesheet.css b/dcs/src/main/resources/dcs-webapps/master/css/stylesheet.css
deleted file mode 100644
index 55ad4a15f0..0000000000
--- a/dcs/src/main/resources/dcs-webapps/master/css/stylesheet.css
+++ /dev/null
@@ -1,191 +0,0 @@
-@charset "UTF-8";
-/* --- CSS Document by Codify Design Studio - codifydesign.com --- */
-
-body {
- margin: 0px;
- color: #777777;
- font-size: 12px;
- font-family: Arial;
- font-family: Georgia, "Times New Roman", Times, serif;
- background-color: #FFFFFF;
-}
-
-h1 {
- color: #25476C;
- font-size: 24px;
- font-weight: normal;
- margin: 3px 0px 3px 0px;
-}
-h2 {
- color: #214265;
- font-size: 14px;
- font-weight: normal;
- margin: 0px 0px 15px 0px;
-}
-h3 {
- color: #244669;
- margin-top: 12px;
- margin-bottom: 0px;
-}
-p {
- margin-top: 0px;
- margin-right: 0px;
- margin-bottom: 12px;
- margin-left: 0px;
-}
-a {
- color: #36C;
-}
-a:visited { color: black; }
-a:hover { color: #5872f4; }
-
-.container {
- width: auto;
- margin-left: auto;
- margin-right: auto;
-}
-
-
-/* BANNER AREA */
-.bannerArea {
- width: 100%;
- height: 50px;
- background-color: #3BAAE3;
- background-repeat: repeat-x;
- background-position: 50% 50%;
-}
-
-.bannerArea .container {
- width: auto;
- height:83px;
- background-repeat: no-repeat;
- padding-right: 10px;
- padding-left: 10px;
-}
-
-.bannernav {
- padding-top: 20px;
- float: right;
- color: #112c4c;
- font-size: 10px;
- font-family: Arial, Helvetica, Verdana, sans-serif;
-}
-
-.bannernav a { color: #ffffff; text-decoration: none; }
-.bannernav a:visited { color: #ffffff; }
-.bannernav a:hover {
- color: #112c4c;
- text-decoration: none;
- font-family: Arial, Helvetica, Verdana, sans-serif;
-}
-
-.toplogo {
- padding-left: 5px;
- padding-top: 5px;
-}
-
-
-/* TOP NAVIGATION AREA */
-.topnavigationArea {
- background-repeat:repeat-x;
- background-color: #436d9c;
- background-image: url(images/topnav_bkgd.jpg);
-}
-.topnavigationArea .container {
- background-image: url(images/topnav_bkgd.jpg);
- background-repeat:repeat-x;
- background-color: #436d9c;
- width: 980px;
-}
-
-.topnavigationgroup {
- margin: 0px;
- float: right;
- padding-right: 1px;
-}
-.topnavigationgroup ul {
- margin: 0px;
-}
-
-.topnavigationitem {
- max-width: 167px;
- margin-left: 0px;
- padding-left: 10px;
- padding-top: 9px;
- padding-right: 10px;
- padding-bottom: 9px;
- border-left-style: solid;
- border-left-width: 1px;
- border-right-style: none;
- border-right-width: 1px;
- color: white;
- font-size: 14px;
- font-weight: bold;
- font-family: Arial, Helvetica, Verdana, sans-serif;
- float: left;
-}
-
-/* CONTENT AREA */
-.contentArea {
- width: 100%;
- background-color: #f0eff0;
- background-image: url(images/content_bkgd_tile.jpg);
- background-repeat: repeat-x;
- height: 100%;
-}
-
-
-.contentArea .container {
- padding-top: 30px;
- padding-bottom: 30px;
- background-repeat: repeat;
- background-color: #f2f2f2;
- background-image: url(images/content_bkgd_tile.jpg);
- padding-left: 10px;
- padding-right: 10px;
-}
-
-.contentleft {
- width: auto;
- padding-left: 40px;
- padding-right: 40px;
- float: left;
- border-right-style: solid;
- border-right-width: 1px;
-}
-
-.contentright {
- padding-top: 20px;
- float: left;
- padding-left: 30px;
-
- width: 210px;
-}
-
-.imageright {
- margin-left: 5px;
- float: right;
-}
-
-
-/* FOOTER AREA */
-.footerArea {
- width: 100%;
- background-color: #DEEDF7;
- background-repeat: repeat-x;
- background-image: url(images/ui-bg_highlight-soft_33_003147_1x100.png);
- background-position: 50% 50%;
-}
-
-.footerArea .container {
- height: 40px;
- background-repeat: repeat-x;
-}
-
-.copyright {
- padding-left: 40px;
- padding-top: 12px;
- color: #FFF;
- font-size: 12px;
- font-family: Arial, Helvetica, Verdana, sans-serif;
-}
diff --git a/docs/client_install/source/Trafodion_Client_Installation_Guide.xml b/docs/client_install/source/Trafodion_Client_Installation_Guide.xml
index 85ac48b67a..947d709e82 100755
--- a/docs/client_install/source/Trafodion_Client_Installation_Guide.xml
+++ b/docs/client_install/source/Trafodion_Client_Installation_Guide.xml
@@ -24,7 +24,7 @@
-->
-
Trafodion Client Installation Guide
@@ -35,7 +35,7 @@
Last updated: 04/21/2015
-2015Hewlett-Packard Development Company,
+2015Apache Software Foundation (ASF),
L.P.
@@ -2899,7 +2899,7 @@ role="select:quotedtitle page"/>
>basicsql.cpp:
-
diff --git a/docs/command_interface/source/Trafodion_Command_Interface_Guide.xml b/docs/command_interface/source/Trafodion_Command_Interface_Guide.xml
index bb668eea9d..9dadb5a9d6 100755
--- a/docs/command_interface/source/Trafodion_Command_Interface_Guide.xml
+++ b/docs/command_interface/source/Trafodion_Command_Interface_Guide.xml
@@ -24,7 +24,7 @@
-->
-
Trafodion Command Interface Guide
@@ -35,8 +35,8 @@
Last updated: 5/28/2014
-2014Hewlett-Packard Development Company,
-L.P.
+2015 Apache Software Foundation (ASF),
+
Legal Notice
@@ -749,8 +749,8 @@ trafci.cmd -h 16.123.456.78:23400 -u user1 ‑p xxxxxx
./trafci.sh -h 16.123.456.78:23400 -u user1 -p xxxxxx
TrafCI launches and prompts you to enter an SQL statement or
an interface command:
-Welcome to Trafodion Command Interface
-Copyright(C) 2013–2014 Hewlett-Packard Development Company, L.P.
+Welcome to Apache Trafodion Command Interface
+Copyright(C) 2015 Apache Software Foundation (ASF)
Connected to Trafodion
@@ -775,8 +775,8 @@ of three retries, before it closes the session.
you to reenter the host name. For example:
trafci –h dd
-Welcome to Trafodion Command Interface
-Copyright(C) 2013–2014 Hewlett-Packard Development Company, L.P.
+Welcome to Apache Trafodion Command Interface
+Copyright(C) 2015 Apache Software Foundation
Unknown Host: dd
@@ -796,8 +796,8 @@ name and password. After three unsuccessful retries, the session is
terminated:
trafci –h 172.16.1.1 –u user1 –p x
-Welcome to Trafodion Command Interface
-Copyright(C) 2013–2014 Hewlett-Packard Development Company, L.P.
+Welcome to Apache Trafodion Command Interface
+Copyright(C) 2015 Apache Software Foundation
*** ERROR[8837] CLI Authentication : User: user1 : invalid username or password [2014-03-12 16:23:44]
@@ -1048,7 +1048,7 @@ Password:
--- SQL operation complete.
-C:\Program Files (x86)\Hewlett-Packard\Trafodion Command Interface\bin>_
+C:\Program Files (x86)\Apache Software Foundation\Apache Trafodion Command Interface\bin>_
Example of Running an Interface Command With ‑q or ‑sql
@@ -1133,8 +1133,8 @@ did not specify ‑h, ‑u, and ‑p, and ‑dsn on the command line), and runs the commands in the script
file:
-Welcome to Trafodion Command Interface
-Copyright(C) 2013–2014 Hewlett-Packard Development Company, L.P.
+Welcome to Apache Trafodion Command Interface
+Copyright(C) 2015 Apache Software Foundation
Host Name/IP Address: 16.123.456.78:23400
User Name: user1
@@ -1204,8 +1204,8 @@ they are ignored.
./trafci.sh ‑version
-Welcome to Trafodion Command Interface
-Copyright(C) 2013–2014 Hewlett-Packard Development Company, L.P.
+Welcome to Apache Trafodion Command Interface
+Copyright(C) 2015 Apache Software Foundation
Trafodion JDBC Type 4 Driver : Traf_JDBC_Type4_Build_40646
Trafodion Command Interface : TrafCI_Build_40646
@@ -2790,7 +2790,7 @@ at a command prompt. For example:
Example of a Perl Program (sample.pl)
-use lib 'C:\\Program Files (x86)\\Hewlett-Packard\\Trafodion Command Interface\\lib\\perl';
+use lib 'C:\\Program Files (x86)\\Apache Software Foundation\\Apache Trafodion Command Interface\\lib\\perl';
use Session;
# create a new session
@@ -2815,7 +2815,7 @@ $sess->disconnect();
import os
import sys
## Modify this path
-sys.path.append("C:\\Program Files (x86)\\Hewlett-Packard\\Trafodion Command Interface\\lib\\python")
+sys.path.append("C:\\Program Files (x86)\\Apache Software Foundation\\Apache Trafodion Command Interface\\lib\\python")
import Session
# create a new session
@@ -3353,7 +3353,7 @@ names, TrafCI uses the current catalog and schema.
is the directory of the JAR file on the client workstation.
If you omit the directory, TrafCI searches for the JAR file in the
same location as the TrafCI Windows launch file, trafci.cmd, in C:\Program Files (x86)\Hewlett-Packard\Trafodion
+>, in C:\Program Files (x86)\Apache Software Foundation\Apache Trafodion
Command Interface\bin.
jar-file-name
@@ -4732,7 +4732,7 @@ the folder.
Volume in drive C is E-Client
Volume Serial Number is DC4F-5B3B
-Directory of c:\Program Files (x86)\Hewlett-Packard\Trafodion Command Interface\bin
+Directory of c:\Program Files (x86)\Apache Software Foundation\Apache Trafodion Command Interface\bin
05/11/2014 01:17 PM <DIR>
05/11/2014 01:17 PM <DIR>
@@ -4974,7 +4974,7 @@ names, TrafCI uses the current catalog and schema.
is the directory of the JAR file on the client workstation.
If you omit the directory, TrafCI searches for the JAR file in the
same location as the TrafCI Windows launch file, trafci.cmd, in C:\Program Files (x86)\Hewlett-Packard\Trafodion
+>, in C:\Program Files (x86)\Apache Software Foundation\Apache Trafodion
Command Interface\bin.
jar-file-name
@@ -9825,9 +9825,9 @@ SQL>
> parameter, the VERSION command displays only TrafCI and the Trafodion
JDBC Type 4 Driver versions.
-C:\Program Files (x86)\Hewlett-Packard\Trafodion Command Interface\bin>trafci -noconnect
-Welcome to Trafodion Command Interface
-Copyright(C) 2013–2014 Hewlett-Packard Development Company, L.P.
+C:\Program Files (x86)\Apache Software Foundation\Apache Trafodion Command Interface\bin>trafci -noconnect
+Welcome to Apache Trafodion Command Interface
+Copyright(C) 2015 Apache Software Foundation
Not connected.
diff --git a/docs/command_interface/source/xml-profile.xml b/docs/command_interface/source/xml-profile.xml
index 0e9b3172fd..4a470f742f 100755
--- a/docs/command_interface/source/xml-profile.xml
+++ b/docs/command_interface/source/xml-profile.xml
@@ -23,7 +23,7 @@
-->
-
diff --git a/docs/odb_manual/Trafodion_odb_User_Guide.xml b/docs/odb_manual/Trafodion_odb_User_Guide.xml
index 3364d1fcb2..fed681e7e8 100755
--- a/docs/odb_manual/Trafodion_odb_User_Guide.xml
+++ b/docs/odb_manual/Trafodion_odb_User_Guide.xml
@@ -1,6 +1,6 @@
-Trafodion odb User GuideTrafodion Release 1.1—April 2015Part Number: T632-110-001Published: April 2015Edition: Trafodion Release 1.1© Copyright 2015 Hewlett-Packard Development Company, L.P.Legal NoticeThe information contained herein is subject to change without notice. This documentation is distributed on an “AS IS” basis, without warranties or conditions of any kind, either express or implied. Nothing herein should be construed as constituting an additional warranty. HP shall not be liable for technical or editorial errors or omissions contained herein.NOTICE REGARDING OPEN SOURCE SOFTWARE: Project Trafodion is licensed under the Apache License, Version 2.0 (the "License"); you may not use software from Project Trafodion except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.Table of Contents TOC \o "1-3" \h \z \u About This Document PAGEREF _Toc417975704 \h 5Intended Audience PAGEREF _Toc417975705 \h 5New and Changed Information in This Edition PAGEREF _Toc417975706 \h 5Document Conventions PAGEREF _Toc417975707 \h 5Publishing History PAGEREF _Toc417975708 \h 5We Encourage Your Comments PAGEREF _Toc417975709 \h 51Introduction PAGEREF _Toc417975710 \h 61.1What is odb PAGEREF _Toc417975711 \h 62Installation and Configuration PAGEREF _Toc417975712 \h 72.1odb Requirements PAGEREF _Toc417975713 \h 72.2Installing and Configuring the Required unixODBC PAGEREF _Toc417975714 \h 72.3Installing odb PAGEREF _Toc417975715 \h 93Basic Concepts PAGEREF _Toc417975716 \h 103.1Getting Help PAGEREF _Toc417975717 \h 103.2Connecting to a Database PAGEREF _Toc417975718 \h 113.3Listing Available ODBC Drivers and Data Sources PAGEREF _Toc417975719 \h 123.4Obtaining Database Information PAGEREF _Toc417975720 \h 123.5Performing Actions on Multiple Database Objects PAGEREF _Toc417975721 \h 133.6Running Commands and Scripts PAGEREF _Toc417975722 \h 143.7Using “here document” Syntax in Shell Scripts PAGEREF _Toc417975723 \h 153.8Running Multiple Commands and Scripts in Parallel PAGEREF _Toc417975724 \h 153.9Limiting the Number of Threads Created by odb PAGEREF _Toc417975725 \h 163.10Changing the Number of Executions Distributed Across Threads PAGEREF _Toc417975726 \h 163.11Learning How Dynamic Load Balancing Works PAGEREF _Toc417975727 \h 173.12Using Variables in odb Scripts PAGEREF _Toc417975728 \h 173.13Understanding Thread ID, Thread Execution#, and Script Command# PAGEREF _Toc417975729 \h 173.14Checking SQL Scripts PAGEREF _Toc417975730 \h 183.15Using Different Data Sources for Different Threads PAGEREF _Toc417975731 \h 183.16Formatting Query Results PAGEREF _Toc417975732 \h 193.17Extracting Tables’ DDL PAGEREF _Toc417975733 \h 194Loading, Extracting, and Copying Data PAGEREF _Toc417975734 \h 214.1Loading Files PAGEREF _Toc417975735 \h 214.2Mapping Source File Fields to Target Table Columns PAGEREF _Toc417975736 \h 244.3Using mapfiles to Ignore and/or Transform Fields When Loading PAGEREF _Toc417975737 \h 274.4Using mapfiles to Load Fixed Format Files PAGEREF _Toc417975738 \h 274.5Generating and Loading Data PAGEREF _Toc417975739 \h 284.6Loading Default Values PAGEREF _Toc417975740 \h 294.7Loading Binary Files PAGEREF _Toc417975741 \h 304.8Reducing the ODBC Buffer Size PAGEREF _Toc417975742 \h 304.9Extracting Tables PAGEREF _Toc417975743 \h 314.10Extracting a List of Tables PAGEREF _Toc417975744 \h 354.11Copying Tables From One Database to Another PAGEREF _Toc417975745 \h 354.12Copying a List of Tables PAGEREF _Toc417975746 \h 384.13Using Case-Sensitive Table and Column Names PAGEREF _Toc417975747 \h 384.14Determining the Appropriate Number of Threads for Load/Extract/Copy/Diff PAGEREF _Toc417975748 \h 394.15Integrating With Hadoop PAGEREF _Toc417975749 \h 395Comparing Tables (Technology Preview) PAGEREF _Toc417975750 \h 406Using odb as a Query Driver (Technology Preview) PAGEREF _Toc417975751 \h 446.1Getting CSV Output PAGEREF _Toc417975752 \h 446.2Assigning a Label to a Query PAGEREF _Toc417975753 \h 456.3Running All Scripts With a Given Path PAGEREF _Toc417975754 \h 456.4Randomizing Execution Order PAGEREF _Toc417975755 \h 466.5Defining a Timeout PAGEREF _Toc417975756 \h 466.6Simulating User Thinking Time PAGEREF _Toc417975757 \h 466.7Starting Threads Gracefully PAGEREF _Toc417975758 \h 466.8Re-looping a Given Workload PAGEREF _Toc417975759 \h 477Using odb as a SQL Interpreter (Technology Preview) PAGEREF _Toc417975760 \h 487.1Running Commands When the Interpreter Starts PAGEREF _Toc417975761 \h 517.2Customizing the Interpreter Prompt PAGEREF _Toc417975762 \h 52A.Warnings, Limits, and Troubleshooting PAGEREF _Toc417975763 \h 53B.How odb Is Coded and Tested PAGEREF _Toc417975764 \h 54About This DocumentThis manual describes how to use odb, a multi-threaded, ODBC-based command-line tool, to perform various operations on a Trafodion database.Note: In Trafodion Release 1.1, only loading, extracting, and copying data operations are production ready, meaning that that have been fully tested and are ready to be used in a production environment. Other features are designated as “Technology Preview,” meaning that they have not been fully tested and are not ready for production use.Intended AudienceThis manual is intended for database administrators and other users who want to run scripts that operate on a Trafodion database, primarily for parallel data loading.New and Changed Information in This EditionThis manual is new.Document ConventionsThe manual uses the following typographic conventions:ConventionDescriptionExampleMonospace dark grayCommand promptmfelici ~odb $Monospace orangeTyped command./odb64luo -hMonospace blueCommand output[0.0.0]--- 1 row(s) selected in 0.015sMonospace grayComments and clarifications# this will drop all views in your schemaPublishing HistoryPart NumberProduct VersionPublication DateT632-110-001Trafodion Release 1.1.0April 2015We Encourage Your CommentsThe Trafodion community encourages your comments concerning this document. We are committed to providing documentation that meets your needs. Send any errors found, suggestions for improvement, or compliments to:trafodion-documentation@lists.launchpad.net Include the document title, part number, and any comment, error found, or suggestion for improvement you have concerning this document.IntroductionWhat is odbodb is a platform independent, multi-threaded, ODBC command-line tool you can use as a:Parallel data loader/extractorQuery driver (Technology Preview)SQL interpreter (Technology Preview)odb is written in ANSI C. In Trafodion Release 1.1, odb is available only in a 64-bit version for the Linux platform, linked to the unixODBC driver manager.odb executables use the following naming convention, odbAABCC, where:AA can be 64 (bit) (32 bit is not currently supported).B identifies the platform/compiler:l = Linux/gccw = Windows/MS Visual Studio (not yet tested)a = IBM AIX/gcc (not currently supported)x = IBM/AIX xlc (not currently supported)h = HP-UX/ANSI C (not currently supported)c = Cygwin/gcc (not currently supported)m = Mac OSX/gcc (not currently supported)CC identifies the ODBC Driver Manager to which odb was linked:uo = unixODBC Driver Managerms = Microsoft ODBC Driver Manager (not yet tested)td = Teradata ODBC Driver Manager (not currently supported)io = iODBC Driver Manager (not currently supported)dd = DataDirect Driver Manager (not currently supported)So, for example:odb64luo is the 64-bit executable for Linux linked with the unixODBC Driver Managerodb32cms is the 32-bit executable for Cygwin linked with MS Windows Driver Manager (not currently supported)odb64mio is the 64-bit executable for Mac OSX linked to the “native” iODBC Driver Manager (not currently supported)This document contains examples run with the odb64luo executable.In the future, this document might contain examples prepared with different executables. All odb executables have exactly the same features unless otherwise specified.Installation and Configurationodb RequirementsA general requirement is the ODBC library for the database you want to connect to and:PlatformRequirementsLinuxUNIX (not currently supported)Mac OS/X (not currently supported)pthread libraries (Generally installed by default)Windows (not yet tested)Microsoft Visual C++ 2010 Redistributable Package (x86)Note: In Trafodion Release 1.1, odb is available only for the Linux platform.Installing and Configuring the Required unixODBCThis section explains how to install and configure unixODBC, which is required for Trafodion Release 1.1.Grab the source code tarball from http://www.unixodbc.org. Use at least version 2.3.x.Unpack the tarball:$ tar xzvf unixODBC-2.3.1.tar.gzConfigure unixODBC installation:$ cd unixODBC-2.3.1$ ./configure --disable-gui --enable-threads --disable-driversThis will install unixODBC under /usr/local and requires root access. If you don’t have root privileges or you want to install unixODBC somewhere else you have to add --prefix=<installation_path> to the configure command here above. For example:$ ./configure --prefix=/home/mauro/uodbc --disable-gui --enable-threads --disable-driversCompile unixODBC sources:$ makeInstall unixODBC:$ make installNow, in order to configure unixODBC, we have to:Define a few environment variables.Define our Data Sources.Let’s start with the environment variables (which you can add to your profile script):First you have to set the ODBCHOME variable to the unixODBC installation dir (the one configured via --prefix here above). For example:export ODBCHOME=/home/mauro/uodbcThen you have to configure the system data sources directory (the one containing odbc.ini and odbcinst.ini). Normally this is the etc/ directory under $ODBCHOME:export ODBCSYSINI=${ODBCHOME}/etcThen you have to configure the ODBCINI variable to the full path of the odbc.ini file:export ODBCINI=${ODBCSYSINI}/odbc.iniFinally… do not forget to add unixODBC lib directory to your LD_LIBRARY_PATH (Linux) or LIBPATH (IBM AIX) or SHLIB_PATH (HP/UX):export LD_LIBRARY_PATH=${ODBCHOME}/libNow we have to configure our data sources in odbc.ini:[<DATA_SOURCE_NAME>]Description = DSN DescriptionDriver = <odbcinst.ini corresponding section>...Other (Driver specific) parameters...and odbcinst.ini:[<Driver name in odbc.ini>]Description = Driver descriptionDriver = <ODBC driverFileUsage = 1UsageCount = 1Note: The Trafodion ODBC driver requires an environment variable, AppUnicodeType, to be specified in odbcinst.ini. This variable must be set to utf16.If you are using Vertica, the Vertica ODBC driver requires an additional section named [Driver] in odbc.ini with specific settings for unixODBC (odbcinst library location and UTF level supported by unixODBC). See the Vertica Programmer’s Guide – Additional ODBC Driver Configuration Settings. See also the following example: Example:$ cat odbc.ini[ODBC Data Sources]VMFELICI = VerticaMachine1traf = Trafodion database[VMFELICI]Description = Vertica Machine 1Driver = VODBCDatabase = vertica01_machineServername = server_nameUID =PWD =Port = 1111ConnSettings =DriverStringConversions = NONESSLKeyFile = /<dir-name>client.keySSLCertFile = /<dir-name>/client.crt[Driver] << This section is required by Vertica’s ODBC Driver Locale = en_USODBCInstLib = /<dir-name>/uodbc/lib/libodbcinst.soErrorMessagesPath = /opt/vertica/lib64/DriverManagerEncoding = UTF-16LogPath = /tmpLogNameSpace =LogLevel = 0[traf]Description = traf DSNDriver = TrafodionCatalog = TRAFODIONSchema = QADataLang = 0FetchBufferSize = SYSTEM_DEFAULTServer = TCP:<server-name>:<port-no>SQL_ATTR_CONNECTION_TIMEOUT = SYSTEM_DEFAULTSQL_LOGIN_TIMEOUT = SYSTEM_DEFAULTSQL_QUERY_TIMEOUT = NO_TIMEOUTServiceName = HP_DEFAULT_SERVICE$ cat odbcinst.ini[VODBC]Description = Vertica ODBC DriverDriver = /<dir-name>/vodbc/lib64/libverticaodbc.soFileUsage = 1UsageCount = 1[Trafodion]Description= Trafodion ODBC Stand Alone DriverDriver= /<dir-name>/conn/clients/odbc/libtrafodbc_drvr64.soFileUsage= 1UsageCount= 1AppUnicodeType= utf16[ODBC]Threading = 1Trace = OffTracefile = uodbc.trcAnother important entry in odbcinst.ini is Threading. The following comment (extracted from unixODBC sources DriverManager/__handles.c) explain its meaning: * ... * If compiled with thread support the DM allows four different * thread strategies * * Level 0 - Only the DM internal structures are protected * the driver is assumed to take care of it's self * * Level 1 - The driver is protected down to the statement level * each statement will be protected, and the same for the connect * level for connect functions, note that descriptors are considered * equal to statements when it comes to thread protection. * * Level 2 - The driver is protected at the connection level. only * one thread can be in a particular driver at one time * * Level 3 - The driver is protected at the env level, only one thing * at a time. * * By default the driver open connections with a lock level of 0, * drivers should be expecetd to be thread safe now. * this can be changed by adding the line * * Threading = N * * to the driver entry in odbcinst.ini, where N is the locking level * (0-3)Installing odbSee the Trafodion Client Installation Guide (1.1) for details.Basic ConceptsGetting HelpThe following command shows the odb help:mfelici ~/Devel/odb $ ./odb64luo -h odb version 1.1.0Build: linux, amd64, gcc generic m64, uodbc, mreadline, dynamic gzip, dynamic libhdfs, dynamic libxml2 [Mar 30 2015 00:29:25] -h: print this help -version: print odb version and exit -lsdrv: list available drivers @ Driver Manager level -lsdsn: list available Data SourcesConnection related options. You can connect using either: -u User: (default $ODB_USER variable) -p Password: (default $ODB_PWD variable) -d Data_Source_Name: (default $ODB_DSN variable) -ca Connection_Attributes (normally used instead of -d DSN) -U sets SQL_TXN_READ_UNCOMMITTED isolation level -ndsn [+]<number>: adds 1 to <number> to DSN -nps <nbytes>[:<nbytes>]: specify source[:target] network packet sizeSQL interpreter options: -I [$ODB_INI SECTION]: interactive mode shell -noconnect: do not connect on startupGeneral options: -q [cmd|res|all|off]: do not print commands/results/both -i [TYPE[MULT,WIDE_MULT]:CATALOG.SCHEMA[.TABLE]]: lists following object types: (t)ables, (v)iews, s(y)nonyns, (s)chemas, (c)atalogs, syst(e)m tables (l)ocal temp, (g)lobal temp, (m)at views, (M)mat view groups, (a)lias (A)ll object types, (T)table desc, (D)table DDL, (U) table DDL with multipliers -r #rowset: rowset to be used insert/selects (default 100) -soe: Stop On Error (script execution/loading task) -N : Null run. Doesn't SQLExecute statements -v : be verbose -vv : Print execution table -noschema : do not use schemas: CAT.OBJ instead of CAT.SCH.OBJ -nocatalog : do not use catalogs: SCH.OBJ instead of CAT.SCH.OBJ -nocatnull : like -nocatalog but uses NULL instead of empty CAT strings -ucs2toutf8 : set UCS-2 to UTF-8 conversion in odb -var var_name var_value: set user defined variables -ksep char/code: Thousands Separator Character (default ',') -dsep char/code: Decimal Separator Character (default '.')SQL execution options [connection required]: -x [#inst:]'command': runs #inst (default 1) command instances -f [#inst:]'script': runs #inst (default 1) script instances -P script_path_regexp: runs in parallel scripts_path_regexp if script_path_regexp ends with / all files in that dir -S script_path_regexp: runs serially scripts_path_regexp if script_path_regexp ends with / all files in that dir -L #loops: runs everything #loops times -T max_threads: max number of execution threads -dlb: use Dynamic Load Balancing -timeout #seconds: stops everything after #seconds (no Win32) -delay #ms: delay (ms) before starting next thread -ldelay #ms: delay (ms) before starting next loop in a thread -ttime #ms[:ms]: delay (ms) before starting next command in a thread random delay if a [min:max] range is specified -F #records: max rows to fetch -c : output in csv format -b : print start time in the headers when CSV output -pcn: Print Column Names -plm: Print Line Mode -fs char/code: Field Sep <char> ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> -rs char/code: Rec Sep <char> ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> -sq char/code: String Qualifier (default none) -ec char/code: Escape Character (default '\') -ns nullstring: print nullstring when a field is NULL -trim: Trim leading/trailing white spaces from txt cols -drs: describe result set (#cols, data types...) for each Q) -hint: do not remove C style comments (treat them as hints) -casesens: set case sensitive DB -Z : shuffle the execution table randomizing Qs start orderData loading options [connection required]: -l src=[-]file:tgt=table[:map=mapfile][:fs=fieldsep][:rs=recsep][:soe] [:skip=linestoskip][:ns=nullstring][:ec=eschar][:sq=stringqualifier] [:pc=padchar][:em=embedchar][:errmax=#max_err][:commit=auto|end|#rows|x#rs] [:rows=#rowset][:norb][:full][:max=#max_rec][:truncate][:show][:bpc=#][:bpwc=#] [:nomark][:parallel=number][:iobuff=#size][:buffsz=#size]][:fieldtrunc={0-4}] [:pre={@sqlfile}|{[sqlcmd]}][:post={@sqlfile}|{[sqlcmd]}][:ifempty] [:direct][:bad=[+]badfile][:tpar=#tables][:maxlen=#bytes][:time][:loadcmd=IN|UP|UL] [:xmltag=[+]element][:xmlord][:xmldump] Defaults/notes: * src file: local file or {hdfs,mapr}[@host,port[,huser]].<HDFS_PATH> * fs: default ','. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * rs: default '\n'. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * ec: default '\'. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * pc: no default. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * direct: only for Vertica databases * bpc: default 1,bpwc: default 4 * loadcmd: default IN. only for Trafodion databasesData extraction options [connection required]: -e {src={table|-file}|sql=<custom sql>}:tgt=[+]file[:pwhere=where_cond] [:fs=fieldsep][:rs=recsep][:sq=stringqualifier][:ec=escape_char][:soe] [:ns=nullstring][es=emptystring][:rows=#rowset][:nomark][:binary][:bpc=#][:bpwc=#] [:max=#max_rec][:[r]trim[+]][:cast][:multi][parallel=number][:gzip[=lev]] [:splitby=column][:uncommitted][:iobuff=#size][hblock=#size][:ucs2toutf8] [:pre={@sqlfile}|{[sqlcmd]}[:mpre={@sqlfile}|{[sqlcmd]}[:post={@sqlfile}|{[sqlcmd]}] [tpar=#tables][:time][:cols=[-]columns]][:maxlen=#bytes][:xml] Defaults/notes: * tgt file: local file or {hdfs,mapr}.[@host,port[,huser]].<HDFS_PATH> * fs: default ','. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * rs: default '\n'. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * ec: default '\'. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * sq: no default. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * gzip compressione level between 0 and 9 * bpc: default 1,bpwc: default 4 Data copy options [connection required]: -cp src={table|-file:tgt=schema[.table][pwhere=where_cond][:soe][:roe=#][:roedel=#ms] [:truncate][:rows=#rowset][:nomark][:max=#max_rec][:bpc=#][:bpwc=#][:[r]trim[+]] [:parallel=number][errmax=#max_err][:commit=auto|end|#rows|x#rs][:time] ][:cast] [:direct][:uncommitted][:norb][:splitby=column][:pre={@sqlfile}|{[sqlcmd]}] [:post={@sqlfile}|{[sqlcmd]}][:mpre={@sqlfile}|{[sqlcmd]}][:ifempty] [:loaders=#loaders][:tpar=#tables][:cols=[-]columns][:errdmp=file] ][:loadcmd=IN|UP|UL] [sql={[sqlcmd]|@sqlfile|-file}[:bind=auto|char|cdef][:seq=field#[,start]] [tmpre={@sqlfile}|{[sqlcmd]}][:ucs2toutf8=[skip,force,cpucs2,qmark]] Defaults/notes: * loaders: default 2 load threads for each 'extractor' * direct: only work if target database is Vertica * ucs2toutf8: default is 'skip' * roe: default 3 if no arguments * bpc: default 1,bpwc: default 4 * loadcmd: default IN. only for Trafodion databasesData pipe options [connection required]: -pipe sql={[sqlcmd]|@sqlscript|-file}:tgtsql={@sqlfile|[sqlcmd]}[:soe] [:rows=#rowset][:nomark][:max=#max_rec][:bpc=#][:bpwc=#][:errdmp=file] [:parallel=number][errmax=#max_err][:commit=auto|end|#rows|x#rs][:time] [:pre={@sqlfile}|{[sqlcmd]}][:post={@sqlfile}|{[sqlcmd]}] [:mpre={@sqlfile}|{[sqlcmd]}][:tmpre={@sqlfile}|{[sqlcmd]}] [:loaders=#loaders][:tpar=#tables][:bind=auto|char|cdef] Defaults/notes: * loaders: default 1 load threads for each extraction thread * bpc: default 1,bpwc: default 4 Table diff options [connection required]: -diff src={table|-file}:tgt=table:[key=columns][:output=[+]file][:pwhere=where_cond] [:pwhere=where_cond][:nomark][:rows=#rowset][:odad][:fs=fieldsep][:time][trim[+]] [:rs=recsep][:quick][:splitby=column][:parallel=number][:max=#max_rec] [:print=[I][D][C]][:ns=nullstring][:es=emptystring][:bpc=#][:bpwc=#][:uncommitted] [:pre={@sqlfile}|{[sqlcmd]}][:post={@sqlfile}|{[sqlcmd]}][tpar=#tables] Defaults/notes: * bpc: default 1,bpwc: default 4 * print: default is Inserted Deleted Changed Connecting to a Databaseodb uses standard ODBC APIs to connect to a database. Normally you have to provide the following information: user, password and ODBC data source. Example:$ ./odb64luo –u user –p password –d dsn ... You can provide Driver specific connection attributes using -ca command line option. Note: Command-line passwords are protected against “ps -ef" sniffing attacks under *nix. You can safely pass your password via –p. An alternative approach is to use environment variables or the odb password prompt (see below)odb will use the following environment variables (if defined):VariableMeaningCorresponding Command-Line OptionODB_USERUser name to use for database connections-u <user>ODB_PWDPassword for database connections-p <passwd> ODB_DSNDSN for database connection-d <dsn> ODB_INIInit file for interactive shellODB_HISThistory file name to save command history on exitNote: Command-line options take precedence over environment variables.Listing Available ODBC Drivers and Data SourcesYou can list available drivers with -lsdrv:felici ~/Devel/odb $ ./odb64luo -lsdrvTrafodion - Description=Trafodion ODBC Stand Alone Driver...You can list locally configured data sources with -lsdsn:felici ~/Devel/odb $ ./odb64luo -lsdsntraf - TrafodionVMFELICI – Vertica ...Obtaining Database InformationUsing the -i option you can get information about the database you’re connecting to as well as the ODBC driver. It’s a simple way to check your credentials and database connection. Example:ncsi@ldratc16 ~/mauro/odb $ ./odb64luo -u xxx -p xxx -d traf -iodb [2015-04-20 21:20:47]: starting ODBC connection(s)... 0[odb version 1.1.0]Build: linux, amd64, gcc generic m64, uodbc, mreadline, dynamic gzip, dynamic libhdfs, dynamic libxml2 [Apr 8 2015 16:47:49]DBMS product name (SQL_DBMS_NAME) : HP DatabaseDBMS product version (SQL_DBMS_VER) : 01.01.0000Database name (SQL_DATABASE_NAME) : TRAFODIONServer name (SQL_SERVER_NAME) : --name--Data source name (SQL_DATA_SOURCE_NAME) : trafData source RO (SQL_DATA_SOURCE_READ_ONLY) : NODBC Driver name (SQL_DRIVER_NAME) : libhpodbc64.soODBC Driver version (SQL_DRIVER_VER) : 03.00.0000ODBC Driver level (SQL_DRIVER_ODBC_VER) : 03.51ODBC Driver Manager version (SQL_DM_VER) : 03.52.0002.0002ODBC Driver Manager level (SQL_ODBC_VER) : 03.52Connection Packet Size (SQL_ATTR_PACKET_SIZE): 0odb [2015-04-20 21:20:48]: exiting. Session Elapsed time 0.229 seconds (00:00:00.229)Listing Database ObjectsIn the previous section, we have used the -i option without any argument. This option accepts arguments with the following syntax: [TYPE:][CATALOG.SCHEMA][.OBJECT], where type can be:Type Meaning<missing>All database object typesA:All database object typest:Tablesv:Viewsa:Aliasesy:Synonymsl:Local Temporaryg:Global Temporarym:Materialized viewsM:Materialized view groupss:Schemasc:CatalogsT:Table descriptionsD:Table DDLU[x,y]:Table DDL multiplying wide columns by Y and non-wide columns by XExamples:ExampleAction-i c:List all catalogs-i s:List all schemas-i TRAFODION.MFTESTList all objects in TRAFODION.MFTEST schema-i t:TRAFODION.MFTESTList all tables in TRAFODION.MFTEST -i t:TRAFODION.MFTEST.A%List all tables in TRAFODION.MFTEST schema staring with A-i v:TRAFODION.MFTESTList all views in TRAFODION.MFTEST-i v:TRAFODION.MFTEST.%_VList all views in TRAFODION.MFTEST ending with _V-i T:TRAFODION.MFTEST.STG%Describe all tables starting with STG in TRAFODION.MFTESTExtended examples:ncsi@ldratc16 ~/mauro/odb $ ./odb64luo -u MFELICI -p xxx -d MFELICI -i T:TRAFODION.MAURIZIO.T%odb [2011-12-07 14:43:51]: starting (1) ODBC connection(s)... 1Describing: TRAFODION.MAURIZIO.T1+------+--------------+----+-------+-----+|COLUMN|TYPE |NULL|DEFAULT|INDEX|+------+--------------+----+-------+-----+|ID |INTEGER SIGNED|YES | | ||NAME |CHAR(10) |YES | | ||LASTN |VARCHAR(20) |YES | | |+------+--------------+----+-------+-----+Describing: TRAFODION.MAURIZIO.T11+------+--------------+----+-------+-------+|COLUMN|TYPE |NULL|DEFAULT|INDEX |+------+--------------+----+-------+-------+|ID |INTEGER SIGNED|NO | |T11 1 U||NAME |CHAR(10) |YES | | |+------+--------------+----+-------+-------+The INDEX column (when using type T) contains the following information:name of the INDEX (in Trafodion indexes having the same name as the table are Primary Keys)ordinal number to identify the order of that field in the index(U)nique o (M)ultiple values allowed (+) means that more than one index includes that fieldPerforming Actions on Multiple Database Objectsodb uses extended SQL syntax to execute actions on multiple objects: &<type>:<path> - where <type> is one of the object types listed in the previous section.Examples:ExampleActiondelete from &t:MF%Purge ALL tables (t:) staring with “MF” drop view &v:mftest.%vwDrop ALL views (v:) ending with _VW in the schema MFTESTUPDATE STATISTICS FOR TABLE &t:TRAFODION.MFTEST.%Update Stats for ALL tables in TRAFODION.MFTESTYou can use this extended SQL syntax in the SQL Interpreter or generic SQL scripts. Running Commands and Scripts–x switch can be used to run generic SQL commands. You can also use –f to run SQL scripts:-x "SQL command" to run a specific SQL command-f <script> to run a script fileFor example:felici ~/Devel/odb $ ./odb64luo -x "select count(*) from customer"150000[0.0.0]--- 1 row(s) selected in 0.137s (prep 0.000s, exec 0.137s, 1st fetch 0.000s, fetch 0.000s)The meaning of [0.0.0] will be explained later.felici ~/Devel/odb $ cat script.sql SELECT COUNT(*) FROM T1;-- This is a commentSELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY) AS SUM_QTY, SUM(L_EXTENDEDPRICE) AS SUM_BASE_PRICE, SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)) AS SUM_DISC_PRICE, SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)*(1+L_TAX)) AS SUM_CHARGE, AVG(L_QUANTITY) AS AVG_QTY, AVG(L_EXTENDEDPRICE) AS AVG_PRICE, AVG(L_DISCOUNT) AS AVG_DISC, COUNT(*) AS COUNT_ORDERFROM LINEITEMWHERE L_SHIPDATE <= DATE '1998-12-01' - INTERVAL '90' DAYGROUP BY L_RETURNFLAG, L_LINESTATUSORDER BY L_RETURNFLAG, L_LINESTATUS;felici ~/Devel/odb $ ./odb64luo -f script.sql [0.0.0]Executing: 'SELECT COUNT(*) FROM T1;'5[0.0.0]--- 1 row(s) selected in 0.015s (prep 0.000s, exec 0.015s, 1st fetch -0.000s, fetch -0.000s)[0.0.1]Executing: 'SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY) AS SUM_QTY, SUM(L_EXTENDEDPRICE) AS SUM_BASE_PRICE, SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)) AS SUM_DISC_PRICE, SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)*(1+L_TAX)) AS SUM_CHARGE, AVG(L_QUANTITY) AS AVG_QTY, AVG(L_EXTENDEDPRICE) AS AVG_PRICE, AVG(L_DISCOUNT) AS AVG_DISC, COUNT(*) AS COUNT_ORDER FROM LINEITEM WHERE L_SHIPDATE <= DATE '1998-12-01' - INTERVAL '90' DAY GROUP BY L_RETURNFLAG, L_LINESTATUS ORDER BY L_RETURNFLAG, L_LINESTATUS;'A,F,37734107.00,56586554400.73,53758257134.8700,55909065222.827692,25.522006,38273.129735,0.049985,1478493...R,F,37719753.00,56568041380.90,53741292684.6040,55889619119.831932,25.505794,38250.854626,0.050009,1478870[0.0.1]--- 4 row(s) selected in 21.344s (prep 0.000s, exec 21.344s, 1st fetch 0.000s, fetch 0.000s)You can use the -q switch to omit selected output components. So, for example, –q cmd won’t print the commands being executed:felici ~/Devel/odb $ ./odb64luo -f script.sql –q cmd5[0.0.0]--- 1 row(s) selected in 0.015s (prep 0.000s, exec 0.015s, 1st fetch -0.000s, fetch -0.000s)A,F,37734107.00,56586554400.73,53758257134.8700,55909065222.827692,25.522006,38273.129735,0.049985,1478493...R,F,37719753.00,56568041380.90,53741292684.6040,55889619119.831932,25.505794,38250.854626,0.050009,1478870[0.0.1]--- 4 row(s) selected in 21.344s (prep 0.000s, exec 21.344s, 1st fetch 0.000s, fetch 0.000s)While -q res won’t print the results:felici ~/Devel/odb $ ./odb64luo -f script.sql –q res[0.0.0]Executing: 'SELECT COUNT(*) FROM T1;'[0.0.0]--- 1 row(s) selected in 0.015s (prep 0.000s, exec 0.015s, 1st fetch -0.000s, fetch -0.000s)[0.0.1]Executing: 'SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY) AS SUM_QTY, SUM(L_EXTENDEDPRICE) AS SUM_BASE_PRICE, SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)) AS SUM_DISC_PRICE, SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)*(1+L_TAX)) AS SUM_CHARGE, AVG(L_QUANTITY) AS AVG_QTY, AVG(L_EXTENDEDPRICE) AS AVG_PRICE, AVG(L_DISCOUNT) AS AVG_DISC, COUNT(*) AS COUNT_ORDER FROM LINEITEM WHERE L_SHIPDATE <= DATE '1998-12-01' - INTERVAL '90' DAY GROUP BY L_RETURNFLAG, L_LINESTATUS ORDER BY L_RETURNFLAG, L_LINESTATUS;'[0.0.1]--- 4 row(s) selected in 21.344s (prep 0.000s, exec 21.344s, 1st fetch 0.000s, fetch 0.000s)And finally -q all (or just -q) won’t print neither the commands nor the results:felici ~/Devel/odb $ ./odb64luo -f script.sql -q all[0.0.0]--- 1 row(s) selected in 0.015s (prep 0.000s, exec 0.015s, 1st fetch -0.000s, fetch -0.000s)[0.0.1]--- 4 row(s) selected in 21.344s (prep 0.000s, exec 21.344s, 1st fetch 0.000s, fetch 0.000s)This is often used with odb as query driver. Note: Even when odb doesn’t print query results (-q res), the result set will be fetched and data is transferred from the database server to the client. In other words “-q res” is somehow similar (but not exactly equivalent) to a /dev/null output redirection.A special file name you can use with -f is “-“(dash). It means: read the script to be executed from the standard input. For example the following command will copy table definitions from one system to another recreating, on the target system, the same table structures as in the source system:$ odb64luo –u u1 -p p1 -d SRC -i t:TRAFODION.CIV04 -x "SHOWDDL &1" | odb64luo –u u2 –p p2 -d TGT -f -Using “here document” Syntax in Shell ScriptsWe often need to embed SQL commands in shell scripts. In these cases you can use ”-f -“ (read commands from standard input) odb syntax. Example:odb64luo -f - <<-EOF 2>&1 | tee -a ${LOG} drop table &t:TRAFODION.maurizio.ml%; create table ml2 ( id integer, fname char(10), bdate date, lname char(10) default 'Felici', comment char(20), city char(10) ) no partitions;EOFRunning Multiple Commands and Scripts in Parallelodb uses threads to run multiple commands in parallel. Each command (-x) or script (-f) will be executed, independently from the others, using a different thread. So, for example:felici ~/Devel/odb $ ./odb64luo -x "select count(*) from types" -f script1.sqlwill use two independent threads executed in parallel. The first thread will run“select count(*) from types” and the other “script1.sql”. You can also run multiple copies of the same command by adding <num>: before -x or -f arguments. So, for example, the following command will run 3 instances of " select count(*) from types ", 5 instances of “script1.sql” and 3 instances of “script2.sql” in parallel using 3 + 5 + 3 = 11 threads in total:felici ~/Devel/odb $ ./odb64luo -x 3:"select count(*) from types" -f 5:script1.sql -f 3:script2.sql -q[1.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[0.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[2.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[4.0.0]--- 1 row(s) selected in 0.001s (prep 0.000s, exec 0.001s, fetch 0.000s/0.000s)[6.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[5.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[3.0.0]--- 1 row(s) selected in 0.001s (prep 0.000s, exec 0.001s, fetch 0.000s/0.000s)[8.0.0]--- 1 row(s) selected in 0.001s (prep 0.000s, exec 0.001s, fetch 0.000s/0.000s)[7.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[9.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[10.0.0]--- 1 row(s) selected in 0.001s (prep 0.000s, exec 0.001s, fetch 0.000s/0.000s)The number highlighted in red here above is the thread ID. Thread IDs are assigned by odb starting from zero.You can limit the maximum number of threads with -T option. The following example will run the same 11 commands/scripts limiting the number of threads (and ODBC connections) to 4:felici ~/Devel/odb $ ./odb64luo -x 3:"select count(*) from types" -f 5:script1.sql -f 3:script2.sql -q -T 4[1.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[0.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[2.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[1.1.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[2.1.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[0.1.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[3.0.0]--- 1 row(s) selected in 0.001s (prep 0.000s, exec 0.001s, fetch 0.000s/0.000s)[2.2.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[3.1.0]--- 1 row(s) selected in 0.001s (prep 0.000s, exec 0.001s, fetch 0.000s/0.000s)[0.2.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[1.2.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)As you can see the thread ID is now in the 0-3 range because the 11 executions were queued into 4 threads. As we will see in the next sessions odb offers several alternatives to queue M executions in N (<M) threads.Limiting the Number of Threads Created by odbBy default odb will create as many threads as the numbers of executions. So, the following command:mauro@newton ~/src/C/odb $ ./odb64luo -f script1.sql -f 3:script2.sql -x 3:"<mysqlcmd>"It will create 1 + 3 + 3 = 7 threads. Each thread will start its own ODBC connection. You can limit the max number of threads using –T. So the following:mauro@newton ~/src/C/odb $ ./odb64luo -f script1.sql -f 3:script2.sql -x 3:"<mysqlcmd>" -T 2It will create just two threads to execute the seven commands/scripts.odb will never create more threads than needed:felici ~/Devel/odb $ ./odb64luo -f 2:script1.sql -f 3:script2.sql -T 8 -c -qodb [main(1017)] - Warning: won't be created more thread (8) then needed (5).Changing the Number of Executions Distributed Across ThreadsBy defaults executions are distributed in round-robin across threads. Let's say we run:mauro@newton ~/src/C/odb $ ./odb64luo -f script1.sql -f 3:script2.sql -x 3:"<mysqlcmd>" -T 3Then, the execution queue will be the following:Thread 1Thread 2Thread33rd executionmysqlcmd2nd executionScript2.sqlmysqlcmdmysqlcmd1st executionScript1.sqlScript2.sqlScript2.sqlThis (standard) behavior can be modified using the following options:–Z (shuffle). This will option will randomize the execution orderfactor sign with –P option. See REF _Ref311450589 \h Running All Scripts With a Given Path.-dlb (Dynamic Load Balancing). See REF _Ref315957664 \h Learning How Dynamic Load Balancing Works.Learning How Dynamic Load Balancing WorksAs discussed in the previous section, executions are normally pre-assigned to threads using a simple round-robin algorithm. This way the total elapsed time for each thread depends on the complexity of “its own” executions. Suppose you have two threads and two executions per thread:Thread 1Thread 22nd executionScript1.2Script2.21st executionScript1.1Script2.1If thread 2.1 and 2.2 require a very short time to be executed you can have a situation where Thread2 has nothing to do (it will be terminated) while Thread1 is still busy with “its own” Script1.1 and Script1.2.In some cases, for example during data extractions (see REF _Ref416261416 \h Loading Binary Files), we might want to keep all threads busy at any given time. In these cases we can use Dynamic Load Balancing (-dlb). With Dynamic Load Balancing jobs are not “pre-assigned” to threads when odb starts; each thread will pick the next job to run from the job list at run-time.Using Variables in odb Scriptsodb let you to use two kinds of variables:Internal variables defined through the “set param” command and identified by the ampersand character;Environment variables defined at operating system level and identified by a dollar sign;You can mix internal and environment variables in your scripts. If a variable is not expanded to a valid Internal/Environment variable the text will remain unchanged. So, for example, if you have a script like this:felici ~/Devel/odb $ cat scr.sqlset param region1 ASIA-- region1 is defined as an internal odb parameterselect * from tpch.region where r_name = '®ion1';-- region2 is defined as an environment variableselect * from tpch.region where r_name = '$region2';-- you can mix internal and environment variablesselect * from tpch.region where r_name = '$region2' or r_name = '®ion1';-- region3 variable does not exists so it won’t be not expandedselect * from tpch.region where r_name = '®ion3';After you define region2 at operating system level:felici ~/Devel/odb $ export region2=AMERICAYou will get the following result:felici ~/Devel/odb $ ./odb64luo -u mauro -p xx -d pglocal -f scr.sqlodb [2011-12-12 08:01:31]: starting (1) ODBC connection(s)... 1[0.0.0]Executing: 'select * from tpch.region where r_name = 'ASIA';'2,ASIA ,ges. thinly even pinto beans ca[0.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, 1st fetch 0.000s, fetch 0.000s)[0.0.1]Executing: 'select * from tpch.region where r_name = 'AMERICA';'1,AMERICA ,hs use ironic, even requests. s[0.0.1]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, 1st fetch 0.000s, fetch 0.000s)[0.0.2]Executing: 'select * from tpch.region where r_name = 'AMERICA' or r_name = 'ASIA';'1,AMERICA ,hs use ironic, even requests. s2,ASIA ,ges. thinly even pinto beans ca[0.0.2]--- 2 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, 1st fetch 0.000s, fetch 0.000s)[0.0.3]Executing: 'select * from tpch.region where r_name = '®ion3';'[0.0.3]--- 0 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, 1st fetch 0.000s, fetch 0.000s)Understanding Thread ID, Thread Execution#, and Script Command#Suppose we have a script containing two commands:mauriziof@traf64-dl2a ~/odb $ cat script.sql SELECT COUNT(*) FROM ORDERS;SELECT COUNT(*) FROM SUPPLIER;If we run this script ten times using two threads, we will get:mauriziof@traf64-dl2a ~/odb $ ./odb64luo -f 10:script.sql -q -T 2[0.0.0]--- 1 row(s) selected in 0.102s (prep 0.022s, exec 0.044s, 1st fetch 0.037s, fetch 0.037s)[1.0.0]--- 1 row(s) selected in 0.125s (prep 0.022s, exec 0.068s, 1st fetch 0.036s, fetch 0.036s)[0.0.1]--- 1 row(s) selected in 0.520s (prep 0.022s, exec 0.048s, 1st fetch 0.450s, fetch 0.450s)[1.0.1]--- 1 row(s) selected in 0.564s (prep 0.017s, exec 0.480s, 1st fetch 0.067s, fetch 0.067s)[0.1.0]--- 1 row(s) selected in 0.131s (prep 0.022s, exec 0.060s, 1st fetch 0.048s, fetch 0.048s)[0.1.1]--- 1 row(s) selected in 0.086s (prep 0.022s, exec 0.057s, 1st fetch 0.007s, fetch 0.007s)[1.1.0]--- 1 row(s) selected in 0.136s (prep 0.035s, exec 0.058s, 1st fetch 0.042s, fetch 0.042s)[0.2.0]--- 1 row(s) selected in 0.123s (prep 0.029s, exec 0.068s, 1st fetch 0.026s, fetch 0.026s)[1.1.1]--- 1 row(s) selected in 0.119s (prep 0.016s, exec 0.082s, 1st fetch 0.021s, fetch 0.021s)[0.2.1]--- 1 row(s) selected in 0.089s (prep 0.031s, exec 0.054s, 1st fetch 0.004s, fetch 0.004s)[1.2.0]--- 1 row(s) selected in 0.138s (prep 0.023s, exec 0.041s, 1st fetch 0.073s, fetch 0.073s)[0.3.0]--- 1 row(s) selected in 0.144s (prep 0.038s, exec 0.045s, 1st fetch 0.061s, fetch 0.061s)[1.2.1]--- 1 row(s) selected in 0.127s (prep 0.016s, exec 0.041s, 1st fetch 0.070s, fetch 0.070s)[0.3.1]--- 1 row(s) selected in 0.136s (prep 0.033s, exec 0.056s, 1st fetch 0.048s, fetch 0.048s)[1.3.0]--- 1 row(s) selected in 0.131s (prep 0.023s, exec 0.037s, 1st fetch 0.071s, fetch 0.071s)[0.4.0]--- 1 row(s) selected in 0.111s (prep 0.033s, exec 0.045s, 1st fetch 0.033s, fetch 0.033s)[0.4.1]--- 1 row(s) selected in 0.076s (prep 0.033s, exec 0.037s, 1st fetch 0.005s, fetch 0.006s)[1.3.1]--- 1 row(s) selected in 0.098s (prep 0.016s, exec 0.065s, 1st fetch 0.017s, fetch 0.017s)[1.4.0]--- 1 row(s) selected in 0.133s (prep 0.023s, exec 0.074s, 1st fetch 0.035s, fetch 0.035s)[1.4.1]--- 1 row(s) selected in 0.098s (prep 0.017s, exec 0.064s, 1st fetch 0.016s, fetch 0.016s) The numbers between square brackets have the following meaning:The first digit is the thread ID. As we have two threads in our example this ID is either 0 or 1;The second digit is the execution# for a given thread. As we have ten script executions for two threads, each thread will have to execute this script five times. So the execution#, in our example, is between 0 and 4;The third (last) digit is the command# in a given script. Our script contains two commands so this value is 0 or 1.So, for example, [0.3.1] means that the first thread (thread id=0) was executing its fourth job (thread execution#=3) and - more specifically - the second command in that script (script command#=1). Checking SQL ScriptsYou can check commands and SQL scripts with odb using the -N (null run) flag. This will just “prepare” (compile) the commands without executing them and fetching the results.Using Different Data Sources for Different ThreadsNormally all ODBC connections started by odb will use the same Data Source. However, there could be special cases where you want to use different DSN for different threads. In these cases you can use the –ndsn <number> option. This will append to the Data Source name specified via –d a suffix from 1 to -ndsn argument. So, for example:$ ./odb64luo ... –d MYDSN –ndsn 4It will use the following (round-robin) DSN/thread association: MYDSN1 for the first thread, MYDSN2 for the second thread and so on. The fifth thread (if any) will use MYDSN1 again. You can use a sequential DSN/thread association by using a ‘+’ sign in front of the –ndsn argument. So, for example, if you have 16 threads and –d MYDSN:Thread IDDSN with –ndsn 8DSN with –ndsn +80MYDSN1MYDSN11MYDSN2MYDSN12MYDSN3MYDSN23MYDSN4MYDSN24MYDSN5MYDSN35MYDSN6MYDSN36MYDSN7MYDSN47MYDSN8MYDSN48MYDSN1MYDSN59MYDSN2MYDSN510MYDSN3MYDSN611MYDSN4MYDSN612MYDSN5MYDSN713MYDSN6MYDSN714MYDSN7MYDSN815MYDSN8MYDSN8This technique has been used to maximize extraction throughput from a multi-segment Trafodion system. Each (local) Data Source was “linked” to a corresponding remote Data Source extracting its own data through its own network interface card.Formatting Query ResultsNormally odb prints query results using a very basic CSV format. Example:$ ./odb64luo -x "select s_suppkey, s_name, s_phone from tpch.supplier limit 5"1,Supplier#000000001,27-918-335-17362,Supplier#000000002,15-679-861-22593,Supplier#000000003,11-383-516-11994,Supplier#000000004,25-843-787-74795,Supplier#000000005,21-151-690-3663Adding the option -pad you will get a table format like this:$ ./odb64luo -x "select s_suppkey, s_name, s_phone from tpch.supplier limit 5" -pad s_suppkey|s_name |s_phone---------------+-------------------------+--------------- 1|Supplier#000000001 |27-918-335-1736 2|Supplier#000000002 |15-679-861-2259 3|Supplier#000000003 |11-383-516-1199 4|Supplier#000000004 |25-843-787-7479 5|Supplier#000000005 |21-151-690-3663Extracting Tables’ DDLYou can extract (multiple) tables’ DDL using either -i D… or -i U… options. For example:$ ./odb64luo -u xxx -p xxx -d traf -i D:TRAFODION.SEABASE.REGIONSodb [2015-04-20 21:25:35]: starting ODBC connection(s)... 0Connected to HP DatabaseCREATE TABLE TRAFODION.SEABASE."REGIONS" ( REGION_ID INTEGER NOT NULL,REGION_NAME VARCHAR(25));Sometimes you might want to extract the DDL for multiple objects, In this case you can use % wildcard. For example, the following command will extract the DDL for all tables in schema tpch starting with “P”:$ ./odb64luo -u xxx -p xxx -d traf -i D:TRAFODION.TPCH.P%odb [2015-04-20 21:33:43]: starting ODBC connection(s)... 0Connected to HP DatabaseCREATE TABLE TRAFODION.TPCH."PART" ( P_PARTKEY BIGINT NOT NULL,P_NAME VARCHAR(55) NOT NULL,P_MFGR CHAR(25) NOT NULL,P_BRAND CHAR(10) NOT NULL,P_TYPE VARCHAR(25) NOT NULL,P_SIZE INTEGER NOT NULL,P_CONTAINER CHAR(10) NOT NULL,P_RETAILPRICE BIGINT NOT NULL,P_COMMENT VARCHAR(23) NOT NULL,PRIMARY KEY (P_PARTKEY));CREATE TABLE TRAFODION.TPCH."PARTSUPP" ( PS_PARTKEY BIGINT NOT NULL,PS_SUPPKEY BIGINT NOT NULL,PS_AVAILQTY INTEGER NOT NULL,PS_SUPPLYCOST BIGINT NOT NULL,PS_COMMENT VARCHAR(199) NOT NULL,PRIMARY KEY (PS_PARTKEY,PS_SUPPKEY));odb [2015-04-20 21:33:45]: exiting. Session Elapsed time 2.069 seconds (00:00:02.069)When porting DDLs from one database to another we should consider possible differences in text column length semantic: certain database use “character oriented” text columns length while other use a “byte oriented” semantic. You can ask odb to multiply text column length when printing DDL using the switch –U[non-wide_char_multiplier,wide_char_multiplier]. For example: $ ./odb64luo -u xxx -p xxx -d traf -i U2,4:TRAFODION.TPCH.P%odb [2015-04-20 21:35:17]: starting ODBC connection(s)... 0Connected to HP DatabaseCREATE TABLE TRAFODION.TPCH."PART" ( P_PARTKEY BIGINT NOT NULL,P_NAME VARCHAR(110) NOT NULL,P_MFGR CHAR(50) NOT NULL,P_BRAND CHAR(20) NOT NULL,P_TYPE VARCHAR(50) NOT NULL,P_SIZE INTEGER NOT NULL,P_CONTAINER CHAR(20) NOT NULL,P_RETAILPRICE BIGINT NOT NULL,P_COMMENT VARCHAR(46) NOT NULL,PRIMARY KEY (P_PARTKEY));CREATE TABLE TRAFODION.TPCH."PARTSUPP" ( PS_PARTKEY BIGINT NOT NULL,PS_SUPPKEY BIGINT NOT NULL,PS_AVAILQTY INTEGER NOT NULL,PS_SUPPLYCOST BIGINT NOT NULL,PS_COMMENT VARCHAR(398) NOT NULL,PRIMARY KEY (PS_PARTKEY,PS_SUPPKEY));odb [2015-04-20 21:35:18]: exiting. Session Elapsed time 1.620 seconds (00:00:01.620)That command will multiply by 2 the length of “non-wide” text fields and by 4 the length of wide text fields.Loading, Extracting, and Copying DataLoading FilesYou can load a data file using -l option. For example:$ odb64luo –u user –p xx –d dsn –l src=customer.tbl:tgt=TRAFODION.MAURIZIO.CUSTOMER:fs=\|:rows=1000:loadcmd=UL:truncate:parallel=4This command will:load the file named customer.tbl (src=customer.tbl)in the table TRAFODION.MAURIZIO.CUSTOMER (tgt=TRAFODION.MAURIZIO.CUSTOMER)using “|” (vertical bar) as a field separator (fs=\|)using 1000 rows as rowset buffer (rows=1000)using UPSERT USING LOAD syntax to achieve better throughput as described in https://wiki.trafodion.org/wiki/index.php/Data_Loading (Trafodion only)truncating the target table before loading (truncate)using 4 parallel threads to load the target table (parallel=4)This is a complete list of the data loading operators: -l src=[-]file:tgt=table[:map=mapfile][:fs=fieldsep][:rs=recsep][:soe] [:skip=linestoskip][:ns=nullstring][:ec=eschar][:sq=stringqualifier] [:pc=padchar][:em=embedchar][:errmax=#max_err][:commit=auto|end|#rows|x#rs] [:rows=#rowset][:norb][:full][:max=#max_rec][:truncate][:show][:bpc=#][:bpwc=#] [:nomark][:parallel=number][:iobuff=#size][:buffsz=#size]][:fieldtrunc={0-4}] [:pre={@sqlfile}|{[sqlcmd]}][:post={@sqlfile}|{[sqlcmd]}][:ifempty] [:direct][:bad=[+]badfile][:tpar=#tables][:maxlen=#bytes][:time] [:xmltag=[+]element][:xmlord][:xmldump][:loadcmd=IN|UP|UL]The following table describes each data loading operator:Load optionMeaningsrc=<file>Input file. You can use the following keywords for this field:%t will be expanded to the (lower case) table name%T will be expanded to the (upper case) table name%s/%S will be expanded to the schema name%c/%C will be expanded to the catalog namestdin will load reading from the standard input-<file> to load all files listed in <file>hdfs][@host,port[,user]].<hdfspath> to load files from Hadoop File System (via libhdfs.so)mapr][@host,port[,user]].<maprpath> to load files from MapR File System (via libMapRClient.so)tgt=<CAT.SCH.TAB>This is the target tablefs=<char>|<code>This is the field separator. You can define the field separator:- as normal character (for example fs=,)- as ASCII decimal (for example fs=44 - 44 means comma)- as ASCII octal value (for example fs=054 – 054 means comma)- as ASCII hex value (for example fs=x2C – x2C means comma)Default field separator is ‘,’ (comma)rs=<char>|<code>This is the record separator. You can define the record separator the same way as the field separator. Default record separator is ‘\n’ (new line)pc=<char|code>Pad character used when loading fixed format files. You can use the same notation as the field separator.map=<mapfile>This will use mapfile to map source file to target table columns (see paragraph REF _Ref311455289 \r \* MERGEFORMAT 4.2)skip=numThis will skip a given number of lines when loading. This can be useful to skip headers in the source file.max=numThis is the max number of records to load. Default is to load all records in the input filens=<nullstring>odb will insert NULL when it finds nullstring in the input file. By default the nullstring is the empty stringsq=<char>|<code>This is the string qualifier character used to enclose strings. You can define the escape character the same way as the field separator.ec=<char>|<code>This is the character used as escape character. You can define the escape character the same way as the field separator. Default is “\” (back slash). rows=<num>|k<num>|m<num>This defines the size of the I/O buffer for each loading thread. You can define the size of this buffer in two different ways:- number of rows (for example: rows=100 means 100 rows as IO buffer)- buffer size in kB or MB (for example: rows=k512 (512 kB buffer) or rows=m20 (20MB buffer))Default value is 100.bad=[+]fileWhere to write rejected rows. If you omit this parameter rejected rows will be printed to standard error together with the error returned by the ODBC Driver.If you add a +sign in front of the file-name, odb will append instead of create the “bad file”.truncateWill truncate the target table before loadingifemptyWill load the target table only if it contains no recordsnorbWill load WITH NO ROLLBACKnomarkWon’t print the number of records loaded so far during loadssoeStop On Error. Will stop as soon as odb will encounter an errorparallel=numNumber of loading threads. odb will use:- one thread to read from the input file and - as many threads as the parallel argument to write via ODBC. This option is database independent.errmax=numodb will print up to num error messages per rowset. Normally used with soe to limit the number of error messages printed to the standard errorcommit=auto|end|#rows|x#rsThis defines how odb will commit the inserts. You have three choices:- auto will commit every single insert (see also rows load operator). This is the default.- end will commit when all rows (assigned to a given thread) have been inserted- #rows will commit every #rows inserted rows- x#rs will commit every #rs rowsets (see :rows)directThis will add /*+ DIRECT */ hint to the insert statement. To be used with Vertica databases in order to store inserted rows “directly” into the ROS (see Vertica’s documentation)fieldtrunc={0-4}This defines how odb will manage fields longer than the destination target column:- fieldtrunc=0 (default): will truncate input string, print a warning and load the truncated field if the target column is a text field;- fieldtrunc=1: like fieldtrunc=0 but no warning message is printed;- fieldtrunc=2: print an error message and does NOT load the row- fieldtrunc=3: like fieldtrunc=0 but tries to load the field even if the target column is NOT a text field- fieldtrunc=4: like fieldtrunc=3 but no warnings are printed.Be aware: last two options could bring to unwanted resilts because, for example, an input string like “2001-10-2345” will be loaded as a valid 2001-10-23 if the target field is a DATE.em=<char>|<code>This is the character used to embed binary files (see paragraph REF _Ref336620054 \r \h 4.6). You can define the embed character the same way as the field separator. No default value. pre={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd (enclosed between square brackets) on the target system immediately before loading the target table. You can, for example, CREATE the target table before loading it.Target table won’t be loaded if SQL execution fails and Stop On Error is setpost={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd (enclosed between square brackets) on the target system immediately after the target table has been loaded. You can, for example, update database stats after loading a table.tpar=numodb will load num tables in parallel when src is a list of files to be loadedshowodb will print what would be loaded in each column but no data is actually loaded. This I useful if you want to see how the input file “fits” into the target tables and is normally used to analyze the first few rows of CSV files (use :max).This option will force:- parallel to 1- rows to 1- ifempty to false- truncate to falsemaxlen=#bytesodb will limit the amount of memory allocated in the ODBC buffers for CHAR/VARCHAR fields to #bytestimeodb will print a “timeline” (milliseconds from start) for each insertbpc=#Bytes allocated in the ODBC buffer for each (non wide) CHAR/VARCHAR column length unit (default 1)bwpc=#Bytes allocated in the ODBC buffer for each (wide) CHAR/VARCHAR column length unit (default 4)Xmltag=[+]tagInput file is XML. Load all XML nodes under the one specified with this option. If a plus sign is specified, odb will load node attributes values. xmlordBy default odb will match target table columns with XML node or attributes using their names. If this option is specified odb will load the first node/attribute to the first column, the second node/attribute to the second column and so on without checking node/attribute namesxmldumpUsing this option odb won’t load the XML file content. XML attribute/tage names are printed to standard output so you can check what is going to be loaded.loadcmdSQL operation to be used for load, default is INSERT. UPSERT and UPSERT USING LOAD are also available for Trafodion. You can load multiple files using different -l options. By default odb will create as many threads (and ODBC connections) as the sum of parallel load threads. You can limit this number using -T option. So, for example: $ odb64luo –u user –p xx –d dsn –T 5 \ -l src=./data/%t.tbl.gz:tgt=TRAFODION.MAURO.CUSTOMER:fs=\|:rows=m2:truncate:norb:parallel=4 \ -l src=./data/%t.tbl.gz:tgt=TRAFODION.MAURO.ORDERS:fs=\|:rows=1000:truncate:norb:parallel=4 \ -l src=./data/%t.tbl.gz:tgt=TRAFODION.MAURO.LINEITEM:fs=\|:rows=m10:truncate:norb:parallel=4 Will truncate and load (CUSTOMER, ORDERS and LINEITEM) tables. Input files have the same name as the target tables – in lower cases). Loads will be distributed among available threads this way: Load OrderThread 0Thread 1Thread2Thread3Thread43rd Read lineitem.tblLoad TRAFODION.MAURO.LINEITEMLoad TRAFODION.MAURO.LINEITEMLoad TRAFODION.MAURO.LINEITEMLoad TRAFODION.MAURO.LINEITEM2nd Read orders.tblLoad TRAFODION.MAURO.ORDERSLoad TRAFODION.MAURO.ORDERSLoad TRAFODION.MAURO.ORDERSLoad TRAFODION.MAURO.ORDERS1st Read customer.tblLoad TRAFODION.MAURO.CUSTOMERLoad TRAFODION.MAURO.CUSTOMERLoad TRAFODION.MAURO.CUSTOMERLoad TRAFODION.MAURO.CUSTOMERIf you want to load more than one table in parallel you should use a number of threads defined as: (parallel + 1) * tables_to_load_in_parallel Note: You can load gzipped files without any special option. odb will automatically check input files and decompress them on the fly when needed.Also note that, even using one single loading thread (parallel=1), odb is faster than without parallel. This because if we do not specify parallel, odb will use one thread to both read from file and write into the target table:Read buffer #1 > Write Buffer #1 > Read Buffer #2 > Write Buffer #2 > Read Buffer #3 > Write Buffer #3 ...With parallel=1 we have one thread to read from file and one to write:Read buffer #1 > Read Buffer #2 > Read Buffer #3 > ... Write Buffer #1 > Write Buffer #2 > Write Buffer #3 ...Now reading from file is – normally – much faster than writing via ODBC so a single “reading thread” can serve different “loading threads”. One could ask: what the “right” number of loading threads is? In order to define the right number of loading threads you should run a few test and monitor the “Wait Cycles” reported by odb. Wait Cycles represent the number of times the “reading thread” had to wait for one “loading thread” to become available. When you have high “Wait Cycles/Total Cycles” ratio... it’s better to increase the number of writers. When the “Wait Cycles/Total Cycles” is less than 5%, adding more loading threads is useless or counterproductive. Mapping Source File Fields to Target Table Columnsodb, by default, assumes that input files contain as many fields as the target table columns. Also: file fields and target table columns are in the same order. This means that the first field in the input file will be loaded in the first table column, second input field will go to the second column and so on.If this basic assumption is not true and you need more flexibility to “link” input fields to target table columns, odb provides mapping/transformation capabilities though “mapfiles”. By specifying map=<mapfile> load option you can:Associate any input file field to any table columnSkip input file fieldsGenerate sequencesInsert constants Transform dates/timestamp formatsExtract substringsReplace input file strings. For example: insert ‘Maurizio Felici’ when you read ‘MF’Generate random values… and much moreA generic mapfile contains:Comments (line starting with ‘#’)Mappings to link input file fields to the corresponding target table columns. Mappings use the following syntax:<colname>:<field>[:transformation operator]. Where:<colname> is the target table column name.<field> is one of the following:The ordinal position (starting from zero) of the input file field. So: first input field is ‘0’ (zero), second input field is ‘1’ and so onCONST:<CONSTANT> to load a constant valueSEQ:<START> to generate/load a sequence starting from <START>IRAND:<MIN>:<MAX> to generate/load a random integer between <MIN> and <MAX>DRAND:<MIN_YEAR>:<MAX_YEAR> to generate/load a random date (YYYY-MM-DD) between <MIN_YEAR> and <MAX_YEAR>TMRAND: to generate/load a random time (hh:mm:ss) between 00:00:00 and 23:59:59TSRAND: to generate/load a random timestamp (YYYY-MM-DD hh:mm:ss) between midnight UTC – 01 Jan 1970 and the current timestampCRAND:<LENGTH> will generate/load a string of <LENGTH> characters randomly selected in the following ranges: a-z, A-Z, 0-9NRAND:<PREC>:<SCALE> will generate/load a random NUMERIC field with precision <PREC> and scale <SCALE>DSRAND:<file> will select and load a random line from <file>TXTRAND:<MIN_LENGTH>:<MAX_LENGTH>:<file>: will select and load a random portion of test from <file> with length between <MIN_LENGTH> and <MAX_LENGTH>LSTRAND:<VALUE1,VALUE2,…> will select and load a random value from <VALUE1,VALUE2,…>EMRAND:<MIN_ULENGTH>:<MAX_ULENGTH>:<MIN_DLENGTH>:<MAX_DLENGTH>:<SUFFIX1,SUFFIX2,…> will generate and load a string made of local@domain.suffix where:local is a string of random characters (a-z, A-Z, 0-9) with length between <MIN_ULENGTH> and <MAX_ULENGTH>domain is a string of random characters (a-z, A-Z, 0-9) with length between <MIN_DLENGTH> and <MAX_DLENGTH>suffix is a randomly selected suffix from <SUFFIX1,SUFFIX2,…>CDATE: to load the current date (YYYY-MM-DD)CTIME: to load the current time (hh:mm:ss)CTSTAMP: to load the current timestamp (YYYY-MM-SS hh:mm:ss)FIXED:<START>:<LENGTH> to load fixed format fields made of <LENGTH> characters starting at <START>. Note: <START> starts from zero.EMPTYASNULL: will load empty strings in the input file as NULLs (default is to load empty string as… empty strings)EMPTYASCONST:<CONSTANT>: will load empty fields in the input file as <CONSTANT>NULL: will insert NULL The final, optional, transformation operators can be:SUBSTR:<START>:<END>. For example if you have an input field containing “__Tib:student” a transformation rule like SUBSTR:3:6 will load “Tib” in the database. TSCONV:<FORMAT>. This operator convert timestamps from the input file format defined through <FORMAT> to YYYY-MM-DD HH:MM:SS before loading. The input format is defined through any combination of the following characters:CharMeaningb abbreviated month nameB full month named day of the monthH hour (24 hour format)m month numberM MinuteS Secondy year (four digits)D##decimal digits. ignore a single char _ ignore up to the next digitDCONV:<FORMAT>. This operator convert dates from the input file format defined through <FORMAT> to YYYY-MM-DD (see TSCONV operator). Example: DCONV:B.d.y will convert “August,24 1991” to 1991-08-24TCONV:<FORMAT>. This operator convert times from the input file format defined through <FORMAT> to HH:MM:SS (see TSCONV operator). REPLACE:<READ>:<WRITTEN>. This operator will load the string <WRITTEN> when the input file fields contains <READ>. If the input file string doesn’t match <READ> it will be loaded as is (see paragraph REF _Ref318964245 \r \h 4.3)TOUPPER. This operator converts the string read from the input file to uppercase before loading. Example: proGRAmMEr PROGRAMMERTOLOWER. This operator converts the string read from the input file to lowercase before loading. Example: proGRAmMEr programmerFIRSTUP. This operator converts the first character of the string read from the input file to uppercase and the remaining characters to lowercase before loading. Example: proGRAmMEr ProgrammerTRANSLIT:<LIST OF CHARS>:<LIST OF CHARS>. This opertator let you to delete or change any character with another. Examples:WORK:7:translit:Gp:HP will load the seventh input field into the target column named WORK and will replace all “G” with “H” and all “p” with “P”WORK:7:translit:Gp\r:HP\d behaves like the previous example but will also delete all “carriage returns” (\r)CSUBSTR. This operator is somehow similar to SUBSTR but instead of using fixed position to extract substrings will use delimiting characters. For example, suppose your input fields (comma is the field separator) are: … other fields…,name_Maurizio.programmer,…other fields … other fields…,_name_Lucia.housewife, …other fields… … other fields…,first_name_Giovanni.farmer,… other fields… … other fields…,_Antonella,… other fields… … other fields…,Martina,…other fields… … other fields…,Marco.student, …other fields … Using a transformation like: NAME:4:CSUBSTR:95:46 (where 95 is the ASCII code for “_” and 46 is the ASCII code for “.”) will result in loading the following values into the target (NAME) column:MaurizioLuciaGiovanniAntonellaMartinaMarcoCOMP. This operator will transform a packed binary COMP into a target database number. For example: hex 80 00 00 7b will be loaded as -123COMP3:PRECISION:SCALE. This operator will transform a packed binary COMP-3 format into a target database number. For example: hex 12 34 56 78 90 12 34 56 78 9b will be loaded as -1234567890123456.789ZONED:PRECISION:SCALE. This operator will transform a packed binary ZONED format into a target database number. For example: hex 31 32 33 34 35 36 will be loaded as +.123456Using mapfiles to Ignore and/or Transform Fields When LoadingLet me use an example to explain mapfile usage to skip/transform or generate fields. Suppose you have a target table like this:+------+---------------+----+-------+------------+|COLUMN|TYPE |NULL|DEFAULT|INDEX |+------+---------------+----+-------+------------+|ID |INTEGER SIGNED |NO | |mf_pkey 1 U ||NAME |CHAR(10) |YES | | ||AGE |SMALLINT SIGNED|YES | | ||BDATE |DATE |YES | | |+------+---------------+----+-------+------------+And an input file like this:uno,00,51,due,Maurizio,tre,07 Mar 1959, ignore,remaining, fieldsuno,00,46,due,Lucia,tre,13 Oct 1964, ignore, thisuno,00,34,due,Giovanni,tre,30 Mar 1976uno,00,48,due,Antonella,tre,24 Apr 1962242062034925BDATE00BDATE-29908517780AGE00AGE131000523495NAME00NAMEYou want to load the highlighted fields into the appropriate column, generate a unique key for ID and ignore the non-highlighted fields. In addition: you need to convert date format and replace all occurrencies of “Lucia” with “Lucy”. Well, you will need a mapfile like this:felici ~/Devel/odb $ cat test/load_map/ml1.map# Map file to load TRAFODION.MFTEST.FRIENDS from friends.datID:seq:1 This will insert into ID column a sequence starting from 1NAME:4:REPLACE:Lucia:Lucy This will load field #4 into NAME and replace all occurrences of Lucia with LucyAGE:2 This will load field #2 (they start from zero) into AGEBDATE:6:DCONV:d.b.y This will load field #6 into BDATE converting date format from “dd mmm yyyy”Then we will load this way:$ odb64luo –u user –p xx –d dsn \ -l src=frends.dat:tgt=TRAFODION.MFTEST.FRIENDS:map=ml1.map:fs=, Using mapfiles to Load Fixed Format FilesSuppose you have a target table like this:+------+---------------+----+-------+------------+|COLUMN|TYPE |NULL|DEFAULT|INDEX |+------+---------------+----+-------+------------+|NAME |CHAR(10) |YES | | ||JOB |CHAR(10) |YES | | ||BDATE |DATE |YES | | |+------+---------------+----+-------+------------+And an input file like this:GiovanniXXX30 Mar 1976YFarmer Lucia XXX13 Oct 1964YHousewifeMartina XXX28 Oct 1991Y? Marco XXX06 Nov 1994Y? MaurizioXXX07 Mar 1959YProgrammer31051527305NAME00NAME115443027305BDATE00BDATE204597027305JOB00JOBYou want to load the highlighted fields into the appropriate columns and to convert date format. Null values in the input file are represented by question marks. In this case you can use a mapfile like this:felici ~/Devel/odb $ cat test/fixed/ff.mapNAME:FIXED:0:8 insert into NAME characters starting at position 0, length 8BDATE:FIXED:11:11:DCONV:d.b.y insert into BDATE characters starting at col 11, length 11 and convert dateJOB:FIXED:23:10 insert into JOB characters starting at position 23, length 10Then we will load this way:$ odb64luo –u user –p xx –d dsn \ -l src=frends1.dat:tgt=TRAFODION.MFTEST.FRIENDS1:map=ff.map:ns=\?:pc=32 Where: pc=32 identify the pad character in the input file (space = ASCII 32) and ns=? defines the null string in the input file.Generating and Loading Dataodb can generate and load data for testing purposes. Let’s discuss odb capabilities in this area through an example. Suppose you want to fill with test data a table like this:MFELICI [MAURIZIO] (03:37:06) SQL> ls -D personCREATE TABLE TRAFODION.MAURIZIO."PERSON" ( PID BIGINT SIGNED NOT NULL ,FNAME CHAR(20) NOT NULL ,LNAME CHAR(20) NOT NULL ,COUNTRY VARCHAR(40) NOT NULL ,CITY VARCHAR(40) NOT NULL ,BDATE DATE NOT NULL ,SEX CHAR(1) NOT NULL ,EMAIL VARCHAR(40) NOT NULL ,SALARY NUMERIC SIGNED(9,2) NOT NULL ,EMPL VARCHAR(40) NOT NULL ,NOTES VARCHAR(80) ,LOADTS TIMESTAMP(0) ,PRIMARY KEY (PID));You can use a mapfile like this:felici ~/Devel/odb $ cat person.mapPID:SEQ:100FNAME:DSRAND:datasets/first_names.txtLNAME:DSRAND:datasets/last_names.txtCOUNTRY:DSRAND:datasets/countries.txtCITY:DSRAND:datasets/cities.txtBDATE:DRAND:1800:2012SEX:LSTRAND:M,F,UEMAIL:EMRAND:3:12:5:8:com,edu,org,netSALARY:NRAND:9:2EMPL:DSRAND:datasets/fortune500.txtNOTES:TXTRAND:20:80:datasets/lorem_ipsum.txtLOADTS:CTSTAMPwhere:PID:SEQ:100 will load a sequence starting from 100 into PIDFNAME:DSRAND:datasets/first_names.txt will load FNAME with a randomly selected value from first_names.txt. There are plenty of sample datasets available to generate all sort of data using realistic valuesLNAME:DSRAND:datasets/last_names.txt will load LNAME with a random value from last_names.txtCOUNTRY:DSRAND:datasets/countries.txt will load COUNTRY with a random value from countries.txtCITY:DSRAND:datasets/cities.txt will load CITY with a random value from cities.txtBDATE:DRAND:1800:2012 will generate and load into BDATE a random date between 1800-01-01 and 2012-12-31SEX:LSTRAND:M,F,U will load SEX with a random value in the M, F, U rangeEMAIL:EMRAND:3:12:5:8:com,edu,org,net will generate and load a local@domain.suffix email addresses where:local is made of 3 to 12 random charactersdomain is made of 5 to 8 random characterssuffix is com, ord, edu or netSALARY:NRAND:9:2 will generate and load a random NUMERIC(9,2)EMPL:DSRAND:datasets/fortune500.txt will load EMPL with a random value from fortune500.txtNOTES:TXTRAND:20:80:datasets/lorem_ipsum.txt will load NOTES with a random section of lorem_ipsum.txt with length between 20 and 80 charactersAnd, finally, LOADTS:CTSTAMP will load the current timestamp into LOADTSWe will generate and load test data with a command like this:$ ./ odb64luo -l src=nofile:tgt=traf.maurizio.person:max=1000000:map=person.map:rows=5000:parallel=8:loadcmd=ULPlease note the “src=nofile” (it means “there is no input file”) and “max=1000000” (generate and load one million rows). The above command has generated and loaded 1 Ml rows of “realistic” data in about ten seconds:[0] odb Loading statistics: [0] Target table: TRAFODION.MAURIZIO.PERSON [0] Source: nofile [0] Pre-loading time: 2.911 s [0] Loading time: 7.466 s [0] Total records read: 1,000,000 [0] Total records inserted: 1,000,000 [0] Total number of columns: 12 [0] Total bytes read: 3,963 [0] Average input row size: 0.0 B [0] ODBC row size: 323 B (data) + 88 B (len ind) [0] Rowset size: 5,000 [0] Rowset buffer size: 2,006.83 KiB [0] Load Performances (real data): 0.518 KiB/s [0] Load Performances (ODBC): 42,243.161 KiB/s [0] Reader Total/Wait Cycles: 200/16Loading Default ValuesThe simpler way to load database generated defaults is to ignore the associated columns in the map file. For example, suppose you have a table like this under Trafodion:create table TRAFODION.maurizio.dtest ( id largeint generated by default as identity not null, fname char(10), lname char(10) default 'Felici', bdate date, comment varchar(100) );If you have an input file containing:ignoreme,Maurizio,xyz,commentM, ignore,remaining, fieldsignoreme,Lucia,xyz,commentL, ignore, thisignoreme,Giovanni,xyz,commentGignoreme,Antonella,xyz,commentA-2667097790FNAME00FNAME1858645-3810COMMENT00COMMENTand a map-file like this:FNAME:1BDATE:CDATECOMMENT:4Then:First column (ID) will be loaded with its default value (not in the map file)Second column (FNAME) will be loaded with the second input field from file (FNAME:1)Third column (LNAME) will be loaded with its default value (not in the map file)Fourth column (BDATE) will be loaded with the Current Data generated by odb (BDATE:CDATE)And the Fifth column (COMMENT) will be loaded with the fifth column in the input file (COMMENT:4)Loading Binary FilesAssuming your backend database (and your ODBC Driver) supports BLOB data types, or equivalent, you can use odb to directly load binary (or any other) files into a database column using the [:em=char] symbol to identify the file to be loaded into that specific database field. Example. Suppose you have a table like this (MySQL):create table pers.myphotos ( id integer, image mediumblob, phts timestamp);You can load a file like this:$ cat myphotos.csv001,@/home/mauro/images/image1.jpg,2012-10-21 07:31:21002,@/home/mauro/images/image2.jpg,2012-10-21 07:31:21003,@/home/mauro/images/image3.jpg,2012-10-21 07:31:21by running a command like this:$ odb64luo –u user –p xx –d dsn -l src=myphotos.csv:tgt=pers.myphotos:em=\@odb will consider the string following the “em” character as the path of the file to be loaded in that specific field.Note: odb will not load rows where the size of the input file is greater than the target database column Reducing the ODBC Buffer Sizeodb will allocate memory for the ODBC buffers during load/extract operations based on the max possible length of the source/target columns. So, if you have a column defined as VARCHAR(2000), odb will allocate enough space for 2,000 characters in the ODBC buffer. Now, if you know in advance that you will never load/extract 2,000 characters you can limit the amount of space allocated by odb. This will reduce memory usage and increase performances because of the reduced network traffic.Suppose we have a table like this:felici ~/Devel/odb $ ./odb64luo -u xxx -p xxx -d traf -i D:TRAFODION.USR.TMXodb [2015-04-20 21:41:38]: starting ODBC connection(s)... 0Connected to HP DatabaseCREATE TABLE TRAFODION.USR."TMX" ( ID INTEGER NOT NULL,NAME VARCHAR(400),PRIMARY KEY (ID));And an input file containing:felici ~/Devel/odb $ cat tmx.dat1,Maurizio2,Lucia3,Martina4,Giovanni5,Marco6,Roland7,Randy8,Paul9,Josef10,Some other nameThe max length of the second field in this file is:felici ~/Devel/odb $ awk -F\, 'BEGIN{max=0} {if(NF==2){len=length($i);if(len>max)max=len}} END{print max}' tmx.dat15In this case you can use :maxlen=15 to limit the amount of the ODBC buffer:felici ~/Devel/odb $ ./odb64luo -u xxx -p xxx -d traf -l src=tmx.dat:tgt=usr.tmx:truncate:maxlen=15odb [2015-04-20 21:46:11]: starting ODBC connection(s)... 0Connected to HP Database[0.0.0]--- 0 row(s) deleted in 0.052s (prep 0.012s, exec 0.040s, fetch 0.000s/0.000s)[0] 10 records inserted [commit][0] odb version 1.1.0 Load(2) statistics:[0] Target table: (null).USR.TMX[0] Source: tmx.dat[0] Pre-loading time: 1.254 s (00:00:01.254)[0] Loading time: 0.026 s(00:00:00.026)[0] Total records read: 10[0] Total records inserted: 10[0] Total number of columns: 2[0] Total bytes read: 99[0] Average input row size: 9.9 B[0] ODBC row size: 26 B (data) + 16 B (len ind)[0] Rowset size: 100[0] Rowset buffer size: 4.10 KiB[0] Load throughput (real data): 3.718 KiB/s[0] Load throughput (ODBC): 9.766 KiB/sodb [2015-04-20 21:46:12]: exiting. Session Elapsed time 1.294 seconds (00:00:01.294)If you do not specify this parameter odb will allocate the buffer for the max possible length of each field:felici ~/Devel/odb $ ./odb64luo -u xxx -p xxx -d traf -l src=tmx.dat:tgt=usr.tmx:truncateodb [2015-04-20 21:47:13]: starting ODBC connection(s)... 0Connected to HP Database[0.0.0]--- 10 row(s) deleted in 0.107s (prep 0.012s, exec 0.095s, fetch 0.000s/0.000s)[0] 10 records inserted [commit][0] odb version 1.1.0 Load(2) statistics:[0] Target table: (null).USR.TMX[0] Source: tmx.dat[0] Pre-loading time: 1.330 s (00:00:01.330)[0] Loading time: 0.032 s(00:00:00.032)[0] Total records read: 10[0] Total records inserted: 10[0] Total number of columns: 2[0] Total bytes read: 99[0] Average input row size: 9.9 B[0] ODBC row size: 411 B (data) + 16 B (len ind)[0] Rowset size: 100[0] Rowset buffer size: 41.70 KiB[0] Load throughput (real data): 3.021 KiB/s[0] Load throughput (ODBC): 125.427 KiB/sodb [2015-04-20 21:47:14]: exiting. Session Elapsed time 1.373 seconds (00:00:01.373)Extracting TablesYou can use odb to extract tables from a database and write them to standard files (or named pipes). This is a table extraction example:$ odb64luo –u user –p xx –d dsn –T 3 \ -e src=TRAFODION.MAURIZIO.LIN%:tgt=${DATA}/ext_%t.csv.gz:rows=m10:fs=\|:trim:gzip: \ -e src=TRAFODION.MAURIZIO.REGION:tgt=${DATA}/ext_%t.csv.gz:rows=m10:fs=\|:trim:gzip \ -e src=TRAFODION.MAURIZIO.NATION:tgt=${DATA}/ext_%t.csv.gz:rows=m10:fs=\|:trim:gzip \ The command here above will:extract tables REGION, NATION and all tables starting with LIN from TRAFODION.MAURIZIO schema, data will be saved into files ext_%t.csv.gz (%t will be expanded to the real table name), output file will be gzipped on the fly (uncompressed data will never land to disk),text fields will be trimmedthe IO buffer is 10 MBthe extraction process will use three threads (ODBC connection)This is a complete list of the extraction options: -e {src={table|-file}|sql=<custom sql>}:tgt=[+]file[:pwhere=where_cond] [:fs=fieldsep][:rs=recsep][:sq=stringqualifier][:ec=escape_char][:soe] [:ns=nullstring][es=emptystring][:rows=#rowset][:nomark][:binary][:fwc] [:max=#max_rec][:trim=[cCvVdt]][:rtrim][:cast][:multi][:efs=string] [:parallel=number][:gzip][:gzpar=wb??][:uncommitted][:splitby=column] [:pre={@sqlfile}|{[sqlcmd]}[:mpre={@sqlfile}|{[sqlcmd]}[:post={@sqlfile}|{[sqlcmd]}] [tpar=#tables][:time][:nts][:cols=[-]columns]][:maxlen=#bytes][:xml]The following table describes each extract operator:Extract optionMeaningsrc=<CAT.SCH.TAB>|-fileThis filed is used to define the source table(s). You can use here:- a single table name (for example TRAFODION.MFTEST.LINEITEM)- a group of tables (for example TRAFODION.MFTEST.LIN%)- a file containing a list of tables to extract (‘-‘ should precede the filename)sql=<sql>This is a custom SQL command you can use to extract data. This is alternative to src=tgt=[+]fileOutput file. You can use the following keywords for this field:%t/%T will be expanded to the (lower/upper case) table name%s/%S will be expanded to the (lower/upper case) schema name%c/%C will be expanded to the (lower/upper case) catalog name%d will be expanded to the extraction date (YYYYMMDD format)%D will be expanded to the extraction date (YYYY-MM-DD format)%m will be expanded to the extraction time (hhmmss format)%M will be expanded to the extraction time (hh:mm:ss format)stdout will print the extracted records to the standard output.If you add a +sign in front of the file-name, odb will append instead of create it.hdfs./<hdfspath>/<file> to write exported table under Hadoop File Systemfs=<char>|<code>This is the field separator. You can define the field separator:- as normal character (for example fs=,)- as ASCII decimal (for example fs=44 - 44 means comma)- as ASCII octal value (for example fs=054 – 054 means comma)- as ASCII hex value (for example fs=x2C – x2C means comma)Default field separator is ‘,’ (comma)rs=<char>|<code>This is the record separator. You can define the record separator the same way as the field separator. Default record separator is ‘\n’ (new line)max=numThis is the max number of records to extract. Default is to extract all recordssq=<char>|<code>This is the string qualifier character used to enclose strings. You can define the string qualifier the same way as the field separatorec=<char>|<code>This is the character used as escape character. You can define the escape character the same way as the field separator. Default is “\” (back slash). rows=<num>|k<num>|m<num>This defines the size of the I/O buffer for each extraction thread. You can define the size of this buffer in two different ways:- number of rows (for example: rows=100 means 100 rows as IO buffer)- buffer size in kB or MB (for example: rows=k512 (512 kB buffer) or rows=m20 (20MB buffer))ns=<nullstring>This is how odb will represent NULL values in the output file. Default is the empty string (two field separators one after the other)es=<emptystring>This is how odb will represent VARCHAR empty strings (NOT NULL with zero length) values in the output file. Default is the empty string (two field separators one after the other)gzpar=<params>This are extra parameters you can pass to tune the gzip compression algorithm. Examples:- gzpar=wb9: max compression (slower)- gzpar=wb1: basic compression (faster)- gzpar=wb6h: Huffman compression only- gzpar=wb6R: Run-length encoding onlytrim[=<params>]Accept the following optional parameters:ctrims leading spaces from CHAR fieldsCtrims trailing spaces from CHA R fields NOTEREF _Ref317771400 \h \* MERGEFORMAT 2vtrims leading spaces from VARCHAR fieldsVtrims trailing spaces from VARCHAR fieldsdtrims trailing zeros after decimal sign. Example: 12.3000 will be extracted as 12.3. ttrims decimal portion from TIME/TIMESTAMP fields. For example: 1999-12-19 12:00:21.345 will be extracted as 1999-12-19 12:00:21trim examples::trim=cC trims leading/trailing spaces from CHAR fields:trim=cCd trims leading/trailing spaces from CHARs and trailing decimal zerosIf you do not specify any argument for this operator odb will use “cCvV”. In other words :trim: is a shortcut for :trim=cCvV:nomarkWon’t print the number of records extracted so far by each threadsoeStop On Error. Will stop as soon as odb will encounter an errorparallel=numodb will use as many threads as the parallel argument to extract data from partitioned source tables. You have to use splitby.Each thread will take care of a specific range of the source table partitions. For example if you specify parallel=4 and the source table is made of 32 partitions, odb will start four threads (four ODBC connections):- thread 0 will extract partitions 0-7- thread 1 will extract partitions 8-15- thread 2 will extract partitions 16-23- thread 3 will extract partitions 24-31multiThis option can be used in conjunction with parallel operator to write as many ouput files as the number of extraction threads. Output file names are built adding four digits at the end of the file identified by the tgt operator.For example, with src=trafodion.mauro.orders:tgt=%t.csv:parallel=4:multi odb will write into the following output files:- orders.csv.0001- orders.csv.0002- orders.csv.0003- orders.csv.0004pwhere=<where condition>This option is used in conjunction with parallel and will limit the extraction to records satisfying the where condition. Note: The where condition is limited to columns in the source table.For example: you want to extract records with TRANS_TS > 1999-12-12 09:00:00 from the source table TRAFODION.MAURO.MFORDERS using 8 parallel streams to a single, gzipped, file having the same name as the source table:src=trafodion.mauro.mforders:tgt=%t.gz:gzip:parallel=8:pwhere=[TRANS_TS > TIMESTAMP ‘1999-12-12 09:00:00’]... You can enclose the where condition between square brackets to avoid a misinterpretation of the characters in the where condition.errmax=numodb will print up to num error messages per rowset. Normally used with soe to limit the number of error messages printed to the standard erroruncommittedWill add FOR READ UNCOMMITTED ACCESS to the select(s) command(s)rtrimThis will RTRIM() CHAR columns on the server. From a functional point of view this is equivalent to trim=C but rtrim is executed on the server so it saves both client CPU cycles and network bandwidth.castIt performs a (server side) cast to VARCHAR for all non-text columns. Main scope of this operator is to "move" CPU cycles from the client to the database server. It increases network traffic. To be used when: - the extraction process is CPU bound on the client AND - network has a lot of available bandwidth AND - database server CPUs are not "under pressure".Tests extracting a table full of NUMERIC(18,4), INT and DATES shows: - client CPU cycles down ~50% on the client - network traffic up ~40%splitby=<column>This operator let you to use parallel extract from any database. <column> has to be a SINGLE, numeric column. odb will calculate min()/max() value for <column> and assign to each <parallel> thread the extraction of the rows in its "bucket". For example, if you have: ...:splitby=emp_id:parallel=4...with min(emp_id)=1 and max(emp_id)=1000, the four threads will extract the following rows: thread #0 emp_id >=1 and emp_id < 251 thread #1 emp_id >=251 and emp_id < 501 thread #2 emp_id >=501 and emp_id < 751 thread #3 emp_id >=751 and emp_id < 1001 (odb uses max(emp_id) + 1)If the values are not equally distributed data extraction will be deskewed. pre={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd SQL command (enclosed between square brackets) on the source system immediately before table extraction.Source table won’t be extracted if SQL execution fails and Stop On Error is setmpre={@sqlfile}|{[sqlcmd]}Each odb thread will run either sqlfile script or sqlcmd SQL command (enclosed between square brackets)on the source system immediately before table extraction. You can use mpre to set database specific features for each extraction thread. Examples:1) you want Trafodion to ignore missing stats warning. Then you can run via mpre a sql script containing:control query default HIST_MISSING_STATS_WARNING_LEVEL '0';2) you want Oracle to extract dates in the YYYY-MM-DD hh:mm:ss format. Then you can run via mpre a script containing:ALTER SESSION SET NLS_DATE_FORMAT='YYYY-MM-DD HH:MI:SS'post={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd SQL command (enclosed between square brackets) on the source system immediately after table extraction. tpar=numodb will extract num tables in parallel when src is a list of files to be loadedmaxlen=#bytesodb will limit the amount of memory allocated in the ODBC buffers for CHAR/VARCHAR fields to #bytesxmlWill write output file in XML format timeodb will print a “timeline” (milliseconds from start)Extracting a List of TablesYou can use odb to extract all tables listed in a file. Example:felici ~/Devel/odb $ cat tlist.txt# List of tables to extractsrc=TRAFODION.MAURIZIO.ORDERSsrc=TRAFODION.MAURIZIO.CUSTOMERsrc=TRAFODION.MAURIZIO.PARTsrc=TRAFODION.MAURIZIO.LINEITEMYou can extract all these tables by running:$ odb64luo –u user –p xx –d dsn -e src=-tlist.txt:tgt=%t_%d%m:rows=m20:sq=\”Please note the src=-tlist.txt. Copying Tables From One Database to Anotherodb can directly copy tables from one data-source to another (for example from Trafodion to Teradata or vice-versa). Using this option data will never land to disk. Target table has to be be created in advance and should have a compatible structure. Here we have a complete list of the odb’s copy operators: -cp src={table|-file:tgt=schema[.table][pwhere=where_cond][:soe][:nts] [:truncate][:rows=#rowset][:nomark][:max=#max_rec][:fwc][:bpwc=#] [:parallel=number][errmax=#max_err][:commit=auto|end|#rows|x#rs][:time] [:direct][:uncommitted][:norb][:splitby=column][:pre={@sqlfile}|{[sqlcmd]}] [:post={@sqlfile}|{[sqlcmd]}][:mpre={@sqlfile}|{[sqlcmd]}][:ifempty] [:loaders=#loaders][:tpar=#tables][:cols=[-]columns] [sql={[sqlcmd]|@sqlfile|-file}[:bind=auto|char|cdef] [tmpre={@sqlfile}|{[sqlcmd]}][seq=field#[,start]]odb copy’s operators:Copy operatorMeaningsrc=<CAT.SCH.TAB>|-fileThis filed is to define the source table(s). You can use here:- a single table (for example: TRAFODION.MFTEST.LINEITEM)- a group of tables (for example: TRAFODION.MFTEST.LIN%)- a file containing a list of tables to copy (‘-‘ should precede the filename)tgt=<CAT.SCH.TAB>Target table(s). You can use the following keywords for this field:%t/%T will be expanded to the (lower/upper case) source table name%s/%S will be expanded to the (lower/upper case) source schema name%c/%C will be expanded to the (lower/upper case) source catalog namesql={[sqlcmd]|@sqlfile|-file}odb use a generic SQL– instead of a “real” table – as source.max=numThis is the max number of records to copy. Default is to copy all records in the source tablerows=<num>|k<num>|m<num>This defines the size of the I/O buffer for each copy thread. You can define the size of this buffer in two different ways:- number of rows (for example: rows=100 means 100 rows as IO buffer)- buffer size in kB or MB (for example: rows=k512 (512 kB buffer) or rows=m20 (20MB buffer))truncateWill truncate the target table before loadingifemptyWill load the target table only if emptynomarkWon’t print the number of records loaded so far during loadssoeStop On Error. Will stop as soon as odb will encounter an errorparallel=numodb will use as many threads as the parallel argument to extract data from partitioned source tables PLUS an equivalent number of threads to write to the target table. For example if you specify parallel=4 and the source table is made of 32 partitions, odb will start four threads (four ODBC connections) to read from the source table PLUS four threads (four ODBC connections) to write to the target table:- thread 0 will extract partitions 0-7 from source- thread 1 will write data extracted from thread 0 to target- thread 2 will extract partitions 8-15 from source- thread 3 will write data extracted from thread 2 to target- thread 4 will extract partitions 16-23 from source- thread 5 will write data extracted from thread 4 to target- thread 6 will extract partitions 24-31 from source- thread 7 will write data extracted from thread 6 to targetYou have to specify splitby.pwhere=<where condition>This option is used in conjunction with parallel to copy only records satisfying the where condition. Note: The where condition is limited to columns in the source table.For example: you want to copy records with TRANS_TS > 1999-12-12 09:00:00 from the source table TRAFODION.MAURO.MFORDERS using 8 parallel streams to a target table having the same name as the source table:src=trafodion.mauro.mforders:tgt=trafodion.dest_schema.%t:parallel=8:pwhere=[TRANS_TS > TIMESTAMP ‘1999-12-12 09:00:00’]... You can enclose the where condition between square brackets to avoid a misinterpretation of the characters in the where condition.commit=auto|end|#rows|x#rsThis defines how odb will commit the inserts. You have three choices:- auto will commit every single insert (see also rows load operator). This is the default.- end will commit when all rows (assigned to a given thread) have been inserted- #rows will commit every #rows copied rows- x#rs will commit every #rs rowsets copied (see :rows)directThis will add /*+ DIRECT */ hint to the insert statement. To be used with Vertica databases in order to store inserted rows “directly” into the ROS (see Vertica’s documentation)errmax=numodb will print up to num error messages per rowset. Normally used with soe to limit the number of error messages printed to the standard erroruncommittedWill add FOR READ UNCOMMITTED ACCESS to the select(s) command(s)splitby=<column>This operator let you to use parallel copy from any database. <column> has to be a SINGLE, numeric column. odb will calculate min()/max() value for <column> and assign to each <parallel> thread the extraction of the rows in its "bucket". For example, if you have: ...:splitby=emp_id:parallel=4...with min(emp_id)=1 and max(emp_id)=1000, the four threads will extract the following rows: thread #0 emp_id >=1 and emp_id < 251 thread #1 emp_id >=251 and emp_id < 501 thread #2 emp_id >=501 and emp_id < 751 thread #3 emp_id >=751 and emp_id < 1001 (odb uses max(emp_id) + 1)If the values are not equally distributed data extraction will be deskewed. pre={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd (enclosed between square brackets) on the target system immediately before loading the target table. You can, for example, CREATE the target table before loading it.Target table won’t be loaded if SQL execution fails and Stop On Error is setmpre={@sqlfile}|{[sqlcmd]}Each odb thread will run either sqlfile script or sqlcmd (enclosed between square brackets) on the source system immediately before loading the target table. You can use mpre to set database specific features for each thread.tmpre={@sqlfile}|{[sqlcmd]}Each odb thread will run either sqlfile script or sqlcmd (enclosed between square brackets) on the target system immediately before loading the target table. You can use mpre to set database specific features for each thread.post={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd (enclosed between square brackets) on the target system immediately after the target table has been loaded. You can, for example, update database stats after loading a table.tpar=numodb will copy num tables in parallel when src is a list of files to be loadedloaders=numodb will use num load threads for each extract thread (default is 2 loaders per extractor) fwcForce Wide Characters. odb will consider SQL_CHAR/SQL_VARCHAR fileds as they were defined SQL_WCHAR/SQL_WVARCHARbpwc=#odb internally allocates 4 bytes/char foe SQL_WCHAR/SQL_WVARCHAR columns. You can modify the number of bytes allocated for each char using this parameterbind=auto|char|cdefodb can bind columns to ODBC buffer as characters (char) or “C Default” data types (cdef). The default (auto) will use cdef if SRC/TGT use the same database or char if SRC/TGT databases differ seq=field#[,start]odb will add a sequence when loading the target system on column number field#. You can optionally define the sequence start value (default 1)timeodb will print a “timeline” (milliseconds from start)When copying data from one data source to another, odb will need user/password/dsn for both source and target system. User credentials and DSN for the target system are specified this way:$ odb64luo –u src_user:tgt_user –p src_pwd:tgt:pwd –d src_dsn:tgt_dsn ... -cp src=...:tgt=...Copying a List of TablesYou can use odb to copy a list of tables from one database to another. Example:felici ~/Devel/odb $ cat tlist.txt# List of tables to extractsrc=TRAFODION.MAURIZIO.ORDERSsrc=TRAFODION.MAURIZIO.CUSTOMERsrc=TRAFODION.MAURIZIO.PARTsrc=TRAFODION.MAURIZIO.LINEITEMYou can extract all these tables by running:$ odb64luo –u user1:user2 –p xx:yy –d dsn1:dsn2 \ -cp src=-tlist.txt:tgt=tpch.stg_%t:rows=m2:truncate:parallel=4 -T 8Please note the src=-tlist.txt. Will copy:SourceTargetTRAFODION.MAURIZIO.ORDERStpch.stg_ordersTRAFODION.MAURIZIO.CUSTOMERtpch.stg_customerTRAFODION.MAURIZIO.PARTtpch.stg_partTRAFODION.MAURIZIO.LINEITEMtpch.stg_lineitemYou can optionally define any other “command line” options in the input file. For example you can use different “splitby columns”:For example:felici ~/Devel/odb $ cat tlist2.txt# List of tables to extract and their “splitby columns”src=TRAFODION.MAURIZIO.ORDERS:splitby=O_ORDERKEYsrc=TRAFODION.MAURIZIO.CUSTOMER:splitby=C_CUSTOMERKEYsrc=TRAFODION.MAURIZIO.PART:splitby=P_PARTKEYsrc=TRAFODION.MAURIZIO.LINEITEM:splitby=L_PARTKEYUsing Case-Sensitive Table and Column NamesThe possibility to use case sensitive table/column names depends on your database configuration. odb will maintain table/column case sensitiveness when they are enclosed in double quotes. This will create a TRAFODION.MAURIZIO.Names table made of three columns: “name”, “NAME” and “Name” create table trafodion.maurizio."Names" ( "name" char(10), "NAME" char(10), "Name" char(10) ) no partitions;Double quotes have to be escaped under *nix. A few examples:felici ~/Devel/odb $ ./odb64luo -i T:trafodion.maurizio.\"Names\"felici ~/Devel/odb $ ./odb64luo -x "select * from trafodion.maurizio.\"Names\""felici ~/Devel/odb $ ./odb64luo -l src=names.txt:tgt=trafodion.maurizio.\"Names\":map=names.map:pc=32You can omit double quotes around column names when using mapfiles.Determining the Appropriate Number of Threads for Load/Extract/Copy/DiffIf you have to load/extract or copy multiple tables in parallel the best option is to use the options :tpar=number and :parallel=number. The first option (:tpar) defines how many tables have to be copied/extracted in parallel; the second option defines how many “data streams” to use for each table. This way odb will automatically allocate and start the “right” number of threads.My rule of thumb when copying/loading or extracting tables is to use as many “data streams” as:min(number of middletier CPUs, number of source CPUs, number of target CPUs)The number of threads started for each “data stream” depend on the operation type: OperationTotal threadsExplanationExample with parallel=4Loadparallel + 1One thread to read from file + one thread per parallel to load5ExtractparallelOne thread per parallel to extract4Copyparallel * (1+loaders)Two threads per parallel: read from source and write to target12 (if loaders=2)Diffparallel * 3Three threads per parallel: read from source, read from target, compare12 Integrating With HadoopThere are basically two ways to integrate a generic database with Hadoop using odb:First Option is to use HIVE (Hadoop DWH) and its ODBC Driver. In this case odb can access HIVE like any other “normal” relational database. So – for example – you can copy to from HIVE and other databases using odb’s copy optionSecond option if to add the hdfs. prefix to the input or output file during loads/extracts. In this case the file will be read/written from/to Hadoop. odb interacts directly with the HDFS file system using libhdfs. This option is currently available only under Linux.Comparing Tables (Technology Preview)You can use odb to compare two tables with the same structure on different databases. In order to compare the two tables odb will:extract source/target tables ordered by Primary Key or any other set of columns (see key option here below) compare source/target ODBC buffers without “unpacking” them into columns/rows. Each “comparison stream” is made of three threads:One reading from the source tableOne reading from the target tableOne comparing the source/target buffersThese three threads work in parallel: the “compare” thread will check buffer N while the other two threads extract the net block of data from the source/target database in parallel.You can have multiple “triplets” working in parallel on different section of the table using the splitby operator. For example the following command:[dbadmin@n021 mftest]$ odb64luo -u MFELICI:maurizio.felici@hp.com -d MFELICI:VMFELICI -p xx:yy –diff src=trafodion.maurizio.lineitem:tgt=mftest.lineitem:key=l_orderkey,l_linenumber:output=lineitem.diff:rows=m2:print=IDC:splitby=l_orderkey:parallel=8Compare two tables using 8 streams (parallel=8) made of three threads each. The comparison threads use double buffering and advanced memory comparison techniques. odb can provide the following information in output as a CSV file:Missing rows on target (‘D’ – deleted – rows) based on the key columnsNew rows on target (‘I’ – inserted – rows) based on the key columnsChanged rows (same key columns but with different values in other fields). For these rows odb can print the original source version (‘C’ rows) and/or the modified target version (‘U’ rows).Here you have an example of the odb output when comparing two tables:[dbadmin@n021 mftest]$ cat lineitem.diffDTYPE,L_ORDERKEY,L_LINENUMBER,L_SUPPKEY,L_PARTKEY,L_QUANTITY,L_EXTENDEDPRICE,L_DISCOUNT,L_TAX,L_RETURNFLAG,L_LINESTATUS,L_SHIPDATE,L_COMMITDATE,L_RECEIPTDATE,L_SHIPINSTRUCT,L_SHIPMODE,L_COMMENTD,4532896,1,5974,100953,42.00,82065.90,0.03,0.00,R,F,1994-12-15,1995-01-17,1995-01-07,COLLECT COD,TRUCK,leep across the caD,4532896,2,2327,102326,48.00,63759.36,0.07,0.05,A,F,1995-02-18,1994-12-10,1995-03-12,TAKE BACK RETURN,RAIL,usly regular platelets. carefulD,4532896,3,612,193054,12.00,13764.60,0.05,0.02,R,F,1994-11-17,1994-11-23,1994-12-06,COLLECT COD,SHIP,s haggle quickly. ideas after theD,4532896,4,9867,47362,36.00,47136.96,0.10,0.06,A,F,1995-01-05,1994-11-29,1995-01-06,COLLECT COD,RAIL,s haggle carefully boD,4532896,5,9576,2075,19.00,18564.33,0.00,0.05,R,F,1994-11-26,1995-01-17,1994-12-03,COLLECT COD,TRUCK,en sauternes integrate blithely alonD,4532896,6,1016,68509,9.00,13297.50,0.07,0.00,R,F,1995-02-16,1995-01-05,1995-02-24,TAKE BACK RETURN,RAIL,ily above the blithelC,1652227,3,2298,87281,28.00,35511.84,0.06,0.05,R,F,1993-05-04,1993-03-12,1993-05-12,TAKE BACK RETURN,MAIL,lly final accoU,1652227,3,2298,87281,99.99,35511.84,0.06,0.05,R,F,1993-05-04,1993-03-12,1993-05-12,TAKE BACK RETURN,MAIL,lly final accoD,3456226,1,8161,148160,22.00,26579.52,0.06,0.02,A,F,1994-06-26,1994-06-08,1994-07-10,DELIVER IN PERSON,FOB,uriously. furioD,3456226,2,6293,108762,20.00,35415.20,0.10,0.05,R,F,1994-05-07,1994-06-03,1994-05-15,NONE,RAIL,ously bold requests along the bD,3456226,3,4542,159511,33.00,51826.83,0.05,0.03,A,F,1994-07-04,1994-05-15,1994-07-26,NONE,FOB,wake carefully alD,3456226,4,154,95135,33.00,37294.29,0.04,0.08,A,F,1994-05-27,1994-05-10,1994-06-14,DELIVER IN PERSON,AIR,ests. unusual dependencies wake fluffilyD,3456226,5,9027,126514,31.00,47755.81,0.08,0.01,R,F,1994-06-13,1994-06-18,1994-07-10,TAKE BACK RETURN,FOB,according to the carefully regular instructD,3456226,6,8477,110943,14.00,27355.16,0.03,0.01,R,F,1994-07-03,1994-05-28,1994-07-13,TAKE BACK RETURN,FOB,onic accounts. ironic, pendD,3456226,7,1773,4272,34.00,39993.18,0.08,0.00,A,F,1994-05-01,1994-05-29,1994-05-15,TAKE BACK RETURN,MAIL,ounts are finally caD,3456227,7,3722,101211,22.00,26668.62,0.02,0.01,N,O,1997-12-16,1998-02-05,1997-12-19,NONE,TRUCK,uriously even platelets are fuI,3456227,8,3722,101211,22.00,26668.62,0.02,0.01,N,O,1997-12-16,1998-02-05,1997-12-19,NONE,TRUCK,uriously even platelets are fuI,9999999,1,8161,148160,22.00,26579.52,0.06,0.02,A,F,1994-06-26,1994-06-08,1994-07-10,DELIVER IN PERSON,FOB,uriously. furioI,9999999,2,6293,108762,20.00,35415.20,0.10,0.05,R,F,1994-05-07,1994-06-03,1994-05-15,NONE,RAIL,ously bold requests along the bI,9999999,3,4542,159511,33.00,51826.83,0.05,0.03,A,F,1994-07-04,1994-05-15,1994-07-26,NONE,FOB,wake carefully alI,9999999,4,154,95135,33.00,37294.29,0.04,0.08,A,F,1994-05-27,1994-05-10,1994-06-14,DELIVER IN PERSON,AIR,ests. unusual dependencies wake fluffilyI,9999999,5,9027,126514,31.00,47755.81,0.08,0.01,R,F,1994-06-13,1994-06-18,1994-07-10,TAKE BACK RETURN,FOB,according to the carefully regular instructI,9999999,6,8477,110943,14.00,27355.16,0.03,0.01,R,F,1994-07-03,1994-05-28,1994-07-13,TAKE BACK RETURN,FOB,onic accounts. ironic, pendI,9999999,7,1773,4272,34.00,39993.18,0.08,0.00,A,F,1994-05-01,1994-05-29,1994-05-15,TAKE BACK RETURN,MAIL,ounts are finally caAs you can see the first column defines the type of difference.Here we have a complete list of the odb’s diff operators: -diff src={table|-file}:tgt=table:[key=columns][:output=[+]file][:pwhere=where_cond] [:pwhere=where_cond][:nomark][:rows=#rowset][:odad][:fs=fieldsep][:time] [:rs=recsep][:quick][:splitby=column][:parallel=number][:max=#max_rec] [:print=[I][D][C]][:ns=nullstring][:es=emptystring][:fwc][:uncommitted] [:pre={@sqlfile}|{[sqlcmd]}][:post={@sqlfile}|{[sqlcmd]}][tpar=#tables]odb copy’s operators:Diff operatorMeaningsrc=<CAT.SCH.TAB>|-fileThis field definse the source table(s). You can use here:- a single table (for example: TRAFODION.MFTEST.LINEITEM)- a file containing a list of tables to compare (‘-‘ should precede the filename)tgt=<CAT.SCH.TAB>This is the target table. key=column[,column,...]This option is used to define how to order records extracted from both source and target table. If you do not specify any key column, odb will use the Primary Keyoutput=[+]fileThis is the output file where the differences will be reported. You can use stdout to print odb output on the standard output. A + sign in front of the file-name will tell odb to append to an existing file.Default value: stdoutfs=<char>|<code>This is the field separator of the output file. You can define the field separator:- as normal character (for example fs=,)- as ASCII decimal (for example fs=44 - 44 means comma)- as ASCII octal value (for example fs=054 – 054 means comma)- as ASCII hex value (for example fs=x2C – x2C means comma)Default field separator is ‘,’ (comma)rs=<char>|<code>This is the record separator used in the output file. You can define the record separator the same way as the field separator. Default record separator is ‘\n’ (new line)max=numThis is the max number of records to compare. Default is to compare all recordsrows=<num>|k<num>|m<num>This defines the size of the I/O buffer for each extraction thread. You can define the size of this buffer in two different ways:- number of rows (for example: rows=100 means 100 rows as IO buffer)- buffer size in kB or MB (for example: rows=k512 (512 kB buffer) or rows=m20 (20MB buffer))ns=<nullstring>This is how odb will represent NULL values in the output file. Default is the empty string (two field separators one after the other)es=<emptystring>This is how odb will represent VARCHAR empty strings (NOT NULL with zero length) values in the output file. Default is the empty string (two field separators one after the other)nomarkWon’t print the number of records extracted so far by each threadsoeStop On Error. Will stop as soon as odb will encounter an errorparallel=numodb will use as many “threads triplets” (extract from source, extract from target, compare) as the parallel argument.Each thread will take care of a specific range of the source table data defined through the splitby optionuncommittedWill add FOR READ UNCOMMITTED ACCESS to the select(s) command(s)splitby=<column>This operator let you to use parallel extract from any database. <column> has to be a SINGLE, numeric column (or expression). odb will calculate min()/max() value for <column> and assign to each <parallel> thread the extraction of the rows in its "bucket". For example, if you have: ...:splitby=emp_id:parallel=4...with min(emp_id)=1 and max(emp_id)=1000, the four threads will extract the following rows: thread #0 emp_id >=1 and emp_id < 251 thread #1 emp_id >=251 and emp_id < 501 thread #2 emp_id >=501 and emp_id < 751 thread #3 emp_id >=751 and emp_id < 1001 (odb uses max(emp_id) + 1)If the values are not equally distributed data extraction will be deskewed. print=[I][C][D]This operator is used to specify which rows will be printed in the output file:‘I’ will print the new rows on target (based on key)‘D’ will print the missing rows on target (based on key)‘C’ will print the source rows with the same key columns but differences in other fieldsDefault value for print is IDCpre={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd SQL command (enclosed between square brackets) on the target system immediately before reading the target table. post={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd SQL command (enclosed between square brackets) on the target system immediately after the target table has been compared. tpar=numNumber of tables to compare in parallel when you have a list of tables in inputloaders=numodb will use num load threads for each extract thread (default is 2 loaders per extractor) pwhere=<where condition>This option is used in conjunction with parallel to “diff” only records satisfying the where condition. For example: you want to compare rows with TRANS_TS > 1999-12-12 09:00:00 from the source table TRAFODION.MAURO.MFORDERS using 8 parallel streams to a target table having the same name as the source table:src=trafodion.mauro.mforders:tgt=trafodion.dest_schema.%t:parallel=8:pwhere=[TRANS_TS > TIMESTAMP ‘1999-12-12 09:00:00’]... You can enclose the where condition between square brackets to avoid a misinterpretation of the characters in the where condition.quickThis option will limit the comparison to the columns in the key option (PK by default). This is a fast way to check for new/missing records but it won’t find rows with differences in “non-key” columnstimeodb will print a “timeline” (milliseconds from start)Using odb as a Query Driver (Technology Preview)Getting CSV OutputIt’s often handy, while running performance tests to get a CSV output ready to be imported into your spreadsheet. You can easily get this kind of output with –c odb option. Let’s analyze the following command:$ ./odb64luo -u mauro -p xxx -d pglocal -x 3:"select count(*) from tpch.region" -f 5:Q01.sql -f 3:Q02.sql -T 4 -q –cThis will run:Three copies of the select count(*): -x 3:”select count(*) from tpch.region” Five copies of Q01.sql: -f 5:Q01.sqlThree copies of Q02: -f 3:Q02.sqlQueuing the resulting 11 executions into 4 threads: -T 4Omitting query text and query results (-q is equivalent to –q all): -qPrinting a CSV output: –cProducing the following output:odb [2011-12-12 08:08:43]: starting (4) threads...Thread id,Proc id,Thread Exec#,Script Cmd#,File,Label,Command,Rows,Rsds,Prepare(s),Exec(s),1st Fetch(s),Fetch(s),Total(s),STimeline,ETimeline1,1,0,0,(none),,"select count(*) from tpch.region",1,20,0.000,0.109,0.000,0.000,0.109,94,2030,0,0,0,(none),,"select count(*) from tpch.region",1,20,0.000,0.125,0.000,0.000,0.125,94,2192,2,0,0,(none),,"select count(*) from tpch.region",1,20,0.000,0.109,0.000,0.000,0.109,110,2192,6,1,0,Q01.sql,,"SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY)>",4,234,0.000,136.297,0.000,0.000,136.297,141,1364382,10,2,0,Q02.sql,,"SELECT S_ACCTBAL, S_NAME, N_NAME, P_PARTKEY, P_MF>",0,274,0.000,0.468,0.000,0.016,0.484,136438,1369220,4,1,0,Q01.sql,,"SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY)>",4,234,0.000,139.667,0.016,0.016,139.683,0,1396830,8,2,0,Q02.sql,,"SELECT S_ACCTBAL, S_NAME, N_NAME, P_PARTKEY, P_MFG>",0,274,0.000,0.015,0.000,0.000,0.015,139683,1396981,5,1,0,Q01.sql,,"SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY)>",4,234,0.000,144.347,0.015,0.015,144.362,141,1445031,9,2,0,Q02.sql,,"SELECT S_ACCTBAL, S_NAME, N_NAME, P_PARTKEY, P_MFG>",0,274,0.000,0.000,0.000,0.016,0.016,144503,1445193,3,0,0,Q01.sql,,"SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY)>",4,234,0.000,144.394,0.016,0.016,144.410,390,1448003,7,1,0,Q01.sql,,"SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY)>",4,234,0.000,69.373,0.000,0.000,69.373,144800,214173odb statistics: Init timestamp: 2011-12-12 08:08:42 Start timestamp: 2011-12-12 08:08:43 End timestamp: 2011-12-12 08:12:17 Elapsed [Start->End] (s): 214.173CSV output columns have the following meaning:ColumnMeaningThread IDThread ID. As we limited the number of threads to 4, thread id values are 0, 1, 2, 3Proc IDExecution number. As we have 11 executions in the 0-10 rangeThread Exec#This is the progressive number (starting from 0) of execution for a specific thread Script Cmd#In case your script contains multiple SQL statement they will be numbered starting from zeroFileThis is the script file name or “(null)” for –x commandsLabelThis is the label assigned though “set qlabel” in the scriptsCommandFirst 30 characters of the SQL command. It will end with “>” if command text was truncatedRowsThe number of returned rows (not printed if you used –q)RsdsThis is the Record Set Display Size. It gives you an idea of “how big” the result set isPrepare(s)Prepare (compile) time in secondsExec(s)Execution time in seconds1st Fetch(s)Time needed to fetch the first row in secondsFetch(s)Total Fetch time in secondsTotal(s)Total query elapsed time from prepare to fetch in secondsStimelineQueries start timeline in milliseconds EtimelineQueries end timeline in milliseconds Assigning a Label to a QuerySometimes it’s not easy to recognize a query by reading the first 30 characters. odb lets you to assign a label to a generic query using set qlabel <label>. Have a look to the following script:felici ~/Devel/odb $ cat script.sql-- Trafodion TPC-H Query 1SET QLABEL Q01SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY) AS SUM_QTY,...-- TPC-H/TPC-R Minimum Cost Supplier Query (Q2)SET QLABEL Q02SELECT S_ACCTBAL, S_NAME,...When you run this script you will have the Query Label in the CSV output:felici ~/Devel/odb $ ./odb64luo -u mauro -p xxx -d pglocal -f script.sql -q -codb [2011-12-12 09:06:28]: starting (1) threads...Thread id,Proc id,Thread Exec#,Script Cmd#,File,Label,Command,Rows,Rsds,Prepare(s),Exec(s),1st Fetch(s),Fetch(s),Total(s),STimeline,ETimeline0,0,0,0,script.sql,Q01,"SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY)>",4,234,0.000,43.102,0.000,0.000,43.102,0,431020,0,0,1,script.sql,Q02,"SELECT S_ACCTBAL, S_NAME, N_NAME, P_PARTKEY, P_MFG>",0,274,0.000,0.016,0.000,0.000,0.016,43102,43118odb statistics: Init timestamp: 2011-12-12 09:06:28 Start timestamp: 2011-12-12 09:06:28 End timestamp: 2011-12-12 09:07:11 Elapsed [Start->End] (s): 43.118Running All Scripts With a Given PathUsing -S <path> or -P <path> options you can run all scripts with a given path (for example all files in a directory) either serially (-S) or in parallel (-P). Both options let you to use multiplying factors to run all scripts multiple times. This multiplying factors are defined with a <number>: preceding the script path.Examples:odb Command Line Actionodb64luo -S ./test/queries/*.sql -c -qExecutes serially all scripts with extension .sql under ./test/queries/ providing CSV type output (-c) and omitting query output (-q)odb64luo -P test/queries/* -T 50 -c -qRuns in parallel all files under test/queries/ using 50 threads (ODBC connections) (-T 50), with CSV output (-c) and omitting query output (-q)odb64luo -P 3: test/queries/* -T 3 -c -qRuns in parallel three times (3:) all files under test/queries/ using 3 threads (ODBC connections) (-T 3), with CSV output (-c) and omitting query output (-q). Scripts will be assigned to threads using standard assignmentodb64luo -P -3: test/queries/* -T 3 -c -qRuns in parallel three times (-3:) all files under test/queries/ using 3 threads (ODBC connections) (-T 3), with CSV type output (-c) and omitting query output (-q). Scripts will be assigned to threads using round-robin assignmentTo understand the difference between “standard” and “round-robin” assignments imagine you have four scripts in the target path. This is how the executions will be assigned to threads: Standard Assignment (es. -P 3:)Round-Robin Assignment (es. -P -3:)Thread 1Thread 2Thread 3Thread1Thread 2Thread3nth execution……......4th executionScript4.sqlScript4.sql…Script2.sqlScript3.sql...3rd executionScript3.sqlScript3.sqlScript3.sqlScript3.sqlScript4.sqlScript1.sql2nd executionScript2.sqlScript2.sqlScript2.sqlScript4.sqlScript1.sqlScript2.sql1st executionScript1.sqlScript1.sqlScript1.sqlScript1.sqlScript2.sqlScript3.sqlRandomizing Execution OrderYou can use the -Z option to shuffle the odb internal execution table. This way the execution order is not predictable. Examples:odb Command Line Actionodb64luo... -S 3: test/queries/* -Z -c –qwill execute three times (3:) all files in the test/queries directory serially (-S) and in random order (-Z) odb64luo... -P 3: test/queries/* -Z –T 5 -c -qwill execute three times (3:) all files in the test/queries directory in parallel (-P), using five threads (-T 5) and in random order (-Z) Defining a TimeoutYou can stop odb after a given timeout (assuming the execution is not already completed) using -maxtime <seconds> option.Example:felici ~/Devel/odb $ ./odb64luo -S /home/mauro/scripts/*.sql –maxtime 7200 It will execute, serially, all scripts with extension .sql under /home/mauro/scripts/; if the execution - after two hours (7200 seconds) is not completed, odb will stop.Simulating User Thinking TimeYou can simulate user thinking time using the -ttime <delay> option. This will introduce a <delay> millisecond pause between two consecutive executions in the same thread. Example:mauro@newton ~/src/C/odb $ ./odb64luo -f 5:script1.sql -c -q -ttime 75 -T 2It will run five times script1.sql using 2 threads. Each thread will wait 75 milliseconds before starting the next execution within a thread. You can also use a random thinking time in a given min:max range. For example the following command will start commands within a thread with a random delay between 50 and 500 milliseconds:mauro@newton ~/src/C/odb $ ./odb64luo -f 5:script1.sql -c -q -ttime 50:500 -T 2Starting Threads GracefullyYou might want to wait a little before starting the next thread. This can be obtained using the -delay option. Example:mauro@newton ~/src/C/odb $ ./odb64luo -f 5:script1.sql -c -q -delay 200 -T 2It will run five times script1.sql using 2 threads. Each thread will be started 200 milliseconds after the other.Note: -delay introduces a delay during threads start-up while –ttime introduce a delay between one command and another within the same thread.Re-looping a Given WorkloadUsing -L option you can re-loop the workload defined through -x, -f, -P, -S commands a given number of times. Each thread will re-loop the same number of times. Example:mauro@newton ~/src/C/odb $ ./odb64luo -f 5:script1.sql -c -q -M 75 -T 2 -L 3will re-loop three times (-L 3) the same five executions, using two threads (-T 2) with a 75 millisecond pause (-M 75) between two consecutive executions in the same thread.Using odb as a SQL Interpreter (Technology Preview)To start the odb SQL Interpreter you have to use -I (uppercase i) switch with an optional argument. Example:$ odb64luo –u user –p xx –d dsn –I MFTESTThe optional -I argument (MFTEST here above) is used to specify the .odbrc section containing commands to be automatically executed when odb starts (see REF _Ref316800680 \h Running Commands When the Interpreter Starts).Main odb SQL Interpreter features:It uses mreadline library to manage command line editing and history. History will keep track of the whole command, not just… lines. So if you enter a SQL command in more than one line:S01_Maurizio@TRAFODION64[MFTEST]SQL> selectS01_Maurizio@TRAFODION64[MFTEST]...> count(*)S01_Maurizio@TRAFODION64[MFTEST]...> fromS01_Maurizio@TRAFODION64[MFTEST]...> t1;When you press the up arrow key the whole command (up to semi-colon) will be ready for editing and/or re-run. mreadline provides several useful extra features:CTRL-V to edit the current command using your preferred editor ($EDITOR is used). When the editing session is closed the current command is automatically updatedCTRL-U/CTRL-L to change the command caseCTRL-X to kill the current commandSee the online help for the other mreadline commandsHistory is saved when you exit the SQL Interpreter in a file identified by the ODB_HIST environment variable. You can change the number of commands saved in the history file (default 100):S01_Maurizio@TRAFODION64[MFTEST]SQL> set hist 200Customizable prompt. You can personalize your prompt through the set prompt command. Under Unix/Linux/Cygwin you can use the standard ANSI codes to create color prompts (see REF _Ref316800847 \h Customizing the Interpreter Prompt). It’s possible to run multi-threaded odb instances from within the single-threaded Interpreter with the odb keyword. This will run another odb instance using the same credentials, data source, and connection attributes used to start the interpreter:S01_Maurizio@TRAFODION64[MFTEST]SQL> odb -l src=myfile:tgt=mytable:parallel=8:... S01_Maurizio@TRAFODION64[MFTEST]SQL> odb -e src=mytable:tgt=myfile:parallel=8:...It’s possible to define aliases. Example:root@MFDB[MFDB]SQL> set alias count "select row count from &1;"then, when you call the alias "count" the first argument will be substituted to &1. You can use up to nine positional parameters (&1 to &9)You can run operating system commands with !commandYou can run scripts with @scriptYou can spool to file with set spool <myfile> and stop spooling with set spool offYou can switch to a special "prepare only" mode with set prepare on .This way commands you type will be just prepared (not executed)Different databases use different commands to set default schema(s):- Trafodion: set schema <name>;- MySQL: use <name>;- Postgresql/Vertica: set search_path to <name1,name2,…>;- Teradata: set database <name>; set chsch <command> is used to define database specific commands to change your schema. When odb recognize the change schema command it will update accordingly internal catalog (if any) and schema names;To list database objects, you can use ls command. Few examples:S01_Maurizio@MFTEST[MFTEST]SQL>ls . << list all objects in the current schema TABLE : CITIES TABLE : CUSTOMER TABLE : LINEITEM TABLE : NATION TABLE : ORDERS TABLE : PART TABLE : PARTSUPP TABLE : REGION TABLE : SUPPLIER TABLE : T1 VIEW : V_CITIESS01_Maurizio@MFTEST[MFTEST]SQL>ls -t %S << list tables (-t) ending with SCITIES ORDERSS01_Maurizio@MFTEST[MFTEST]SQL>ls -v << list views (-v)V_CITIES S01_Maurizio@MFTEST[MFTEST]SQL>ls -s << list schemas (-s)... and so on ...To get tables DDL, you can use either ls –T <table> or ls -D <table>. Examples:mauro pglocal[PUBLIC] (09:12:56) SQL> ls -T tpch.ordersDescribing: postgres.TPCH.orders+---------------+-------------+----+-------+---------------+|COLUMN |TYPE |NULL|DEFAULT|INDEX |+---------------+-------------+----+-------+---------------+|o_orderkey |int8 |NO | |orders_pkey 1 U||o_custkey |int8 |NO | | ||o_orderstatus |bpchar(1) |NO | | ||o_totalprice |numeric(15,2)|NO | | ||o_orderdate |date |NO | | ||o_orderpriority|bpchar(15) |NO | | ||o_clerk |bpchar(15) |NO | | ||o_shippriority |int4 |NO | | ||o_comment |varchar(80) |NO | | |+---------------+-------------+----+-------+---------------+mauro pglocal[PUBLIC] (09:13:20) SQL> ls -D tpch.ordersCREATE TABLE postgres.TPCH.orders ( o_orderkey int8 ,o_custkey int8 ,o_orderstatus bpchar(1) ,o_totalprice numeric(15,2) ,o_orderdate date ,o_orderpriority bpchar(15) ,o_clerk bpchar(15) ,o_shippriority int4 ,o_comment varchar(80) ,primary key (o_orderkey));You can define your own variables or use odb internal variables or environment variables directly from the InterpreterYou can “set pad fit” to automatically shrink CHAR/VARCHAR fields in order to fit one record in one line. Line length is defined through “set scols #”. Each record will be printed in one line truncating the length of CHAR/VARCHAR fields proportionally to their original display size length. In case of field truncation a ‘>’ character will be printed at the end of the truncated string.Example:MFELICI [MAURIZIO] (03:30:32) SQL> select [first 5] * from part; P_PARTKEY|P_NAME |P_MFGR |P_BRAND|P_TYPE | P_SIZE|P_CONTAINER| P_RETAILPRICE|P_COMMENT ---------+-------------------------------+---------------+-------+---------------+----------+-------+----------------+-------------- 33|maroon beige mint cyan peru |Manufacturer#2>|Brand#>|ECONOMY PLATED>| 16|LG PKG>| 933.03|ly eve 39|rose dodger lace peru floral |Manufacturer#5>|Brand#>|SMALL POLISHED>| 43|JUMBO >| 939.03|se slowly abo> 60|sky burnished salmon navajo hot|Manufacturer#1>|Brand#>|LARGE POLISHED>| 27|JUMBO >| 960.06| integ 81|misty salmon cornflower dark f>|Manufacturer#5>|Brand#>|ECONOMY BRUSHE>| 21|MED BA>| 981.08|ove the furious 136|cornsilk blush powder tan rose |Manufacturer#2>|Brand#>|SMALL PLATED S>| 2|WRAP B>| 1036.13|kages print c>You can “set plm” to print one field per row. This is useful when you have to carefully analyze few records. Example:MFELICI [MAURIZIO] (03:38:12) SQL> set plm onMFELICI [MAURIZIO] (03:38:12) SQL> select * from part where p_partkey = 136;P_PARTKEY :136P_NAME :cornsilk blush powder tan roseP_MFGR :Manufacturer#2P_BRAND :Brand#22P_TYPE :SMALL PLATED STEELP_SIZE :2P_CONTAINER :WRAP BAGP_RETAILPRICE:1036.13P_COMMENT :kages print carefullyCheck the rest on your own.This is the current odb SQL Interpreter help on line:mauro pglocal[PUBLIC] (06:51:20) SQL> helpmauro pglocal[public] (16:57:52) SQL> helpAll the following are case insensitive: h | help : print this help i | info : print database info q | quit : exit SQL Interpreter c | connect { no | [user[/pswd][;opts;...] (re/dis)connect using previous or new user odb odb_command : will run an odb instance using the same DSN/credentials ls -[type] [pattern] : list objects. Type=(t)ables, (v)iews, s(y)nonyns, (s)chemas : (c)atalogs, syst(e)m tables, (l)ocal temp, (g)lobal temp : (m)at views, (M)mat view groups, (a)lias, (A)ll object types : (D)table DDL, (T)table desc print <string> : print <string> !cmd : execute the operating system cmd @file [&0]... [&9] : execute the sql script in file set : show all settings set alias [name] [cmd|-]: show/set/change/delete aliases set chsch [cmd] : show/set change schema command set cols [#cols] : show/set ls number of columns set cwd [<directory>] : show/set current working directory set drs [on|off] : show/enable/disable describe result set mode set fs [<char>] : show/set file field separator set hist [#lines] : show/set lines saved in the history file set maxfetch [#rows] : show/set max lines to be fetched (-1 = unlimited) set nocatalog [on|off] : show/enable/disable "no catalog" database mode) set nocatnull [on|off] : show/enable/disable "no catalog as null" database mode) set noschema [on|off] : show/enable/disable "no schema" database mode) set nullstr [<string>] : show/set string used to display NULLs (* to make it Null) set pad [fit|full|off] : show/set column padding set param name [value|-]: show/set/change/delete a parameter set pcn [on|off] : show/enable/disable printing column names set plm [on|off] : show/enable/disable print list mode (one col/row) set prepare [on|off] : show/enable/disable 'prepare only' mode set prompt [string] : show/set prompt string set query_timeout [s] : show/set query timeout in seconds (def = 0 no timeout) set quiet [cmd|res|all|off] : show/enable/disable quiet mode set rowset [#] : show/set rowset used to fetch rows set soe [on|off] : show/enable/disable Stop On Error mode set spool [<file>|off] : show/enable/disable spooling output on <file> <SQL statement>; : everything ending with ';' is sent to the databasemreadline keys: Control-A : move to beginning of line Control-P : history Previous Control-E : move to end of line Up Arrow : history Previous Control-B : move cursor Back Control-N : history Next Left Arrow : move cursor Back Down Arrow : history Next Control-F : move cursor Forward Control-W ; history List Right Arrow: move cursor Forward Control-R : Redraw Control-D : input end (exit) - DEL right Control-V : Edit current line Control-L : Lowercase Line Control-X : Kill line Control-U : Uppercase Line #Control-G : load history entry #Running Commands When the Interpreter StartsWhen the odb SQL Interpreter starts it looks for the Initialization File. This Initialization File is made of Sections containing the commands to be executed. To find the Initialization File, odb will:first check the ODB_INI environment variableIf this variable is not set, odb willlook for a file named “.odbrc” (*nix) or “_odbrc” (Windows) under your HOME directoryThe Initialization File contains Sections identified by names between square brackets. For example, the following section is named MFTEST:[MFTEST]set pcn onset pad fitset fs |set cols 3 30set editor "vim -n --noplugin"set efile /home/felici/.odbedit.sqlset prompt "%U %D [%S] (%T) %M> "set alias count "select row count from &1;"set alias size "select sum(current_eof) from table (disk label statistics (&1) );set alias ll "select left(object_name, 40) as object_name, sum(row_count) as nrows, count(partition_num) as Nparts, sum(current_eof) as eof from table(disk label statistics( using (select * from (get tables in schema &catalog.&schema, no header, return full names) s(b) ))) group by object_name order by object_name;"set schema TRAFODION.MAURIZIO;odb SQL Interpreter will automatically run all commands in the section identified by the -I argument (for example -I MFTEST). A section named DEFAULT will be executed when -I has no arguments.Customizing the Interpreter PromptYou can define your prompt through the set prompt command when running the SQL Interpreter. set prompt can be executed interactively or included in your ($ODB_INI) "init file". set prompt recognize and expand the following variables:%U User name%D Data Source name%S Schema name%T Current Time %M odb mode:SQL when running sql commandsPRE if you're in "prepare only" modeSPO if you are spooling output somewhere NDC (No Database Connection)So, for example:set prompt "Prompt for %U connected via %D to %S in %M mode > "will generate a prompt like this:Prompt for S01_Maurizio connected via CIV to CIV03 in SQL mode >Under Cygwin, Unix and Linux (and probably under Windows too using ANSI.SYS driver - not tested) you can use standard ANSI escape color codes. For example:set prompt “^A^[[01;32m^A%U@%D^A^[[01;34m^A[%S]^A^[[00m^A (%T) %M> “Where:^A is a "real" Control-A (ASCII 001 and 002) before and after each color code sequence;^[ is a "real" Escape Character and the meaning of the ANSI color codes are:^[[01;32m means green^[[01;34m means blue^[[00m means reset.This is how my standard prompt looks:Warnings, Limits, and Troubleshootingodb uses Condition Variables to synchronize threads during copy and parallel load operations.Most of the memory allocation operations are dynamic. So, for example, you can execute an SQL command as long as you want (in previous version of odb maximum SQL command length was 8192 bytes). However, I found it convenient to "hard code" the following limits :#define MAX_VNLEN 32 /* Max variable name length */#define MAXCOL_LEN 128 /* Max column name length */#define MAXOBJ_LEN 128 /* Max catalog/schema/table name length */#define MAX_CLV 64 /* Max command line variables (-var) */Some Linux/UNIX systems (notably the Linux Loader) have huge default stack size. Due to this insane value, you can have errors like this when starting tens/hundreds of threads:Error starting cmd thread #: cannot allocate memoryIf you get this error check your default stack size:s19user@traf64-dl2a ~/mf $ ulimit -acore file size (blocks, -c) 0data seg size (kbytes, -d) unlimitedmax nice (-e) 0file size (blocks, -f) unlimitedpending signals (-i) 137215max locked memory (kbytes, -l) 32max memory size (kbytes, -m) unlimitedopen files (-n) 65536pipe size (512 bytes, -p) 8POSIX message queues (bytes, -q) 819200max rt priority (-r) 0stack size (kbytes, -s) 204800 << this is the problemcpu time (seconds, -t) unlimitedmax user processes (-u) 2047virtual memory (kbytes, -v) unlimitedfile locks (-x) unlimitedand reset it to sane values (value will be reset to the initial value when you start a new session). For example:s19user@traf64-dl2a ~/mf $ ulimit -s 4096If you get errors like this:C:\Users\felici>odb64luo -u xx -p yy -d oraxe -l src=region.tbl.gz:tgt=region:fs=^|:truncateodb [2012-10-11 13:27:22]: starting ODBC connection(s)... 0[0] odb(5020) - [Oracle][ODBC]Optional feature not implemented. (State: HYC00 Native Err: 0)try adding -nocatnull to your command line. When the backend database doesn’t use catalogs you should use an empty string as catalog name. Some buggy ODBC Drivers unfortunately want NULL here – instead of empty strings as it should be.You can have errors loading TIME(N) fields with N>0 under Trafodion because the ODBC Driver does not manage the field display size when N>0.If you have problems starting odb on Unix/Linux check:the shared library dependencies with ldd <odb_executable_name>;and the shared lib path defined in the following environment variables used by the shared library loader:Linux: LD_LIBRARY_PATHIBM IAX: LIBPATH (not currently supported)HP/UX: SHLIB_PATH (not currently supported)How odb Is Coded and TestedHow odb Is CodedFor Trafodion Release 1.1, odb is coded in "ANSI C" (K&R programming style) and is compiled in a 64-bit version on the Linux platform, linked to the unixODBC driver manager. Other platforms, compilers, and ODBC libraries have not yet been tested. PlatformCompilerODBC Libraries NoteLinuxgccunixODBC (supported), iODBC (not currently supported), Data Direct (not currently supported), Teradata (not currently supported)64 bit (32 bit is not currently supported)MS-Windows (not tested)Visual Studio (not tested)MS-Windows (not tested)64 bitIBM AIX (not currently supported)gccunixODBC, iODBC, Data Direct32 and 64 bitIBM AIX (not currently supported)xlcTo be determined32 and 64 bitCygwin (not currently supported)gccMS-Windows32 bitHP/UX (not currently supported)ANSI CunixODBC, iODBC64 bitMac (not currently supported)gcciODBC64 bitC compilers are set with “all warnings” enabled and odb has to compile, on each platform, with no errors (of course) AND no warnings. Tools used to code odb:vim (http://www.vim.org) as editor (or Visual Studio embedded editor)splint (http://www.splint.org) to statically check the source codeHow odb Is TestedFor Trafodion Release 1.1, the info, load, extract, and copy operations of odb have been fully tested.Previously, odb had been tested using a set of 137 standard tests to check functionalities and identify memory/thread issues. Nevertheless, there could be bugs somewhere.Useful odb CommandsUpdating Missing StatisticsYou can update missing statistics with a single line command like this: $ ./odb64luo -u xyx -p yyy -d zzz -N -S test/queries/Q*.sql 2>&1 | sed -n 's/^.*WARNING\[600[7-8]\].*column\? \(.*\) from table \(.*\) were.*$/UPDATE STATISTICS FOR TABLE \2 ON \1;/p' | ./odb64luo -u xyz -p xyz -d mft -f –Where:./odb64luo -u xx -p xyz -d xx -N -S test/queries/Q*.sql 2>&1 will just prepare queries (the -N option will prevent their real execution) and write warnings/errors on the standard output.sed -n 's/^.*WARNING\[600[7-8]\].*column\? \(.*\) from table \(.*\) were.*$/UPDATE STATISTICS FOR TABLE \2 ON \1;/p' will transform the warnings into real UPDATE STATISTICS statements../odb64luo -u xyz -p xyz -d mft -f – will run the UPDATE STATISTICS statements built with the previous sed (-f – means that commands are provided via standard input).Getting All DDL Statements in a Given Schema$ odb64luo -u xx -p xx -d xx -i t:TRAFODION.HP_MIGR.% -x "showddl &1" -T 1 > ddl.sqlAliases for the SQL InterpreterYou can copy and paste these aliases in your $ODB_INI file:set alias count "select count(*) from &1;"Simple ODBC Port Scanner#!/usr/bin/bash# Simple ODBC Port Scanner# Config Parameters:FOUND="invalid username" # Message to grepPSTART=18600 # Initial Port to checkPEND=18700 # Final Port to checkDRV="{Trafodion}" # ODBC DriverHOST="abc123.ca.cname.com" # Fully Qualified Host Name or IPDSN="DSN=MFELICI" # You can use SERVERDSN hereMISC="CERTIFICATEDIR=C:\Users\felici" # Other params# Body:for port in `seq $PSTART $PEND` ; do echo -ne "\rNow Testing port $port" ./odb64luo -u x -p x -ca "DRIVER=$DRV;SERVER=TCP:$HOST/$port;$DSN;$MISC" -i \ 2>&1 | grep -q "$FOUND" test $? == 0 && { echo "... found port $port on $HOST"; }doneecho -e "\rScan completed for $HOST on range $PSTART:$PEND" Be aware: this is case sensitive. The following characters are considered “spaces”: blank, tab, new line, carriage return, form feed, vertical tab Note: When the source table column is defined as NOT NULL and the specific field contains only blanks, odb will leave in the output file one single blank. This will help to distinguish between NULL fields (<field_sep><field_sep>) and NOT NULL fields containing all blanks (<field_sep><blank><field_sep>)odb User GuidePage PAGE 1/ NUMPAGES 1iVBORw0KGgoAAAANSUhEUgAAAeAAAAA1CAIAAAAWMq/qAAAAAXNSR0IArs4c6QAADwJJREFUeF7t
+Trafodion odb User GuideTrafodion Release 1.1—April 2015Part Number: T632-110-001Published: April 2015Edition: Trafodion Release 1.1© Copyright 2015 Apache Software Foundation (ASF)Legal NoticeThe information contained herein is subject to change without notice. This documentation is distributed on an “AS IS” basis, without warranties or conditions of any kind, either express or implied. Nothing herein should be construed as constituting an additional warranty. HP shall not be liable for technical or editorial errors or omissions contained herein.NOTICE REGARDING OPEN SOURCE SOFTWARE: Project Trafodion is licensed under the Apache License, Version 2.0 (the "License"); you may not use software from Project Trafodion except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.Table of Contents TOC \o "1-3" \h \z \u About This Document PAGEREF _Toc417975704 \h 5Intended Audience PAGEREF _Toc417975705 \h 5New and Changed Information in This Edition PAGEREF _Toc417975706 \h 5Document Conventions PAGEREF _Toc417975707 \h 5Publishing History PAGEREF _Toc417975708 \h 5We Encourage Your Comments PAGEREF _Toc417975709 \h 51Introduction PAGEREF _Toc417975710 \h 61.1What is odb PAGEREF _Toc417975711 \h 62Installation and Configuration PAGEREF _Toc417975712 \h 72.1odb Requirements PAGEREF _Toc417975713 \h 72.2Installing and Configuring the Required unixODBC PAGEREF _Toc417975714 \h 72.3Installing odb PAGEREF _Toc417975715 \h 93Basic Concepts PAGEREF _Toc417975716 \h 103.1Getting Help PAGEREF _Toc417975717 \h 103.2Connecting to a Database PAGEREF _Toc417975718 \h 113.3Listing Available ODBC Drivers and Data Sources PAGEREF _Toc417975719 \h 123.4Obtaining Database Information PAGEREF _Toc417975720 \h 123.5Performing Actions on Multiple Database Objects PAGEREF _Toc417975721 \h 133.6Running Commands and Scripts PAGEREF _Toc417975722 \h 143.7Using “here document” Syntax in Shell Scripts PAGEREF _Toc417975723 \h 153.8Running Multiple Commands and Scripts in Parallel PAGEREF _Toc417975724 \h 153.9Limiting the Number of Threads Created by odb PAGEREF _Toc417975725 \h 163.10Changing the Number of Executions Distributed Across Threads PAGEREF _Toc417975726 \h 163.11Learning How Dynamic Load Balancing Works PAGEREF _Toc417975727 \h 173.12Using Variables in odb Scripts PAGEREF _Toc417975728 \h 173.13Understanding Thread ID, Thread Execution#, and Script Command# PAGEREF _Toc417975729 \h 173.14Checking SQL Scripts PAGEREF _Toc417975730 \h 183.15Using Different Data Sources for Different Threads PAGEREF _Toc417975731 \h 183.16Formatting Query Results PAGEREF _Toc417975732 \h 193.17Extracting Tables’ DDL PAGEREF _Toc417975733 \h 194Loading, Extracting, and Copying Data PAGEREF _Toc417975734 \h 214.1Loading Files PAGEREF _Toc417975735 \h 214.2Mapping Source File Fields to Target Table Columns PAGEREF _Toc417975736 \h 244.3Using mapfiles to Ignore and/or Transform Fields When Loading PAGEREF _Toc417975737 \h 274.4Using mapfiles to Load Fixed Format Files PAGEREF _Toc417975738 \h 274.5Generating and Loading Data PAGEREF _Toc417975739 \h 284.6Loading Default Values PAGEREF _Toc417975740 \h 294.7Loading Binary Files PAGEREF _Toc417975741 \h 304.8Reducing the ODBC Buffer Size PAGEREF _Toc417975742 \h 304.9Extracting Tables PAGEREF _Toc417975743 \h 314.10Extracting a List of Tables PAGEREF _Toc417975744 \h 354.11Copying Tables From One Database to Another PAGEREF _Toc417975745 \h 354.12Copying a List of Tables PAGEREF _Toc417975746 \h 384.13Using Case-Sensitive Table and Column Names PAGEREF _Toc417975747 \h 384.14Determining the Appropriate Number of Threads for Load/Extract/Copy/Diff PAGEREF _Toc417975748 \h 394.15Integrating With Hadoop PAGEREF _Toc417975749 \h 395Comparing Tables (Technology Preview) PAGEREF _Toc417975750 \h 406Using odb as a Query Driver (Technology Preview) PAGEREF _Toc417975751 \h 446.1Getting CSV Output PAGEREF _Toc417975752 \h 446.2Assigning a Label to a Query PAGEREF _Toc417975753 \h 456.3Running All Scripts With a Given Path PAGEREF _Toc417975754 \h 456.4Randomizing Execution Order PAGEREF _Toc417975755 \h 466.5Defining a Timeout PAGEREF _Toc417975756 \h 466.6Simulating User Thinking Time PAGEREF _Toc417975757 \h 466.7Starting Threads Gracefully PAGEREF _Toc417975758 \h 466.8Re-looping a Given Workload PAGEREF _Toc417975759 \h 477Using odb as a SQL Interpreter (Technology Preview) PAGEREF _Toc417975760 \h 487.1Running Commands When the Interpreter Starts PAGEREF _Toc417975761 \h 517.2Customizing the Interpreter Prompt PAGEREF _Toc417975762 \h 52A.Warnings, Limits, and Troubleshooting PAGEREF _Toc417975763 \h 53B.How odb Is Coded and Tested PAGEREF _Toc417975764 \h 54About This DocumentThis manual describes how to use odb, a multi-threaded, ODBC-based command-line tool, to perform various operations on a Trafodion database.Note: In Trafodion Release 1.1, only loading, extracting, and copying data operations are production ready, meaning that that have been fully tested and are ready to be used in a production environment. Other features are designated as “Technology Preview,” meaning that they have not been fully tested and are not ready for production use.Intended AudienceThis manual is intended for database administrators and other users who want to run scripts that operate on a Trafodion database, primarily for parallel data loading.New and Changed Information in This EditionThis manual is new.Document ConventionsThe manual uses the following typographic conventions:ConventionDescriptionExampleMonospace dark grayCommand promptmfelici ~odb $Monospace orangeTyped command./odb64luo -hMonospace blueCommand output[0.0.0]--- 1 row(s) selected in 0.015sMonospace grayComments and clarifications# this will drop all views in your schemaPublishing HistoryPart NumberProduct VersionPublication DateT632-110-001Trafodion Release 1.1.0April 2015We Encourage Your CommentsThe Trafodion community encourages your comments concerning this document. We are committed to providing documentation that meets your needs. Send any errors found, suggestions for improvement, or compliments to:trafodion-documentation@lists.launchpad.net Include the document title, part number, and any comment, error found, or suggestion for improvement you have concerning this document.IntroductionWhat is odbodb is a platform independent, multi-threaded, ODBC command-line tool you can use as a:Parallel data loader/extractorQuery driver (Technology Preview)SQL interpreter (Technology Preview)odb is written in ANSI C. In Trafodion Release 1.1, odb is available only in a 64-bit version for the Linux platform, linked to the unixODBC driver manager.odb executables use the following naming convention, odbAABCC, where:AA can be 64 (bit) (32 bit is not currently supported).B identifies the platform/compiler:l = Linux/gccw = Windows/MS Visual Studio (not yet tested)a = IBM AIX/gcc (not currently supported)x = IBM/AIX xlc (not currently supported)h = HP-UX/ANSI C (not currently supported)c = Cygwin/gcc (not currently supported)m = Mac OSX/gcc (not currently supported)CC identifies the ODBC Driver Manager to which odb was linked:uo = unixODBC Driver Managerms = Microsoft ODBC Driver Manager (not yet tested)td = Teradata ODBC Driver Manager (not currently supported)io = iODBC Driver Manager (not currently supported)dd = DataDirect Driver Manager (not currently supported)So, for example:odb64luo is the 64-bit executable for Linux linked with the unixODBC Driver Managerodb32cms is the 32-bit executable for Cygwin linked with MS Windows Driver Manager (not currently supported)odb64mio is the 64-bit executable for Mac OSX linked to the “native” iODBC Driver Manager (not currently supported)This document contains examples run with the odb64luo executable.In the future, this document might contain examples prepared with different executables. All odb executables have exactly the same features unless otherwise specified.Installation and Configurationodb RequirementsA general requirement is the ODBC library for the database you want to connect to and:PlatformRequirementsLinuxUNIX (not currently supported)Mac OS/X (not currently supported)pthread libraries (Generally installed by default)Windows (not yet tested)Microsoft Visual C++ 2010 Redistributable Package (x86)Note: In Trafodion Release 1.1, odb is available only for the Linux platform.Installing and Configuring the Required unixODBCThis section explains how to install and configure unixODBC, which is required for Trafodion Release 1.1.Grab the source code tarball from http://www.unixodbc.org. Use at least version 2.3.x.Unpack the tarball:$ tar xzvf unixODBC-2.3.1.tar.gzConfigure unixODBC installation:$ cd unixODBC-2.3.1$ ./configure --disable-gui --enable-threads --disable-driversThis will install unixODBC under /usr/local and requires root access. If you don’t have root privileges or you want to install unixODBC somewhere else you have to add --prefix=<installation_path> to the configure command here above. For example:$ ./configure --prefix=/home/mauro/uodbc --disable-gui --enable-threads --disable-driversCompile unixODBC sources:$ makeInstall unixODBC:$ make installNow, in order to configure unixODBC, we have to:Define a few environment variables.Define our Data Sources.Let’s start with the environment variables (which you can add to your profile script):First you have to set the ODBCHOME variable to the unixODBC installation dir (the one configured via --prefix here above). For example:export ODBCHOME=/home/mauro/uodbcThen you have to configure the system data sources directory (the one containing odbc.ini and odbcinst.ini). Normally this is the etc/ directory under $ODBCHOME:export ODBCSYSINI=${ODBCHOME}/etcThen you have to configure the ODBCINI variable to the full path of the odbc.ini file:export ODBCINI=${ODBCSYSINI}/odbc.iniFinally… do not forget to add unixODBC lib directory to your LD_LIBRARY_PATH (Linux) or LIBPATH (IBM AIX) or SHLIB_PATH (HP/UX):export LD_LIBRARY_PATH=${ODBCHOME}/libNow we have to configure our data sources in odbc.ini:[<DATA_SOURCE_NAME>]Description = DSN DescriptionDriver = <odbcinst.ini corresponding section>...Other (Driver specific) parameters...and odbcinst.ini:[<Driver name in odbc.ini>]Description = Driver descriptionDriver = <ODBC driverFileUsage = 1UsageCount = 1Note: The Trafodion ODBC driver requires an environment variable, AppUnicodeType, to be specified in odbcinst.ini. This variable must be set to utf16.If you are using Vertica, the Vertica ODBC driver requires an additional section named [Driver] in odbc.ini with specific settings for unixODBC (odbcinst library location and UTF level supported by unixODBC). See the Vertica Programmer’s Guide – Additional ODBC Driver Configuration Settings. See also the following example: Example:$ cat odbc.ini[ODBC Data Sources]VMFELICI = VerticaMachine1traf = Trafodion database[VMFELICI]Description = Vertica Machine 1Driver = VODBCDatabase = vertica01_machineServername = server_nameUID =PWD =Port = 1111ConnSettings =DriverStringConversions = NONESSLKeyFile = /<dir-name>client.keySSLCertFile = /<dir-name>/client.crt[Driver] << This section is required by Vertica’s ODBC Driver Locale = en_USODBCInstLib = /<dir-name>/uodbc/lib/libodbcinst.soErrorMessagesPath = /opt/vertica/lib64/DriverManagerEncoding = UTF-16LogPath = /tmpLogNameSpace =LogLevel = 0[traf]Description = traf DSNDriver = TrafodionCatalog = TRAFODIONSchema = QADataLang = 0FetchBufferSize = SYSTEM_DEFAULTServer = TCP:<server-name>:<port-no>SQL_ATTR_CONNECTION_TIMEOUT = SYSTEM_DEFAULTSQL_LOGIN_TIMEOUT = SYSTEM_DEFAULTSQL_QUERY_TIMEOUT = NO_TIMEOUTServiceName = HP_DEFAULT_SERVICE$ cat odbcinst.ini[VODBC]Description = Vertica ODBC DriverDriver = /<dir-name>/vodbc/lib64/libverticaodbc.soFileUsage = 1UsageCount = 1[Trafodion]Description= Trafodion ODBC Stand Alone DriverDriver= /<dir-name>/conn/clients/odbc/libtrafodbc_drvr64.soFileUsage= 1UsageCount= 1AppUnicodeType= utf16[ODBC]Threading = 1Trace = OffTracefile = uodbc.trcAnother important entry in odbcinst.ini is Threading. The following comment (extracted from unixODBC sources DriverManager/__handles.c) explain its meaning: * ... * If compiled with thread support the DM allows four different * thread strategies * * Level 0 - Only the DM internal structures are protected * the driver is assumed to take care of it's self * * Level 1 - The driver is protected down to the statement level * each statement will be protected, and the same for the connect * level for connect functions, note that descriptors are considered * equal to statements when it comes to thread protection. * * Level 2 - The driver is protected at the connection level. only * one thread can be in a particular driver at one time * * Level 3 - The driver is protected at the env level, only one thing * at a time. * * By default the driver open connections with a lock level of 0, * drivers should be expecetd to be thread safe now. * this can be changed by adding the line * * Threading = N * * to the driver entry in odbcinst.ini, where N is the locking level * (0-3)Installing odbSee the Trafodion Client Installation Guide (1.1) for details.Basic ConceptsGetting HelpThe following command shows the odb help:mfelici ~/Devel/odb $ ./odb64luo -h odb version 1.1.0Build: linux, amd64, gcc generic m64, uodbc, mreadline, dynamic gzip, dynamic libhdfs, dynamic libxml2 [Mar 30 2015 00:29:25] -h: print this help -version: print odb version and exit -lsdrv: list available drivers @ Driver Manager level -lsdsn: list available Data SourcesConnection related options. You can connect using either: -u User: (default $ODB_USER variable) -p Password: (default $ODB_PWD variable) -d Data_Source_Name: (default $ODB_DSN variable) -ca Connection_Attributes (normally used instead of -d DSN) -U sets SQL_TXN_READ_UNCOMMITTED isolation level -ndsn [+]<number>: adds 1 to <number> to DSN -nps <nbytes>[:<nbytes>]: specify source[:target] network packet sizeSQL interpreter options: -I [$ODB_INI SECTION]: interactive mode shell -noconnect: do not connect on startupGeneral options: -q [cmd|res|all|off]: do not print commands/results/both -i [TYPE[MULT,WIDE_MULT]:CATALOG.SCHEMA[.TABLE]]: lists following object types: (t)ables, (v)iews, s(y)nonyns, (s)chemas, (c)atalogs, syst(e)m tables (l)ocal temp, (g)lobal temp, (m)at views, (M)mat view groups, (a)lias (A)ll object types, (T)table desc, (D)table DDL, (U) table DDL with multipliers -r #rowset: rowset to be used insert/selects (default 100) -soe: Stop On Error (script execution/loading task) -N : Null run. Doesn't SQLExecute statements -v : be verbose -vv : Print execution table -noschema : do not use schemas: CAT.OBJ instead of CAT.SCH.OBJ -nocatalog : do not use catalogs: SCH.OBJ instead of CAT.SCH.OBJ -nocatnull : like -nocatalog but uses NULL instead of empty CAT strings -ucs2toutf8 : set UCS-2 to UTF-8 conversion in odb -var var_name var_value: set user defined variables -ksep char/code: Thousands Separator Character (default ',') -dsep char/code: Decimal Separator Character (default '.')SQL execution options [connection required]: -x [#inst:]'command': runs #inst (default 1) command instances -f [#inst:]'script': runs #inst (default 1) script instances -P script_path_regexp: runs in parallel scripts_path_regexp if script_path_regexp ends with / all files in that dir -S script_path_regexp: runs serially scripts_path_regexp if script_path_regexp ends with / all files in that dir -L #loops: runs everything #loops times -T max_threads: max number of execution threads -dlb: use Dynamic Load Balancing -timeout #seconds: stops everything after #seconds (no Win32) -delay #ms: delay (ms) before starting next thread -ldelay #ms: delay (ms) before starting next loop in a thread -ttime #ms[:ms]: delay (ms) before starting next command in a thread random delay if a [min:max] range is specified -F #records: max rows to fetch -c : output in csv format -b : print start time in the headers when CSV output -pcn: Print Column Names -plm: Print Line Mode -fs char/code: Field Sep <char> ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> -rs char/code: Rec Sep <char> ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> -sq char/code: String Qualifier (default none) -ec char/code: Escape Character (default '\') -ns nullstring: print nullstring when a field is NULL -trim: Trim leading/trailing white spaces from txt cols -drs: describe result set (#cols, data types...) for each Q) -hint: do not remove C style comments (treat them as hints) -casesens: set case sensitive DB -Z : shuffle the execution table randomizing Qs start orderData loading options [connection required]: -l src=[-]file:tgt=table[:map=mapfile][:fs=fieldsep][:rs=recsep][:soe] [:skip=linestoskip][:ns=nullstring][:ec=eschar][:sq=stringqualifier] [:pc=padchar][:em=embedchar][:errmax=#max_err][:commit=auto|end|#rows|x#rs] [:rows=#rowset][:norb][:full][:max=#max_rec][:truncate][:show][:bpc=#][:bpwc=#] [:nomark][:parallel=number][:iobuff=#size][:buffsz=#size]][:fieldtrunc={0-4}] [:pre={@sqlfile}|{[sqlcmd]}][:post={@sqlfile}|{[sqlcmd]}][:ifempty] [:direct][:bad=[+]badfile][:tpar=#tables][:maxlen=#bytes][:time][:loadcmd=IN|UP|UL] [:xmltag=[+]element][:xmlord][:xmldump] Defaults/notes: * src file: local file or {hdfs,mapr}[@host,port[,huser]].<HDFS_PATH> * fs: default ','. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * rs: default '\n'. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * ec: default '\'. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * pc: no default. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * direct: only for Vertica databases * bpc: default 1,bpwc: default 4 * loadcmd: default IN. only for Trafodion databasesData extraction options [connection required]: -e {src={table|-file}|sql=<custom sql>}:tgt=[+]file[:pwhere=where_cond] [:fs=fieldsep][:rs=recsep][:sq=stringqualifier][:ec=escape_char][:soe] [:ns=nullstring][es=emptystring][:rows=#rowset][:nomark][:binary][:bpc=#][:bpwc=#] [:max=#max_rec][:[r]trim[+]][:cast][:multi][parallel=number][:gzip[=lev]] [:splitby=column][:uncommitted][:iobuff=#size][hblock=#size][:ucs2toutf8] [:pre={@sqlfile}|{[sqlcmd]}[:mpre={@sqlfile}|{[sqlcmd]}[:post={@sqlfile}|{[sqlcmd]}] [tpar=#tables][:time][:cols=[-]columns]][:maxlen=#bytes][:xml] Defaults/notes: * tgt file: local file or {hdfs,mapr}.[@host,port[,huser]].<HDFS_PATH> * fs: default ','. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * rs: default '\n'. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * ec: default '\'. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * sq: no default. Also <ASCII_dec> 0<ASCII_OCT> X<ASCII_HEX> * gzip compressione level between 0 and 9 * bpc: default 1,bpwc: default 4 Data copy options [connection required]: -cp src={table|-file:tgt=schema[.table][pwhere=where_cond][:soe][:roe=#][:roedel=#ms] [:truncate][:rows=#rowset][:nomark][:max=#max_rec][:bpc=#][:bpwc=#][:[r]trim[+]] [:parallel=number][errmax=#max_err][:commit=auto|end|#rows|x#rs][:time] ][:cast] [:direct][:uncommitted][:norb][:splitby=column][:pre={@sqlfile}|{[sqlcmd]}] [:post={@sqlfile}|{[sqlcmd]}][:mpre={@sqlfile}|{[sqlcmd]}][:ifempty] [:loaders=#loaders][:tpar=#tables][:cols=[-]columns][:errdmp=file] ][:loadcmd=IN|UP|UL] [sql={[sqlcmd]|@sqlfile|-file}[:bind=auto|char|cdef][:seq=field#[,start]] [tmpre={@sqlfile}|{[sqlcmd]}][:ucs2toutf8=[skip,force,cpucs2,qmark]] Defaults/notes: * loaders: default 2 load threads for each 'extractor' * direct: only work if target database is Vertica * ucs2toutf8: default is 'skip' * roe: default 3 if no arguments * bpc: default 1,bpwc: default 4 * loadcmd: default IN. only for Trafodion databasesData pipe options [connection required]: -pipe sql={[sqlcmd]|@sqlscript|-file}:tgtsql={@sqlfile|[sqlcmd]}[:soe] [:rows=#rowset][:nomark][:max=#max_rec][:bpc=#][:bpwc=#][:errdmp=file] [:parallel=number][errmax=#max_err][:commit=auto|end|#rows|x#rs][:time] [:pre={@sqlfile}|{[sqlcmd]}][:post={@sqlfile}|{[sqlcmd]}] [:mpre={@sqlfile}|{[sqlcmd]}][:tmpre={@sqlfile}|{[sqlcmd]}] [:loaders=#loaders][:tpar=#tables][:bind=auto|char|cdef] Defaults/notes: * loaders: default 1 load threads for each extraction thread * bpc: default 1,bpwc: default 4 Table diff options [connection required]: -diff src={table|-file}:tgt=table:[key=columns][:output=[+]file][:pwhere=where_cond] [:pwhere=where_cond][:nomark][:rows=#rowset][:odad][:fs=fieldsep][:time][trim[+]] [:rs=recsep][:quick][:splitby=column][:parallel=number][:max=#max_rec] [:print=[I][D][C]][:ns=nullstring][:es=emptystring][:bpc=#][:bpwc=#][:uncommitted] [:pre={@sqlfile}|{[sqlcmd]}][:post={@sqlfile}|{[sqlcmd]}][tpar=#tables] Defaults/notes: * bpc: default 1,bpwc: default 4 * print: default is Inserted Deleted Changed Connecting to a Databaseodb uses standard ODBC APIs to connect to a database. Normally you have to provide the following information: user, password and ODBC data source. Example:$ ./odb64luo –u user –p password –d dsn ... You can provide Driver specific connection attributes using -ca command line option. Note: Command-line passwords are protected against “ps -ef" sniffing attacks under *nix. You can safely pass your password via –p. An alternative approach is to use environment variables or the odb password prompt (see below)odb will use the following environment variables (if defined):VariableMeaningCorresponding Command-Line OptionODB_USERUser name to use for database connections-u <user>ODB_PWDPassword for database connections-p <passwd> ODB_DSNDSN for database connection-d <dsn> ODB_INIInit file for interactive shellODB_HISThistory file name to save command history on exitNote: Command-line options take precedence over environment variables.Listing Available ODBC Drivers and Data SourcesYou can list available drivers with -lsdrv:felici ~/Devel/odb $ ./odb64luo -lsdrvTrafodion - Description=Trafodion ODBC Stand Alone Driver...You can list locally configured data sources with -lsdsn:felici ~/Devel/odb $ ./odb64luo -lsdsntraf - TrafodionVMFELICI – Vertica ...Obtaining Database InformationUsing the -i option you can get information about the database you’re connecting to as well as the ODBC driver. It’s a simple way to check your credentials and database connection. Example:ncsi@ldratc16 ~/mauro/odb $ ./odb64luo -u xxx -p xxx -d traf -iodb [2015-04-20 21:20:47]: starting ODBC connection(s)... 0[odb version 1.1.0]Build: linux, amd64, gcc generic m64, uodbc, mreadline, dynamic gzip, dynamic libhdfs, dynamic libxml2 [Apr 8 2015 16:47:49]DBMS product name (SQL_DBMS_NAME) : HP DatabaseDBMS product version (SQL_DBMS_VER) : 01.01.0000Database name (SQL_DATABASE_NAME) : TRAFODIONServer name (SQL_SERVER_NAME) : --name--Data source name (SQL_DATA_SOURCE_NAME) : trafData source RO (SQL_DATA_SOURCE_READ_ONLY) : NODBC Driver name (SQL_DRIVER_NAME) : libhpodbc64.soODBC Driver version (SQL_DRIVER_VER) : 03.00.0000ODBC Driver level (SQL_DRIVER_ODBC_VER) : 03.51ODBC Driver Manager version (SQL_DM_VER) : 03.52.0002.0002ODBC Driver Manager level (SQL_ODBC_VER) : 03.52Connection Packet Size (SQL_ATTR_PACKET_SIZE): 0odb [2015-04-20 21:20:48]: exiting. Session Elapsed time 0.229 seconds (00:00:00.229)Listing Database ObjectsIn the previous section, we have used the -i option without any argument. This option accepts arguments with the following syntax: [TYPE:][CATALOG.SCHEMA][.OBJECT], where type can be:Type Meaning<missing>All database object typesA:All database object typest:Tablesv:Viewsa:Aliasesy:Synonymsl:Local Temporaryg:Global Temporarym:Materialized viewsM:Materialized view groupss:Schemasc:CatalogsT:Table descriptionsD:Table DDLU[x,y]:Table DDL multiplying wide columns by Y and non-wide columns by XExamples:ExampleAction-i c:List all catalogs-i s:List all schemas-i TRAFODION.MFTESTList all objects in TRAFODION.MFTEST schema-i t:TRAFODION.MFTESTList all tables in TRAFODION.MFTEST -i t:TRAFODION.MFTEST.A%List all tables in TRAFODION.MFTEST schema staring with A-i v:TRAFODION.MFTESTList all views in TRAFODION.MFTEST-i v:TRAFODION.MFTEST.%_VList all views in TRAFODION.MFTEST ending with _V-i T:TRAFODION.MFTEST.STG%Describe all tables starting with STG in TRAFODION.MFTESTExtended examples:ncsi@ldratc16 ~/mauro/odb $ ./odb64luo -u MFELICI -p xxx -d MFELICI -i T:TRAFODION.MAURIZIO.T%odb [2011-12-07 14:43:51]: starting (1) ODBC connection(s)... 1Describing: TRAFODION.MAURIZIO.T1+------+--------------+----+-------+-----+|COLUMN|TYPE |NULL|DEFAULT|INDEX|+------+--------------+----+-------+-----+|ID |INTEGER SIGNED|YES | | ||NAME |CHAR(10) |YES | | ||LASTN |VARCHAR(20) |YES | | |+------+--------------+----+-------+-----+Describing: TRAFODION.MAURIZIO.T11+------+--------------+----+-------+-------+|COLUMN|TYPE |NULL|DEFAULT|INDEX |+------+--------------+----+-------+-------+|ID |INTEGER SIGNED|NO | |T11 1 U||NAME |CHAR(10) |YES | | |+------+--------------+----+-------+-------+The INDEX column (when using type T) contains the following information:name of the INDEX (in Trafodion indexes having the same name as the table are Primary Keys)ordinal number to identify the order of that field in the index(U)nique o (M)ultiple values allowed (+) means that more than one index includes that fieldPerforming Actions on Multiple Database Objectsodb uses extended SQL syntax to execute actions on multiple objects: &<type>:<path> - where <type> is one of the object types listed in the previous section.Examples:ExampleActiondelete from &t:MF%Purge ALL tables (t:) staring with “MF” drop view &v:mftest.%vwDrop ALL views (v:) ending with _VW in the schema MFTESTUPDATE STATISTICS FOR TABLE &t:TRAFODION.MFTEST.%Update Stats for ALL tables in TRAFODION.MFTESTYou can use this extended SQL syntax in the SQL Interpreter or generic SQL scripts. Running Commands and Scripts–x switch can be used to run generic SQL commands. You can also use –f to run SQL scripts:-x "SQL command" to run a specific SQL command-f <script> to run a script fileFor example:felici ~/Devel/odb $ ./odb64luo -x "select count(*) from customer"150000[0.0.0]--- 1 row(s) selected in 0.137s (prep 0.000s, exec 0.137s, 1st fetch 0.000s, fetch 0.000s)The meaning of [0.0.0] will be explained later.felici ~/Devel/odb $ cat script.sql SELECT COUNT(*) FROM T1;-- This is a commentSELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY) AS SUM_QTY, SUM(L_EXTENDEDPRICE) AS SUM_BASE_PRICE, SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)) AS SUM_DISC_PRICE, SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)*(1+L_TAX)) AS SUM_CHARGE, AVG(L_QUANTITY) AS AVG_QTY, AVG(L_EXTENDEDPRICE) AS AVG_PRICE, AVG(L_DISCOUNT) AS AVG_DISC, COUNT(*) AS COUNT_ORDERFROM LINEITEMWHERE L_SHIPDATE <= DATE '1998-12-01' - INTERVAL '90' DAYGROUP BY L_RETURNFLAG, L_LINESTATUSORDER BY L_RETURNFLAG, L_LINESTATUS;felici ~/Devel/odb $ ./odb64luo -f script.sql [0.0.0]Executing: 'SELECT COUNT(*) FROM T1;'5[0.0.0]--- 1 row(s) selected in 0.015s (prep 0.000s, exec 0.015s, 1st fetch -0.000s, fetch -0.000s)[0.0.1]Executing: 'SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY) AS SUM_QTY, SUM(L_EXTENDEDPRICE) AS SUM_BASE_PRICE, SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)) AS SUM_DISC_PRICE, SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)*(1+L_TAX)) AS SUM_CHARGE, AVG(L_QUANTITY) AS AVG_QTY, AVG(L_EXTENDEDPRICE) AS AVG_PRICE, AVG(L_DISCOUNT) AS AVG_DISC, COUNT(*) AS COUNT_ORDER FROM LINEITEM WHERE L_SHIPDATE <= DATE '1998-12-01' - INTERVAL '90' DAY GROUP BY L_RETURNFLAG, L_LINESTATUS ORDER BY L_RETURNFLAG, L_LINESTATUS;'A,F,37734107.00,56586554400.73,53758257134.8700,55909065222.827692,25.522006,38273.129735,0.049985,1478493...R,F,37719753.00,56568041380.90,53741292684.6040,55889619119.831932,25.505794,38250.854626,0.050009,1478870[0.0.1]--- 4 row(s) selected in 21.344s (prep 0.000s, exec 21.344s, 1st fetch 0.000s, fetch 0.000s)You can use the -q switch to omit selected output components. So, for example, –q cmd won’t print the commands being executed:felici ~/Devel/odb $ ./odb64luo -f script.sql –q cmd5[0.0.0]--- 1 row(s) selected in 0.015s (prep 0.000s, exec 0.015s, 1st fetch -0.000s, fetch -0.000s)A,F,37734107.00,56586554400.73,53758257134.8700,55909065222.827692,25.522006,38273.129735,0.049985,1478493...R,F,37719753.00,56568041380.90,53741292684.6040,55889619119.831932,25.505794,38250.854626,0.050009,1478870[0.0.1]--- 4 row(s) selected in 21.344s (prep 0.000s, exec 21.344s, 1st fetch 0.000s, fetch 0.000s)While -q res won’t print the results:felici ~/Devel/odb $ ./odb64luo -f script.sql –q res[0.0.0]Executing: 'SELECT COUNT(*) FROM T1;'[0.0.0]--- 1 row(s) selected in 0.015s (prep 0.000s, exec 0.015s, 1st fetch -0.000s, fetch -0.000s)[0.0.1]Executing: 'SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY) AS SUM_QTY, SUM(L_EXTENDEDPRICE) AS SUM_BASE_PRICE, SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)) AS SUM_DISC_PRICE, SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)*(1+L_TAX)) AS SUM_CHARGE, AVG(L_QUANTITY) AS AVG_QTY, AVG(L_EXTENDEDPRICE) AS AVG_PRICE, AVG(L_DISCOUNT) AS AVG_DISC, COUNT(*) AS COUNT_ORDER FROM LINEITEM WHERE L_SHIPDATE <= DATE '1998-12-01' - INTERVAL '90' DAY GROUP BY L_RETURNFLAG, L_LINESTATUS ORDER BY L_RETURNFLAG, L_LINESTATUS;'[0.0.1]--- 4 row(s) selected in 21.344s (prep 0.000s, exec 21.344s, 1st fetch 0.000s, fetch 0.000s)And finally -q all (or just -q) won’t print neither the commands nor the results:felici ~/Devel/odb $ ./odb64luo -f script.sql -q all[0.0.0]--- 1 row(s) selected in 0.015s (prep 0.000s, exec 0.015s, 1st fetch -0.000s, fetch -0.000s)[0.0.1]--- 4 row(s) selected in 21.344s (prep 0.000s, exec 21.344s, 1st fetch 0.000s, fetch 0.000s)This is often used with odb as query driver. Note: Even when odb doesn’t print query results (-q res), the result set will be fetched and data is transferred from the database server to the client. In other words “-q res” is somehow similar (but not exactly equivalent) to a /dev/null output redirection.A special file name you can use with -f is “-“(dash). It means: read the script to be executed from the standard input. For example the following command will copy table definitions from one system to another recreating, on the target system, the same table structures as in the source system:$ odb64luo –u u1 -p p1 -d SRC -i t:TRAFODION.CIV04 -x "SHOWDDL &1" | odb64luo –u u2 –p p2 -d TGT -f -Using “here document” Syntax in Shell ScriptsWe often need to embed SQL commands in shell scripts. In these cases you can use ”-f -“ (read commands from standard input) odb syntax. Example:odb64luo -f - <<-EOF 2>&1 | tee -a ${LOG} drop table &t:TRAFODION.maurizio.ml%; create table ml2 ( id integer, fname char(10), bdate date, lname char(10) default 'Felici', comment char(20), city char(10) ) no partitions;EOFRunning Multiple Commands and Scripts in Parallelodb uses threads to run multiple commands in parallel. Each command (-x) or script (-f) will be executed, independently from the others, using a different thread. So, for example:felici ~/Devel/odb $ ./odb64luo -x "select count(*) from types" -f script1.sqlwill use two independent threads executed in parallel. The first thread will run“select count(*) from types” and the other “script1.sql”. You can also run multiple copies of the same command by adding <num>: before -x or -f arguments. So, for example, the following command will run 3 instances of " select count(*) from types ", 5 instances of “script1.sql” and 3 instances of “script2.sql” in parallel using 3 + 5 + 3 = 11 threads in total:felici ~/Devel/odb $ ./odb64luo -x 3:"select count(*) from types" -f 5:script1.sql -f 3:script2.sql -q[1.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[0.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[2.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[4.0.0]--- 1 row(s) selected in 0.001s (prep 0.000s, exec 0.001s, fetch 0.000s/0.000s)[6.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[5.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[3.0.0]--- 1 row(s) selected in 0.001s (prep 0.000s, exec 0.001s, fetch 0.000s/0.000s)[8.0.0]--- 1 row(s) selected in 0.001s (prep 0.000s, exec 0.001s, fetch 0.000s/0.000s)[7.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[9.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[10.0.0]--- 1 row(s) selected in 0.001s (prep 0.000s, exec 0.001s, fetch 0.000s/0.000s)The number highlighted in red here above is the thread ID. Thread IDs are assigned by odb starting from zero.You can limit the maximum number of threads with -T option. The following example will run the same 11 commands/scripts limiting the number of threads (and ODBC connections) to 4:felici ~/Devel/odb $ ./odb64luo -x 3:"select count(*) from types" -f 5:script1.sql -f 3:script2.sql -q -T 4[1.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[0.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[2.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[1.1.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[2.1.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[0.1.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[3.0.0]--- 1 row(s) selected in 0.001s (prep 0.000s, exec 0.001s, fetch 0.000s/0.000s)[2.2.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[3.1.0]--- 1 row(s) selected in 0.001s (prep 0.000s, exec 0.001s, fetch 0.000s/0.000s)[0.2.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)[1.2.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, fetch 0.000s/0.000s)As you can see the thread ID is now in the 0-3 range because the 11 executions were queued into 4 threads. As we will see in the next sessions odb offers several alternatives to queue M executions in N (<M) threads.Limiting the Number of Threads Created by odbBy default odb will create as many threads as the numbers of executions. So, the following command:mauro@newton ~/src/C/odb $ ./odb64luo -f script1.sql -f 3:script2.sql -x 3:"<mysqlcmd>"It will create 1 + 3 + 3 = 7 threads. Each thread will start its own ODBC connection. You can limit the max number of threads using –T. So the following:mauro@newton ~/src/C/odb $ ./odb64luo -f script1.sql -f 3:script2.sql -x 3:"<mysqlcmd>" -T 2It will create just two threads to execute the seven commands/scripts.odb will never create more threads than needed:felici ~/Devel/odb $ ./odb64luo -f 2:script1.sql -f 3:script2.sql -T 8 -c -qodb [main(1017)] - Warning: won't be created more thread (8) then needed (5).Changing the Number of Executions Distributed Across ThreadsBy defaults executions are distributed in round-robin across threads. Let's say we run:mauro@newton ~/src/C/odb $ ./odb64luo -f script1.sql -f 3:script2.sql -x 3:"<mysqlcmd>" -T 3Then, the execution queue will be the following:Thread 1Thread 2Thread33rd executionmysqlcmd2nd executionScript2.sqlmysqlcmdmysqlcmd1st executionScript1.sqlScript2.sqlScript2.sqlThis (standard) behavior can be modified using the following options:–Z (shuffle). This will option will randomize the execution orderfactor sign with –P option. See REF _Ref311450589 \h Running All Scripts With a Given Path.-dlb (Dynamic Load Balancing). See REF _Ref315957664 \h Learning How Dynamic Load Balancing Works.Learning How Dynamic Load Balancing WorksAs discussed in the previous section, executions are normally pre-assigned to threads using a simple round-robin algorithm. This way the total elapsed time for each thread depends on the complexity of “its own” executions. Suppose you have two threads and two executions per thread:Thread 1Thread 22nd executionScript1.2Script2.21st executionScript1.1Script2.1If thread 2.1 and 2.2 require a very short time to be executed you can have a situation where Thread2 has nothing to do (it will be terminated) while Thread1 is still busy with “its own” Script1.1 and Script1.2.In some cases, for example during data extractions (see REF _Ref416261416 \h Loading Binary Files), we might want to keep all threads busy at any given time. In these cases we can use Dynamic Load Balancing (-dlb). With Dynamic Load Balancing jobs are not “pre-assigned” to threads when odb starts; each thread will pick the next job to run from the job list at run-time.Using Variables in odb Scriptsodb let you to use two kinds of variables:Internal variables defined through the “set param” command and identified by the ampersand character;Environment variables defined at operating system level and identified by a dollar sign;You can mix internal and environment variables in your scripts. If a variable is not expanded to a valid Internal/Environment variable the text will remain unchanged. So, for example, if you have a script like this:felici ~/Devel/odb $ cat scr.sqlset param region1 ASIA-- region1 is defined as an internal odb parameterselect * from tpch.region where r_name = '®ion1';-- region2 is defined as an environment variableselect * from tpch.region where r_name = '$region2';-- you can mix internal and environment variablesselect * from tpch.region where r_name = '$region2' or r_name = '®ion1';-- region3 variable does not exists so it won’t be not expandedselect * from tpch.region where r_name = '®ion3';After you define region2 at operating system level:felici ~/Devel/odb $ export region2=AMERICAYou will get the following result:felici ~/Devel/odb $ ./odb64luo -u mauro -p xx -d pglocal -f scr.sqlodb [2011-12-12 08:01:31]: starting (1) ODBC connection(s)... 1[0.0.0]Executing: 'select * from tpch.region where r_name = 'ASIA';'2,ASIA ,ges. thinly even pinto beans ca[0.0.0]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, 1st fetch 0.000s, fetch 0.000s)[0.0.1]Executing: 'select * from tpch.region where r_name = 'AMERICA';'1,AMERICA ,hs use ironic, even requests. s[0.0.1]--- 1 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, 1st fetch 0.000s, fetch 0.000s)[0.0.2]Executing: 'select * from tpch.region where r_name = 'AMERICA' or r_name = 'ASIA';'1,AMERICA ,hs use ironic, even requests. s2,ASIA ,ges. thinly even pinto beans ca[0.0.2]--- 2 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, 1st fetch 0.000s, fetch 0.000s)[0.0.3]Executing: 'select * from tpch.region where r_name = '®ion3';'[0.0.3]--- 0 row(s) selected in 0.000s (prep 0.000s, exec 0.000s, 1st fetch 0.000s, fetch 0.000s)Understanding Thread ID, Thread Execution#, and Script Command#Suppose we have a script containing two commands:mauriziof@traf64-dl2a ~/odb $ cat script.sql SELECT COUNT(*) FROM ORDERS;SELECT COUNT(*) FROM SUPPLIER;If we run this script ten times using two threads, we will get:mauriziof@traf64-dl2a ~/odb $ ./odb64luo -f 10:script.sql -q -T 2[0.0.0]--- 1 row(s) selected in 0.102s (prep 0.022s, exec 0.044s, 1st fetch 0.037s, fetch 0.037s)[1.0.0]--- 1 row(s) selected in 0.125s (prep 0.022s, exec 0.068s, 1st fetch 0.036s, fetch 0.036s)[0.0.1]--- 1 row(s) selected in 0.520s (prep 0.022s, exec 0.048s, 1st fetch 0.450s, fetch 0.450s)[1.0.1]--- 1 row(s) selected in 0.564s (prep 0.017s, exec 0.480s, 1st fetch 0.067s, fetch 0.067s)[0.1.0]--- 1 row(s) selected in 0.131s (prep 0.022s, exec 0.060s, 1st fetch 0.048s, fetch 0.048s)[0.1.1]--- 1 row(s) selected in 0.086s (prep 0.022s, exec 0.057s, 1st fetch 0.007s, fetch 0.007s)[1.1.0]--- 1 row(s) selected in 0.136s (prep 0.035s, exec 0.058s, 1st fetch 0.042s, fetch 0.042s)[0.2.0]--- 1 row(s) selected in 0.123s (prep 0.029s, exec 0.068s, 1st fetch 0.026s, fetch 0.026s)[1.1.1]--- 1 row(s) selected in 0.119s (prep 0.016s, exec 0.082s, 1st fetch 0.021s, fetch 0.021s)[0.2.1]--- 1 row(s) selected in 0.089s (prep 0.031s, exec 0.054s, 1st fetch 0.004s, fetch 0.004s)[1.2.0]--- 1 row(s) selected in 0.138s (prep 0.023s, exec 0.041s, 1st fetch 0.073s, fetch 0.073s)[0.3.0]--- 1 row(s) selected in 0.144s (prep 0.038s, exec 0.045s, 1st fetch 0.061s, fetch 0.061s)[1.2.1]--- 1 row(s) selected in 0.127s (prep 0.016s, exec 0.041s, 1st fetch 0.070s, fetch 0.070s)[0.3.1]--- 1 row(s) selected in 0.136s (prep 0.033s, exec 0.056s, 1st fetch 0.048s, fetch 0.048s)[1.3.0]--- 1 row(s) selected in 0.131s (prep 0.023s, exec 0.037s, 1st fetch 0.071s, fetch 0.071s)[0.4.0]--- 1 row(s) selected in 0.111s (prep 0.033s, exec 0.045s, 1st fetch 0.033s, fetch 0.033s)[0.4.1]--- 1 row(s) selected in 0.076s (prep 0.033s, exec 0.037s, 1st fetch 0.005s, fetch 0.006s)[1.3.1]--- 1 row(s) selected in 0.098s (prep 0.016s, exec 0.065s, 1st fetch 0.017s, fetch 0.017s)[1.4.0]--- 1 row(s) selected in 0.133s (prep 0.023s, exec 0.074s, 1st fetch 0.035s, fetch 0.035s)[1.4.1]--- 1 row(s) selected in 0.098s (prep 0.017s, exec 0.064s, 1st fetch 0.016s, fetch 0.016s) The numbers between square brackets have the following meaning:The first digit is the thread ID. As we have two threads in our example this ID is either 0 or 1;The second digit is the execution# for a given thread. As we have ten script executions for two threads, each thread will have to execute this script five times. So the execution#, in our example, is between 0 and 4;The third (last) digit is the command# in a given script. Our script contains two commands so this value is 0 or 1.So, for example, [0.3.1] means that the first thread (thread id=0) was executing its fourth job (thread execution#=3) and - more specifically - the second command in that script (script command#=1). Checking SQL ScriptsYou can check commands and SQL scripts with odb using the -N (null run) flag. This will just “prepare” (compile) the commands without executing them and fetching the results.Using Different Data Sources for Different ThreadsNormally all ODBC connections started by odb will use the same Data Source. However, there could be special cases where you want to use different DSN for different threads. In these cases you can use the –ndsn <number> option. This will append to the Data Source name specified via –d a suffix from 1 to -ndsn argument. So, for example:$ ./odb64luo ... –d MYDSN –ndsn 4It will use the following (round-robin) DSN/thread association: MYDSN1 for the first thread, MYDSN2 for the second thread and so on. The fifth thread (if any) will use MYDSN1 again. You can use a sequential DSN/thread association by using a ‘+’ sign in front of the –ndsn argument. So, for example, if you have 16 threads and –d MYDSN:Thread IDDSN with –ndsn 8DSN with –ndsn +80MYDSN1MYDSN11MYDSN2MYDSN12MYDSN3MYDSN23MYDSN4MYDSN24MYDSN5MYDSN35MYDSN6MYDSN36MYDSN7MYDSN47MYDSN8MYDSN48MYDSN1MYDSN59MYDSN2MYDSN510MYDSN3MYDSN611MYDSN4MYDSN612MYDSN5MYDSN713MYDSN6MYDSN714MYDSN7MYDSN815MYDSN8MYDSN8This technique has been used to maximize extraction throughput from a multi-segment Trafodion system. Each (local) Data Source was “linked” to a corresponding remote Data Source extracting its own data through its own network interface card.Formatting Query ResultsNormally odb prints query results using a very basic CSV format. Example:$ ./odb64luo -x "select s_suppkey, s_name, s_phone from tpch.supplier limit 5"1,Supplier#000000001,27-918-335-17362,Supplier#000000002,15-679-861-22593,Supplier#000000003,11-383-516-11994,Supplier#000000004,25-843-787-74795,Supplier#000000005,21-151-690-3663Adding the option -pad you will get a table format like this:$ ./odb64luo -x "select s_suppkey, s_name, s_phone from tpch.supplier limit 5" -pad s_suppkey|s_name |s_phone---------------+-------------------------+--------------- 1|Supplier#000000001 |27-918-335-1736 2|Supplier#000000002 |15-679-861-2259 3|Supplier#000000003 |11-383-516-1199 4|Supplier#000000004 |25-843-787-7479 5|Supplier#000000005 |21-151-690-3663Extracting Tables’ DDLYou can extract (multiple) tables’ DDL using either -i D… or -i U… options. For example:$ ./odb64luo -u xxx -p xxx -d traf -i D:TRAFODION.SEABASE.REGIONSodb [2015-04-20 21:25:35]: starting ODBC connection(s)... 0Connected to HP DatabaseCREATE TABLE TRAFODION.SEABASE."REGIONS" ( REGION_ID INTEGER NOT NULL,REGION_NAME VARCHAR(25));Sometimes you might want to extract the DDL for multiple objects, In this case you can use % wildcard. For example, the following command will extract the DDL for all tables in schema tpch starting with “P”:$ ./odb64luo -u xxx -p xxx -d traf -i D:TRAFODION.TPCH.P%odb [2015-04-20 21:33:43]: starting ODBC connection(s)... 0Connected to HP DatabaseCREATE TABLE TRAFODION.TPCH."PART" ( P_PARTKEY BIGINT NOT NULL,P_NAME VARCHAR(55) NOT NULL,P_MFGR CHAR(25) NOT NULL,P_BRAND CHAR(10) NOT NULL,P_TYPE VARCHAR(25) NOT NULL,P_SIZE INTEGER NOT NULL,P_CONTAINER CHAR(10) NOT NULL,P_RETAILPRICE BIGINT NOT NULL,P_COMMENT VARCHAR(23) NOT NULL,PRIMARY KEY (P_PARTKEY));CREATE TABLE TRAFODION.TPCH."PARTSUPP" ( PS_PARTKEY BIGINT NOT NULL,PS_SUPPKEY BIGINT NOT NULL,PS_AVAILQTY INTEGER NOT NULL,PS_SUPPLYCOST BIGINT NOT NULL,PS_COMMENT VARCHAR(199) NOT NULL,PRIMARY KEY (PS_PARTKEY,PS_SUPPKEY));odb [2015-04-20 21:33:45]: exiting. Session Elapsed time 2.069 seconds (00:00:02.069)When porting DDLs from one database to another we should consider possible differences in text column length semantic: certain database use “character oriented” text columns length while other use a “byte oriented” semantic. You can ask odb to multiply text column length when printing DDL using the switch –U[non-wide_char_multiplier,wide_char_multiplier]. For example: $ ./odb64luo -u xxx -p xxx -d traf -i U2,4:TRAFODION.TPCH.P%odb [2015-04-20 21:35:17]: starting ODBC connection(s)... 0Connected to HP DatabaseCREATE TABLE TRAFODION.TPCH."PART" ( P_PARTKEY BIGINT NOT NULL,P_NAME VARCHAR(110) NOT NULL,P_MFGR CHAR(50) NOT NULL,P_BRAND CHAR(20) NOT NULL,P_TYPE VARCHAR(50) NOT NULL,P_SIZE INTEGER NOT NULL,P_CONTAINER CHAR(20) NOT NULL,P_RETAILPRICE BIGINT NOT NULL,P_COMMENT VARCHAR(46) NOT NULL,PRIMARY KEY (P_PARTKEY));CREATE TABLE TRAFODION.TPCH."PARTSUPP" ( PS_PARTKEY BIGINT NOT NULL,PS_SUPPKEY BIGINT NOT NULL,PS_AVAILQTY INTEGER NOT NULL,PS_SUPPLYCOST BIGINT NOT NULL,PS_COMMENT VARCHAR(398) NOT NULL,PRIMARY KEY (PS_PARTKEY,PS_SUPPKEY));odb [2015-04-20 21:35:18]: exiting. Session Elapsed time 1.620 seconds (00:00:01.620)That command will multiply by 2 the length of “non-wide” text fields and by 4 the length of wide text fields.Loading, Extracting, and Copying DataLoading FilesYou can load a data file using -l option. For example:$ odb64luo –u user –p xx –d dsn –l src=customer.tbl:tgt=TRAFODION.MAURIZIO.CUSTOMER:fs=\|:rows=1000:loadcmd=UL:truncate:parallel=4This command will:load the file named customer.tbl (src=customer.tbl)in the table TRAFODION.MAURIZIO.CUSTOMER (tgt=TRAFODION.MAURIZIO.CUSTOMER)using “|” (vertical bar) as a field separator (fs=\|)using 1000 rows as rowset buffer (rows=1000)using UPSERT USING LOAD syntax to achieve better throughput as described in https://wiki.trafodion.org/wiki/index.php/Data_Loading (Trafodion only)truncating the target table before loading (truncate)using 4 parallel threads to load the target table (parallel=4)This is a complete list of the data loading operators: -l src=[-]file:tgt=table[:map=mapfile][:fs=fieldsep][:rs=recsep][:soe] [:skip=linestoskip][:ns=nullstring][:ec=eschar][:sq=stringqualifier] [:pc=padchar][:em=embedchar][:errmax=#max_err][:commit=auto|end|#rows|x#rs] [:rows=#rowset][:norb][:full][:max=#max_rec][:truncate][:show][:bpc=#][:bpwc=#] [:nomark][:parallel=number][:iobuff=#size][:buffsz=#size]][:fieldtrunc={0-4}] [:pre={@sqlfile}|{[sqlcmd]}][:post={@sqlfile}|{[sqlcmd]}][:ifempty] [:direct][:bad=[+]badfile][:tpar=#tables][:maxlen=#bytes][:time] [:xmltag=[+]element][:xmlord][:xmldump][:loadcmd=IN|UP|UL]The following table describes each data loading operator:Load optionMeaningsrc=<file>Input file. You can use the following keywords for this field:%t will be expanded to the (lower case) table name%T will be expanded to the (upper case) table name%s/%S will be expanded to the schema name%c/%C will be expanded to the catalog namestdin will load reading from the standard input-<file> to load all files listed in <file>hdfs][@host,port[,user]].<hdfspath> to load files from Hadoop File System (via libhdfs.so)mapr][@host,port[,user]].<maprpath> to load files from MapR File System (via libMapRClient.so)tgt=<CAT.SCH.TAB>This is the target tablefs=<char>|<code>This is the field separator. You can define the field separator:- as normal character (for example fs=,)- as ASCII decimal (for example fs=44 - 44 means comma)- as ASCII octal value (for example fs=054 – 054 means comma)- as ASCII hex value (for example fs=x2C – x2C means comma)Default field separator is ‘,’ (comma)rs=<char>|<code>This is the record separator. You can define the record separator the same way as the field separator. Default record separator is ‘\n’ (new line)pc=<char|code>Pad character used when loading fixed format files. You can use the same notation as the field separator.map=<mapfile>This will use mapfile to map source file to target table columns (see paragraph REF _Ref311455289 \r \* MERGEFORMAT 4.2)skip=numThis will skip a given number of lines when loading. This can be useful to skip headers in the source file.max=numThis is the max number of records to load. Default is to load all records in the input filens=<nullstring>odb will insert NULL when it finds nullstring in the input file. By default the nullstring is the empty stringsq=<char>|<code>This is the string qualifier character used to enclose strings. You can define the escape character the same way as the field separator.ec=<char>|<code>This is the character used as escape character. You can define the escape character the same way as the field separator. Default is “\” (back slash). rows=<num>|k<num>|m<num>This defines the size of the I/O buffer for each loading thread. You can define the size of this buffer in two different ways:- number of rows (for example: rows=100 means 100 rows as IO buffer)- buffer size in kB or MB (for example: rows=k512 (512 kB buffer) or rows=m20 (20MB buffer))Default value is 100.bad=[+]fileWhere to write rejected rows. If you omit this parameter rejected rows will be printed to standard error together with the error returned by the ODBC Driver.If you add a +sign in front of the file-name, odb will append instead of create the “bad file”.truncateWill truncate the target table before loadingifemptyWill load the target table only if it contains no recordsnorbWill load WITH NO ROLLBACKnomarkWon’t print the number of records loaded so far during loadssoeStop On Error. Will stop as soon as odb will encounter an errorparallel=numNumber of loading threads. odb will use:- one thread to read from the input file and - as many threads as the parallel argument to write via ODBC. This option is database independent.errmax=numodb will print up to num error messages per rowset. Normally used with soe to limit the number of error messages printed to the standard errorcommit=auto|end|#rows|x#rsThis defines how odb will commit the inserts. You have three choices:- auto will commit every single insert (see also rows load operator). This is the default.- end will commit when all rows (assigned to a given thread) have been inserted- #rows will commit every #rows inserted rows- x#rs will commit every #rs rowsets (see :rows)directThis will add /*+ DIRECT */ hint to the insert statement. To be used with Vertica databases in order to store inserted rows “directly” into the ROS (see Vertica’s documentation)fieldtrunc={0-4}This defines how odb will manage fields longer than the destination target column:- fieldtrunc=0 (default): will truncate input string, print a warning and load the truncated field if the target column is a text field;- fieldtrunc=1: like fieldtrunc=0 but no warning message is printed;- fieldtrunc=2: print an error message and does NOT load the row- fieldtrunc=3: like fieldtrunc=0 but tries to load the field even if the target column is NOT a text field- fieldtrunc=4: like fieldtrunc=3 but no warnings are printed.Be aware: last two options could bring to unwanted resilts because, for example, an input string like “2001-10-2345” will be loaded as a valid 2001-10-23 if the target field is a DATE.em=<char>|<code>This is the character used to embed binary files (see paragraph REF _Ref336620054 \r \h 4.6). You can define the embed character the same way as the field separator. No default value. pre={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd (enclosed between square brackets) on the target system immediately before loading the target table. You can, for example, CREATE the target table before loading it.Target table won’t be loaded if SQL execution fails and Stop On Error is setpost={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd (enclosed between square brackets) on the target system immediately after the target table has been loaded. You can, for example, update database stats after loading a table.tpar=numodb will load num tables in parallel when src is a list of files to be loadedshowodb will print what would be loaded in each column but no data is actually loaded. This I useful if you want to see how the input file “fits” into the target tables and is normally used to analyze the first few rows of CSV files (use :max).This option will force:- parallel to 1- rows to 1- ifempty to false- truncate to falsemaxlen=#bytesodb will limit the amount of memory allocated in the ODBC buffers for CHAR/VARCHAR fields to #bytestimeodb will print a “timeline” (milliseconds from start) for each insertbpc=#Bytes allocated in the ODBC buffer for each (non wide) CHAR/VARCHAR column length unit (default 1)bwpc=#Bytes allocated in the ODBC buffer for each (wide) CHAR/VARCHAR column length unit (default 4)Xmltag=[+]tagInput file is XML. Load all XML nodes under the one specified with this option. If a plus sign is specified, odb will load node attributes values. xmlordBy default odb will match target table columns with XML node or attributes using their names. If this option is specified odb will load the first node/attribute to the first column, the second node/attribute to the second column and so on without checking node/attribute namesxmldumpUsing this option odb won’t load the XML file content. XML attribute/tage names are printed to standard output so you can check what is going to be loaded.loadcmdSQL operation to be used for load, default is INSERT. UPSERT and UPSERT USING LOAD are also available for Trafodion. You can load multiple files using different -l options. By default odb will create as many threads (and ODBC connections) as the sum of parallel load threads. You can limit this number using -T option. So, for example: $ odb64luo –u user –p xx –d dsn –T 5 \ -l src=./data/%t.tbl.gz:tgt=TRAFODION.MAURO.CUSTOMER:fs=\|:rows=m2:truncate:norb:parallel=4 \ -l src=./data/%t.tbl.gz:tgt=TRAFODION.MAURO.ORDERS:fs=\|:rows=1000:truncate:norb:parallel=4 \ -l src=./data/%t.tbl.gz:tgt=TRAFODION.MAURO.LINEITEM:fs=\|:rows=m10:truncate:norb:parallel=4 Will truncate and load (CUSTOMER, ORDERS and LINEITEM) tables. Input files have the same name as the target tables – in lower cases). Loads will be distributed among available threads this way: Load OrderThread 0Thread 1Thread2Thread3Thread43rd Read lineitem.tblLoad TRAFODION.MAURO.LINEITEMLoad TRAFODION.MAURO.LINEITEMLoad TRAFODION.MAURO.LINEITEMLoad TRAFODION.MAURO.LINEITEM2nd Read orders.tblLoad TRAFODION.MAURO.ORDERSLoad TRAFODION.MAURO.ORDERSLoad TRAFODION.MAURO.ORDERSLoad TRAFODION.MAURO.ORDERS1st Read customer.tblLoad TRAFODION.MAURO.CUSTOMERLoad TRAFODION.MAURO.CUSTOMERLoad TRAFODION.MAURO.CUSTOMERLoad TRAFODION.MAURO.CUSTOMERIf you want to load more than one table in parallel you should use a number of threads defined as: (parallel + 1) * tables_to_load_in_parallel Note: You can load gzipped files without any special option. odb will automatically check input files and decompress them on the fly when needed.Also note that, even using one single loading thread (parallel=1), odb is faster than without parallel. This because if we do not specify parallel, odb will use one thread to both read from file and write into the target table:Read buffer #1 > Write Buffer #1 > Read Buffer #2 > Write Buffer #2 > Read Buffer #3 > Write Buffer #3 ...With parallel=1 we have one thread to read from file and one to write:Read buffer #1 > Read Buffer #2 > Read Buffer #3 > ... Write Buffer #1 > Write Buffer #2 > Write Buffer #3 ...Now reading from file is – normally – much faster than writing via ODBC so a single “reading thread” can serve different “loading threads”. One could ask: what the “right” number of loading threads is? In order to define the right number of loading threads you should run a few test and monitor the “Wait Cycles” reported by odb. Wait Cycles represent the number of times the “reading thread” had to wait for one “loading thread” to become available. When you have high “Wait Cycles/Total Cycles” ratio... it’s better to increase the number of writers. When the “Wait Cycles/Total Cycles” is less than 5%, adding more loading threads is useless or counterproductive. Mapping Source File Fields to Target Table Columnsodb, by default, assumes that input files contain as many fields as the target table columns. Also: file fields and target table columns are in the same order. This means that the first field in the input file will be loaded in the first table column, second input field will go to the second column and so on.If this basic assumption is not true and you need more flexibility to “link” input fields to target table columns, odb provides mapping/transformation capabilities though “mapfiles”. By specifying map=<mapfile> load option you can:Associate any input file field to any table columnSkip input file fieldsGenerate sequencesInsert constants Transform dates/timestamp formatsExtract substringsReplace input file strings. For example: insert ‘Maurizio Felici’ when you read ‘MF’Generate random values… and much moreA generic mapfile contains:Comments (line starting with ‘#’)Mappings to link input file fields to the corresponding target table columns. Mappings use the following syntax:<colname>:<field>[:transformation operator]. Where:<colname> is the target table column name.<field> is one of the following:The ordinal position (starting from zero) of the input file field. So: first input field is ‘0’ (zero), second input field is ‘1’ and so onCONST:<CONSTANT> to load a constant valueSEQ:<START> to generate/load a sequence starting from <START>IRAND:<MIN>:<MAX> to generate/load a random integer between <MIN> and <MAX>DRAND:<MIN_YEAR>:<MAX_YEAR> to generate/load a random date (YYYY-MM-DD) between <MIN_YEAR> and <MAX_YEAR>TMRAND: to generate/load a random time (hh:mm:ss) between 00:00:00 and 23:59:59TSRAND: to generate/load a random timestamp (YYYY-MM-DD hh:mm:ss) between midnight UTC – 01 Jan 1970 and the current timestampCRAND:<LENGTH> will generate/load a string of <LENGTH> characters randomly selected in the following ranges: a-z, A-Z, 0-9NRAND:<PREC>:<SCALE> will generate/load a random NUMERIC field with precision <PREC> and scale <SCALE>DSRAND:<file> will select and load a random line from <file>TXTRAND:<MIN_LENGTH>:<MAX_LENGTH>:<file>: will select and load a random portion of test from <file> with length between <MIN_LENGTH> and <MAX_LENGTH>LSTRAND:<VALUE1,VALUE2,…> will select and load a random value from <VALUE1,VALUE2,…>EMRAND:<MIN_ULENGTH>:<MAX_ULENGTH>:<MIN_DLENGTH>:<MAX_DLENGTH>:<SUFFIX1,SUFFIX2,…> will generate and load a string made of local@domain.suffix where:local is a string of random characters (a-z, A-Z, 0-9) with length between <MIN_ULENGTH> and <MAX_ULENGTH>domain is a string of random characters (a-z, A-Z, 0-9) with length between <MIN_DLENGTH> and <MAX_DLENGTH>suffix is a randomly selected suffix from <SUFFIX1,SUFFIX2,…>CDATE: to load the current date (YYYY-MM-DD)CTIME: to load the current time (hh:mm:ss)CTSTAMP: to load the current timestamp (YYYY-MM-SS hh:mm:ss)FIXED:<START>:<LENGTH> to load fixed format fields made of <LENGTH> characters starting at <START>. Note: <START> starts from zero.EMPTYASNULL: will load empty strings in the input file as NULLs (default is to load empty string as… empty strings)EMPTYASCONST:<CONSTANT>: will load empty fields in the input file as <CONSTANT>NULL: will insert NULL The final, optional, transformation operators can be:SUBSTR:<START>:<END>. For example if you have an input field containing “__Tib:student” a transformation rule like SUBSTR:3:6 will load “Tib” in the database. TSCONV:<FORMAT>. This operator convert timestamps from the input file format defined through <FORMAT> to YYYY-MM-DD HH:MM:SS before loading. The input format is defined through any combination of the following characters:CharMeaningb abbreviated month nameB full month named day of the monthH hour (24 hour format)m month numberM MinuteS Secondy year (four digits)D##decimal digits. ignore a single char _ ignore up to the next digitDCONV:<FORMAT>. This operator convert dates from the input file format defined through <FORMAT> to YYYY-MM-DD (see TSCONV operator). Example: DCONV:B.d.y will convert “August,24 1991” to 1991-08-24TCONV:<FORMAT>. This operator convert times from the input file format defined through <FORMAT> to HH:MM:SS (see TSCONV operator). REPLACE:<READ>:<WRITTEN>. This operator will load the string <WRITTEN> when the input file fields contains <READ>. If the input file string doesn’t match <READ> it will be loaded as is (see paragraph REF _Ref318964245 \r \h 4.3)TOUPPER. This operator converts the string read from the input file to uppercase before loading. Example: proGRAmMEr PROGRAMMERTOLOWER. This operator converts the string read from the input file to lowercase before loading. Example: proGRAmMEr programmerFIRSTUP. This operator converts the first character of the string read from the input file to uppercase and the remaining characters to lowercase before loading. Example: proGRAmMEr ProgrammerTRANSLIT:<LIST OF CHARS>:<LIST OF CHARS>. This opertator let you to delete or change any character with another. Examples:WORK:7:translit:Gp:HP will load the seventh input field into the target column named WORK and will replace all “G” with “H” and all “p” with “P”WORK:7:translit:Gp\r:HP\d behaves like the previous example but will also delete all “carriage returns” (\r)CSUBSTR. This operator is somehow similar to SUBSTR but instead of using fixed position to extract substrings will use delimiting characters. For example, suppose your input fields (comma is the field separator) are: … other fields…,name_Maurizio.programmer,…other fields … other fields…,_name_Lucia.housewife, …other fields… … other fields…,first_name_Giovanni.farmer,… other fields… … other fields…,_Antonella,… other fields… … other fields…,Martina,…other fields… … other fields…,Marco.student, …other fields … Using a transformation like: NAME:4:CSUBSTR:95:46 (where 95 is the ASCII code for “_” and 46 is the ASCII code for “.”) will result in loading the following values into the target (NAME) column:MaurizioLuciaGiovanniAntonellaMartinaMarcoCOMP. This operator will transform a packed binary COMP into a target database number. For example: hex 80 00 00 7b will be loaded as -123COMP3:PRECISION:SCALE. This operator will transform a packed binary COMP-3 format into a target database number. For example: hex 12 34 56 78 90 12 34 56 78 9b will be loaded as -1234567890123456.789ZONED:PRECISION:SCALE. This operator will transform a packed binary ZONED format into a target database number. For example: hex 31 32 33 34 35 36 will be loaded as +.123456Using mapfiles to Ignore and/or Transform Fields When LoadingLet me use an example to explain mapfile usage to skip/transform or generate fields. Suppose you have a target table like this:+------+---------------+----+-------+------------+|COLUMN|TYPE |NULL|DEFAULT|INDEX |+------+---------------+----+-------+------------+|ID |INTEGER SIGNED |NO | |mf_pkey 1 U ||NAME |CHAR(10) |YES | | ||AGE |SMALLINT SIGNED|YES | | ||BDATE |DATE |YES | | |+------+---------------+----+-------+------------+And an input file like this:uno,00,51,due,Maurizio,tre,07 Mar 1959, ignore,remaining, fieldsuno,00,46,due,Lucia,tre,13 Oct 1964, ignore, thisuno,00,34,due,Giovanni,tre,30 Mar 1976uno,00,48,due,Antonella,tre,24 Apr 1962242062034925BDATE00BDATE-29908517780AGE00AGE131000523495NAME00NAMEYou want to load the highlighted fields into the appropriate column, generate a unique key for ID and ignore the non-highlighted fields. In addition: you need to convert date format and replace all occurrencies of “Lucia” with “Lucy”. Well, you will need a mapfile like this:felici ~/Devel/odb $ cat test/load_map/ml1.map# Map file to load TRAFODION.MFTEST.FRIENDS from friends.datID:seq:1 This will insert into ID column a sequence starting from 1NAME:4:REPLACE:Lucia:Lucy This will load field #4 into NAME and replace all occurrences of Lucia with LucyAGE:2 This will load field #2 (they start from zero) into AGEBDATE:6:DCONV:d.b.y This will load field #6 into BDATE converting date format from “dd mmm yyyy”Then we will load this way:$ odb64luo –u user –p xx –d dsn \ -l src=frends.dat:tgt=TRAFODION.MFTEST.FRIENDS:map=ml1.map:fs=, Using mapfiles to Load Fixed Format FilesSuppose you have a target table like this:+------+---------------+----+-------+------------+|COLUMN|TYPE |NULL|DEFAULT|INDEX |+------+---------------+----+-------+------------+|NAME |CHAR(10) |YES | | ||JOB |CHAR(10) |YES | | ||BDATE |DATE |YES | | |+------+---------------+----+-------+------------+And an input file like this:GiovanniXXX30 Mar 1976YFarmer Lucia XXX13 Oct 1964YHousewifeMartina XXX28 Oct 1991Y? Marco XXX06 Nov 1994Y? MaurizioXXX07 Mar 1959YProgrammer31051527305NAME00NAME115443027305BDATE00BDATE204597027305JOB00JOBYou want to load the highlighted fields into the appropriate columns and to convert date format. Null values in the input file are represented by question marks. In this case you can use a mapfile like this:felici ~/Devel/odb $ cat test/fixed/ff.mapNAME:FIXED:0:8 insert into NAME characters starting at position 0, length 8BDATE:FIXED:11:11:DCONV:d.b.y insert into BDATE characters starting at col 11, length 11 and convert dateJOB:FIXED:23:10 insert into JOB characters starting at position 23, length 10Then we will load this way:$ odb64luo –u user –p xx –d dsn \ -l src=frends1.dat:tgt=TRAFODION.MFTEST.FRIENDS1:map=ff.map:ns=\?:pc=32 Where: pc=32 identify the pad character in the input file (space = ASCII 32) and ns=? defines the null string in the input file.Generating and Loading Dataodb can generate and load data for testing purposes. Let’s discuss odb capabilities in this area through an example. Suppose you want to fill with test data a table like this:MFELICI [MAURIZIO] (03:37:06) SQL> ls -D personCREATE TABLE TRAFODION.MAURIZIO."PERSON" ( PID BIGINT SIGNED NOT NULL ,FNAME CHAR(20) NOT NULL ,LNAME CHAR(20) NOT NULL ,COUNTRY VARCHAR(40) NOT NULL ,CITY VARCHAR(40) NOT NULL ,BDATE DATE NOT NULL ,SEX CHAR(1) NOT NULL ,EMAIL VARCHAR(40) NOT NULL ,SALARY NUMERIC SIGNED(9,2) NOT NULL ,EMPL VARCHAR(40) NOT NULL ,NOTES VARCHAR(80) ,LOADTS TIMESTAMP(0) ,PRIMARY KEY (PID));You can use a mapfile like this:felici ~/Devel/odb $ cat person.mapPID:SEQ:100FNAME:DSRAND:datasets/first_names.txtLNAME:DSRAND:datasets/last_names.txtCOUNTRY:DSRAND:datasets/countries.txtCITY:DSRAND:datasets/cities.txtBDATE:DRAND:1800:2012SEX:LSTRAND:M,F,UEMAIL:EMRAND:3:12:5:8:com,edu,org,netSALARY:NRAND:9:2EMPL:DSRAND:datasets/fortune500.txtNOTES:TXTRAND:20:80:datasets/lorem_ipsum.txtLOADTS:CTSTAMPwhere:PID:SEQ:100 will load a sequence starting from 100 into PIDFNAME:DSRAND:datasets/first_names.txt will load FNAME with a randomly selected value from first_names.txt. There are plenty of sample datasets available to generate all sort of data using realistic valuesLNAME:DSRAND:datasets/last_names.txt will load LNAME with a random value from last_names.txtCOUNTRY:DSRAND:datasets/countries.txt will load COUNTRY with a random value from countries.txtCITY:DSRAND:datasets/cities.txt will load CITY with a random value from cities.txtBDATE:DRAND:1800:2012 will generate and load into BDATE a random date between 1800-01-01 and 2012-12-31SEX:LSTRAND:M,F,U will load SEX with a random value in the M, F, U rangeEMAIL:EMRAND:3:12:5:8:com,edu,org,net will generate and load a local@domain.suffix email addresses where:local is made of 3 to 12 random charactersdomain is made of 5 to 8 random characterssuffix is com, ord, edu or netSALARY:NRAND:9:2 will generate and load a random NUMERIC(9,2)EMPL:DSRAND:datasets/fortune500.txt will load EMPL with a random value from fortune500.txtNOTES:TXTRAND:20:80:datasets/lorem_ipsum.txt will load NOTES with a random section of lorem_ipsum.txt with length between 20 and 80 charactersAnd, finally, LOADTS:CTSTAMP will load the current timestamp into LOADTSWe will generate and load test data with a command like this:$ ./ odb64luo -l src=nofile:tgt=traf.maurizio.person:max=1000000:map=person.map:rows=5000:parallel=8:loadcmd=ULPlease note the “src=nofile” (it means “there is no input file”) and “max=1000000” (generate and load one million rows). The above command has generated and loaded 1 Ml rows of “realistic” data in about ten seconds:[0] odb Loading statistics: [0] Target table: TRAFODION.MAURIZIO.PERSON [0] Source: nofile [0] Pre-loading time: 2.911 s [0] Loading time: 7.466 s [0] Total records read: 1,000,000 [0] Total records inserted: 1,000,000 [0] Total number of columns: 12 [0] Total bytes read: 3,963 [0] Average input row size: 0.0 B [0] ODBC row size: 323 B (data) + 88 B (len ind) [0] Rowset size: 5,000 [0] Rowset buffer size: 2,006.83 KiB [0] Load Performances (real data): 0.518 KiB/s [0] Load Performances (ODBC): 42,243.161 KiB/s [0] Reader Total/Wait Cycles: 200/16Loading Default ValuesThe simpler way to load database generated defaults is to ignore the associated columns in the map file. For example, suppose you have a table like this under Trafodion:create table TRAFODION.maurizio.dtest ( id largeint generated by default as identity not null, fname char(10), lname char(10) default 'Felici', bdate date, comment varchar(100) );If you have an input file containing:ignoreme,Maurizio,xyz,commentM, ignore,remaining, fieldsignoreme,Lucia,xyz,commentL, ignore, thisignoreme,Giovanni,xyz,commentGignoreme,Antonella,xyz,commentA-2667097790FNAME00FNAME1858645-3810COMMENT00COMMENTand a map-file like this:FNAME:1BDATE:CDATECOMMENT:4Then:First column (ID) will be loaded with its default value (not in the map file)Second column (FNAME) will be loaded with the second input field from file (FNAME:1)Third column (LNAME) will be loaded with its default value (not in the map file)Fourth column (BDATE) will be loaded with the Current Data generated by odb (BDATE:CDATE)And the Fifth column (COMMENT) will be loaded with the fifth column in the input file (COMMENT:4)Loading Binary FilesAssuming your backend database (and your ODBC Driver) supports BLOB data types, or equivalent, you can use odb to directly load binary (or any other) files into a database column using the [:em=char] symbol to identify the file to be loaded into that specific database field. Example. Suppose you have a table like this (MySQL):create table pers.myphotos ( id integer, image mediumblob, phts timestamp);You can load a file like this:$ cat myphotos.csv001,@/home/mauro/images/image1.jpg,2012-10-21 07:31:21002,@/home/mauro/images/image2.jpg,2012-10-21 07:31:21003,@/home/mauro/images/image3.jpg,2012-10-21 07:31:21by running a command like this:$ odb64luo –u user –p xx –d dsn -l src=myphotos.csv:tgt=pers.myphotos:em=\@odb will consider the string following the “em” character as the path of the file to be loaded in that specific field.Note: odb will not load rows where the size of the input file is greater than the target database column Reducing the ODBC Buffer Sizeodb will allocate memory for the ODBC buffers during load/extract operations based on the max possible length of the source/target columns. So, if you have a column defined as VARCHAR(2000), odb will allocate enough space for 2,000 characters in the ODBC buffer. Now, if you know in advance that you will never load/extract 2,000 characters you can limit the amount of space allocated by odb. This will reduce memory usage and increase performances because of the reduced network traffic.Suppose we have a table like this:felici ~/Devel/odb $ ./odb64luo -u xxx -p xxx -d traf -i D:TRAFODION.USR.TMXodb [2015-04-20 21:41:38]: starting ODBC connection(s)... 0Connected to HP DatabaseCREATE TABLE TRAFODION.USR."TMX" ( ID INTEGER NOT NULL,NAME VARCHAR(400),PRIMARY KEY (ID));And an input file containing:felici ~/Devel/odb $ cat tmx.dat1,Maurizio2,Lucia3,Martina4,Giovanni5,Marco6,Roland7,Randy8,Paul9,Josef10,Some other nameThe max length of the second field in this file is:felici ~/Devel/odb $ awk -F\, 'BEGIN{max=0} {if(NF==2){len=length($i);if(len>max)max=len}} END{print max}' tmx.dat15In this case you can use :maxlen=15 to limit the amount of the ODBC buffer:felici ~/Devel/odb $ ./odb64luo -u xxx -p xxx -d traf -l src=tmx.dat:tgt=usr.tmx:truncate:maxlen=15odb [2015-04-20 21:46:11]: starting ODBC connection(s)... 0Connected to HP Database[0.0.0]--- 0 row(s) deleted in 0.052s (prep 0.012s, exec 0.040s, fetch 0.000s/0.000s)[0] 10 records inserted [commit][0] odb version 1.1.0 Load(2) statistics:[0] Target table: (null).USR.TMX[0] Source: tmx.dat[0] Pre-loading time: 1.254 s (00:00:01.254)[0] Loading time: 0.026 s(00:00:00.026)[0] Total records read: 10[0] Total records inserted: 10[0] Total number of columns: 2[0] Total bytes read: 99[0] Average input row size: 9.9 B[0] ODBC row size: 26 B (data) + 16 B (len ind)[0] Rowset size: 100[0] Rowset buffer size: 4.10 KiB[0] Load throughput (real data): 3.718 KiB/s[0] Load throughput (ODBC): 9.766 KiB/sodb [2015-04-20 21:46:12]: exiting. Session Elapsed time 1.294 seconds (00:00:01.294)If you do not specify this parameter odb will allocate the buffer for the max possible length of each field:felici ~/Devel/odb $ ./odb64luo -u xxx -p xxx -d traf -l src=tmx.dat:tgt=usr.tmx:truncateodb [2015-04-20 21:47:13]: starting ODBC connection(s)... 0Connected to HP Database[0.0.0]--- 10 row(s) deleted in 0.107s (prep 0.012s, exec 0.095s, fetch 0.000s/0.000s)[0] 10 records inserted [commit][0] odb version 1.1.0 Load(2) statistics:[0] Target table: (null).USR.TMX[0] Source: tmx.dat[0] Pre-loading time: 1.330 s (00:00:01.330)[0] Loading time: 0.032 s(00:00:00.032)[0] Total records read: 10[0] Total records inserted: 10[0] Total number of columns: 2[0] Total bytes read: 99[0] Average input row size: 9.9 B[0] ODBC row size: 411 B (data) + 16 B (len ind)[0] Rowset size: 100[0] Rowset buffer size: 41.70 KiB[0] Load throughput (real data): 3.021 KiB/s[0] Load throughput (ODBC): 125.427 KiB/sodb [2015-04-20 21:47:14]: exiting. Session Elapsed time 1.373 seconds (00:00:01.373)Extracting TablesYou can use odb to extract tables from a database and write them to standard files (or named pipes). This is a table extraction example:$ odb64luo –u user –p xx –d dsn –T 3 \ -e src=TRAFODION.MAURIZIO.LIN%:tgt=${DATA}/ext_%t.csv.gz:rows=m10:fs=\|:trim:gzip: \ -e src=TRAFODION.MAURIZIO.REGION:tgt=${DATA}/ext_%t.csv.gz:rows=m10:fs=\|:trim:gzip \ -e src=TRAFODION.MAURIZIO.NATION:tgt=${DATA}/ext_%t.csv.gz:rows=m10:fs=\|:trim:gzip \ The command here above will:extract tables REGION, NATION and all tables starting with LIN from TRAFODION.MAURIZIO schema, data will be saved into files ext_%t.csv.gz (%t will be expanded to the real table name), output file will be gzipped on the fly (uncompressed data will never land to disk),text fields will be trimmedthe IO buffer is 10 MBthe extraction process will use three threads (ODBC connection)This is a complete list of the extraction options: -e {src={table|-file}|sql=<custom sql>}:tgt=[+]file[:pwhere=where_cond] [:fs=fieldsep][:rs=recsep][:sq=stringqualifier][:ec=escape_char][:soe] [:ns=nullstring][es=emptystring][:rows=#rowset][:nomark][:binary][:fwc] [:max=#max_rec][:trim=[cCvVdt]][:rtrim][:cast][:multi][:efs=string] [:parallel=number][:gzip][:gzpar=wb??][:uncommitted][:splitby=column] [:pre={@sqlfile}|{[sqlcmd]}[:mpre={@sqlfile}|{[sqlcmd]}[:post={@sqlfile}|{[sqlcmd]}] [tpar=#tables][:time][:nts][:cols=[-]columns]][:maxlen=#bytes][:xml]The following table describes each extract operator:Extract optionMeaningsrc=<CAT.SCH.TAB>|-fileThis filed is used to define the source table(s). You can use here:- a single table name (for example TRAFODION.MFTEST.LINEITEM)- a group of tables (for example TRAFODION.MFTEST.LIN%)- a file containing a list of tables to extract (‘-‘ should precede the filename)sql=<sql>This is a custom SQL command you can use to extract data. This is alternative to src=tgt=[+]fileOutput file. You can use the following keywords for this field:%t/%T will be expanded to the (lower/upper case) table name%s/%S will be expanded to the (lower/upper case) schema name%c/%C will be expanded to the (lower/upper case) catalog name%d will be expanded to the extraction date (YYYYMMDD format)%D will be expanded to the extraction date (YYYY-MM-DD format)%m will be expanded to the extraction time (hhmmss format)%M will be expanded to the extraction time (hh:mm:ss format)stdout will print the extracted records to the standard output.If you add a +sign in front of the file-name, odb will append instead of create it.hdfs./<hdfspath>/<file> to write exported table under Hadoop File Systemfs=<char>|<code>This is the field separator. You can define the field separator:- as normal character (for example fs=,)- as ASCII decimal (for example fs=44 - 44 means comma)- as ASCII octal value (for example fs=054 – 054 means comma)- as ASCII hex value (for example fs=x2C – x2C means comma)Default field separator is ‘,’ (comma)rs=<char>|<code>This is the record separator. You can define the record separator the same way as the field separator. Default record separator is ‘\n’ (new line)max=numThis is the max number of records to extract. Default is to extract all recordssq=<char>|<code>This is the string qualifier character used to enclose strings. You can define the string qualifier the same way as the field separatorec=<char>|<code>This is the character used as escape character. You can define the escape character the same way as the field separator. Default is “\” (back slash). rows=<num>|k<num>|m<num>This defines the size of the I/O buffer for each extraction thread. You can define the size of this buffer in two different ways:- number of rows (for example: rows=100 means 100 rows as IO buffer)- buffer size in kB or MB (for example: rows=k512 (512 kB buffer) or rows=m20 (20MB buffer))ns=<nullstring>This is how odb will represent NULL values in the output file. Default is the empty string (two field separators one after the other)es=<emptystring>This is how odb will represent VARCHAR empty strings (NOT NULL with zero length) values in the output file. Default is the empty string (two field separators one after the other)gzpar=<params>This are extra parameters you can pass to tune the gzip compression algorithm. Examples:- gzpar=wb9: max compression (slower)- gzpar=wb1: basic compression (faster)- gzpar=wb6h: Huffman compression only- gzpar=wb6R: Run-length encoding onlytrim[=<params>]Accept the following optional parameters:ctrims leading spaces from CHAR fieldsCtrims trailing spaces from CHA R fields NOTEREF _Ref317771400 \h \* MERGEFORMAT 2vtrims leading spaces from VARCHAR fieldsVtrims trailing spaces from VARCHAR fieldsdtrims trailing zeros after decimal sign. Example: 12.3000 will be extracted as 12.3. ttrims decimal portion from TIME/TIMESTAMP fields. For example: 1999-12-19 12:00:21.345 will be extracted as 1999-12-19 12:00:21trim examples::trim=cC trims leading/trailing spaces from CHAR fields:trim=cCd trims leading/trailing spaces from CHARs and trailing decimal zerosIf you do not specify any argument for this operator odb will use “cCvV”. In other words :trim: is a shortcut for :trim=cCvV:nomarkWon’t print the number of records extracted so far by each threadsoeStop On Error. Will stop as soon as odb will encounter an errorparallel=numodb will use as many threads as the parallel argument to extract data from partitioned source tables. You have to use splitby.Each thread will take care of a specific range of the source table partitions. For example if you specify parallel=4 and the source table is made of 32 partitions, odb will start four threads (four ODBC connections):- thread 0 will extract partitions 0-7- thread 1 will extract partitions 8-15- thread 2 will extract partitions 16-23- thread 3 will extract partitions 24-31multiThis option can be used in conjunction with parallel operator to write as many ouput files as the number of extraction threads. Output file names are built adding four digits at the end of the file identified by the tgt operator.For example, with src=trafodion.mauro.orders:tgt=%t.csv:parallel=4:multi odb will write into the following output files:- orders.csv.0001- orders.csv.0002- orders.csv.0003- orders.csv.0004pwhere=<where condition>This option is used in conjunction with parallel and will limit the extraction to records satisfying the where condition. Note: The where condition is limited to columns in the source table.For example: you want to extract records with TRANS_TS > 1999-12-12 09:00:00 from the source table TRAFODION.MAURO.MFORDERS using 8 parallel streams to a single, gzipped, file having the same name as the source table:src=trafodion.mauro.mforders:tgt=%t.gz:gzip:parallel=8:pwhere=[TRANS_TS > TIMESTAMP ‘1999-12-12 09:00:00’]... You can enclose the where condition between square brackets to avoid a misinterpretation of the characters in the where condition.errmax=numodb will print up to num error messages per rowset. Normally used with soe to limit the number of error messages printed to the standard erroruncommittedWill add FOR READ UNCOMMITTED ACCESS to the select(s) command(s)rtrimThis will RTRIM() CHAR columns on the server. From a functional point of view this is equivalent to trim=C but rtrim is executed on the server so it saves both client CPU cycles and network bandwidth.castIt performs a (server side) cast to VARCHAR for all non-text columns. Main scope of this operator is to "move" CPU cycles from the client to the database server. It increases network traffic. To be used when: - the extraction process is CPU bound on the client AND - network has a lot of available bandwidth AND - database server CPUs are not "under pressure".Tests extracting a table full of NUMERIC(18,4), INT and DATES shows: - client CPU cycles down ~50% on the client - network traffic up ~40%splitby=<column>This operator let you to use parallel extract from any database. <column> has to be a SINGLE, numeric column. odb will calculate min()/max() value for <column> and assign to each <parallel> thread the extraction of the rows in its "bucket". For example, if you have: ...:splitby=emp_id:parallel=4...with min(emp_id)=1 and max(emp_id)=1000, the four threads will extract the following rows: thread #0 emp_id >=1 and emp_id < 251 thread #1 emp_id >=251 and emp_id < 501 thread #2 emp_id >=501 and emp_id < 751 thread #3 emp_id >=751 and emp_id < 1001 (odb uses max(emp_id) + 1)If the values are not equally distributed data extraction will be deskewed. pre={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd SQL command (enclosed between square brackets) on the source system immediately before table extraction.Source table won’t be extracted if SQL execution fails and Stop On Error is setmpre={@sqlfile}|{[sqlcmd]}Each odb thread will run either sqlfile script or sqlcmd SQL command (enclosed between square brackets)on the source system immediately before table extraction. You can use mpre to set database specific features for each extraction thread. Examples:1) you want Trafodion to ignore missing stats warning. Then you can run via mpre a sql script containing:control query default HIST_MISSING_STATS_WARNING_LEVEL '0';2) you want Oracle to extract dates in the YYYY-MM-DD hh:mm:ss format. Then you can run via mpre a script containing:ALTER SESSION SET NLS_DATE_FORMAT='YYYY-MM-DD HH:MI:SS'post={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd SQL command (enclosed between square brackets) on the source system immediately after table extraction. tpar=numodb will extract num tables in parallel when src is a list of files to be loadedmaxlen=#bytesodb will limit the amount of memory allocated in the ODBC buffers for CHAR/VARCHAR fields to #bytesxmlWill write output file in XML format timeodb will print a “timeline” (milliseconds from start)Extracting a List of TablesYou can use odb to extract all tables listed in a file. Example:felici ~/Devel/odb $ cat tlist.txt# List of tables to extractsrc=TRAFODION.MAURIZIO.ORDERSsrc=TRAFODION.MAURIZIO.CUSTOMERsrc=TRAFODION.MAURIZIO.PARTsrc=TRAFODION.MAURIZIO.LINEITEMYou can extract all these tables by running:$ odb64luo –u user –p xx –d dsn -e src=-tlist.txt:tgt=%t_%d%m:rows=m20:sq=\”Please note the src=-tlist.txt. Copying Tables From One Database to Anotherodb can directly copy tables from one data-source to another (for example from Trafodion to Teradata or vice-versa). Using this option data will never land to disk. Target table has to be be created in advance and should have a compatible structure. Here we have a complete list of the odb’s copy operators: -cp src={table|-file:tgt=schema[.table][pwhere=where_cond][:soe][:nts] [:truncate][:rows=#rowset][:nomark][:max=#max_rec][:fwc][:bpwc=#] [:parallel=number][errmax=#max_err][:commit=auto|end|#rows|x#rs][:time] [:direct][:uncommitted][:norb][:splitby=column][:pre={@sqlfile}|{[sqlcmd]}] [:post={@sqlfile}|{[sqlcmd]}][:mpre={@sqlfile}|{[sqlcmd]}][:ifempty] [:loaders=#loaders][:tpar=#tables][:cols=[-]columns] [sql={[sqlcmd]|@sqlfile|-file}[:bind=auto|char|cdef] [tmpre={@sqlfile}|{[sqlcmd]}][seq=field#[,start]]odb copy’s operators:Copy operatorMeaningsrc=<CAT.SCH.TAB>|-fileThis filed is to define the source table(s). You can use here:- a single table (for example: TRAFODION.MFTEST.LINEITEM)- a group of tables (for example: TRAFODION.MFTEST.LIN%)- a file containing a list of tables to copy (‘-‘ should precede the filename)tgt=<CAT.SCH.TAB>Target table(s). You can use the following keywords for this field:%t/%T will be expanded to the (lower/upper case) source table name%s/%S will be expanded to the (lower/upper case) source schema name%c/%C will be expanded to the (lower/upper case) source catalog namesql={[sqlcmd]|@sqlfile|-file}odb use a generic SQL– instead of a “real” table – as source.max=numThis is the max number of records to copy. Default is to copy all records in the source tablerows=<num>|k<num>|m<num>This defines the size of the I/O buffer for each copy thread. You can define the size of this buffer in two different ways:- number of rows (for example: rows=100 means 100 rows as IO buffer)- buffer size in kB or MB (for example: rows=k512 (512 kB buffer) or rows=m20 (20MB buffer))truncateWill truncate the target table before loadingifemptyWill load the target table only if emptynomarkWon’t print the number of records loaded so far during loadssoeStop On Error. Will stop as soon as odb will encounter an errorparallel=numodb will use as many threads as the parallel argument to extract data from partitioned source tables PLUS an equivalent number of threads to write to the target table. For example if you specify parallel=4 and the source table is made of 32 partitions, odb will start four threads (four ODBC connections) to read from the source table PLUS four threads (four ODBC connections) to write to the target table:- thread 0 will extract partitions 0-7 from source- thread 1 will write data extracted from thread 0 to target- thread 2 will extract partitions 8-15 from source- thread 3 will write data extracted from thread 2 to target- thread 4 will extract partitions 16-23 from source- thread 5 will write data extracted from thread 4 to target- thread 6 will extract partitions 24-31 from source- thread 7 will write data extracted from thread 6 to targetYou have to specify splitby.pwhere=<where condition>This option is used in conjunction with parallel to copy only records satisfying the where condition. Note: The where condition is limited to columns in the source table.For example: you want to copy records with TRANS_TS > 1999-12-12 09:00:00 from the source table TRAFODION.MAURO.MFORDERS using 8 parallel streams to a target table having the same name as the source table:src=trafodion.mauro.mforders:tgt=trafodion.dest_schema.%t:parallel=8:pwhere=[TRANS_TS > TIMESTAMP ‘1999-12-12 09:00:00’]... You can enclose the where condition between square brackets to avoid a misinterpretation of the characters in the where condition.commit=auto|end|#rows|x#rsThis defines how odb will commit the inserts. You have three choices:- auto will commit every single insert (see also rows load operator). This is the default.- end will commit when all rows (assigned to a given thread) have been inserted- #rows will commit every #rows copied rows- x#rs will commit every #rs rowsets copied (see :rows)directThis will add /*+ DIRECT */ hint to the insert statement. To be used with Vertica databases in order to store inserted rows “directly” into the ROS (see Vertica’s documentation)errmax=numodb will print up to num error messages per rowset. Normally used with soe to limit the number of error messages printed to the standard erroruncommittedWill add FOR READ UNCOMMITTED ACCESS to the select(s) command(s)splitby=<column>This operator let you to use parallel copy from any database. <column> has to be a SINGLE, numeric column. odb will calculate min()/max() value for <column> and assign to each <parallel> thread the extraction of the rows in its "bucket". For example, if you have: ...:splitby=emp_id:parallel=4...with min(emp_id)=1 and max(emp_id)=1000, the four threads will extract the following rows: thread #0 emp_id >=1 and emp_id < 251 thread #1 emp_id >=251 and emp_id < 501 thread #2 emp_id >=501 and emp_id < 751 thread #3 emp_id >=751 and emp_id < 1001 (odb uses max(emp_id) + 1)If the values are not equally distributed data extraction will be deskewed. pre={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd (enclosed between square brackets) on the target system immediately before loading the target table. You can, for example, CREATE the target table before loading it.Target table won’t be loaded if SQL execution fails and Stop On Error is setmpre={@sqlfile}|{[sqlcmd]}Each odb thread will run either sqlfile script or sqlcmd (enclosed between square brackets) on the source system immediately before loading the target table. You can use mpre to set database specific features for each thread.tmpre={@sqlfile}|{[sqlcmd]}Each odb thread will run either sqlfile script or sqlcmd (enclosed between square brackets) on the target system immediately before loading the target table. You can use mpre to set database specific features for each thread.post={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd (enclosed between square brackets) on the target system immediately after the target table has been loaded. You can, for example, update database stats after loading a table.tpar=numodb will copy num tables in parallel when src is a list of files to be loadedloaders=numodb will use num load threads for each extract thread (default is 2 loaders per extractor) fwcForce Wide Characters. odb will consider SQL_CHAR/SQL_VARCHAR fileds as they were defined SQL_WCHAR/SQL_WVARCHARbpwc=#odb internally allocates 4 bytes/char foe SQL_WCHAR/SQL_WVARCHAR columns. You can modify the number of bytes allocated for each char using this parameterbind=auto|char|cdefodb can bind columns to ODBC buffer as characters (char) or “C Default” data types (cdef). The default (auto) will use cdef if SRC/TGT use the same database or char if SRC/TGT databases differ seq=field#[,start]odb will add a sequence when loading the target system on column number field#. You can optionally define the sequence start value (default 1)timeodb will print a “timeline” (milliseconds from start)When copying data from one data source to another, odb will need user/password/dsn for both source and target system. User credentials and DSN for the target system are specified this way:$ odb64luo –u src_user:tgt_user –p src_pwd:tgt:pwd –d src_dsn:tgt_dsn ... -cp src=...:tgt=...Copying a List of TablesYou can use odb to copy a list of tables from one database to another. Example:felici ~/Devel/odb $ cat tlist.txt# List of tables to extractsrc=TRAFODION.MAURIZIO.ORDERSsrc=TRAFODION.MAURIZIO.CUSTOMERsrc=TRAFODION.MAURIZIO.PARTsrc=TRAFODION.MAURIZIO.LINEITEMYou can extract all these tables by running:$ odb64luo –u user1:user2 –p xx:yy –d dsn1:dsn2 \ -cp src=-tlist.txt:tgt=tpch.stg_%t:rows=m2:truncate:parallel=4 -T 8Please note the src=-tlist.txt. Will copy:SourceTargetTRAFODION.MAURIZIO.ORDERStpch.stg_ordersTRAFODION.MAURIZIO.CUSTOMERtpch.stg_customerTRAFODION.MAURIZIO.PARTtpch.stg_partTRAFODION.MAURIZIO.LINEITEMtpch.stg_lineitemYou can optionally define any other “command line” options in the input file. For example you can use different “splitby columns”:For example:felici ~/Devel/odb $ cat tlist2.txt# List of tables to extract and their “splitby columns”src=TRAFODION.MAURIZIO.ORDERS:splitby=O_ORDERKEYsrc=TRAFODION.MAURIZIO.CUSTOMER:splitby=C_CUSTOMERKEYsrc=TRAFODION.MAURIZIO.PART:splitby=P_PARTKEYsrc=TRAFODION.MAURIZIO.LINEITEM:splitby=L_PARTKEYUsing Case-Sensitive Table and Column NamesThe possibility to use case sensitive table/column names depends on your database configuration. odb will maintain table/column case sensitiveness when they are enclosed in double quotes. This will create a TRAFODION.MAURIZIO.Names table made of three columns: “name”, “NAME” and “Name” create table trafodion.maurizio."Names" ( "name" char(10), "NAME" char(10), "Name" char(10) ) no partitions;Double quotes have to be escaped under *nix. A few examples:felici ~/Devel/odb $ ./odb64luo -i T:trafodion.maurizio.\"Names\"felici ~/Devel/odb $ ./odb64luo -x "select * from trafodion.maurizio.\"Names\""felici ~/Devel/odb $ ./odb64luo -l src=names.txt:tgt=trafodion.maurizio.\"Names\":map=names.map:pc=32You can omit double quotes around column names when using mapfiles.Determining the Appropriate Number of Threads for Load/Extract/Copy/DiffIf you have to load/extract or copy multiple tables in parallel the best option is to use the options :tpar=number and :parallel=number. The first option (:tpar) defines how many tables have to be copied/extracted in parallel; the second option defines how many “data streams” to use for each table. This way odb will automatically allocate and start the “right” number of threads.My rule of thumb when copying/loading or extracting tables is to use as many “data streams” as:min(number of middletier CPUs, number of source CPUs, number of target CPUs)The number of threads started for each “data stream” depend on the operation type: OperationTotal threadsExplanationExample with parallel=4Loadparallel + 1One thread to read from file + one thread per parallel to load5ExtractparallelOne thread per parallel to extract4Copyparallel * (1+loaders)Two threads per parallel: read from source and write to target12 (if loaders=2)Diffparallel * 3Three threads per parallel: read from source, read from target, compare12 Integrating With HadoopThere are basically two ways to integrate a generic database with Hadoop using odb:First Option is to use HIVE (Hadoop DWH) and its ODBC Driver. In this case odb can access HIVE like any other “normal” relational database. So – for example – you can copy to from HIVE and other databases using odb’s copy optionSecond option if to add the hdfs. prefix to the input or output file during loads/extracts. In this case the file will be read/written from/to Hadoop. odb interacts directly with the HDFS file system using libhdfs. This option is currently available only under Linux.Comparing Tables (Technology Preview)You can use odb to compare two tables with the same structure on different databases. In order to compare the two tables odb will:extract source/target tables ordered by Primary Key or any other set of columns (see key option here below) compare source/target ODBC buffers without “unpacking” them into columns/rows. Each “comparison stream” is made of three threads:One reading from the source tableOne reading from the target tableOne comparing the source/target buffersThese three threads work in parallel: the “compare” thread will check buffer N while the other two threads extract the net block of data from the source/target database in parallel.You can have multiple “triplets” working in parallel on different section of the table using the splitby operator. For example the following command:[dbadmin@n021 mftest]$ odb64luo -u MFELICI:maurizio.felici@hp.com -d MFELICI:VMFELICI -p xx:yy –diff src=trafodion.maurizio.lineitem:tgt=mftest.lineitem:key=l_orderkey,l_linenumber:output=lineitem.diff:rows=m2:print=IDC:splitby=l_orderkey:parallel=8Compare two tables using 8 streams (parallel=8) made of three threads each. The comparison threads use double buffering and advanced memory comparison techniques. odb can provide the following information in output as a CSV file:Missing rows on target (‘D’ – deleted – rows) based on the key columnsNew rows on target (‘I’ – inserted – rows) based on the key columnsChanged rows (same key columns but with different values in other fields). For these rows odb can print the original source version (‘C’ rows) and/or the modified target version (‘U’ rows).Here you have an example of the odb output when comparing two tables:[dbadmin@n021 mftest]$ cat lineitem.diffDTYPE,L_ORDERKEY,L_LINENUMBER,L_SUPPKEY,L_PARTKEY,L_QUANTITY,L_EXTENDEDPRICE,L_DISCOUNT,L_TAX,L_RETURNFLAG,L_LINESTATUS,L_SHIPDATE,L_COMMITDATE,L_RECEIPTDATE,L_SHIPINSTRUCT,L_SHIPMODE,L_COMMENTD,4532896,1,5974,100953,42.00,82065.90,0.03,0.00,R,F,1994-12-15,1995-01-17,1995-01-07,COLLECT COD,TRUCK,leep across the caD,4532896,2,2327,102326,48.00,63759.36,0.07,0.05,A,F,1995-02-18,1994-12-10,1995-03-12,TAKE BACK RETURN,RAIL,usly regular platelets. carefulD,4532896,3,612,193054,12.00,13764.60,0.05,0.02,R,F,1994-11-17,1994-11-23,1994-12-06,COLLECT COD,SHIP,s haggle quickly. ideas after theD,4532896,4,9867,47362,36.00,47136.96,0.10,0.06,A,F,1995-01-05,1994-11-29,1995-01-06,COLLECT COD,RAIL,s haggle carefully boD,4532896,5,9576,2075,19.00,18564.33,0.00,0.05,R,F,1994-11-26,1995-01-17,1994-12-03,COLLECT COD,TRUCK,en sauternes integrate blithely alonD,4532896,6,1016,68509,9.00,13297.50,0.07,0.00,R,F,1995-02-16,1995-01-05,1995-02-24,TAKE BACK RETURN,RAIL,ily above the blithelC,1652227,3,2298,87281,28.00,35511.84,0.06,0.05,R,F,1993-05-04,1993-03-12,1993-05-12,TAKE BACK RETURN,MAIL,lly final accoU,1652227,3,2298,87281,99.99,35511.84,0.06,0.05,R,F,1993-05-04,1993-03-12,1993-05-12,TAKE BACK RETURN,MAIL,lly final accoD,3456226,1,8161,148160,22.00,26579.52,0.06,0.02,A,F,1994-06-26,1994-06-08,1994-07-10,DELIVER IN PERSON,FOB,uriously. furioD,3456226,2,6293,108762,20.00,35415.20,0.10,0.05,R,F,1994-05-07,1994-06-03,1994-05-15,NONE,RAIL,ously bold requests along the bD,3456226,3,4542,159511,33.00,51826.83,0.05,0.03,A,F,1994-07-04,1994-05-15,1994-07-26,NONE,FOB,wake carefully alD,3456226,4,154,95135,33.00,37294.29,0.04,0.08,A,F,1994-05-27,1994-05-10,1994-06-14,DELIVER IN PERSON,AIR,ests. unusual dependencies wake fluffilyD,3456226,5,9027,126514,31.00,47755.81,0.08,0.01,R,F,1994-06-13,1994-06-18,1994-07-10,TAKE BACK RETURN,FOB,according to the carefully regular instructD,3456226,6,8477,110943,14.00,27355.16,0.03,0.01,R,F,1994-07-03,1994-05-28,1994-07-13,TAKE BACK RETURN,FOB,onic accounts. ironic, pendD,3456226,7,1773,4272,34.00,39993.18,0.08,0.00,A,F,1994-05-01,1994-05-29,1994-05-15,TAKE BACK RETURN,MAIL,ounts are finally caD,3456227,7,3722,101211,22.00,26668.62,0.02,0.01,N,O,1997-12-16,1998-02-05,1997-12-19,NONE,TRUCK,uriously even platelets are fuI,3456227,8,3722,101211,22.00,26668.62,0.02,0.01,N,O,1997-12-16,1998-02-05,1997-12-19,NONE,TRUCK,uriously even platelets are fuI,9999999,1,8161,148160,22.00,26579.52,0.06,0.02,A,F,1994-06-26,1994-06-08,1994-07-10,DELIVER IN PERSON,FOB,uriously. furioI,9999999,2,6293,108762,20.00,35415.20,0.10,0.05,R,F,1994-05-07,1994-06-03,1994-05-15,NONE,RAIL,ously bold requests along the bI,9999999,3,4542,159511,33.00,51826.83,0.05,0.03,A,F,1994-07-04,1994-05-15,1994-07-26,NONE,FOB,wake carefully alI,9999999,4,154,95135,33.00,37294.29,0.04,0.08,A,F,1994-05-27,1994-05-10,1994-06-14,DELIVER IN PERSON,AIR,ests. unusual dependencies wake fluffilyI,9999999,5,9027,126514,31.00,47755.81,0.08,0.01,R,F,1994-06-13,1994-06-18,1994-07-10,TAKE BACK RETURN,FOB,according to the carefully regular instructI,9999999,6,8477,110943,14.00,27355.16,0.03,0.01,R,F,1994-07-03,1994-05-28,1994-07-13,TAKE BACK RETURN,FOB,onic accounts. ironic, pendI,9999999,7,1773,4272,34.00,39993.18,0.08,0.00,A,F,1994-05-01,1994-05-29,1994-05-15,TAKE BACK RETURN,MAIL,ounts are finally caAs you can see the first column defines the type of difference.Here we have a complete list of the odb’s diff operators: -diff src={table|-file}:tgt=table:[key=columns][:output=[+]file][:pwhere=where_cond] [:pwhere=where_cond][:nomark][:rows=#rowset][:odad][:fs=fieldsep][:time] [:rs=recsep][:quick][:splitby=column][:parallel=number][:max=#max_rec] [:print=[I][D][C]][:ns=nullstring][:es=emptystring][:fwc][:uncommitted] [:pre={@sqlfile}|{[sqlcmd]}][:post={@sqlfile}|{[sqlcmd]}][tpar=#tables]odb copy’s operators:Diff operatorMeaningsrc=<CAT.SCH.TAB>|-fileThis field definse the source table(s). You can use here:- a single table (for example: TRAFODION.MFTEST.LINEITEM)- a file containing a list of tables to compare (‘-‘ should precede the filename)tgt=<CAT.SCH.TAB>This is the target table. key=column[,column,...]This option is used to define how to order records extracted from both source and target table. If you do not specify any key column, odb will use the Primary Keyoutput=[+]fileThis is the output file where the differences will be reported. You can use stdout to print odb output on the standard output. A + sign in front of the file-name will tell odb to append to an existing file.Default value: stdoutfs=<char>|<code>This is the field separator of the output file. You can define the field separator:- as normal character (for example fs=,)- as ASCII decimal (for example fs=44 - 44 means comma)- as ASCII octal value (for example fs=054 – 054 means comma)- as ASCII hex value (for example fs=x2C – x2C means comma)Default field separator is ‘,’ (comma)rs=<char>|<code>This is the record separator used in the output file. You can define the record separator the same way as the field separator. Default record separator is ‘\n’ (new line)max=numThis is the max number of records to compare. Default is to compare all recordsrows=<num>|k<num>|m<num>This defines the size of the I/O buffer for each extraction thread. You can define the size of this buffer in two different ways:- number of rows (for example: rows=100 means 100 rows as IO buffer)- buffer size in kB or MB (for example: rows=k512 (512 kB buffer) or rows=m20 (20MB buffer))ns=<nullstring>This is how odb will represent NULL values in the output file. Default is the empty string (two field separators one after the other)es=<emptystring>This is how odb will represent VARCHAR empty strings (NOT NULL with zero length) values in the output file. Default is the empty string (two field separators one after the other)nomarkWon’t print the number of records extracted so far by each threadsoeStop On Error. Will stop as soon as odb will encounter an errorparallel=numodb will use as many “threads triplets” (extract from source, extract from target, compare) as the parallel argument.Each thread will take care of a specific range of the source table data defined through the splitby optionuncommittedWill add FOR READ UNCOMMITTED ACCESS to the select(s) command(s)splitby=<column>This operator let you to use parallel extract from any database. <column> has to be a SINGLE, numeric column (or expression). odb will calculate min()/max() value for <column> and assign to each <parallel> thread the extraction of the rows in its "bucket". For example, if you have: ...:splitby=emp_id:parallel=4...with min(emp_id)=1 and max(emp_id)=1000, the four threads will extract the following rows: thread #0 emp_id >=1 and emp_id < 251 thread #1 emp_id >=251 and emp_id < 501 thread #2 emp_id >=501 and emp_id < 751 thread #3 emp_id >=751 and emp_id < 1001 (odb uses max(emp_id) + 1)If the values are not equally distributed data extraction will be deskewed. print=[I][C][D]This operator is used to specify which rows will be printed in the output file:‘I’ will print the new rows on target (based on key)‘D’ will print the missing rows on target (based on key)‘C’ will print the source rows with the same key columns but differences in other fieldsDefault value for print is IDCpre={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd SQL command (enclosed between square brackets) on the target system immediately before reading the target table. post={@sqlfile}|{[sqlcmd]}odb will run a single instance of either sqlfile script or sqlcmd SQL command (enclosed between square brackets) on the target system immediately after the target table has been compared. tpar=numNumber of tables to compare in parallel when you have a list of tables in inputloaders=numodb will use num load threads for each extract thread (default is 2 loaders per extractor) pwhere=<where condition>This option is used in conjunction with parallel to “diff” only records satisfying the where condition. For example: you want to compare rows with TRANS_TS > 1999-12-12 09:00:00 from the source table TRAFODION.MAURO.MFORDERS using 8 parallel streams to a target table having the same name as the source table:src=trafodion.mauro.mforders:tgt=trafodion.dest_schema.%t:parallel=8:pwhere=[TRANS_TS > TIMESTAMP ‘1999-12-12 09:00:00’]... You can enclose the where condition between square brackets to avoid a misinterpretation of the characters in the where condition.quickThis option will limit the comparison to the columns in the key option (PK by default). This is a fast way to check for new/missing records but it won’t find rows with differences in “non-key” columnstimeodb will print a “timeline” (milliseconds from start)Using odb as a Query Driver (Technology Preview)Getting CSV OutputIt’s often handy, while running performance tests to get a CSV output ready to be imported into your spreadsheet. You can easily get this kind of output with –c odb option. Let’s analyze the following command:$ ./odb64luo -u mauro -p xxx -d pglocal -x 3:"select count(*) from tpch.region" -f 5:Q01.sql -f 3:Q02.sql -T 4 -q –cThis will run:Three copies of the select count(*): -x 3:”select count(*) from tpch.region” Five copies of Q01.sql: -f 5:Q01.sqlThree copies of Q02: -f 3:Q02.sqlQueuing the resulting 11 executions into 4 threads: -T 4Omitting query text and query results (-q is equivalent to –q all): -qPrinting a CSV output: –cProducing the following output:odb [2011-12-12 08:08:43]: starting (4) threads...Thread id,Proc id,Thread Exec#,Script Cmd#,File,Label,Command,Rows,Rsds,Prepare(s),Exec(s),1st Fetch(s),Fetch(s),Total(s),STimeline,ETimeline1,1,0,0,(none),,"select count(*) from tpch.region",1,20,0.000,0.109,0.000,0.000,0.109,94,2030,0,0,0,(none),,"select count(*) from tpch.region",1,20,0.000,0.125,0.000,0.000,0.125,94,2192,2,0,0,(none),,"select count(*) from tpch.region",1,20,0.000,0.109,0.000,0.000,0.109,110,2192,6,1,0,Q01.sql,,"SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY)>",4,234,0.000,136.297,0.000,0.000,136.297,141,1364382,10,2,0,Q02.sql,,"SELECT S_ACCTBAL, S_NAME, N_NAME, P_PARTKEY, P_MF>",0,274,0.000,0.468,0.000,0.016,0.484,136438,1369220,4,1,0,Q01.sql,,"SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY)>",4,234,0.000,139.667,0.016,0.016,139.683,0,1396830,8,2,0,Q02.sql,,"SELECT S_ACCTBAL, S_NAME, N_NAME, P_PARTKEY, P_MFG>",0,274,0.000,0.015,0.000,0.000,0.015,139683,1396981,5,1,0,Q01.sql,,"SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY)>",4,234,0.000,144.347,0.015,0.015,144.362,141,1445031,9,2,0,Q02.sql,,"SELECT S_ACCTBAL, S_NAME, N_NAME, P_PARTKEY, P_MFG>",0,274,0.000,0.000,0.000,0.016,0.016,144503,1445193,3,0,0,Q01.sql,,"SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY)>",4,234,0.000,144.394,0.016,0.016,144.410,390,1448003,7,1,0,Q01.sql,,"SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY)>",4,234,0.000,69.373,0.000,0.000,69.373,144800,214173odb statistics: Init timestamp: 2011-12-12 08:08:42 Start timestamp: 2011-12-12 08:08:43 End timestamp: 2011-12-12 08:12:17 Elapsed [Start->End] (s): 214.173CSV output columns have the following meaning:ColumnMeaningThread IDThread ID. As we limited the number of threads to 4, thread id values are 0, 1, 2, 3Proc IDExecution number. As we have 11 executions in the 0-10 rangeThread Exec#This is the progressive number (starting from 0) of execution for a specific thread Script Cmd#In case your script contains multiple SQL statement they will be numbered starting from zeroFileThis is the script file name or “(null)” for –x commandsLabelThis is the label assigned though “set qlabel” in the scriptsCommandFirst 30 characters of the SQL command. It will end with “>” if command text was truncatedRowsThe number of returned rows (not printed if you used –q)RsdsThis is the Record Set Display Size. It gives you an idea of “how big” the result set isPrepare(s)Prepare (compile) time in secondsExec(s)Execution time in seconds1st Fetch(s)Time needed to fetch the first row in secondsFetch(s)Total Fetch time in secondsTotal(s)Total query elapsed time from prepare to fetch in secondsStimelineQueries start timeline in milliseconds EtimelineQueries end timeline in milliseconds Assigning a Label to a QuerySometimes it’s not easy to recognize a query by reading the first 30 characters. odb lets you to assign a label to a generic query using set qlabel <label>. Have a look to the following script:felici ~/Devel/odb $ cat script.sql-- Trafodion TPC-H Query 1SET QLABEL Q01SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY) AS SUM_QTY,...-- TPC-H/TPC-R Minimum Cost Supplier Query (Q2)SET QLABEL Q02SELECT S_ACCTBAL, S_NAME,...When you run this script you will have the Query Label in the CSV output:felici ~/Devel/odb $ ./odb64luo -u mauro -p xxx -d pglocal -f script.sql -q -codb [2011-12-12 09:06:28]: starting (1) threads...Thread id,Proc id,Thread Exec#,Script Cmd#,File,Label,Command,Rows,Rsds,Prepare(s),Exec(s),1st Fetch(s),Fetch(s),Total(s),STimeline,ETimeline0,0,0,0,script.sql,Q01,"SELECT L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY)>",4,234,0.000,43.102,0.000,0.000,43.102,0,431020,0,0,1,script.sql,Q02,"SELECT S_ACCTBAL, S_NAME, N_NAME, P_PARTKEY, P_MFG>",0,274,0.000,0.016,0.000,0.000,0.016,43102,43118odb statistics: Init timestamp: 2011-12-12 09:06:28 Start timestamp: 2011-12-12 09:06:28 End timestamp: 2011-12-12 09:07:11 Elapsed [Start->End] (s): 43.118Running All Scripts With a Given PathUsing -S <path> or -P <path> options you can run all scripts with a given path (for example all files in a directory) either serially (-S) or in parallel (-P). Both options let you to use multiplying factors to run all scripts multiple times. This multiplying factors are defined with a <number>: preceding the script path.Examples:odb Command Line Actionodb64luo -S ./test/queries/*.sql -c -qExecutes serially all scripts with extension .sql under ./test/queries/ providing CSV type output (-c) and omitting query output (-q)odb64luo -P test/queries/* -T 50 -c -qRuns in parallel all files under test/queries/ using 50 threads (ODBC connections) (-T 50), with CSV output (-c) and omitting query output (-q)odb64luo -P 3: test/queries/* -T 3 -c -qRuns in parallel three times (3:) all files under test/queries/ using 3 threads (ODBC connections) (-T 3), with CSV output (-c) and omitting query output (-q). Scripts will be assigned to threads using standard assignmentodb64luo -P -3: test/queries/* -T 3 -c -qRuns in parallel three times (-3:) all files under test/queries/ using 3 threads (ODBC connections) (-T 3), with CSV type output (-c) and omitting query output (-q). Scripts will be assigned to threads using round-robin assignmentTo understand the difference between “standard” and “round-robin” assignments imagine you have four scripts in the target path. This is how the executions will be assigned to threads: Standard Assignment (es. -P 3:)Round-Robin Assignment (es. -P -3:)Thread 1Thread 2Thread 3Thread1Thread 2Thread3nth execution……......4th executionScript4.sqlScript4.sql…Script2.sqlScript3.sql...3rd executionScript3.sqlScript3.sqlScript3.sqlScript3.sqlScript4.sqlScript1.sql2nd executionScript2.sqlScript2.sqlScript2.sqlScript4.sqlScript1.sqlScript2.sql1st executionScript1.sqlScript1.sqlScript1.sqlScript1.sqlScript2.sqlScript3.sqlRandomizing Execution OrderYou can use the -Z option to shuffle the odb internal execution table. This way the execution order is not predictable. Examples:odb Command Line Actionodb64luo... -S 3: test/queries/* -Z -c –qwill execute three times (3:) all files in the test/queries directory serially (-S) and in random order (-Z) odb64luo... -P 3: test/queries/* -Z –T 5 -c -qwill execute three times (3:) all files in the test/queries directory in parallel (-P), using five threads (-T 5) and in random order (-Z) Defining a TimeoutYou can stop odb after a given timeout (assuming the execution is not already completed) using -maxtime <seconds> option.Example:felici ~/Devel/odb $ ./odb64luo -S /home/mauro/scripts/*.sql –maxtime 7200 It will execute, serially, all scripts with extension .sql under /home/mauro/scripts/; if the execution - after two hours (7200 seconds) is not completed, odb will stop.Simulating User Thinking TimeYou can simulate user thinking time using the -ttime <delay> option. This will introduce a <delay> millisecond pause between two consecutive executions in the same thread. Example:mauro@newton ~/src/C/odb $ ./odb64luo -f 5:script1.sql -c -q -ttime 75 -T 2It will run five times script1.sql using 2 threads. Each thread will wait 75 milliseconds before starting the next execution within a thread. You can also use a random thinking time in a given min:max range. For example the following command will start commands within a thread with a random delay between 50 and 500 milliseconds:mauro@newton ~/src/C/odb $ ./odb64luo -f 5:script1.sql -c -q -ttime 50:500 -T 2Starting Threads GracefullyYou might want to wait a little before starting the next thread. This can be obtained using the -delay option. Example:mauro@newton ~/src/C/odb $ ./odb64luo -f 5:script1.sql -c -q -delay 200 -T 2It will run five times script1.sql using 2 threads. Each thread will be started 200 milliseconds after the other.Note: -delay introduces a delay during threads start-up while –ttime introduce a delay between one command and another within the same thread.Re-looping a Given WorkloadUsing -L option you can re-loop the workload defined through -x, -f, -P, -S commands a given number of times. Each thread will re-loop the same number of times. Example:mauro@newton ~/src/C/odb $ ./odb64luo -f 5:script1.sql -c -q -M 75 -T 2 -L 3will re-loop three times (-L 3) the same five executions, using two threads (-T 2) with a 75 millisecond pause (-M 75) between two consecutive executions in the same thread.Using odb as a SQL Interpreter (Technology Preview)To start the odb SQL Interpreter you have to use -I (uppercase i) switch with an optional argument. Example:$ odb64luo –u user –p xx –d dsn –I MFTESTThe optional -I argument (MFTEST here above) is used to specify the .odbrc section containing commands to be automatically executed when odb starts (see REF _Ref316800680 \h Running Commands When the Interpreter Starts).Main odb SQL Interpreter features:It uses mreadline library to manage command line editing and history. History will keep track of the whole command, not just… lines. So if you enter a SQL command in more than one line:S01_Maurizio@TRAFODION64[MFTEST]SQL> selectS01_Maurizio@TRAFODION64[MFTEST]...> count(*)S01_Maurizio@TRAFODION64[MFTEST]...> fromS01_Maurizio@TRAFODION64[MFTEST]...> t1;When you press the up arrow key the whole command (up to semi-colon) will be ready for editing and/or re-run. mreadline provides several useful extra features:CTRL-V to edit the current command using your preferred editor ($EDITOR is used). When the editing session is closed the current command is automatically updatedCTRL-U/CTRL-L to change the command caseCTRL-X to kill the current commandSee the online help for the other mreadline commandsHistory is saved when you exit the SQL Interpreter in a file identified by the ODB_HIST environment variable. You can change the number of commands saved in the history file (default 100):S01_Maurizio@TRAFODION64[MFTEST]SQL> set hist 200Customizable prompt. You can personalize your prompt through the set prompt command. Under Unix/Linux/Cygwin you can use the standard ANSI codes to create color prompts (see REF _Ref316800847 \h Customizing the Interpreter Prompt). It’s possible to run multi-threaded odb instances from within the single-threaded Interpreter with the odb keyword. This will run another odb instance using the same credentials, data source, and connection attributes used to start the interpreter:S01_Maurizio@TRAFODION64[MFTEST]SQL> odb -l src=myfile:tgt=mytable:parallel=8:... S01_Maurizio@TRAFODION64[MFTEST]SQL> odb -e src=mytable:tgt=myfile:parallel=8:...It’s possible to define aliases. Example:root@MFDB[MFDB]SQL> set alias count "select row count from &1;"then, when you call the alias "count" the first argument will be substituted to &1. You can use up to nine positional parameters (&1 to &9)You can run operating system commands with !commandYou can run scripts with @scriptYou can spool to file with set spool <myfile> and stop spooling with set spool offYou can switch to a special "prepare only" mode with set prepare on .This way commands you type will be just prepared (not executed)Different databases use different commands to set default schema(s):- Trafodion: set schema <name>;- MySQL: use <name>;- Postgresql/Vertica: set search_path to <name1,name2,…>;- Teradata: set database <name>; set chsch <command> is used to define database specific commands to change your schema. When odb recognize the change schema command it will update accordingly internal catalog (if any) and schema names;To list database objects, you can use ls command. Few examples:S01_Maurizio@MFTEST[MFTEST]SQL>ls . << list all objects in the current schema TABLE : CITIES TABLE : CUSTOMER TABLE : LINEITEM TABLE : NATION TABLE : ORDERS TABLE : PART TABLE : PARTSUPP TABLE : REGION TABLE : SUPPLIER TABLE : T1 VIEW : V_CITIESS01_Maurizio@MFTEST[MFTEST]SQL>ls -t %S << list tables (-t) ending with SCITIES ORDERSS01_Maurizio@MFTEST[MFTEST]SQL>ls -v << list views (-v)V_CITIES S01_Maurizio@MFTEST[MFTEST]SQL>ls -s << list schemas (-s)... and so on ...To get tables DDL, you can use either ls –T <table> or ls -D <table>. Examples:mauro pglocal[PUBLIC] (09:12:56) SQL> ls -T tpch.ordersDescribing: postgres.TPCH.orders+---------------+-------------+----+-------+---------------+|COLUMN |TYPE |NULL|DEFAULT|INDEX |+---------------+-------------+----+-------+---------------+|o_orderkey |int8 |NO | |orders_pkey 1 U||o_custkey |int8 |NO | | ||o_orderstatus |bpchar(1) |NO | | ||o_totalprice |numeric(15,2)|NO | | ||o_orderdate |date |NO | | ||o_orderpriority|bpchar(15) |NO | | ||o_clerk |bpchar(15) |NO | | ||o_shippriority |int4 |NO | | ||o_comment |varchar(80) |NO | | |+---------------+-------------+----+-------+---------------+mauro pglocal[PUBLIC] (09:13:20) SQL> ls -D tpch.ordersCREATE TABLE postgres.TPCH.orders ( o_orderkey int8 ,o_custkey int8 ,o_orderstatus bpchar(1) ,o_totalprice numeric(15,2) ,o_orderdate date ,o_orderpriority bpchar(15) ,o_clerk bpchar(15) ,o_shippriority int4 ,o_comment varchar(80) ,primary key (o_orderkey));You can define your own variables or use odb internal variables or environment variables directly from the InterpreterYou can “set pad fit” to automatically shrink CHAR/VARCHAR fields in order to fit one record in one line. Line length is defined through “set scols #”. Each record will be printed in one line truncating the length of CHAR/VARCHAR fields proportionally to their original display size length. In case of field truncation a ‘>’ character will be printed at the end of the truncated string.Example:MFELICI [MAURIZIO] (03:30:32) SQL> select [first 5] * from part; P_PARTKEY|P_NAME |P_MFGR |P_BRAND|P_TYPE | P_SIZE|P_CONTAINER| P_RETAILPRICE|P_COMMENT ---------+-------------------------------+---------------+-------+---------------+----------+-------+----------------+-------------- 33|maroon beige mint cyan peru |Manufacturer#2>|Brand#>|ECONOMY PLATED>| 16|LG PKG>| 933.03|ly eve 39|rose dodger lace peru floral |Manufacturer#5>|Brand#>|SMALL POLISHED>| 43|JUMBO >| 939.03|se slowly abo> 60|sky burnished salmon navajo hot|Manufacturer#1>|Brand#>|LARGE POLISHED>| 27|JUMBO >| 960.06| integ 81|misty salmon cornflower dark f>|Manufacturer#5>|Brand#>|ECONOMY BRUSHE>| 21|MED BA>| 981.08|ove the furious 136|cornsilk blush powder tan rose |Manufacturer#2>|Brand#>|SMALL PLATED S>| 2|WRAP B>| 1036.13|kages print c>You can “set plm” to print one field per row. This is useful when you have to carefully analyze few records. Example:MFELICI [MAURIZIO] (03:38:12) SQL> set plm onMFELICI [MAURIZIO] (03:38:12) SQL> select * from part where p_partkey = 136;P_PARTKEY :136P_NAME :cornsilk blush powder tan roseP_MFGR :Manufacturer#2P_BRAND :Brand#22P_TYPE :SMALL PLATED STEELP_SIZE :2P_CONTAINER :WRAP BAGP_RETAILPRICE:1036.13P_COMMENT :kages print carefullyCheck the rest on your own.This is the current odb SQL Interpreter help on line:mauro pglocal[PUBLIC] (06:51:20) SQL> helpmauro pglocal[public] (16:57:52) SQL> helpAll the following are case insensitive: h | help : print this help i | info : print database info q | quit : exit SQL Interpreter c | connect { no | [user[/pswd][;opts;...] (re/dis)connect using previous or new user odb odb_command : will run an odb instance using the same DSN/credentials ls -[type] [pattern] : list objects. Type=(t)ables, (v)iews, s(y)nonyns, (s)chemas : (c)atalogs, syst(e)m tables, (l)ocal temp, (g)lobal temp : (m)at views, (M)mat view groups, (a)lias, (A)ll object types : (D)table DDL, (T)table desc print <string> : print <string> !cmd : execute the operating system cmd @file [&0]... [&9] : execute the sql script in file set : show all settings set alias [name] [cmd|-]: show/set/change/delete aliases set chsch [cmd] : show/set change schema command set cols [#cols] : show/set ls number of columns set cwd [<directory>] : show/set current working directory set drs [on|off] : show/enable/disable describe result set mode set fs [<char>] : show/set file field separator set hist [#lines] : show/set lines saved in the history file set maxfetch [#rows] : show/set max lines to be fetched (-1 = unlimited) set nocatalog [on|off] : show/enable/disable "no catalog" database mode) set nocatnull [on|off] : show/enable/disable "no catalog as null" database mode) set noschema [on|off] : show/enable/disable "no schema" database mode) set nullstr [<string>] : show/set string used to display NULLs (* to make it Null) set pad [fit|full|off] : show/set column padding set param name [value|-]: show/set/change/delete a parameter set pcn [on|off] : show/enable/disable printing column names set plm [on|off] : show/enable/disable print list mode (one col/row) set prepare [on|off] : show/enable/disable 'prepare only' mode set prompt [string] : show/set prompt string set query_timeout [s] : show/set query timeout in seconds (def = 0 no timeout) set quiet [cmd|res|all|off] : show/enable/disable quiet mode set rowset [#] : show/set rowset used to fetch rows set soe [on|off] : show/enable/disable Stop On Error mode set spool [<file>|off] : show/enable/disable spooling output on <file> <SQL statement>; : everything ending with ';' is sent to the databasemreadline keys: Control-A : move to beginning of line Control-P : history Previous Control-E : move to end of line Up Arrow : history Previous Control-B : move cursor Back Control-N : history Next Left Arrow : move cursor Back Down Arrow : history Next Control-F : move cursor Forward Control-W ; history List Right Arrow: move cursor Forward Control-R : Redraw Control-D : input end (exit) - DEL right Control-V : Edit current line Control-L : Lowercase Line Control-X : Kill line Control-U : Uppercase Line #Control-G : load history entry #Running Commands When the Interpreter StartsWhen the odb SQL Interpreter starts it looks for the Initialization File. This Initialization File is made of Sections containing the commands to be executed. To find the Initialization File, odb will:first check the ODB_INI environment variableIf this variable is not set, odb willlook for a file named “.odbrc” (*nix) or “_odbrc” (Windows) under your HOME directoryThe Initialization File contains Sections identified by names between square brackets. For example, the following section is named MFTEST:[MFTEST]set pcn onset pad fitset fs |set cols 3 30set editor "vim -n --noplugin"set efile /home/felici/.odbedit.sqlset prompt "%U %D [%S] (%T) %M> "set alias count "select row count from &1;"set alias size "select sum(current_eof) from table (disk label statistics (&1) );set alias ll "select left(object_name, 40) as object_name, sum(row_count) as nrows, count(partition_num) as Nparts, sum(current_eof) as eof from table(disk label statistics( using (select * from (get tables in schema &catalog.&schema, no header, return full names) s(b) ))) group by object_name order by object_name;"set schema TRAFODION.MAURIZIO;odb SQL Interpreter will automatically run all commands in the section identified by the -I argument (for example -I MFTEST). A section named DEFAULT will be executed when -I has no arguments.Customizing the Interpreter PromptYou can define your prompt through the set prompt command when running the SQL Interpreter. set prompt can be executed interactively or included in your ($ODB_INI) "init file". set prompt recognize and expand the following variables:%U User name%D Data Source name%S Schema name%T Current Time %M odb mode:SQL when running sql commandsPRE if you're in "prepare only" modeSPO if you are spooling output somewhere NDC (No Database Connection)So, for example:set prompt "Prompt for %U connected via %D to %S in %M mode > "will generate a prompt like this:Prompt for S01_Maurizio connected via CIV to CIV03 in SQL mode >Under Cygwin, Unix and Linux (and probably under Windows too using ANSI.SYS driver - not tested) you can use standard ANSI escape color codes. For example:set prompt “^A^[[01;32m^A%U@%D^A^[[01;34m^A[%S]^A^[[00m^A (%T) %M> “Where:^A is a "real" Control-A (ASCII 001 and 002) before and after each color code sequence;^[ is a "real" Escape Character and the meaning of the ANSI color codes are:^[[01;32m means green^[[01;34m means blue^[[00m means reset.This is how my standard prompt looks:Warnings, Limits, and Troubleshootingodb uses Condition Variables to synchronize threads during copy and parallel load operations.Most of the memory allocation operations are dynamic. So, for example, you can execute an SQL command as long as you want (in previous version of odb maximum SQL command length was 8192 bytes). However, I found it convenient to "hard code" the following limits :#define MAX_VNLEN 32 /* Max variable name length */#define MAXCOL_LEN 128 /* Max column name length */#define MAXOBJ_LEN 128 /* Max catalog/schema/table name length */#define MAX_CLV 64 /* Max command line variables (-var) */Some Linux/UNIX systems (notably the Linux Loader) have huge default stack size. Due to this insane value, you can have errors like this when starting tens/hundreds of threads:Error starting cmd thread #: cannot allocate memoryIf you get this error check your default stack size:s19user@traf64-dl2a ~/mf $ ulimit -acore file size (blocks, -c) 0data seg size (kbytes, -d) unlimitedmax nice (-e) 0file size (blocks, -f) unlimitedpending signals (-i) 137215max locked memory (kbytes, -l) 32max memory size (kbytes, -m) unlimitedopen files (-n) 65536pipe size (512 bytes, -p) 8POSIX message queues (bytes, -q) 819200max rt priority (-r) 0stack size (kbytes, -s) 204800 << this is the problemcpu time (seconds, -t) unlimitedmax user processes (-u) 2047virtual memory (kbytes, -v) unlimitedfile locks (-x) unlimitedand reset it to sane values (value will be reset to the initial value when you start a new session). For example:s19user@traf64-dl2a ~/mf $ ulimit -s 4096If you get errors like this:C:\Users\felici>odb64luo -u xx -p yy -d oraxe -l src=region.tbl.gz:tgt=region:fs=^|:truncateodb [2012-10-11 13:27:22]: starting ODBC connection(s)... 0[0] odb(5020) - [Oracle][ODBC]Optional feature not implemented. (State: HYC00 Native Err: 0)try adding -nocatnull to your command line. When the backend database doesn’t use catalogs you should use an empty string as catalog name. Some buggy ODBC Drivers unfortunately want NULL here – instead of empty strings as it should be.You can have errors loading TIME(N) fields with N>0 under Trafodion because the ODBC Driver does not manage the field display size when N>0.If you have problems starting odb on Unix/Linux check:the shared library dependencies with ldd <odb_executable_name>;and the shared lib path defined in the following environment variables used by the shared library loader:Linux: LD_LIBRARY_PATHIBM IAX: LIBPATH (not currently supported)HP/UX: SHLIB_PATH (not currently supported)How odb Is Coded and TestedHow odb Is CodedFor Trafodion Release 1.1, odb is coded in "ANSI C" (K&R programming style) and is compiled in a 64-bit version on the Linux platform, linked to the unixODBC driver manager. Other platforms, compilers, and ODBC libraries have not yet been tested. PlatformCompilerODBC Libraries NoteLinuxgccunixODBC (supported), iODBC (not currently supported), Data Direct (not currently supported), Teradata (not currently supported)64 bit (32 bit is not currently supported)MS-Windows (not tested)Visual Studio (not tested)MS-Windows (not tested)64 bitIBM AIX (not currently supported)gccunixODBC, iODBC, Data Direct32 and 64 bitIBM AIX (not currently supported)xlcTo be determined32 and 64 bitCygwin (not currently supported)gccMS-Windows32 bitHP/UX (not currently supported)ANSI CunixODBC, iODBC64 bitMac (not currently supported)gcciODBC64 bitC compilers are set with “all warnings” enabled and odb has to compile, on each platform, with no errors (of course) AND no warnings. Tools used to code odb:vim (http://www.vim.org) as editor (or Visual Studio embedded editor)splint (http://www.splint.org) to statically check the source codeHow odb Is TestedFor Trafodion Release 1.1, the info, load, extract, and copy operations of odb have been fully tested.Previously, odb had been tested using a set of 137 standard tests to check functionalities and identify memory/thread issues. Nevertheless, there could be bugs somewhere.Useful odb CommandsUpdating Missing StatisticsYou can update missing statistics with a single line command like this: $ ./odb64luo -u xyx -p yyy -d zzz -N -S test/queries/Q*.sql 2>&1 | sed -n 's/^.*WARNING\[600[7-8]\].*column\? \(.*\) from table \(.*\) were.*$/UPDATE STATISTICS FOR TABLE \2 ON \1;/p' | ./odb64luo -u xyz -p xyz -d mft -f –Where:./odb64luo -u xx -p xyz -d xx -N -S test/queries/Q*.sql 2>&1 will just prepare queries (the -N option will prevent their real execution) and write warnings/errors on the standard output.sed -n 's/^.*WARNING\[600[7-8]\].*column\? \(.*\) from table \(.*\) were.*$/UPDATE STATISTICS FOR TABLE \2 ON \1;/p' will transform the warnings into real UPDATE STATISTICS statements../odb64luo -u xyz -p xyz -d mft -f – will run the UPDATE STATISTICS statements built with the previous sed (-f – means that commands are provided via standard input).Getting All DDL Statements in a Given Schema$ odb64luo -u xx -p xx -d xx -i t:TRAFODION.HP_MIGR.% -x "showddl &1" -T 1 > ddl.sqlAliases for the SQL InterpreterYou can copy and paste these aliases in your $ODB_INI file:set alias count "select count(*) from &1;"Simple ODBC Port Scanner#!/usr/bin/bash# Simple ODBC Port Scanner# Config Parameters:FOUND="invalid username" # Message to grepPSTART=18600 # Initial Port to checkPEND=18700 # Final Port to checkDRV="{Trafodion}" # ODBC DriverHOST="abc123.ca.cname.com" # Fully Qualified Host Name or IPDSN="DSN=MFELICI" # You can use SERVERDSN hereMISC="CERTIFICATEDIR=C:\Users\felici" # Other params# Body:for port in `seq $PSTART $PEND` ; do echo -ne "\rNow Testing port $port" ./odb64luo -u x -p x -ca "DRIVER=$DRV;SERVER=TCP:$HOST/$port;$DSN;$MISC" -i \ 2>&1 | grep -q "$FOUND" test $? == 0 && { echo "... found port $port on $HOST"; }doneecho -e "\rScan completed for $HOST on range $PSTART:$PEND" Be aware: this is case sensitive. The following characters are considered “spaces”: blank, tab, new line, carriage return, form feed, vertical tab Note: When the source table column is defined as NOT NULL and the specific field contains only blanks, odb will leave in the output file one single blank. This will help to distinguish between NULL fields (<field_sep><field_sep>) and NOT NULL fields containing all blanks (<field_sep><blank><field_sep>)odb User GuidePage PAGE 1/ NUMPAGES 1iVBORw0KGgoAAAANSUhEUgAAAeAAAAA1CAIAAAAWMq/qAAAAAXNSR0IArs4c6QAADwJJREFUeF7t
XbGOGzkSpe4TFrjQmWTgfM6cHKQfsGQcMJEjAw4W0GQrJc4cOHDmpOcyKRvgIkcGDqPZH5CwyWZe
G7Am22wX8C/oiqxuNskmm2STrR1pilCgGRWLxdfs6urqaj7G7mcbM3YQn84tXcOmsgEtKTqb4uw4
37DDof7sexgiv9H5Nc7Zu4P2eQ4H74za870+QVhYyW2+OUDbzJMVnZWC80Plb2d1fDJOBk6iqa7u
@@ -178,4 +178,4 @@ QoAQyI4AMap4ID0nRpX/A5otny9j37kTAAAAAElFTkSuQmCC
-DocumentLibraryFormDocumentLibraryFormDocumentLibraryFormMaurizio FeliciVarnau, Steve (Seaquest R&D)22009-12-04T11:37:00Z2015-06-04T21:15:00Z2015-06-04T21:15:00Z
\ No newline at end of file
+DocumentLibraryFormDocumentLibraryFormDocumentLibraryFormMaurizio FeliciVarnau, Steve (Seaquest R&D)22009-12-04T11:37:00Z2015-06-04T21:15:00Z2015-06-04T21:15:00Z
diff --git a/docs/sql_reference/source/Trafodion_SQL_Reference_Manual.xml b/docs/sql_reference/source/Trafodion_SQL_Reference_Manual.xml
index f29b34e75c..351df80dd7 100755
--- a/docs/sql_reference/source/Trafodion_SQL_Reference_Manual.xml
+++ b/docs/sql_reference/source/Trafodion_SQL_Reference_Manual.xml
@@ -23,7 +23,7 @@
-->
-
Trafodion SQL Reference Manual
@@ -34,8 +34,8 @@
Last updated: 4/24/2015
-2015Hewlett-Packard Development Company,
-L.P.
+2015Apache Software Foundation (ASF)
+
Legal Notice
@@ -9197,8 +9197,8 @@ and external objects.
This GET HBASE OBJECTS statement displays the Trafodion
user objects in HBase:
-Trafodion Conversational Interface 1.1.0
-(c) Copyright 2014 Hewlett-Packard Development Company, LP.
+Apache Trafodion Conversational Interface 1.3.0
+(c) Copyright (c) 2015 Apache Software Foundation
>>get user hbase objects;
diff --git a/docs/sql_reference/source/xml-profile.xml b/docs/sql_reference/source/xml-profile.xml
index c4c5340068..c421b534b7 100755
--- a/docs/sql_reference/source/xml-profile.xml
+++ b/docs/sql_reference/source/xml-profile.xml
@@ -23,7 +23,7 @@
-->
-
diff --git a/install/Makefile b/install/Makefile
index fa895d1568..3ddc671ebf 100644
--- a/install/Makefile
+++ b/install/Makefile
@@ -1,8 +1,6 @@
# @@@ START COPYRIGHT @@@
#
-# (C) Copyright 2007-2015 Hewlett-Packard Development Company, L.P.
-#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
diff --git a/install/README.rst b/install/README.rst
deleted file mode 100644
index d1df88df7b..0000000000
--- a/install/README.rst
+++ /dev/null
@@ -1,32 +0,0 @@
-.. # @@@ START COPYRIGHT @@@
- #
- # (C) Copyright 2014 Hewlett-Packard Development Company, L.P.
- #
- # Licensed under the Apache License, Version 2.0 (the "License");
- # you may not use this file except in compliance with the License.
- # You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #
- # @@@ END COPYRIGHT @@@
-
-=============================
-Trafodion Core README
-=============================
-
-Trafodion provides a transactional SQL-on-HBase engine. "Trafodion" is the Welsh word for transactions. Trafodion builds on the scalability, elasticity, and flexibility of Hadoop. It extends it to handle enterprise-class operational workloads such as online transaction processing.
-
-Trafodion is distributed under the terms of the Apache License, Version 2.0.
-
-To learn about Trafodion visit:
- http://trafodion.org
-
-For more information about how to contribute to Trafodion visit the wiki:
- https://wiki.trafodion.org
-
diff --git a/wms/LICENSE.txt b/wms/LICENSE.txt
deleted file mode 100644
index d645695673..0000000000
--- a/wms/LICENSE.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/wms/NOTICE.txt b/wms/NOTICE.txt
deleted file mode 100644
index ecd51a7ada..0000000000
--- a/wms/NOTICE.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).
-
-In addition, this product includes software developed by:
-
-Jamon (http://www.jamon.org/) is a text template engine for Java used by our
-UI. It uses the Mozilla Public License (http://www.mozilla.org/MPL/)
-See the tail of http://www.jamon.org/About.html
-
-JUnit (http://www.junit.org/) included under the Common Public License v1.0. See
-the full text here: http://junit.sourceforge.net/cpl-v10.html
diff --git a/wms/build.xml b/wms/build.xml
index f1f6177a5c..c416292d21 100644
--- a/wms/build.xml
+++ b/wms/build.xml
@@ -286,7 +286,7 @@
use="true"
windowtitle="${Name} ${version} API"
doctitle="${Name} ${version} API"
- bottom="Copyright © ${year} The Hewlett Packard Company"
+ bottom="Copyright © ${year} Apache Software Foundation"
>
From 26031c8a51514ccee3fae58cb644420434f1f416 Mon Sep 17 00:00:00 2001
From: Roberta Marton
Date: Tue, 24 Nov 2015 01:43:47 +0000
Subject: [PATCH 2/3] updated .rat-excludes and RAT_README.txt to include
copyright changes
---
.rat-excludes | 3 +++
RAT_README.txt | 7 +++++++
2 files changed, 10 insertions(+)
diff --git a/.rat-excludes b/.rat-excludes
index be06b1c389..9733293621 100644
--- a/.rat-excludes
+++ b/.rat-excludes
@@ -98,6 +98,9 @@ TCPIPV4.*
TCPIPV6.*
TCPIPV4_os.*
TCPIPV6_os.*
+swscanf.cpp
+swsprintf.cpp
+ulexer.h
diff --git a/RAT_README.txt b/RAT_README.txt
index 6a41a77cfe..158971bbc1 100644
--- a/RAT_README.txt
+++ b/RAT_README.txt
@@ -123,6 +123,13 @@ copyrights which are covered by MIT
$MY_SQROOT/../sql/qmscommon/expat/*
+The following files are open source from The Regents of the University of
+California which are covered by BDS license
+
+core/sql/common/swscanf.cpp
+core/sql/common/swsprintf.cpp
+core/sql/parser/ulexer.h
+
-------------------------------------------------------------------
The following files in dcs directory have their own BSD license
From f66141834f102e9af39cf9baf3bc3f9c27118133 Mon Sep 17 00:00:00 2001
From: Roberta Marton
Date: Tue, 1 Dec 2015 17:55:29 +0000
Subject: [PATCH 3/3] Copied pull request 187 to release1.3 branch
---
install/installer/dcs_installer | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/install/installer/dcs_installer b/install/installer/dcs_installer
index 298003b2e1..902b314f91 100755
--- a/install/installer/dcs_installer
+++ b/install/installer/dcs_installer
@@ -136,7 +136,8 @@ echo "***INFO: untarring build file $DCS_BUILD_FILE"
tar -xzf $DCS_BUILD_FILE
# get the dcs install directory name which is imbedded in the tar file
-DCS_DIR=$(tar -tf $DCS_BUILD_FILE | grep LICENSE | sed -e "s@\(^.*\)/LICENSE.txt@\1@")
+DCS_DIR=$(tar -tf $DCS_BUILD_FILE | grep --max-count=1 bin | sed -e "s@\(^.*\)/bin/@\1@")
+
DCS_DIR=$DCS_INSTALL_PATH/$DCS_DIR
# set env var in sqenvcom.sh for DCS install directory