Skip to content
Browse files

Merge remote-tracking branch 'origin/candidate-3.10.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>
  • Loading branch information...
2 parents 96a147b + 33cdd43 commit 93cb20ad07726a060c3e706f4f974c4c2430f442 @richardkchapman richardkchapman committed Nov 30, 2012
Showing with 1,025 additions and 402 deletions.
  1. +2 −2 docs/BuildTools/fo.xsl
  2. +2 −2 docs/BuildTools/fo.xsl.in
  3. +11 −10 docs/ECLPlayground/ECLPlayground-includer.xml
  4. +7 −5 docs/ECLPluginForEclipse/ECL_Plugin_for_Eclipse.xml
  5. +50 −30 docs/ECLProgrammersGuide/PrGd-Includer.xml
  6. +63 −35 docs/ECLStandardLibraryReference/SLR-includer.xml
  7. +10 −8 docs/HDFSConnector/HDFS_to_HPCC_ConnectorIncluder.xml
  8. +14 −9 docs/HPCCClientTools/ClientTools.xml
  9. +8 −6 docs/HPCCClientTools/TheECLIDEandHPCCClientTools.xml
  10. +8 −6 docs/HPCCDataHandling/DataHandling.xml
  11. +8 −6 docs/HPCCDataTutorial/DataTutorial.xml
  12. +45 −14 docs/IMDB/IMDB.xml
  13. +399 −0 docs/Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml
  14. +11 −5 docs/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml
  15. +51 −60 docs/One-Click_HPCCinAWS/One-Click_RuningHPCCinAmazonWebServicesEC2.xml
  16. +8 −6 docs/RDDERef/RDDERef.xml
  17. +8 −6 docs/RunningHPCCinaVirtualMachine/RunningHPCCinaVirtualMachine.xml
  18. +10 −8 docs/UsingConfigManager/UsingConfigManager.xml
  19. +7 −5 docs/Visualizing/Visualizing.xml
  20. BIN docs/images/AWS_InstantCloud_COVER_registeredTm.jpg
  21. BIN docs/images/AWS_InstantCloud_Login.jpg
  22. BIN docs/images/AWS_One-click_COVER_LN_Rightjustified.jpg
  23. BIN docs/images/AWS_OneClick_003a.jpg
  24. BIN docs/images/AWS_OneClick_005.jpg
  25. BIN docs/images/AWS_OneClick_006.jpg
  26. BIN docs/images/AWS_OneClick_LaunchLog.jpg
  27. BIN docs/images/AWS_OneClick_LaunchNew.jpg
  28. BIN docs/images/AWS_OneClick_Login.jpg
  29. BIN docs/images/AWS_OneClick_Terminated.jpg
  30. BIN docs/images/AWS_OneClick_ViewClusters.jpg
  31. BIN docs/images/AWS_OneClick_ViewClustersTerminate.jpg
  32. BIN docs/images/redswooshWithLogo3.jpg
  33. BIN docs/images/ssl-01.jpg
  34. BIN docs/images/ssl-02.jpg
  35. BIN docs/images/ssl-03.jpg
  36. BIN docs/images/ssl-04.jpg
  37. +10 −7 docs/wip/certify.xml
  38. +181 −106 ecl/hthor/hthor.cpp
  39. +56 −22 ecl/hthor/hthor.ipp
  40. +5 −1 esp/bindings/SOAP/Platform/soapparam.cpp
  41. +24 −20 esp/files/ecl/DENORMALIZE_Example.ecl
  42. +21 −21 esp/files/ecl/MACRO_Example.ecl
  43. +1 −0 roxie/roxiemem/roxierowbuff.hpp
  44. +1 −1 thorlcr/activities/hashdistrib/thhashdistribslave.cpp
  45. +4 −1 thorlcr/activities/lookupjoin/thlookupjoinslave.cpp
View
4 docs/BuildTools/fo.xsl
@@ -332,13 +332,13 @@
<!--NEW-inline_monospace-8-12-->
<xsl:template match="emphasis[@role='code']">
- <fo:inline font-family="courier">
+ <fo:inline font-family="monospace">
<xsl:apply-templates/>
</fo:inline>
</xsl:template>
<xsl:template match="emphasis[@role='codebold']">
- <fo:inline font-family="courier" font-weight="bold">
+ <fo:inline font-family="monospace" font-weight="bold">
<xsl:apply-templates/>
</fo:inline>
</xsl:template>
View
4 docs/BuildTools/fo.xsl.in
@@ -332,13 +332,13 @@
<!--NEW-inline_monospace-8-12-->
<xsl:template match="emphasis[@role='code']">
- <fo:inline font-family="courier">
+ <fo:inline font-family="monospace">
<xsl:apply-templates/>
</fo:inline>
</xsl:template>
<xsl:template match="emphasis[@role='codebold']">
- <fo:inline font-family="courier" font-weight="bold">
+ <fo:inline font-family="monospace" font-weight="bold">
<xsl:apply-templates/>
</fo:inline>
</xsl:template>
View
21 docs/ECLPlayground/ECLPlayground-includer.xml
@@ -23,13 +23,15 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
- <para />
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
+
+ <para> </para>
</legalnotice>
<xi:include href="common/Version.xml" xpointer="FooterInfo"
@@ -51,11 +53,10 @@
</bookinfo>
<chapter id="ECLPlaygroundChap">
- <title>
- <emphasis>ECL Playground</emphasis>
- </title>
+ <title><emphasis>ECL Playground</emphasis></title>
- <xi:include href="ECLPlayground/ECLPlay-Mods/ECL_Playground.xml" xpointer="ECL_Playground"
+ <xi:include href="ECLPlayground/ECLPlay-Mods/ECL_Playground.xml"
+ xpointer="ECL_Playground"
xmlns:xi="http://www.w3.org/2001/XInclude" />
</chapter>
</book>
View
12 docs/ECLPluginForEclipse/ECL_Plugin_for_Eclipse.xml
@@ -24,11 +24,13 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products and
- services may be trademarks or registered trademarks of their respective
- companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
+
+ <para>Other products and services may be trademarks or registered
+ trademarks of their respective companies. All names and example data
+ used in this manual are fictitious. Any similarity to actual persons,
+ living or dead, is purely coincidental.</para>
<para><emphasis role="redbold">DRAFT 004</emphasis></para>
</legalnotice>
View
80 docs/ECLProgrammersGuide/PrGd-Includer.xml
@@ -23,18 +23,19 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
- <para></para>
- </legalnotice>
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
- <xi:include href="common/Version.xml" xpointer="FooterInfo"
- xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <para> </para>
+ </legalnotice>
+ <xi:include href="common/Version.xml" xpointer="FooterInfo"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="common/Version.xml" xpointer="DateVer"
xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -44,7 +45,6 @@
<xi:include href="common/Version.xml" xpointer="Copyright"
xmlns:xi="http://www.w3.org/2001/XInclude" />
-
<mediaobject role="logo">
<imageobject>
<imagedata fileref="images/LN_Rightjustified.jpg" />
@@ -55,81 +55,100 @@
<chapter>
<title>ECL Programming Concepts</title>
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Attribute_Creation.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Attribute_Creation.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Creating_example_data.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Creating_example_data.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Crosstab_reports.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Crosstab_reports.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Efficient_Value_TypeUsage.xml"
xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Using_Group_Function.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Using_Group_Function.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Automated_ECL.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Automated_ECL.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Job_Failure.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Job_Failure.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_NonRandom_Random.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_NonRandom_Random.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Workwith_XML_Data.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Workwith_XML_Data.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Workwith_Blobs.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Workwith_Blobs.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Using_ECL_Keys.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Using_ECL_Keys.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
</chapter>
<chapter>
<title>Working With SuperFiles</title>
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Superfile_Overview.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Superfile_Overview.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_CreateMaintain_Superfiles.xml"
xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Indexing_Superfiles.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Indexing_Superfiles.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Using_Superkeys.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Using_Superkeys.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
</chapter>
<chapter>
<title>Working With Roxie</title>
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Roxie_Overview.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Roxie_Overview.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_SOAPenable_Queries.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_SOAPenable_Queries.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Complex_ROXIE_Queries.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Complex_ROXIE_Queries.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_SOAPCALL_fromTHOR_to_ROXIE.xml"
xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_control_ROXIE_queries.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_control_ROXIE_queries.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_query_libraries.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_query_libraries.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Smart_Stepping.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Smart_Stepping.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
</chapter>
@@ -144,7 +163,8 @@
xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Simple_Random_Samples.xml" xpointer="element(/1)"
+ <xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Simple_Random_Samples.xml"
+ xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLProgrammersGuide/PRG_Mods/PrG_Hex_String_to_Decimal_String.xml"
View
98 docs/ECLStandardLibraryReference/SLR-includer.xml
@@ -23,13 +23,15 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
- <para></para>
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
+
+ <para> </para>
</legalnotice>
<xi:include href="common/Version.xml" xpointer="FooterInfo"
@@ -167,7 +169,8 @@
<chapter id="Remote_File_Support">
<title><emphasis>Remote Files</emphasis></title>
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/RfsQuery.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/RfsQuery.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLStandardLibraryReference/SLR-Mods/RfsAction.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -201,9 +204,11 @@
<xi:include href="ECLStandardLibraryReference/SLR-Mods/AbortDfuWorkunit.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/Copy.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/Copy.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/DeSpray.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/DeSpray.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLStandardLibraryReference/SLR-Mods/RemotePull.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -217,7 +222,8 @@
<xi:include href="ECLStandardLibraryReference/SLR-Mods/SprayVariable.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/SprayXML.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/SprayXML.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLStandardLibraryReference/SLR-Mods/WaitDfuWorkunit.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -238,7 +244,8 @@
<xi:include href="ECLStandardLibraryReference/SLR-Mods/CompareIgnoreCase.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/Contains.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/Contains.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLStandardLibraryReference/SLR-Mods/CountWords.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -249,23 +256,32 @@
<xi:include href="ECLStandardLibraryReference/SLR-Mods/EditDistanceWithinRadius.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/EndsWith.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/EndsWith.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLStandardLibraryReference/SLR-Mods/EqualIgnoreCase.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/ExcludeFirstWord.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/ExcludeLastWord.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/ExcludeNthWord.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/ExcludeFirstWord.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/ExcludeLastWord.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/ExcludeNthWord.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/Extract.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/Extract.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/Filter.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/Filter.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLStandardLibraryReference/SLR-Mods/FilterOut.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/Find.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/Find.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLStandardLibraryReference/SLR-Mods/FindCount.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -285,7 +301,8 @@
<xi:include href="ECLStandardLibraryReference/SLR-Mods/RemoveSuffix.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/Reverse.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/Reverse.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLStandardLibraryReference/SLR-Mods/SplitWords.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -329,7 +346,7 @@
<xi:include href="ECLStandardLibraryReference/SLR-Mods/Day.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
-
+
<xi:include href="ECLStandardLibraryReference/SLR-Mods/DateFromParts.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -341,43 +358,50 @@
<xi:include href="ECLStandardLibraryReference/SLR-Mods/ToGregorianYMD.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
-
</chapter>
<chapter id="Cluster_Handling">
<title><emphasis>Cluster Handling</emphasis></title>
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/Node.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/Node.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/Nodes.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/Nodes.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLStandardLibraryReference/SLR-Mods/LogicalToPhysical.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLStandardLibraryReference/SLR-Mods/DaliServers.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/Group.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/Group.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLStandardLibraryReference/SLR-Mods/GetExpandLogicalFileName.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
-
</chapter>
<chapter id="Job_Handling">
<title><emphasis>Job Handling</emphasis></title>
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/WUID.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/WUID.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/Cluster.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/Cluster.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/JobName.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/JobName.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/JobOwner.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/JobOwner.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/OS.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/OS.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/Platform.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/Platform.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLStandardLibraryReference/SLR-Mods/LogString.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -396,7 +420,8 @@
<chapter id="Logging">
<title><emphasis>Logging</emphasis></title>
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/dbglog.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/dbglog.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="ECLStandardLibraryReference/SLR-Mods/addWorkunitInformation.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -411,7 +436,8 @@
<chapter id="Auditing">
<title><emphasis>Auditing</emphasis></title>
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/Audit.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/Audit.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
</chapter>
<chapter id="Utilities">
@@ -439,9 +465,11 @@
<xi:include href="ECLStandardLibraryReference/SLR-Mods/GetXMLParseTree.xml"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/Sleep.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/Sleep.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="ECLStandardLibraryReference/SLR-Mods/msTick.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+ <xi:include href="ECLStandardLibraryReference/SLR-Mods/msTick.xml"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
</chapter>
<chapter id="Email">
View
18 docs/HDFSConnector/HDFS_to_HPCC_ConnectorIncluder.xml
@@ -23,13 +23,15 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc. </para>
- <para></para>
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
+
+ <para> </para>
</legalnotice>
<xi:include href="common/Version.xml" xpointer="FooterInfo"
@@ -56,10 +58,10 @@
<xi:include href="HDFSConnector/HDFS_Mods/HDFS_Intro.xml"
xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
-
+
<xi:include href="HDFSConnector/HDFS_Mods/HDFS_Install.xml"
xpointer="element(/1)"
- xmlns:xi="http://www.w3.org/2001/XInclude" />
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
</chapter>
<chapter>
View
23 docs/HPCCClientTools/ClientTools.xml
@@ -23,13 +23,15 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
- <para></para>
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
+
+ <para> </para>
</legalnotice>
<xi:include href="common/Version.xml" xpointer="FooterInfo"
@@ -50,10 +52,12 @@
</mediaobject>
</bookinfo>
- <xi:include href="HPCCClientTools/CT_Mods/CT_Overview_withoutIDE.xml" xpointer="CT_Overview"
+ <xi:include href="HPCCClientTools/CT_Mods/CT_Overview_withoutIDE.xml"
+ xpointer="CT_Overview"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="HPCCClientTools/CT_Mods/CT_Comm_Line_ECL.xml" xpointer="Command_Line_ECL"
+ <xi:include href="HPCCClientTools/CT_Mods/CT_Comm_Line_ECL.xml"
+ xpointer="Command_Line_ECL"
xmlns:xi="http://www.w3.org/2001/XInclude" />
<xi:include href="HPCCClientTools/CT_Mods/CT_ECL_CLI.xml" xpointer="ECL_CLI"
@@ -62,6 +66,7 @@
<xi:include href="HPCCClientTools/CT_Mods/ECLCC.xml" xpointer="ECL_Compiler"
xmlns:xi="http://www.w3.org/2001/XInclude" />
- <xi:include href="HPCCClientTools/CT_Mods/CT_Comm_Line_DFU.xml" xpointer="Command_Line_DFU"
+ <xi:include href="HPCCClientTools/CT_Mods/CT_Comm_Line_DFU.xml"
+ xpointer="Command_Line_DFU"
xmlns:xi="http://www.w3.org/2001/XInclude" />
</book>
View
14 docs/HPCCClientTools/TheECLIDEandHPCCClientTools.xml
@@ -23,13 +23,15 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
- <para></para>
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
+
+ <para> </para>
</legalnotice>
<xi:include href="common/Version.xml" xpointer="FooterInfo"
View
14 docs/HPCCDataHandling/DataHandling.xml
@@ -23,13 +23,15 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
- <para></para>
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
+
+ <para> </para>
</legalnotice>
<xi:include href="common/Version.xml" xpointer="FooterInfo"
View
14 docs/HPCCDataTutorial/DataTutorial.xml
@@ -25,13 +25,15 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products and
- services may be trademarks or registered trademarks of their respective
- companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
- <para></para>
+ <para>Other products and services may be trademarks or registered
+ trademarks of their respective companies. All names and example data
+ used in this manual are fictitious. Any similarity to actual persons,
+ living or dead, is purely coincidental.</para>
+
+ <para> </para>
</legalnotice>
<xi:include href="common/Version.xml" xpointer="FooterInfo"
View
59 docs/IMDB/IMDB.xml
@@ -25,13 +25,15 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
- <para></para>
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
+
+ <para> </para>
</legalnotice>
<xi:include href="common/Version.xml" xpointer="FooterInfo"
@@ -499,14 +501,43 @@ Blankline</programlisting></para>
</listitem>
<listitem>
- <para>Repeat the spray steps for the actresses file. (Just replace
- <emphasis>actor.list</emphasis> with the <emphasis>actresses.list
- </emphasis> in both the Local Path, Logical File name, and Label
- field.)</para>
-
- <para>Once the <emphasis role="bold">two</emphasis> sprays are
- complete, we can query the logical files on the HPCC to see the
- files we sprayed.</para>
+ <para>Following the same steps you used to spray the actors file,
+ now spray the actresses file.</para>
+
+ <para><itemizedlist>
+ <listitem>
+ <para>Click on the Spray CSV link.</para>
+ </listitem>
+
+ <listitem>
+ <para>Select mydropzone in the Source <emphasis
+ role="bold">Machine/dropzone</emphasis> drop-list.</para>
+ </listitem>
+
+ <listitem>
+ <para>Enter the file name for the
+ <emphasis>actresses.list</emphasis> file, or use the
+ <emphasis role="bold">Choose File</emphasis> button to
+ select the the <emphasis>actresses.list </emphasis>file from
+ a list of files in the folder.</para>
+ </listitem>
+
+ <listitem>
+ <para>Fill in the Label using the Logical File name desired:
+ : <emphasis
+ role="bold">~thor::in::IMDB::actresses.list</emphasis></para>
+
+ <para>Make sure the <emphasis
+ role="bold">Overwrite</emphasis> and <emphasis
+ role="bold">Replicate </emphasis>boxes are checked (if
+ available).</para>
+ </listitem>
+
+ <listitem>
+ <para>Press the Submit button</para>
+ </listitem>
+ </itemizedlist>After both sprays are complete, we can query the
+ logical files on the HPCC to see the files we sprayed.</para>
</listitem>
<listitem>
View
399 docs/Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml
@@ -0,0 +1,399 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.1.2//EN"
+"http://www.oasis-open.org/docbook/xml/4.1.2/docbookx.dtd">
+<sect1 id="ssl4esp">
+ <title><emphasis role="bold">Configuring ESP Server to use HTTPS
+ (SSL)</emphasis></title>
+
+ <para>The HPCC Enterprise Services Platform server (ESP) supports Secure
+ Sockets Layer (SSL), a protocol used to send and receive private data or
+ documents.</para>
+
+ <para>SSL works by using a private key to encrypt and decrypt data
+ transferred over the SSL connection. By convention, URLs using an SSL
+ connection start with HTTPS instead of HTTP.</para>
+
+ <para>The SSL option in the ESP Server allows secure and encrypted
+ communication between a browser or SOAP client application and the HPCC
+ platform.</para>
+
+ <para>SSL capabilities are configured in the Configuration Manager, but
+ require a certificate be installed on the ESP server. The OpenSSL libraries
+ provide a means to create the necessary certificate files in one of two
+ ways.</para>
+
+ <itemizedlist mark="bullet">
+ <listitem>
+ <para>You can use the OpenSSL libraries to create a private key and a
+ Certificate Signing Request (CSR) to purchase a certificate from a
+ Certificate Issuing Authority (such as, VeriSign).</para>
+ </listitem>
+
+ <listitem>
+ <para>You can use that CSR to generate your own self-signed certificate
+ and then install the certificate and private key to your ESP
+ Server.</para>
+ </listitem>
+ </itemizedlist>
+
+ <para>In either case, once installed and configured, the network traffic is
+ encrypted and secure. The Public and Private Keys use 1024-bit RSA
+ encryption.</para>
+
+ <sect2>
+ <title><emphasis role="bold">Generate an RSA Private
+ Key</emphasis></title>
+
+ <para>Use the OpenSSL toolkit to generate an RSA Private Key and a
+ Certificate Signing Request (CSR). This can also be the basis for a
+ self-signed certificate. Self-signed certificates are useful for internal
+ use or testing.</para>
+
+ <para>In our example, we create a 1024-bit RSA Private Key which is
+ encrypted using Triple-DES encryption and stored in Privacy Enhanced Mail
+ (PEM) format.</para>
+
+ <programlisting>openssl genrsa -des3 -out server.key 1024</programlisting>
+
+ <para>When prompted, provide a passphrase. This is used as the basis for
+ the encryption.</para>
+
+ <para><emphasis role="bold">Remember this passphrase as you will need to
+ enter it into the Configuration Manager later.</emphasis></para>
+ </sect2>
+
+ <sect2 role="brk">
+ <title><emphasis role="bold">Generate a CSR (Certificate Signing
+ Request)</emphasis></title>
+
+ <para>After you have a private key, you can use it to create a Certificate
+ Signing Request (CSR). You can use your CSR to request a signed
+ certificate from a Certificate Authority (such as Verisign or Network
+ Solutions). You can also use the CSR to create a self-signed
+ certificate.</para>
+
+ <programlisting>openssl req -new -key server.key -out server.csr </programlisting>
+
+ <para>Answer the questions when prompted:</para>
+
+ <para><informaltable colsep="1" frame="all" rowsep="1">
+ <tgroup cols="2">
+ <colspec colwidth="243.90pt" />
+
+ <colspec colwidth="234.90pt" />
+
+ <tbody>
+ <row>
+ <entry>Country Name (2 letter code):</entry>
+
+ <entry></entry>
+ </row>
+
+ <row>
+ <entry>State or Province Name (full name):</entry>
+
+ <entry></entry>
+ </row>
+
+ <row>
+ <entry>Locality Name (eg, city) :</entry>
+
+ <entry></entry>
+ </row>
+
+ <row>
+ <entry>Organization Name (eg, company) :</entry>
+
+ <entry></entry>
+ </row>
+
+ <row>
+ <entry>Organizational Unit Name (eg, section) :</entry>
+
+ <entry></entry>
+ </row>
+
+ <row>
+ <entry>Common Name (e.g., server’s hostname):</entry>
+
+ <entry></entry>
+ </row>
+
+ <row>
+ <entry>Email Address :</entry>
+
+ <entry></entry>
+ </row>
+
+ <row>
+ <entry>A challenge password (optional):</entry>
+
+ <entry></entry>
+ </row>
+
+ <row>
+ <entry>An optional company name (optional):</entry>
+
+ <entry></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </informaltable></para>
+ </sect2>
+
+ <sect2>
+ <title><emphasis role="bold">Generate a Self-Signed
+ Certificate</emphasis></title>
+
+ <para>To generate a temporary certificate, which is good for up to 365
+ days, issue the following command:</para>
+
+ <programlisting>openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt </programlisting>
+
+ <para>When prompted, enter the passphrase you used earlier when creating
+ your CSR.</para>
+ </sect2>
+
+ <sect2>
+ <title><emphasis role="bold">Installing the Private Key and Certificate to
+ your ESP Server</emphasis></title>
+
+ <para>You must install the certificate and private key on all ESP server
+ node(s) that will host a service binding using SSL.</para>
+
+ <para>Your PrivateKey and certificate must be copied to
+ /var/lib/HPCCSystems/myesp/.</para>
+
+ <programlisting># For example:
+sudo cp server.crt /var/lib/HPCCSystems/myesp/certificate.cer
+sudo cp server.key /var/lib/HPCCSystems/myesp/privatekey.cer
+</programlisting>
+ </sect2>
+
+ <sect2 role="brk">
+ <title><emphasis role="bold">Configure HTTPS on your ESP
+ Server</emphasis></title>
+
+ <sect3>
+ <title><emphasis role="bold">Start Configuration Manager in Advanced
+ Mode</emphasis></title>
+
+ <orderedlist numeration="arabic">
+ <listitem>
+ <para>Start the Configuration Manager Service on one node (usually
+ the first node is considered the head node and is used for this
+ task, but this is up to you).</para>
+
+ <para><programlisting>sudo /opt/HPCCSystems/sbin/configmgr
+</programlisting></para>
+ </listitem>
+
+ <listitem>
+ <para>Using a Web browser, go to the Configuration Manager's
+ interface.</para>
+
+ <para>Use the url of http://nnn.nnn.nnn.nnn:pppp, where
+ nnn.nnn.nnn.nnn is the IP address of the node running Configuration
+ Manager and pppp is the port (default is 8015).</para>
+
+ <para>The Configuration Manager startup wizard displays.</para>
+ </listitem>
+
+ <listitem>
+ <para>Select <emphasis role="bold">Advanced View</emphasis>.</para>
+ </listitem>
+
+ <listitem>
+ <para>Select an XML file from the drop list.</para>
+
+ <para>This list is populated from versions of an environment XML
+ file in your server's <emphasis
+ role="bold">/etc/HPCCSystems/source/</emphasis> directory.</para>
+
+ <para><emphasis role="bold">Tip</emphasis>: The XML file that
+ matches the active environment.xml is highlighted.</para>
+ </listitem>
+
+ <listitem>
+ <para>Press the <emphasis role="bold">Next</emphasis> button.</para>
+
+ <para>The Configuration Manager Advanced View interface
+ displays.</para>
+ </listitem>
+
+ <listitem>
+ <para>Check the <emphasis role="bold">Write Access </emphasis>box at
+ the top of the page.</para>
+ </listitem>
+ </orderedlist>
+ </sect3>
+
+ <sect3 role="brk">
+ <title><emphasis role="bold">Configure ESP</emphasis></title>
+
+ <orderedlist numeration="arabic">
+ <listitem>
+ <para>Select ESP - MyEsp in the Navigator panel on the left
+ side.</para>
+ </listitem>
+
+ <listitem>
+ <para>Select the <emphasis role="bold">HTTPS</emphasis> tab. <figure>
+ <title>Select HTTPS Tab</title>
+
+ <mediaobject>
+ <imageobject>
+ <imagedata fileref="../../images/ssl-01.jpg" />
+ </imageobject>
+ </mediaobject>
+ </figure></para>
+ </listitem>
+
+ <listitem>
+ <para>In the <emphasis role="bold">passphrase</emphasis> entry
+ control, enter the passphrase you used earlier when you created the
+ private key.</para>
+ </listitem>
+
+ <listitem>
+ <para>When prompted, provide the passphrase again.</para>
+ </listitem>
+
+ <listitem>
+ <para>Click the disk icon to save.</para>
+ </listitem>
+ </orderedlist>
+ </sect3>
+
+ <sect3>
+ <title><emphasis role="bold">Configure one or more SSL-Enabled Service
+ Bindings</emphasis></title>
+
+ <orderedlist numeration="arabic">
+ <listitem>
+ <para>Select the ESP Service Bindings tab.</para>
+ </listitem>
+
+ <listitem>
+ <para>Rt-click on the list of services, then select <emphasis
+ role="bold">Add</emphasis>.</para>
+ </listitem>
+
+ <listitem>
+ <para>Provide a name for the binding (e.g., myws_ecl_ssl)</para>
+ </listitem>
+
+ <listitem>
+ <para>Select myws_ecl from the service drop-list.</para>
+
+ <figure>
+ <title>myws_ecl</title>
+
+ <mediaobject>
+ <imageobject>
+ <imagedata fileref="../../images/ssl-02.jpg" />
+ </imageobject>
+ </mediaobject>
+ </figure>
+ </listitem>
+
+ <listitem>
+ <para>When prompted about the FilesBaseDN, press the <emphasis
+ role="bold">OK</emphasis> button.</para>
+
+ <figure>
+ <title>FilesBaseDN</title>
+
+ <mediaobject>
+ <imageobject>
+ <imagedata fileref="../../images/ssl-03.jpg" />
+ </imageobject>
+ </mediaobject>
+ </figure>
+ </listitem>
+
+ <listitem>
+ <para>Select https from the protocol drop-list.</para>
+
+ <figure>
+ <title>Select HTTPS</title>
+
+ <mediaobject>
+ <imageobject>
+ <imagedata fileref="../../images/ssl-04.jpg" />
+ </imageobject>
+ </mediaobject>
+ </figure>
+
+ <para><emphasis role="bold">Note:</emphasis> If you have not
+ previously edited the port, the change from http to https triggers
+ Configuration Manager to automatically change the port to the
+ default port for https (18002). It only updates automatically if the
+ port has not been edited.</para>
+ </listitem>
+
+ <listitem>
+ <para>Click the disk icon to save</para>
+ </listitem>
+ </orderedlist>
+ </sect3>
+ </sect2>
+
+ <sect2 role="brk">
+ <title><emphasis role="bold">Distribute the environment configuration file
+ to all nodes, Restart, and Certify</emphasis></title>
+
+ <para>Once your environment is set up as desired, you must copy the
+ configuration file out to the other nodes.</para>
+
+ <orderedlist numeration="arabic">
+ <listitem>
+ <para>If it is running, stop the system.</para>
+
+ <para>Make sure system is stopped before attempting to move the
+ environment.xml file.</para>
+ </listitem>
+
+ <listitem>
+ <para>Back up the original environment.xml file</para>
+
+ <programlisting># for example
+sudo -u hpcc cp /etc/HPCCSystems/environment.xml /etc/HPCCSystems/environment.bak
+</programlisting>
+
+ <para>Note: the "live" environment.xml file is located in your
+ <emphasis role="bold">/etc/HPCCSystems/ </emphasis>directory.
+ ConfigManager works on files in <emphasis
+ role="bold">/etc/HPCCSystems/source </emphasis>directory. You must
+ copy the XML file from this location to make an environment.xml file
+ active.</para>
+ </listitem>
+
+ <listitem>
+ <para>Copy the NewEnvironment.xml file from the source directory to
+ the /etc/HPCCSystems and rename the file to environment.xml</para>
+
+ <programlisting># for example
+sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/environment.xml
+</programlisting>
+ </listitem>
+
+ <listitem>
+ <para>Copy the <emphasis
+ role="bold">/etc/HPCCSystems/environment.xml</emphasis> to
+ the<emphasis role="bold"> /etc/HPCCSystems/ </emphasis>on every
+ node.</para>
+
+ <para>You might prefer to use a script to automate this step,
+ especially if you have many nodes. See the Example Scripts section in
+ the Appendix of the Installing and Running the HPCCPlatform
+ manual.</para>
+ </listitem>
+
+ <listitem>
+ <para>Restart the HPCC system and certify the components as
+ usual.</para>
+ </listitem>
+ </orderedlist>
+ </sect2>
+</sect1>
View
16 docs/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml
@@ -27,11 +27,13 @@
message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
+
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
<para></para>
</legalnotice>
@@ -1630,6 +1632,10 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
<xi:include href="Installing_and_RunningTheHPCCPlatform/Inst-Mods/UserSecurityMaint.xml"
xpointer="element(/1)"
xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+ <xi:include href="Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml"
+ xpointer="element(/1)"
+ xmlns:xi="http://www.w3.org/2001/XInclude" />
</chapter>
<chapter>
View
111 docs/One-Click_HPCCinAWS/One-Click_RuningHPCCinAmazonWebServicesEC2.xml
@@ -2,11 +2,12 @@
<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
<book lang="en_US" xml:base="../">
- <title>One-Click<trademark> Thor </trademark> within Amazon Web
- Services</title>
+ <title>HPCC Systems<superscript>®</superscript> Instant Cloud for
+ AWS</title>
<bookinfo>
- <title>One-Click Thor™ within AWS</title>
+ <title>HPCC Systems<superscript>®</superscript> Instant Cloud for
+ AWS</title>
<mediaobject>
<imageobject>
@@ -28,7 +29,8 @@
message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
<para>Amazon Web Services, AWS, Amazon EC2, EC2, Amazon Elastic Compute
Cloud, Amazon S3, Amazon Simple Storage Service, are trademarks,
@@ -58,20 +60,20 @@
<mediaobject role="logo">
<imageobject>
- <imagedata fileref="images/AWS_One-click_COVER_LN_Rightjustified.jpg" />
+ <imagedata fileref="images/AWS_InstantCloud_COVER_registeredTm.jpg" />
</imageobject>
</mediaobject>
</bookinfo>
<chapter>
<title>Introduction</title>
- <para>This guide provides details and guidance in running an HPCC Systems
- Thor cluster inside an Amazon Web Services (AWS) Elastic Cloud (EC2) using
- the One-Click Launch page.</para>
+ <para>This guide provides details and guidance in running an HPCC
+ Systems<superscript>®</superscript> Platform inside an Amazon Web Services
+ (AWS) Elastic Cloud (EC2) using the Instant Cloud for AWS page.</para>
<para>This allows you to <emphasis role="bold">instantiate</emphasis> and
- run HPCC Systems Thor clusters of different sizes on the fly.</para>
+ run HPCC Systems clusters of different sizes on the fly.</para>
<para>This is useful for:</para>
@@ -89,12 +91,12 @@
</listitem>
<listitem>
- <para>Leveraging the HPCC Systems Thor platform without incurring cost
- of hardware and administration</para>
+ <para>Leveraging the HPCC Systems platform without incurring cost of
+ hardware and administration</para>
</listitem>
<listitem>
- <para>Create and use an HPCC Systems Thor cluster immediately without
+ <para>Create and use an HPCC Systems cluster immediately without
purchasing and installing new hardware</para>
</listitem>
</itemizedlist>
@@ -114,11 +116,10 @@
<para>We are working with Amazon to improve the set up processes.
<emphasis role="bold">In the near future, we will have a guide to show how
- to run an HPCC Systems Thor cluster in an Amazon Elastic Map Reduce (EMR)
- type environment.</emphasis> We expect that method to be more robust and
- easier to set up and operate. In addition, HPCC Systems is working on its
- own Enterprise Cloud offering, but it is not yet ready to be
- announced.</para>
+ to run an HPCC Systems cluster in an Amazon Elastic Map Reduce (EMR) type
+ environment.</emphasis> We expect that method to be more robust and easier
+ to set up and operate. In addition, HPCC Systems is working on its own
+ Enterprise Cloud offering, but it is not yet ready to be announced.</para>
<informaltable colsep="1" frame="all" rowsep="1">
<?dbfo keep-together="always"?>
@@ -164,7 +165,7 @@
</listitem>
<listitem>
- <para><emphasis>Mac<superscript>*</superscript> workstation
+ <para><emphasis>Mac<superscript></superscript> workstation
with the Eclipse IDE and the ECL plug-in for Eclipse
(available soon)</emphasis></para>
</listitem>
@@ -196,13 +197,9 @@
</listitem>
</itemizedlist>
- <!--***eclipse info needs to be added-->
+ <!--***eclipse info needs to be added when released-->
- <para><footnote label="*">
- <para>Some required components have not yet been built for the Mac
- platform (for example, the eclcc compiler). For the time being you
- can use a Windows emulator or a Virtual Machine on a Mac.</para>
- </footnote></para>
+ <para></para>
<para><informaltable colsep="1" frame="all" rowsep="1">
<?dbfo keep-together="always"?>
@@ -227,7 +224,7 @@
</chapter>
<chapter id="Collect">
- <title>Using One-Click Launch</title>
+ <title>Using Instant Cloud Launch</title>
<sect1 role="nobrk">
<title>Before you begin</title>
@@ -271,17 +268,17 @@
and <emphasis role="bold">Secret Access Key</emphasis>. This
information is never stored on our system.</para>
- <para>If you don't have it handy, you can click on the <emphasis
- role="bold">AWS Activities &gt;&gt; Access Keys</emphasis> link
- from the top menu to go to that section of the AWS Management
- Console.<emphasis role="bluebold"></emphasis>.</para>
+ <para>If you don't have it handy, you can click on the link under
+ <emphasis role="bold">Can't find your Access Key ID?</emphasis> to
+ go to that section of the AWS Management Console.<emphasis
+ role="bluebold"></emphasis>.</para>
<para><figure>
<title>Login</title>
<mediaobject>
<imageobject>
- <imagedata fileref="images/AWS_OneClick_Login.jpg" />
+ <imagedata fileref="images/AWS_InstantCloud_Login.jpg" />
</imageobject>
</mediaobject>
</figure></para>
@@ -306,7 +303,7 @@
<para>Click on the <emphasis role="bold">Launch Cluster
</emphasis>link at the top.</para>
- <para>The <emphasis role="bold">Launch a New Thor Cluster
+ <para>The <emphasis role="bold">Launch a New HPCC Cluster
</emphasis>window displays.</para>
<para></para>
@@ -315,10 +312,10 @@
</sect1>
<sect1>
- <title>Launch a Thor Cluster</title>
+ <title>Launch a New HPCC Cluster</title>
- <para>In this section, you will launch a set of Ubuntu 10.04 machines to
- use for your HPCC Systems Thor platform. The One-Click page, uses the
+ <para>In this section, you will launch a set of Ubuntu 12.04 machines to
+ use for your HPCC Systems Thor platform. The Instant Cloud page uses the
input you specify to create your cluster for you.</para>
<para>When you press the <emphasis role="bold">Launch Cluster
@@ -339,8 +336,8 @@
</listitem>
<listitem>
- <para>Launches the number of m1.large nodes requested using
- ami-fbbf7892.</para>
+ <para>Launches the number of m1.large nodes requested using the
+ provided AMI (ami-e01698d0).</para>
</listitem>
<listitem>
@@ -352,7 +349,8 @@
</listitem>
<listitem>
- <para>Configures Cluster and Support nodes.</para>
+ <para>Configures the requested Thor Cluster, the requested Roxie
+ Cluster, and the required Support nodes.</para>
</listitem>
<listitem>
@@ -373,7 +371,8 @@
<orderedlist numeration="arabic">
<listitem>
- <para>Specify the number of Thor nodes to instantiate.</para>
+ <para>Specify the number of Thor nodes and the number of Roxie
+ nodes to instantiate.</para>
<para><figure>
<title>Launch a New Thor Cluster</title>
@@ -387,6 +386,14 @@
</listitem>
<listitem>
+ <para>Optionally, specify Snapshot ID(s) to attach data to your
+ landing zone.</para>
+
+ <para>This would be a previously saved "snapshot" of a landing
+ zone data store.</para>
+ </listitem>
+
+ <listitem>
<para>Press the <emphasis role="bold">Launch Cluster</emphasis>
button.</para>
@@ -550,7 +557,7 @@
<mediaobject>
<imageobject>
- <imagedata fileref="images/AWS_OneClick_003.jpg" />
+ <imagedata fileref="images/AWS_OneClick_ViewClustersTerminate.jpg" />
</imageobject>
</mediaobject>
</figure></para>
@@ -566,10 +573,9 @@
<para><orderedlist>
<listitem>
- <para>Open the <emphasis role="bold">Running
- Clusters</emphasis> page using the <emphasis
- role="bluebold">View Clusters</emphasis> Link at the top of a
- page.</para>
+ <para>Open the <emphasis role="bold">View Clusters</emphasis>
+ page using the <emphasis role="bluebold">View
+ Clusters</emphasis> Link at the top of a page.</para>
<figure>
<title>Running Clusters</title>
@@ -612,7 +618,7 @@
system.</para>
<para>Note: This does not remove the keys from your running
- cluster. It only removes it from the One-Click system and
+ cluster. It only removes it from the Instant Cloud system and
prevents further downloads of the key. Once deleted, there is
no way to retrieve the key.</para>
</listitem>
@@ -628,7 +634,7 @@
<para></para>
<sect1 id="ECL-on-a-single-node-system" role="nobrk">
- <title>Running ECL on your HPCC Systems Thor cluster</title>
+ <title>Running ECL on your HPCC Systems cluster</title>
<para>After your platform is running, and you can now create and run
some ECL<footnote>
@@ -1508,21 +1514,6 @@ s3cmd --configure
<chapter id="NextSteps">
<title>Next Steps</title>
- <para>From ECL Watch page, click on the <emphasis role="bold">Resources
- </emphasis> link in the menu on the left side.</para>
-
- <para><figure>
- <title>ECL Watch Resource Page</title>
-
- <mediaobject>
- <imageobject>
- <imagedata fileref="images/AWS_ECLWatchNextSteps.jpg" />
- </imageobject>
- </mediaobject>
- </figure></para>
-
- <para><emphasis role="bold"></emphasis></para>
-
<para>To familiarize yourself with what your system can do, we recommend
following the steps in:<itemizedlist spacing="compact">
<listitem>
View
14 docs/RDDERef/RDDERef.xml
@@ -23,13 +23,15 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
- <para></para>
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
+
+ <para> </para>
</legalnotice>
<xi:include href="common/Version.xml" xpointer="FooterInfo"
View
14 docs/RunningHPCCinaVirtualMachine/RunningHPCCinaVirtualMachine.xml
@@ -25,13 +25,15 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
- <para></para>
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
+
+ <para> </para>
</legalnotice>
<xi:include href="common/Version.xml" xpointer="FooterInfo"
View
18 docs/UsingConfigManager/UsingConfigManager.xml
@@ -3,7 +3,7 @@
"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
<book xml:base="../">
<bookinfo>
- <title> HPCC Configuration Manager</title>
+ <title>HPCC Configuration Manager</title>
<mediaobject>
<imageobject>
@@ -23,13 +23,15 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
- <para></para>
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
+
+ <para> </para>
</legalnotice>
<xi:include href="common/Version.xml" xpointer="FooterInfo"
@@ -535,7 +537,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
Environment heading in the Navigator panel on the left side.</para>
<para><graphic fileref="images/CM-120.jpg" />You can access all
- attributes through the XML view. </para>
+ attributes through the XML view.</para>
<para>If you wish to add an attribute that does not exist, RT-CLICK on
one of the components then you can choose to add an attribute.</para>
View
12 docs/Visualizing/Visualizing.xml
@@ -27,11 +27,13 @@
message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc. </para>
+
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
<para />
</legalnotice>
View
BIN docs/images/AWS_InstantCloud_COVER_registeredTm.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/AWS_InstantCloud_Login.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/AWS_One-click_COVER_LN_Rightjustified.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/AWS_OneClick_003a.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/AWS_OneClick_005.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/AWS_OneClick_006.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/AWS_OneClick_LaunchLog.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/AWS_OneClick_LaunchNew.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/AWS_OneClick_Login.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/AWS_OneClick_Terminated.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/AWS_OneClick_ViewClusters.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/AWS_OneClick_ViewClustersTerminate.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/redswooshWithLogo3.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/ssl-01.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/ssl-02.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/ssl-03.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN docs/images/ssl-04.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
17 docs/wip/certify.xml
@@ -23,15 +23,18 @@
Number in the text of the message.</para>
<para>LexisNexis and the Knowledge Burst logo are registered trademarks
- of Reed Elsevier Properties Inc., used under license. Other products,
- logos, and services may be trademarks or registered trademarks of their
- respective companies. All names and example data used in this manual are
- fictitious. Any similarity to actual persons, living or dead, is purely
- coincidental.</para>
+ of Reed Elsevier Properties Inc., used under license. HPCC Systems is a
+ registered trademark of LexisNexis Risk Data Management Inc.</para>
+
+ <para>Other products, logos, and services may be trademarks or
+ registered trademarks of their respective companies. All names and
+ example data used in this manual are fictitious. Any similarity to
+ actual persons, living or dead, is purely coincidental.</para>
<para><!--Release Information is used for running footers only. The ID attribute in version.xml is FooterInfo
Date is used as the dateline/version number in the title page. The ID attribute in version.xml is DateVer--><!--
-Copyright is used for the copyright line on title page. The ID attribute is CopyRight--></para>
+Copyright is used for the copyright line on title page. The ID attribute is CopyRight-->
+ </para>
</legalnotice>
<xi:include href="common/Version.xml" xpointer="FooterInfo"
@@ -592,7 +595,7 @@ Copyright is used for the copyright line on title page. The ID attribute is Copy
designed to generate a maximum of 400 million records. A
larger number of nodes will result in fewer records per
node, however the code will still work as intended for this
- exercise. </para>
+ exercise.</para>
<?hard-pagebreak ?>
</listitem>
View
287 ecl/hthor/hthor.cpp
@@ -44,16 +44,15 @@
#include "jsmartsock.hpp"
#include "thorstep.hpp"
#include "eclagent.ipp"
+#include "roxierowbuff.hpp"
#define EMPTY_LOOP_LIMIT 1000
static unsigned const hthorReadBufferSize = 0x10000;
-static memsize_t const defaultHThorSpillThreshold = 512*1024*1024; // MORE - should increase on 64-bit platform?
static offset_t const defaultHThorDiskWriteSizeLimit = I64C(10*1024*1024*1024); //10 GB, per Nigel
static size32_t const spillStreamBufferSize = 0x10000;
static int const defaultWorkUnitWriteLimit = 10; //10MB as thor
static unsigned const hthorPipeWaitTimeout = 100; //100ms - fairly arbitrary choice
-static char const * const defaultSortAlgorithm = "stablequick";
using roxiemem::IRowManager;
using roxiemem::OwnedRoxieRow;
@@ -3618,21 +3617,19 @@ CHThorGroupSortActivity::CHThorGroupSortActivity(IAgentContext &_agent, unsigned
void CHThorGroupSortActivity::ready()
{
CHThorSimpleActivityBase::ready();
- eof = false;
- spillThreshold = (memsize_t)agent.queryWorkUnit()->getDebugValueInt64("hthorSpillThreshold", defaultHThorSpillThreshold);
- memsize_t totalMemoryLimit = roxiemem::getTotalMemoryLimit();
- spillThreshold = (memsize_t)std::min(spillThreshold, totalMemoryLimit / 3);
if(!sorter)
createSorter();
}
void CHThorGroupSortActivity::done()
{
if(sorter)
+ {
if(sorterIsConst)
sorter->killSorted();
else
sorter.clear();
+ }
gotSorted = false;
diskReader.clear();
CHThorSimpleActivityBase::done();
@@ -3642,8 +3639,6 @@ const void *CHThorGroupSortActivity::nextInGroup()
{
if(!gotSorted)
getSorted();
- if(eof)
- return NULL;
if(diskReader)
{
@@ -3668,11 +3663,10 @@ const void *CHThorGroupSortActivity::nextInGroup()
void CHThorGroupSortActivity::createSorter()
{
- sorterIsConst = true;
IHThorAlgorithm * algo = static_cast<IHThorAlgorithm *>(helper.selectInterface(TAIalgorithm_1));
if(!algo)
{
- sorter.setown(new CHeapSorter(helper.queryCompare()));
+ sorter.setown(new CHeapSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
return;
}
unsigned flags = algo->getAlgorithmFlags();
@@ -3681,93 +3675,122 @@ void CHThorGroupSortActivity::createSorter()
if(!algoname)
{
if((flags & TAFunstable) != 0)
- sorter.setown(new CQuickSorter(helper.queryCompare()));
+ sorter.setown(new CQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
else
- sorter.setown(new CHeapSorter(helper.queryCompare()));
+ sorter.setown(new CHeapSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
return;
}
if(stricmp(algoname, "quicksort") == 0)
if((flags & TAFstable) != 0)
- sorter.setown(new CStableQuickSorter(helper.queryCompare()));
+ sorter.setown(new CStableQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep, this));
else
- sorter.setown(new CQuickSorter(helper.queryCompare()));
+ sorter.setown(new CQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
else if(stricmp(algoname, "heapsort") == 0)
- sorter.setown(new CHeapSorter(helper.queryCompare()));
+ sorter.setown(new CHeapSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
else if(stricmp(algoname, "insertionsort") == 0)
if((flags & TAFstable) != 0)
- sorter.setown(new CStableInsertionSorter(helper.queryCompare()));
+ sorter.setown(new CStableInsertionSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
else
- sorter.setown(new CInsertionSorter(helper.queryCompare()));
+ sorter.setown(new CInsertionSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
else
{
StringBuffer sb;
sb.appendf("Ignoring unsupported sort order algorithm '%s', using default", algoname);
agent.addWuException(sb.str(),0,ExceptionSeverityWarning,"hthor");
if((flags & TAFunstable) != 0)
- sorter.setown(new CQuickSorter(helper.queryCompare()));
+ sorter.setown(new CQuickSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
else
- sorter.setown(new CHeapSorter(helper.queryCompare()));
+ sorter.setown(new CHeapSorter(helper.queryCompare(), queryRowManager(), InitialSortElements, CommitStep));
}
+ sorter->setActivityId(activityId);
}
void CHThorGroupSortActivity::getSorted()
{
diskMerger.clear();
diskReader.clear();
- memsize_t grpSize = 0;
+ queryRowManager()->addRowBuffer(this);//register for OOM callbacks
const void * next;
while((next = input->nextInGroup()) != NULL)
{
- size32_t nextSize = input->queryOutputMeta()->getRecordSize(next);
- if((grpSize+nextSize) > spillThreshold) //Spill ??
+ if (!sorter->addRow(next))
{
- if(!diskMerger)
{
- StringBuffer fbase;
- agent.getTempfileBase(fbase).appendf(".spill_sort_%p", this);
- PROGLOG("SORT: spilling to disk, filename base %s", fbase.str());
- class CHThorRowLinkCounter : public CSimpleInterface, implements IRowLinkCounter
- {
- public:
- IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
- virtual void releaseRow(const void *row)
- {
- releaseHThorRow(row);
- }
- virtual void linkRow(const void *row)
- {
- linkHThorRow(row);
- }
- };
- Owned<IRowLinkCounter> linker = new CHThorRowLinkCounter();
- Owned<IRowInterfaces> rowInterfaces = createRowInterfaces(input->queryOutputMeta(), activityId, agent.queryCodeContext());
- diskMerger.setown(createDiskMerger(rowInterfaces, linker, fbase.str()));
+ //Unlikely that this code will ever be executed but added for comfort
+ roxiemem::RoxieOutputRowArrayLock block(sorter->getRowArray());
+ sorter->flushRows();
+ sortAndSpillRows();
+ //Ensure new rows are written to the head of the array. It needs to be a separate call because
+ //performSort() cannot shift active row pointer since it can be called from any thread
+ sorter->flushRows();
+ }
+ if (!sorter->addRow(next))
+ {
+ releaseHThorRow(next);
+ throw MakeStringException(0, "Insufficient memory to append sort row");
}
- sorter->performSort();
- sorter->spillSortedToDisk(diskMerger);
- grpSize = 0;
}
- sorter->addRow(next);
- grpSize += nextSize;
}
+ queryRowManager()->removeRowBuffer(this);//unregister for OOM callbacks
+ sorter->flushRows();
if(diskMerger)
{
- sorter->performSort();
- sorter->spillSortedToDisk(diskMerger);
+ sortAndSpillRows();
+ sorter->killSorted();
ICompare *compare = helper.queryCompare();
- assertex(compare);
diskReader.setown(diskMerger->merge(compare));
}
- else if(grpSize)
+ else
{
sorter->performSort();
}
- else
+ gotSorted = true;
+}
+
+//interface roxiemem::IBufferedRowCallback
+unsigned CHThorGroupSortActivity::getPriority() const
+{
+ return 10;
+}
+
+bool CHThorGroupSortActivity::freeBufferedRows(bool critical)
+{
+ roxiemem::RoxieOutputRowArrayLock block(sorter->getRowArray());
+ return sortAndSpillRows();
+}
+
+
+
+bool CHThorGroupSortActivity::sortAndSpillRows()
+{
+ if (0 == sorter->numCommitted())
+ return false;
+ if(!diskMerger)
{
- eof = true;
+ StringBuffer fbase;
+ agent.getTempfileBase(fbase).appendf(".spill_sort_%p", this);
+ PROGLOG("SORT: spilling to disk, filename base %s", fbase.str());
+ class CHThorRowLinkCounter : public CSimpleInterface, implements IRowLinkCounter
+ {
+ public:
+ IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
+ virtual void releaseRow(const void *row)
+ {
+ releaseHThorRow(row);
+ }
+ virtual void linkRow(const void *row)
+ {
+ linkHThorRow(row);
+ }
+ };
+ Owned<IRowLinkCounter> linker = new CHThorRowLinkCounter();
+ Owned<IRowInterfaces> rowInterfaces = createRowInterfaces(input->queryOutputMeta(), activityId, agent.queryCodeContext());
+ diskMerger.setown(createDiskMerger(rowInterfaces, linker, fbase.str()));
}
- gotSorted = true;
+ sorter->performSort();
+ sorter->spillSortedToDisk(diskMerger);
+ return true;