Merge branch 'devel' of https://github.com/arangodb/arangodb into fea…

…ture/planning-query-result-cache

* 'devel' of https://github.com/arangodb/arangodb:
  order of free and free2 changed with use of unordered_multimap vs multimap (#3168)
  jslint (#3167)
  Cleanup after each testrun (#3109)
  Feature/planning 499 different error codes for version check (#3118)
  Remove attribute option for arangoimp  (#3141)
  Bug fix/agency mt fixes (#3158)
  When the user entered non-matching passwords, we need to go back two steps so he can re-enter the first password too. (#3155)
  Changed some tasks to post through Scheduler instead of directly. (#3159)
  Bug fix/scheduling et al (#3161)

.gitignore

@@ -18,6 +18,9 @@ core.*
 *.gcno
 *.gcda
 
+*.deb
+*.rpm
+
 .DS_Store
 *.swp
 *.diff
@@ -110,3 +113,6 @@ data-*
 cluster-init
 
 datafile-*.db
+# by build process
+arangodb-linux-amd64
+last_compiled_version.sha

CHANGELOG

@@ -1,6 +1,20 @@
 devel
 -----
 
+* arangoimp now has a --remove-attribute option
+
+* added V8 context lifetime control options
+  `--javascript.v8-contexts-max-invocations` and `--javascript.v8-contexts-max-age`
+
+  These options allow specifying after how many invocations a used V8 context is 
+  disposed, or after what time a V8 context is disposed automatically after its
+  creation. If either of the two thresholds is reached, a V8 context will be
+  disposed. 
+
+  The default value of `--javascript.v8-contexts-max-invocations` is 0, meaning that
+  the maximum number of invocations per context is unlimited. The default value
+  for `--javascript.v8-contexts-max-age` is 60 seconds.
+
 * fixed wrong ui cluster health information
 
 * fixed issue #3070: Add index in _jobs collection

CMakeLists.txt

@@ -282,9 +282,23 @@ get_filename_component(PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}" REALPATH)
 
 set($ENV{PYTHON_EXECUTABLE} ${PYTHON_EXECUTABLE})
 
+# FIXME the build containers seem to have a
+# /usr/bin/ch(mod|own) to prevent the search
+# to find those files the NO_DEFAULT_PATH
+# argument is passed
 if (NOT WINDOWS)
-  find_program(CHMOD_EXECUTABLE chmod)
-  find_program(CHOWN_EXECUTABLE chown)
+  find_program(
+    CHMOD_EXECUTABLE chmod
+    PATHS "/bin/" "/usr/bin/"
+    NO_DEFAULT_PATH
+  )
+  message(STATUS "chmod found in ${CHMOD_EXECUTABLE}")
+  find_program(
+    CHOWN_EXECUTABLE chown
+    PATHS "/bin" "/usr/bin"
+    NO_DEFAULT_PATH
+  )
+  message(STATUS "chown found in ${CHOWN_EXECUTABLE}")
 endif()
 
 ################################################################################
@@ -894,6 +908,33 @@ if (USE_MAINTAINER_MODE)
   endforeach ()
 
   add_custom_target(errorfiles ALL DEPENDS ${ERROR_FILES_GEN})
+
+  set(EXIT_CODE_FILES
+    lib/Basics/exitcodes.h
+    lib/Basics/exitcodes.cpp
+    js/common/bootstrap/exitcodes.js
+    Installation/Windows/Plugins/exitcodes.nsh
+  )
+
+  set(EXIT_CODE_FILES_GEN)
+  set(EXIT_CODES_DAT lib/Basics/exitcodes.dat)
+
+  foreach (m IN LISTS EXIT_CODE_FILES)
+    add_custom_command(
+      OUTPUT ${CMAKE_SOURCE_DIR}/${m}
+      COMMAND ${PYTHON_EXECUTABLE} ./utils/generateExitCodesFiles.py ./${EXIT_CODES_DAT} ./${m}.tmp
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different ./${m}.tmp ./${m}
+      COMMAND ${CMAKE_COMMAND} -E remove ./${m}.tmp
+      DEPENDS ${CMAKE_SOURCE_DIR}/${EXIT_CODES_DAT}
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      COMMENT  "Building exitcode files ${m}"
+      VERBATIM
+    )
+
+    list(APPEND EXIT_CODE_FILES_GEN ${CMAKE_SOURCE_DIR}/${m})
+  endforeach ()
+
+  add_custom_target(exitcodefiles ALL DEPENDS ${EXIT_CODE_FILES_GEN})
 endif ()
 
 ################################################################################

Documentation/Books/Manual/Administration/Arangoimp.md

@@ -272,6 +272,24 @@ Other common cases are to rename columns in the input file to *_from* and *_to*:
 The *translate* option can be specified multiple types. The source attribute name
 and the target attribute must be separated with a *=*.
 
+
+### Ignoring Attributes 
+
+
+For the CSV and TSV input formats, certain attribute names can be ignored on imports.
+In an ArangoDB cluster there are cases where this can come in handy,
+when your documents already contain a `_key` attribute
+and your collection has a sharding attribute other than `_key`: In the cluster this
+configuration is not supported, because ArangoDB needs to guarantee the uniqueness of the `_key` 
+attribute in *all* shards of the collection.
+
+    > arangoimp --file "data.csv" --type csv --remove-attribute "_key"
+
+The same thing would apply if your data contains an *_id* attribute:
+
+    > arangoimp --file "data.csv" --type csv --remove-attribute "_id"
+
+
 ### Importing into an Edge Collection
 
 arangoimp can also be used to import data into an existing edge collection.

Documentation/Books/Manual/Administration/Configuration/GeneralArangod.md

@@ -496,6 +496,23 @@ value, but it may go up as high as specified via the option `--javascript.v8-con
 When there are unused V8 contexts that linger around and the number of V8 contexts
 is greater than `--javascript.v8-contexts-minimum` the server's garbage collector 
 thread will automatically delete them.
+
+
+`--javascript.v8-contexts-max-invocations`
+
+Specifies the maximum number of invocations after which a used V8 context is 
+disposed. The default value of `--javascript.v8-contexts-max-invocations` is 0, 
+meaning that the maximum number of invocations per context is unlimited. 
+
+`--javascript.v8-contexts-max-age`
+
+Specifies the time duration (in seconds) after which time a V8 context is disposed 
+automatically after its creation. If the time is elapsed, the context will be disposed.
+The default value for `--javascript.v8-contexts-max-age` is 60 seconds.
+
+If both `--javascript.v8-contexts-max-invocations` and `--javascript.v8-contexts-max-age`
+are set, then the context will be destroyed when either of the specified threshold 
+values is reached.
 
 
 ### Garbage collection frequency (time-based)

Installation/Jenkins/build.sh

@@ -656,10 +656,18 @@ if test "${DOWNLOAD_STARTER}" == 1; then
         if test -f "${TN}"; then
             rm -f "${TN}"
         fi
-        curl -LO "${STARTER_URL}"
         FN=$(echo "${STARTER_URL}" |${SED} "s;.*/;;")
-        mv "${FN}" "${BUILD_DIR}/${TN}"
-        chmod a+x "${BUILD_DIR}/${TN}"
+
+        echo $FN
+        if ! test -f "${BUILD_DIR}/${FN}-${STARTER_REV}"; then
+            curl -LO "${STARTER_URL}"
+            cp "${FN}" "${BUILD_DIR}/${TN}"
+            touch "${BUILD_DIR}/${FN}-${STARTER_REV}"
+            chmod a+x "${BUILD_DIR}/${TN}"
+            echo "downloaded ${BUILD_DIR}/${FN}-${STARTER_REV} MD5: $(${MD5} < "${BUILD_DIR}/${TN}")"
+        else
+            echo "using already downloaded ${BUILD_DIR}/${FN}-${STARTER_REV} MD5: $(${MD5} < "${BUILD_DIR}/${TN}")"
+        fi
     fi
     CONFIGURE_OPTIONS+=("-DTHIRDPARTY_BIN=${BUILD_DIR}/${TN}")
 fi

Installation/Windows/Plugins/exitcodes.nsh

@@ -0,0 +1,75 @@
+
+!include "LogicLib.nsh"
+!macro printExitCode exitCode Message
+  Push "${exitCode}"
+  Push "${Message}"
+  Call printExitCode
+!macroend
+Function printExitCode
+pop $1
+pop $2
+${Switch} $0
+
+
+  ${Case} 0 # EXIT_SUCCESS
+    MessageBox MB_ICONEXCLAMATION '$1:$\r$\nsuccess'
+    ; No error has occurred.
+  ${Break}
+
+  ${Case} 1 # EXIT_FAILED
+    MessageBox MB_ICONEXCLAMATION '$1:$\r$\nexit with error'
+    ; Will be returned when a general error occurred.
+  ${Break}
+
+  ${Case} 2 # EXIT_CODE_RESOLVING_FAILED
+    MessageBox MB_ICONEXCLAMATION '$1:$\r$\nexit code resolving failed'
+    ; fill me
+  ${Break}
+
+  ${Case} 5 # EXIT_BINARY_NOT_FOUND
+    MessageBox MB_ICONEXCLAMATION '$1:$\r$\nbinary not found'
+    ; fill me
+  ${Break}
+
+  ${Case} 6 # EXIT_CONFIG_NOT_FOUND
+    MessageBox MB_ICONEXCLAMATION '$1:$\r$\nconfig not found'
+    ; fill me
+  ${Break}
+
+  ${Case} 10 # EXIT_UPGRADE_FAILED
+    MessageBox MB_ICONEXCLAMATION '$1:$\r$\nupgrade failed'
+    ; Will be returned when the database upgrade failed
+  ${Break}
+
+  ${Case} 11 # EXIT_UPGRADE_REQUIRED
+    MessageBox MB_ICONEXCLAMATION '$1:$\r$\ndb upgrade required'
+    ; Will be returned when a database upgrade is required
+  ${Break}
+
+  ${Case} 12 # EXIT_DOWNGRADE_REQUIRED
+    MessageBox MB_ICONEXCLAMATION '$1:$\r$\ndb downgrade required'
+    ; Will be returned when a database upgrade is required
+  ${Break}
+
+  ${Case} 13 # EXIT_VERSION_CHECK_FAILED
+    MessageBox MB_ICONEXCLAMATION '$1:$\r$\nversion check failed'
+    ; Will be returned when there is a version mismatch
+  ${Break}
+
+  ${Case} 20 # EXIT_ALREADY_RUNNING
+    MessageBox MB_ICONEXCLAMATION '$1:$\r$\nalready running'
+    ; Will be returned when arangod is already running according to PID-file
+  ${Break}
+
+  ${Case} 21 # EXIT_COULD_NOT_BIND_PORT
+    MessageBox MB_ICONEXCLAMATION '$1:$\r$\nport blocked'
+    ; Will be returned when endpoint is taken by another process
+  ${Break}
+
+  ${Case} 22 # EXIT_COULD_NOT_LOCK
+    MessageBox MB_ICONEXCLAMATION '$1:$\r$\ncould not lock - another process could be running'
+    ; fill me
+  ${Break}
+
+${EndSwitch}
+FunctionEnd

Installation/Windows/Templates/NSIS.template.in

@@ -5,6 +5,7 @@
 !addincludedir '@CPACK_PLUGIN_PATH@/UAC-plug-in-NSIS'
 !addincludedir '@CPACK_PLUGIN_PATH@/'
 !include "OpenLink.nsh"
+!include "exitcodes.nsh"
 
 ;--------------------------------
 ; Include LogicLib for more readable code
@@ -718,7 +719,10 @@ Function UpgradeExisting
   DetailPrint "Checking whether an existing database needs upgrade: "
   ExecWait "$INSTDIR\${SBIN_DIR}\arangod.exe --server.rest-server false --log.foreground-tty false --database.check-version" $0
   DetailPrint "done Checking whether an existing database needs upgrade: $0"
-  ${If} $0 == 1
+  ${If} $0 != 0
+  ${AndIf} $0 != 11
+    !insertmacro printExitCode $0 "failed to detect whether we need to Upgrade"
+  ${ElseIf} $0 == 11
   ${AndIf} $AUTOMATIC_UPDATE == "1"
     DetailPrint "Yes."
 
@@ -740,8 +744,8 @@ Function UpgradeExisting
     ; Now actually do the upgrade
     ExecWait "$INSTDIR\${SBIN_DIR}\arangod.exe --server.rest-server false  --log.level error --database.auto-upgrade true" $0
     DetailPrint "Done running database upgrade: $0"
-    ${If} $0 == 1
-      MessageBox MB_ICONEXCLAMATION "the Upgrade failed, please do a manual upgrade"
+    ${If} $0 != 0
+      !insertmacro printExitCode $0 "the Upgrade failed, please do a manual upgrade"
       Abort
     ${EndIf}
   ${EndIf}
@@ -757,6 +761,9 @@ Function SetDBPassword
   DetailPrint "Done initializing password: $0"
   ${If} $0 == 0
     return
+  ${Else}
+    !insertmacro printExitCode $0 "Failed to initialize database password.$\r$\nPlease check the windows event log for more details$\r$\n"
+    Abort
   ${EndIf}
   error:
     MessageBox MB_OK "Failed to initialize database password.$\r$\nPlease check the windows event log for details."