Merge branch 'devel' of github.com:arangodb/arangodb into array_indexing

3rdParty/Makefile.v8-windows

@@ -1,5 +1,7 @@
 PDBNAME=vc120.pdb
-
+TARGET=Visual Studio 12
+VS=2013
+VSPATH=C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC
 V8=V8-4.3.61
 
 .PHONY: all install
@@ -8,28 +10,28 @@ all: build64 build32
 
 
 build64:
-	./v8-build.bat x86_amd64   x64    x64  64 
+	./v8-build.bat x86_amd64   x64    x64  64 $(VS) "$(VSPATH)" $(V8)
 
 build32:
-	./v8-build.bat       x86  ia32  Win32  32
+	./v8-build.bat       x86  ia32  Win32  32 $(VS) "$(VSPATH)" $(V8)
 
 
 clean: clean32 clean64
 
 clean64:
-	./v8-clean.bat cmd x86_amd64    x64  x64  64
+	./v8-clean.bat cmd x86_amd64    x64  x64  64 $(VS) "$(VSPATH)" $(V8)
 
 clean32:
-	./v8-clean.bat x86      ia32  Win32  32   32
+	./v8-clean.bat x86      ia32  Win32  32   32 $(VS) "$(VSPATH)" $(V8)
 
 
 distclean: distclean32 distclean64
 
 distclean64:
-	./v8-distclean.bat x86_amd64   x64    x64  64
+	./v8-distclean.bat x86_amd64   x64    x64  64 $(VS) "$(VSPATH)" $(V8)
 
 distclean32:
-	./v8-distclean.bat       x86  ia32  Win32  32
+	./v8-distclean.bat       x86  ia32  Win32  32 $(VS) "$(VSPATH)" $(V8)
 
 
 install:
@@ -40,8 +42,8 @@ install_bits:
 	mkdir -p ../WindowsLibraries/$(BITS)/lib/RelWithDebInfo
 	mkdir -p ../WindowsLibraries/$(BITS)/include/unicode
 	for i in `find $(V8)/build -name $(PDBNAME) | grep $(BITS)`; do \
-		LIBNAME=`echo $$i|sed "s;.*/\(.*\)/$(PDBNAME);\1;"`; \
-		BUILD=`echo $$i|sed "s;$(V8)/build/\(.*\)/obj/$${LIBNAME}/$(PDBNAME);\1;"`; \
+		LIBNAME=`echo $$i|sed 's;.*/\(.*\)/$(PDBNAME);\1;'`; \
+		BUILD=`echo $$i|sed 's;$(V8)/build/\(.*\)/obj/$${LIBNAME}/$(PDBNAME);\1;'`; \
 		cp $$i $(V8)/build/$${BUILD}/lib/$${LIBNAME}.pdb; \
 	done
 

3rdParty/V8-4.3.61/build/gyp/pylib/gyp/MSVSVersion.py

@@ -207,6 +207,24 @@ def _CreateVersion(name, path, sdk_based=False):
   if path:
     path = os.path.normpath(path)
   versions = {
+      '2015': VisualStudioVersion('2015',
+                                  'Visual Studio 2015',
+                                  solution_version='13.00',
+                                  project_version='14.0',
+                                  flat_sln=False,
+                                  uses_vcxproj=True,
+                                  path=path,
+                                  sdk_based=sdk_based,
+                                  default_toolset='v140'),
+      '2015e': VisualStudioVersion('2015e',
+                                   'Visual Studio 2015',
+                                   solution_version='13.00',
+                                   project_version='14.0',
+                                   flat_sln=True,
+                                   uses_vcxproj=True,
+                                   path=path,
+                                   sdk_based=sdk_based,
+                                   default_toolset='v140'),
       '2013': VisualStudioVersion('2013',
                                   'Visual Studio 2013',
                                   solution_version='13.00',
@@ -325,6 +343,7 @@ def _DetectVisualStudioVersions(versions_to_check, force_express):
       '10.0': '2010',
       '11.0': '2012',
       '12.0': '2013',
+      '14.0': '2015',
   }
   versions = []
   for version in versions_to_check:
@@ -379,7 +398,7 @@ def SelectVisualStudioVersion(version='auto'):
   if version == 'auto':
     version = os.environ.get('GYP_MSVS_VERSION', 'auto')
   version_map = {
-    'auto': ('10.0', '12.0', '9.0', '8.0', '11.0'),
+    'auto': ('10.0', '12.0', '9.0', '8.0', '11.0', '14.0'),
     '2005': ('8.0',),
     '2005e': ('8.0',),
     '2008': ('9.0',),
@@ -390,6 +409,8 @@ def SelectVisualStudioVersion(version='auto'):
     '2012e': ('11.0',),
     '2013': ('12.0',),
     '2013e': ('12.0',),
+    '2015': ('14.0',),
+    '2015e': ('14.0',),
   }
   override_path = os.environ.get('GYP_MSVS_OVERRIDE_PATH')
   if override_path:

3rdParty/v8-build.bat

@@ -2,17 +2,6 @@
 :: ==== <CONFIGURATION>
 :: ========================================================================================================
 
-:: Set the version of Visual Studio. This will just add a suffix to the string
-:: of your directories to avoid mixing them up.
-SET VS_VERSION=vs2013
-
-:: Set this to the directory that contains vcvarsall.bat file of the
-:: VC Visual Studio version you want to use for building ICU.
-SET VISUAL_STUDIO_VC="C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC"
-
-:: Set this to the version of ICU you are building
-SET V8_VERSION=4.3.61
-
 :: x86_amd64 or x86
 set ARCHITECTURE=%1
 
@@ -25,13 +14,26 @@ set MSPLATFORM=%3
 :: 64 or 32
 set SUFFIX=%4
 
+:: Set the version of Visual Studio. This will just add a suffix to the string
+:: of your directories to avoid mixing them up.
+SET VS_VERSION=vs%5
+SET MSVS_VERSION=%5
+
+:: Set this to the directory that contains vcvarsall.bat file of the
+:: VC Visual Studio version you want to use for building ICU.
+SET VISUAL_STUDIO_VC=%6
+
+:: Set this to the version of V8 you are building
+SET V8_VERSION=%7
+
+
 :: ========================================================================================================
 :: ==== <BUILD> 
 :: ========================================================================================================
 
 call %VISUAL_STUDIO_VC%\vcvarsall.bat %ARCHITECTURE%
 
-set CMD=-G msvs_version=2013
+set CMD=-G msvs_version=%MSVS_VERSION%
 set CMD=%CMD% -Dtarget_arch=%PLATFORM%
 set CMD=%CMD% -Dcomponent=static_library
 set CMD=%CMD% -Dmode=release
@@ -40,9 +42,9 @@ set CMD=%CMD% -Dv8_use_snapshot=false
 
 echo %CMD%
 
-cd V8-%V8_VERSION%
+cd %V8_VERSION%
 
-set PATH=%~dp0V8-%V8_VERSION%\third_party\python_26;%PATH%
+set PATH=%~dp0%V8_VERSION%\third_party\python_26;%PATH%
 .\third_party\python_26\python.exe build\gyp_v8 %CMD%
 
 cd build

3rdParty/v8-clean.bat

@@ -4,14 +4,14 @@
 
 :: Set the version of Visual Studio. This will just add a suffix to the string
 :: of your directories to avoid mixing them up.
-SET VS_VERSION=vs2013
+SET VS_VERSION=vs%5
 
 :: Set this to the directory that contains vcvarsall.bat file of the
 :: VC Visual Studio version you want to use for building ICU.
-SET VISUAL_STUDIO_VC="C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC"
+SET VISUAL_STUDIO_VC=%6
 
 :: Set this to the version of ICU you are building
-SET V8_VERSION=4.3.61
+SET V8_VERSION=%7
 
 :: x86_amd64 or x86
 set ARCHITECTURE=%1
@@ -29,7 +29,7 @@ set SUFFIX=%4
 :: ==== <CLEAN>
 :: ========================================================================================================
 
-cd V8-%V8_VERSION%
+cd %V8_VERSION%
 cd build
 
 rmdir /S /Q Debug

3rdParty/v8-distclean.bat

@@ -2,17 +2,6 @@
 :: ==== <CONFIGURATION>
 :: ========================================================================================================
 
-:: Set the version of Visual Studio. This will just add a suffix to the string
-:: of your directories to avoid mixing them up.
-SET VS_VERSION=vs2013
-
-:: Set this to the directory that contains vcvarsall.bat file of the
-:: VC Visual Studio version you want to use for building ICU.
-SET VISUAL_STUDIO_VC="C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC"
-
-:: Set this to the version of ICU you are building
-SET V8_VERSION=4.3.61
-
 :: x86_amd64 or x86
 set ARCHITECTURE=%1
 
@@ -25,11 +14,22 @@ set MSPLATFORM=%3
 :: 64 or 32
 set SUFFIX=%4
 
+:: Set the version of Visual Studio. This will just add a suffix to the string
+:: of your directories to avoid mixing them up.
+SET VS_VERSION=vs%5
+
+:: Set this to the directory that contains vcvarsall.bat file of the
+:: VC Visual Studio version you want to use for building ICU.
+SET VISUAL_STUDIO_VC=%6
+
+:: Set this to the version of V8 you are building
+SET V8_VERSION=%7
+
 :: ========================================================================================================
 :: ==== <DISTCLEAN>
 :: ========================================================================================================
 
-cd V8-%V8_VERSION%
+cd %V8_VERSION%
 
 del /f /q "build\all.sln"
 del /f /q "build\all.vcxproj"

CHANGELOG

@@ -1,6 +1,12 @@
 v2.7.0 (XXXX-XX-XX)
 -------------------
 
+* report memory usage for document header data (revision id, pointer to data etc.)
+  in `db.collection.figures()`. The memory used for document headers will now
+  show up in the already existing attribute `indexes.size`. Due to that, the index
+  sizes reported by `figures()` in 2.7 will be higher than those reported by 2.6,
+  but the 2.7 values are more accurate.
+
 * IMPORTANT CHANGE: the filenames in dumps created by arangodump now contain
   not only the name of the dumped collection, but also an additional 32-digit hash
   value. This is done to prevent overwriting dump files in case-insensitive file

Documentation/Books/Makefile

@@ -58,9 +58,9 @@ build-book:
 	done
 	python generateMdFiles.py $(NAME) ppbooks/
 
-	cd ppbooks/$(NAME) && sed -ie 's/VERSION_NUMBER/v$(newVersionNumber)/g' styles/header.js
-	cd ppbooks/$(NAME) && sed -ie 's/VERSION_NUMBER/v$(newVersionNumber)/g' README.md
-	cd ppbooks/$(NAME) && sed -ie 's/VERSION_NUMBER/v$(newVersionNumber)/g' book.json
+	cd ppbooks/$(NAME) && sed -i -e 's/VERSION_NUMBER/v$(newVersionNumber)/g' styles/header.js
+	cd ppbooks/$(NAME) && sed -i -e 's/VERSION_NUMBER/v$(newVersionNumber)/g' README.md
+	cd ppbooks/$(NAME) && sed -i -e 's/VERSION_NUMBER/v$(newVersionNumber)/g' book.json
 
 	test -d books/$(NAME) || mkdir -p books/$(NAME)
 

Documentation/Books/Users/HttpReplications/ReplicationLogger.mdpp

@@ -3,20 +3,28 @@
 Previous versions of ArangoDB allowed starting, stopping and configuring the
 replication logger. These commands are superfluous in ArangoDB 2.2 as all
 data-modification operations are written to the server's write-ahead log and are
-not handled by a separate logger.
+not handled by a separate logger anymore.
 
-The only useful operations remaining in ArangoDB 2.2 are to query the current state
+The only useful operations remaining since ArangoDB 2.2 are to query the current state
 of the logger and to fetch the latest changes written by the logger. The operations
 will return the state and data from the write-ahead log.
 
 <!-- arangod/RestHandler/RestReplicationHandler.cpp -->
 @startDocuBlock JSF_get_api_replication_logger_return_state
 
 To query the latest changes logged by the replication logger, the HTTP interface
-also provides the `logger-follow`.
+also provides the `logger-follow` method.
 
 This method should be used by replication clients to incrementally fetch updates 
 from an ArangoDB database.
 
 <!-- arangod/RestHandler/RestReplicationHandler.cpp -->
 @startDocuBlock JSF_get_api_replication_logger_returns
+
+To check what range of changes is available (identified by tick values), the HTTP
+interface provides the methods `logger-first-tick` and `logger-tick-ranges`.
+
+@startDocuBlock JSF_get_api_replication_logger_first_tick
+
+@startDocuBlock JSF_get_api_replication_logger_tick_ranges
+

Documentation/Books/Users/Replication/Components.mdpp

@@ -33,18 +33,13 @@ Instead, the server will write all data-modification operations into its write-a
 The write-ahead log can be queried by clients, so the need for an extra event log is 
 gone.
 
-!SUBSUBSECTION Starting and Stopping
+!SUBSUBSECTION Checking the state
 
 Starting with version 2.2, ArangoDB will log all data-modification operations in its
 write-ahead log automatically. There is no need to explicitly start or configure the
 replication logger on the master.
 
-The replication logger object is still present in ArangoDB 2.2 with the same methods
-as in previous versions of ArangoDB, but only for compatibility reasons. For example, the
-replication logger has *start* and *stop* methods, which are no-ops in ArangoDB 2.2.
-
-One functionality of the replication logger object remains useful in ArangoDB 2.2, and
-that is to query the current state. The state can be queried using the *state* command:
+To query the current state of the logger, use the *state* command:
 
     require("org/arangodb/replication").logger.state();
 
@@ -67,17 +62,29 @@ The result might look like this:
 }
 ```
 
-In previous versions of ArangoDB, the *running* attribute indicated whether the logger 
+In old versions of ArangoDB, the *running* attribute indicated whether the logger 
 was currently enabled and logged any events. In ArangoDB 2.2 and higher, this attribute
-will always have a value of *true*.
+will always have a value of *true*, as the replication logger cannot be turned off.
 
 The *totalEvents* attribute indicates how many log events have been logged since the start 
 of the ArangoDB server. Finally, the *lastLogTick* value indicates the id of the last 
 operation that was written to the server's write-ahead log. It can be used to determine whether new 
 operations were logged, and is also used by the replication applier for incremental 
 fetching of data.
 
-**Note**: Replication logger state can also be queried via the [HTTP API](../HttpReplications/README.md).
+**Note**: The replication logger state can also be queried via the [HTTP API](../HttpReplications/README.md).
+
+To query which data ranges are still available for replication clients to fetch,
+the logger provides the *firstTick* and *tickRanges* functions:
+
+    require("org/arangodb/replication").logger.firstTick();
+
+This will return the minimum tick value that the server can provide to replication
+clients via its replication APIS. The *tickRanges* function returns the minimum and
+maximum tick values per logfile:
+
+    require("org/arangodb/replication").logger.tickRanges();
+
 
 !SUBSUBSECTION Configuration
 
@@ -251,7 +258,8 @@ The result might look like this:
   "chunkSize" : 0, 
   "autoStart" : false, 
   "adaptivePolling" : true,
-  "includeSystem" : true 
+  "includeSystem" : true,
+  "verbose" : false 
 }
 ```
 
@@ -263,7 +271,8 @@ for the connection via the *username* and *password* attributes.
 require("org/arangodb/replication").applier.properties({ 
   endpoint: "tcp://master.domain.org:8529", 
   username:  "root", 
-  password: "secret"
+  password: "secret",
+  verbose: false
 });
 ```
 
@@ -292,7 +301,13 @@ Set it to *0* to use ArangoDB's built-in default value.
 
 The *includeSystem* attribute controls whether changes to system collections (such as *_graphs* or
 *_users*) should be applied. If set to *true*, changes in these collections will be replicated, 
-otherwise, they will not be replicated.
+otherwise, they will not be replicated. It is often not necessary to replicate data from system
+collections, especially because it may lead to confusion on the slave because the slave needs to 
+have its own system collections in order to start and keep operational.
+
+The *verbose* attribute controls the verbosity of the replication logger. Setting it to `true` will
+make the replication applier write a line to the log for every operation it performs. This should
+only be used for diagnosing replication problems.
 
 The following example will set most of the discussed properties for the current database's applier:
 
@@ -377,7 +392,7 @@ with the command
 require("org/arangodb/replication").applier.start("231848833079705");
 ```
 
-**Note**: The tick values should be handled as strings. Using numeric data types for tick
+**Note**: The tick values should be treated as strings. Using numeric data types for tick
 values is unsafe because they might exceed the 32 bit value and the IEEE754 double accuracy
 ranges.