Merge branch 'dfsg' into debian

* dfsg: (354 commits)
  Update version to 8.7.0 (VowpalWabbit#1926)
  Fix misconfiguration (VowpalWabbit#1925)
  Ataymano ataymano/warnings fixes (VowpalWabbit#1924)
  Update new version script (VowpalWabbit#1922)
  Run clang-format over codebase (VowpalWabbit#1921)
  change semantics of lambda (VowpalWabbit#1920)
  Bremen79 fix save ftrl (VowpalWabbit#1919)
  fix for daemon race condition (VowpalWabbit#1918)
  Revert "Fix to save the state of FTRL models (VowpalWabbit#1912)" (VowpalWabbit#1916)
  fix static library build (VowpalWabbit#1913)
  more warnings (VowpalWabbit#1915)
  Fix to save the state of FTRL models (VowpalWabbit#1912)
  remove warnings (VowpalWabbit#1911)
  fix closing invalid file descriptor with memory_io_buf (VowpalWabbit#1910)
  Optional exception (VowpalWabbit#1906)
  Contextual Memory Tree (VowpalWabbit#1799)
  Coin betting (VowpalWabbit#1903)
  Ataymano/c wrapper fix2 (VowpalWabbit#1859)
  Use Appveyor MSBuildLogger (VowpalWabbit#1904)
  fix for no label confidence (VowpalWabbit#1901)
  ...

.clang-format

@@ -0,0 +1,90 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: DontAlign
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Left
+AlignOperands: false
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:   
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Allman
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializers: BeforeComma
+ColumnLimit:     120
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: true
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+IncludeCategories: 
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IndentCaseLabels: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+ReflowComments:  true
+SortIncludes:    false
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+TabWidth:        8
+UseTab:          Never
+...
+

.gitattributes

@@ -0,0 +1 @@
+*.rc binary

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,32 @@
+---
+name: Bug report
+about: Report a bug in VW
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+#### Describe the bug
+A clear and concise description of what the bug is.
+
+#### To Reproduce
+Steps to reproduce the behavior:
+1. Used this specific command line i.e. `vw -k -d train-sets/0001.dat --progress 10`
+2. Used `model_file`
+3. Used `specific_data_set`
+
+#### Expected behavior
+A clear and concise description of what you expected to happen.
+
+#### Observed Behavior
+How did VW behave? Please include any stack trace, log messages or crash logs.
+
+#### Environment
+What version of VW did you use?
+What OS or language did you use?
+
+e.g. `VW 8.6.1 on Linux with the Python bindings`
+
+#### Additional context
+Add any other context about the problem here.

.gitignore

@@ -16,7 +16,6 @@ libtool
 ltmain.sh
 missing
 vowpalwabbit/Makefile.in
-vowpalwabbit/config.h.in
 config.h
 *.la
 *.lo
@@ -31,6 +30,7 @@ t_*
 bin
 vw
 .*
+!.pipelines
 vw.1
 *.cache
 *.tgz
@@ -108,6 +108,7 @@ python/dist/**
 python/src/**
 python/vowpalwabbit.egg-info/**
 python/.coverage
+python/docs/build
 
 /COPYING
 /README
@@ -120,12 +121,13 @@ vowpalwabbit/vw.VC.db
 /cs/cs/Serializer/VowpalWabbitDefaultMarshallerExt.cs
 /python/pylibvw.pyd
 /python/pylibvw.cp35-win_amd64.pyd
-decision_service/ds.VC.VC.opendb
-decision_service/ds.VC.db
-decision_service/unit_test/dsclient-test.out
-decision_service/rl.VC.VC.opendb
-decision_service/rl.VC.db
+reinforcement_learning/ds.VC.VC.opendb
+reinforcement_learning/ds.VC.db
+reinforcement_learning/rl.VC.VC.opendb
+reinforcement_learning/rl.VC.db
 index.html
 packages-microsoft-prod.deb
-decision_service/unit_test/rlclient-test.out
+reinforcement_learning/unit_test/rlclient-test.out
 cs/azure_service/azure_service.ccproj.user
+
+build

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "rapidjson"]
+	path = rapidjson
+	url = https://github.com/Tencent/rapidjson.git

.pipelines/Dockerfile

@@ -0,0 +1,69 @@
+FROM ubuntu:14.04 AS build
+
+# Upgrade cmake to 3.2
+RUN apt-get update && apt-get install -y software-properties-common python-software-properties debconf-utils
+RUN add-apt-repository -y ppa:george-edison55/cmake-3.x
+RUN apt-get update
+
+# Add Open JDK repo (including license agreement), install Java
+RUN echo "oracle-java11-installer shared/accepted-oracle-license-v1-2 select true" | \
+  debconf-set-selections && \
+  add-apt-repository -y ppa:openjdk-r/ppa && \
+  apt-get update && \
+  apt install -y openjdk-11-jdk
+
+# Install build tools
+RUN apt-get install -y \
+    gcc g++ libboost-all-dev zlib1g-dev help2man make libssl-dev cmake doxygen graphviz \
+    python-setuptools python-dev build-essential maven wget git && \
+  rm -rf /var/lib/apt/lists/*
+
+# ppa for g++ 4.9 (first version that supports complete c++11. i.e. <regex>)
+RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test
+RUN apt-get update
+RUN apt-get install -y gcc-4.9 g++-4.9
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-4.9
+RUN add-apt-repository -y --remove "ubuntu-toolchain-r-test"
+
+# Install cpprestsdk
+RUN git clone https://github.com/Microsoft/cpprestsdk.git casablanca && \
+  cd casablanca/Release && \
+  # Checkout 2.10.1 version of cpprestsdk
+  git checkout e8dda215426172cd348e4d6d455141f40768bf47 && \
+  mkdir build && \
+  cd build && \
+  cmake .. -DBUILD_TESTS=Off -DBUILD_SAMPLES=Off && \
+  make -j 4 && \
+  make install && \
+  cd ../../../ && \
+  rm -rf casablanca
+
+# Install python tools
+RUN easy_install pip && \
+  pip install cpp-coveralls wheel virtualenv pytest readme_renderer pandas && \
+  wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh && \
+  bash miniconda.sh -b -p $HOME/miniconda && \
+  hash -r && \
+  $HOME/miniconda/bin/conda config --set always_yes yes --set changeps1 no && \
+  $HOME/miniconda/bin/conda update -q conda && \
+  $HOME/miniconda/bin/conda create -q -n test-python27 python=2.7 nomkl numpy scipy scikit-learn
+
+# Download maven dependencies
+RUN wget https://raw.githubusercontent.com/VowpalWabbit/vowpal_wabbit/master/java/pom.xml.in && \
+  mvn dependency:resolve -f pom.xml.in && \
+  rm pom.xml.in
+
+# Cleanup
+RUN apt-get clean autoclean && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/{apt,dpkg,cache,log}
+
+# Set environment variables used by build
+ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib"
+ENV JAVA_HOME="/usr/lib/jvm/java-11-openjdk-amd64/"
+
+# Download clang-format 7.0
+RUN wget http://releases.llvm.org/7.0.1/clang+llvm-7.0.1-x86_64-linux-gnu-ubuntu-14.04.tar.xz && \
+	tar xvf clang+llvm-7.0.1-x86_64-linux-gnu-ubuntu-14.04.tar.xz && \
+	mv clang+llvm-7.0.1-x86_64-linux-gnu-ubuntu-14.04/bin/clang-format /usr/local/bin && \
+	rm -rf clang+llvm-7.0.1-x86_64-linux-gnu-ubuntu-14.04/

.pipelines/build-docker-image.yml

@@ -0,0 +1,20 @@
+pool:
+  vmImage: 'ubuntu-16.04'
+
+# Trigger dockerfile to build when updated on master
+trigger:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - Dockerfile
+
+pr: none
+
+steps:
+  - script: docker build -t $(dockerOrgId)/$(imageName):$(Build.BuildNumber) ./.pipelines/
+  - script: docker login -u "$(dockerCredId)" -p "$(pswd)" https://index.docker.io/v1/
+  - script: docker push $(dockerOrgId)/$(imageName):$(Build.BuildNumber)
+
+# To use this pushed docker image on the CI, the image must be tagged as latest.

.pipelines/build-windows-nuget.yml

@@ -0,0 +1,55 @@
+resources:
+- repo: self
+
+pool:
+  name: Hosted VS2017
+  demands: Cmd
+
+steps:
+- task: VisualStudioTestPlatformInstaller@1
+  inputs:
+    packageFeedSelector: 'nugetOrg' # Options: nugetOrg, customFeed, netShare
+    versionSelector: 'latestStable' # Required when packageFeedSelector == NugetOrg || PackageFeedSelector == CustomFeed# Options: latestPreRelease, latestStable, specificVersion
+    #testPlatformVersion: # Required when versionSelector == SpecificVersion
+    #customFeed: # Required when packageFeedSelector == CustomFeed
+    #username: # Optional
+    #password: # Optional
+    #netShare: # Required when packageFeedSelector == NetShare
+- script: CALL .scripts/restore.cmd
+  displayName: 'Restore dependencies'
+  env:
+    nugetPath: $(Build.SourcesDirectory)\vowpalwabbit\.nuget\nuget.exe
+  failOnStderr: true
+- script: CALL .scripts/build.cmd
+  displayName: 'Build vw.sln'
+  env:
+    nugetPath: $(Build.SourcesDirectory)\vowpalwabbit\.nuget\nuget.exe
+  failOnStderr: true
+- script: CALL .scripts/test.cmd
+  displayName: 'Run tests'
+  env:
+    nugetPath: $(Build.SourcesDirectory)\vowpalwabbit\.nuget\nuget.exe
+  failOnStderr: false
+- task: PublishTestResults@2
+  displayName: 'Publish Test Results'
+  inputs:
+    testResultsFormat: VSTest
+    testResultsFiles: '**/vowpalwabbit/out/test/Release/x64/*.trx'
+- script: CALL .scripts/package.cmd
+  displayName: 'Package artifacts'
+  env:
+    Tag: -$(Build.SourceVersion)--$(Build.BuildId)
+    nugetPath: $(Build.SourcesDirectory)\vowpalwabbit\.nuget\nuget.exe
+  failOnStderr: true
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish Artifact: target'
+  inputs:
+    PathtoPublish: $(Build.SourcesDirectory)\vowpalwabbit\out\target
+    ArtifactName: target
+  condition: succeededOrFailed()
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish Artifact: package'
+  inputs:
+    PathtoPublish: $(Build.SourcesDirectory)\vowpalwabbit\out\package
+    ArtifactName: package
+  condition: succeededOrFailed()

.scripts/build.cmd

@@ -0,0 +1,27 @@
+@ECHO OFF
+IF DEFINED DebugBuildScripts (
+    @ECHO ON
+)
+
+SETLOCAL
+
+REM There are some issues around AV in more locked-down environments not releasing certain files in time
+REM This interferes destructively with the MSBuild 2017+ propensity to try to build all possible chains of
+REM dependencies at the same time, causing the clr projects to build multiple times simultaneously.
+REM The fix is to force a particular build order at the solution level. In the meanwhile, to unblock local
+REM command-line builds, set the ForceLinearBuild switch. 
+SET __MULTIBUILD_SWITCH="/m"
+IF DEFINED ForceLinearBuild (
+    SET __MULTIBUILD_SWITCH=""
+)
+
+CALL %~dp0init.cmd
+PUSHD %~dp0
+
+REM TODO: Figure out how to parametrize this script?! (is there a standard, or do we actually need parse args?)
+ECHO Building "%vwRoot%\vowpalwabbit\vw.sln" for Release x64
+"%msbuildPath%" /v:normal %__MULTIBUILD_SWITCH% /nr:false /p:Configuration=Release;Platform=x64 "%vwRoot%\vowpalwabbit\vw.sln" %1
+
+POPD
+
+ENDLOCAL

.scripts/find-vs2017.cmd

@@ -0,0 +1,6 @@
+IF NOT DEFINED VsInstallDir (
+    REM Try to find VS Install
+    FOR /f "usebackq tokens=*" %%i IN (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.Component.MSBuild -property installationPath`) DO (
+        SET "VsInstallDir=%%i"
+    )
+)

.scripts/init.cmd

@@ -0,0 +1,35 @@
+REM Integration points for toolchain customization
+IF NOT DEFINED nugetPath (
+    SET nugetPath=nuget
+)
+
+IF NOT DEFINED msbuildPath (
+    CALL %~dp0find-vs2017.cmd
+)
+
+IF NOT DEFINED vstestPath (
+    CALL %~dp0find-vs2017.cmd
+)
+
+IF NOT DEFINED msbuildPath (
+    IF EXIST "%VsInstallDir%\MSBuild\15.0\Bin\MSBuild.exe" (
+        SET "msBuildPath=%VsInstallDir%\MSBuild\15.0\Bin\MSBuild.exe"
+    ) ELSE (
+        ECHO ERROR: MsBuild couldn't be found
+        EXIT /b 1
+    )
+)
+
+IF NOT DEFINED vstestPath (
+    IF EXIST "%VsInstallDir%\Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe" (
+        SET "vstestPath=%VsInstallDir%\Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe"
+    ) ELSE (
+        ECHO ERROR: vstest.console couldn't be found
+        EXIT /b 1
+    )
+)
+
+REM Repo-specific paths
+IF NOT DEFINED vwRoot (
+    SET vwRoot=%~dp0..
+)