Rebase to master

.travis-opam-coverage.sh

@@ -0,0 +1,35 @@
+# SUMMARY:
+# Builds & tests xapi with coverage in a Ubuntu 16.04 Docker container with
+# OCaml 4.02.3, then uploads the coverage information to coveralls.
+
+set -uex
+
+# Currently there is no way of specifying OPAM depexts for multiple versions of
+# a given disto, and our current depexts only work with Ubuntu >= 16.04, due to
+# a change in packages (libsystemd-dev). Since the build environments of Travis
+# are older then Ubuntu 16.04, we have to run the build in a Docker container
+# with an appropriate Ubuntu version.
+# We need to pass some Travis environment variables to the container to enable
+# uploading to coveralls and detection of Travis CI.
+docker run --rm --volume=$PWD:/mnt --workdir=/mnt \
+  --env "TRAVIS=$TRAVIS" \
+  --env "TRAVIS_JOB_ID=$TRAVIS_JOB_ID" \
+  ocaml/opam:ubuntu-16.04_ocaml-4.02.3 \
+  bash -uex -c '
+sudo apt-get update
+
+# replace the base remote with xs-opam
+opam repository remove default
+opam repository add xs-opam https://github.com/xapi-project/xs-opam.git
+
+# install the dependencies of xapi
+opam pin add --no-action xapi .
+opam depext --yes xapi
+opam install --deps-only xapi
+
+# build and test xapi with coverage, then submit the coverage information to coveralls
+sudo apt-get install --yes wget
+wget https://raw.githubusercontent.com/simonjbeaumont/ocaml-travis-coveralls/master/travis-coveralls.sh
+COV_CONF="./configure" bash -ex travis-coveralls.sh
+'
+

.travis-xenserver-build-env.sh

@@ -0,0 +1,19 @@
+# SUMMARY:
+# Builds and tests xapi using xenserver-build-env, which installs the
+# dependencies as RPMs.
+
+set -uex
+
+wget https://raw.githubusercontent.com/xenserver/xenserver-build-env/master/utils/travis-build-repo.sh
+
+# only run deploy.sh when the build succeeds
+env \
+  CONTAINER_NAME=build-env \
+  OCAMLRUNPARAM=b \
+  REPO_PACKAGE_NAME=xapi \
+  REPO_CONFIGURE_CMD=./configure \
+  REPO_BUILD_CMD=make \
+  REPO_TEST_CMD='make test' \
+  REPO_DOC_CMD='make doc-json' \
+  bash travis-build-repo.sh && \
+  ( ( test $TRAVIS_PULL_REQUEST == "false" && test $TRAVIS_BRANCH == "master" && bash deploy.sh ) || true )

.travis.yml

@@ -1,22 +1,17 @@
 language: c
+sudo: required
 services: docker
-install:
-        - wget https://raw.githubusercontent.com/xenserver/xenserver-build-env/master/utils/travis-build-repo.sh
-script: bash travis-build-repo.sh
-after_success:
-    - test $TRAVIS_PULL_REQUEST == "false" && test $TRAVIS_BRANCH == "master" && bash deploy.sh
-sudo: true
+script: bash ./.travis-$BUILD_METHOD.sh
 env:
     global:
-        - CONTAINER_NAME=build-env
-        - OCAMLRUNPARAM=b
-        - REPO_PACKAGE_NAME=xapi
-        - REPO_CONFIGURE_CMD=./configure
-        - REPO_BUILD_CMD=make
-        - REPO_TEST_CMD='make test'
-        - REPO_DOC_CMD='make doc-json'
+        # for BUILD_METHOD=xenserver-build-env
         - secure: tokxJl2litqu/T6UUwzkLRZzlbxnbYqVG2QRKKQz3tkIXyZHQWTS2NAyH7mwDgdBq2dDVSxAUxS1jWq/vGraX7MmbVz37Pz8wjykoIfIRtQuEx+REDAvAzWSw+1LTpUf7ZcI+F2SpgJrnH87uN5AAc220UqIx8TvAtGrita+2+o=
     matrix:
-        - OCAML_VERSION=4.02
+        - BUILD_METHOD=xenserver-build-env
+        - BUILD_METHOD=opam-coverage
 notifications:
     slack: citrix:BHYQZbI8m036ELU21gZil75Y
+matrix:
+    fast_finish: true
+    allow_failures:
+        - env: BUILD_METHOD=opam-coverage

Makefile

@@ -12,12 +12,12 @@ build: setup.data
 
 doc: setup.data build
 	$(SETUP) -doc $(DOCFLAGS)
-	./jsapi.native -destdir _build/ocaml/doc
+	./jsapi.native -destdir _build/ocaml/doc -templdir ocaml/doc/templates
 
 test: setup.data build
 	$(SETUP) -test $(TESTFLAGS)
 
-all:
+all: setup.ml
 	$(SETUP) -all $(ALLFLAGS)
 
 uninstall: setup.data
@@ -26,10 +26,10 @@ uninstall: setup.data
 reinstall: setup.data
 	$(SETUP) -reinstall $(REINSTALLFLAGS)
 
-clean:
+clean: setup.ml
 	$(SETUP) -clean $(CLEANFLAGS)
 
-distclean:
+distclean: setup.ml
 	$(SETUP) -distclean $(DISTCLEANFLAGS)
 
 setup.data: setup.ml
@@ -38,7 +38,7 @@ setup.data: setup.ml
 setup.ml: _oasis
 	oasis setup -setup-update dynamic
 
-configure:
+configure: setup.ml
 	$(SETUP) -configure $(CONFIGUREFLAGS)
 
 .PHONY: build doc test all install uninstall reinstall clean distclean configure
@@ -95,5 +95,4 @@ install: setup.data rbac_static.csv
 	mkdir -p $(DESTDIR)$(DOCDIR)/html/xenserver
 	cp -r -L _build/ocaml/doc/api $(DESTDIR)$(DOCDIR)/html/xenserver
 	cd ocaml/doc && cp *.js *.html *.css *.png $(DESTDIR)$(DOCDIR)/html/xenserver
-	cp ocaml/doc/xenserver/* $(DESTDIR)$(DOCDIR)/html/xenserver
-
+	cp _build/ocaml/doc/branding.js $(DESTDIR)$(DOCDIR)/html/xenserver/branding.js

README.markdown

@@ -1,6 +1,10 @@
 Xapi Project's XenAPI Management Toolstack
 ==========================================
 
+[![Build Status](https://travis-ci.org/xapi-project/xen-api.svg?branch=master)](https://travis-ci.org/xapi-project/xen-api)
+[![Coverage Status](https://coveralls.io/repos/github/xapi-project/xen-api/badge.svg?branch=master)](https://coveralls.io/github/xapi-project/xen-api?branch=master)
+[![Lines of Code](https://tokei.rs/b1/github/xapi-project/xen-api)](https://github.com/xapi-project/xen-api)
+
 Xen API (or xapi) is a management stack that configures and controls
 Xen-enabled hosts and resource pools, and co-ordinates resources
 within the pool. Xapi exposes the Xen API interface for many

_oasis

@@ -424,7 +424,8 @@ Executable jsapi
     xapi-consts,
     stdext,
     uuid,
-    gzip
+    gzip,
+    mustache
 
 
 ############################################################################

_tags

@@ -0,0 +1,3 @@
+# OASIS_START
+# OASIS_STOP
+<ocaml/xapi/cli_operations.ml>: warn(-52)

ocaml/database/db_cache_impl.ml

@@ -254,14 +254,24 @@ let process_structured_field_locked t (key,value) tblname fld objref proc_fn_sel
     let newval = match proc_fn_selector with
       | AddSet -> add_to_set key existing_str
       | RemoveSet -> remove_from_set key existing_str
-      | AddMap ->
+      | AddMap | AddMapLegacy ->
         begin
           try
-            add_to_map key value existing_str
+            (* We use the idempotent map add if we're using the non-legacy
+               process function, or if the global field 'idempotent_map' has
+               been set. By default, the Db calls on the master use the
+               legacy functions, but those on the slave use the new one.
+               This means xapi code should always assume idempotent_map is
+               true *)
+            let idempotent =
+              (proc_fn_selector = AddMap) || !Db_globs.idempotent_map
+            in
+            add_to_map ~idempotent key value existing_str
           with Duplicate ->
             error "Duplicate key in set or map: table %s; field %s; ref %s; key %s" tblname fld objref key;
             raise (Duplicate_key (tblname,fld,objref,key));
         end
+
       | RemoveMap -> remove_from_map key existing_str in
     write_field_locked t tblname objref fld newval
   with Not_found ->

ocaml/database/db_cache_types.ml

@@ -350,10 +350,10 @@ let remove_from_set key t =
   Schema.Value.Set (List.filter (fun x -> x <> key) t)
 
 exception Duplicate
-let add_to_map key value t =
+let add_to_map ~idempotent key value t =
   let t = Schema.Value.Unsafe_cast.pairs t in
-  if List.mem key (List.map fst t) then raise Duplicate;
-  Schema.Value.Pairs ((key, value) :: t)
+  if List.mem_assoc key t && (not idempotent || List.assoc key t <> value) then raise Duplicate;
+  Schema.Value.Pairs ((key, value) :: List.filter (fun (k, _) -> k <> key) t)
 
 let remove_from_map key t =
   let t = Schema.Value.Unsafe_cast.pairs t in
@@ -500,4 +500,5 @@ type structured_op_t =
   | RemoveSet
   | AddMap
   | RemoveMap
+  | AddMapLegacy
 [@@deriving rpc]

ocaml/database/db_cache_types.mli

@@ -146,7 +146,7 @@ end
 exception Duplicate
 val add_to_set : string -> Schema.Value.t -> Schema.Value.t
 val remove_from_set : string -> Schema.Value.t -> Schema.Value.t
-val add_to_map : string -> string -> Schema.Value.t -> Schema.Value.t
+val add_to_map : idempotent:bool -> string -> string -> Schema.Value.t -> Schema.Value.t
 val remove_from_map : string -> Schema.Value.t -> Schema.Value.t
 
 val set_field : string -> string -> string -> Schema.Value.t -> Database.t -> Database.t
@@ -169,5 +169,6 @@ type structured_op_t =
   | RemoveSet
   | AddMap
   | RemoveMap
+  | AddMapLegacy
 val structured_op_t_of_rpc: Rpc.t -> structured_op_t
 val rpc_of_structured_op_t: structured_op_t -> Rpc.t

ocaml/database/db_globs.ml

@@ -50,6 +50,9 @@ let static_vdis_dir = ref "/etc/xensource/static-vdis"
 (* Note the following has an equivalent in the xapi layer *)
 let http_limit_max_rpc_size = 300 * 1024 (* 300K *)
 
+(* add_to_map is idempotent *)
+let idempotent_map = ref false
+
 (** Dynamic configurations to be read whenever xapi (re)start *)
 
 let permanent_master_failure_retry_interval = ref 60.

ocaml/database/db_rpc_common_v1.ml

@@ -79,7 +79,9 @@ let marshall_structured_op x =
       AddSet -> "addset"
     | RemoveSet -> "removeset"
     | AddMap -> "addmap"
-    | RemoveMap -> "removemap" in
+    | RemoveMap -> "removemap"
+    | AddMapLegacy -> "addmap" (* Nb, we always use 'non-legacy' mode for remote access *)
+  in
   XMLRPC.To.string str
 let unmarshall_structured_op xml =
   match (XMLRPC.From.string xml) with
@@ -311,4 +313,3 @@ let unmarshall_read_records_where_response xml =
            [ref_xml; rec_xml] -> (XMLRPC.From.string ref_xml, unmarshall_read_record_response rec_xml)
          | _ -> raise DB_remote_marshall_error)
       xml_refs_and_recs_list
-

ocaml/database/db_rpc_common_v2.ml

@@ -34,6 +34,16 @@ module Request = struct
     | Read_records_where of string * Db_filter_types.expr
     | Process_structured_field of (string * string) * string * string * string * Db_cache_types.structured_op_t
   [@@deriving rpc]
+
+  (* Make sure the slave only ever uses the idempotent version *)
+  let rpc_of_t t =
+    let t' =
+      match t with
+      | Process_structured_field (a,b,c,d,Db_cache_types.AddMapLegacy) ->
+        Process_structured_field (a,b,c,d,Db_cache_types.AddMap)
+      | x -> x
+    in
+    rpc_of_t t'
 end
 
 module Response = struct