Skip to content

Commit

Permalink
Merge pull request #1665 from rbrw/ticket/master/pdb-2006-move-dual-d…
Browse files Browse the repository at this point in the history 
…b-test-to-postgres

(PDB-2006) Move the facts dual-db test to postgres
  • Loading branch information
@ajroetker
ajroetker committed Oct 8, 2015
2 parents 1d37ccc + cf0d4f0 commit 982d0ec
Show file tree
Hide file tree
Showing 7 changed files with 127 additions and 109 deletions.
4 changes: 4 additions & 0 deletions ext/travisci/test.sh
View file
Expand Up @@ -11,10 +11,14 @@ case "$PDB_TEST_LANG" in
case "$PDB_TEST_DB" in
postgres)
psql -c 'create database puppetdb_test;' -U postgres
psql -c 'create database puppetdb2_test;' -U postgres
PUPPETDB_DBTYPE=postgres \
PUPPETDB_DBUSER=postgres \
PUPPETDB_DBSUBNAME=//127.0.0.1:5432/puppetdb_test \
PUPPETDB_DBPASSWORD= \
PUPPETDB2_DBUSER=postgres \
PUPPETDB2_DBSUBNAME=//127.0.0.1:5432/puppetdb2_test \
PUPPETDB2_DBPASSWORD= \
lein2 test
;;
hsqldb)
Expand Down
8 changes: 4 additions & 4 deletions test/puppetlabs/puppetdb/acceptance/node_ttl.clj
View file
Expand Up @@ -13,7 +13,7 @@
(tu/with-coordinated-fn run-purge-nodes puppetlabs.puppetdb.cli.services/purge-nodes!
(tu/with-coordinated-fn run-expire-nodes puppetlabs.puppetdb.cli.services/auto-expire-nodes!
(svc-utils/call-with-puppetdb-instance
(-> (svc-utils/create-config)
(-> (svc-utils/create-config-and-clear-db!)
(assoc-in [:database :node-ttl] "1s")
(assoc-in [:database :node-purge-ttl] "1s"))
(fn []
Expand All @@ -40,7 +40,7 @@
(deftest test-zero-gc-interval
(with-redefs [puppetlabs.puppetdb.cli.services/purge-nodes! (tu/mock-fn)]
(svc-utils/call-with-puppetdb-instance
(-> (svc-utils/create-config)
(-> (svc-utils/create-config-and-clear-db!)
(assoc-in [:database :node-ttl] "0s")
(assoc-in [:database :report-ttl] "0s")
(assoc-in [:database :node-purge-ttl] "1s")
Expand All @@ -54,7 +54,7 @@
(with-redefs [puppetlabs.puppetdb.cli.services/purge-nodes! (tu/mock-fn)
puppetlabs.puppetdb.cli.services/compress-dlo! (tu/mock-fn)]
(svc-utils/call-with-puppetdb-instance
(-> (svc-utils/create-config)
(-> (svc-utils/create-config-and-clear-db!)
(assoc-in [:database :gc-interval] 0)
(assoc-in [:database :dlo-compression-interval] 1))
(fn []
Expand All @@ -65,7 +65,7 @@
(with-redefs [puppetlabs.puppetdb.cli.services/purge-nodes! (tu/mock-fn)
puppetlabs.puppetdb.cli.services/compress-dlo! (tu/mock-fn)]
(svc-utils/call-with-puppetdb-instance
(-> (svc-utils/create-config)
(-> (svc-utils/create-config-and-clear-db!)
(assoc-in [:database :gc-interval] 0))
(fn []
(Thread/sleep 500)
Expand Down
5 changes: 3 additions & 2 deletions test/puppetlabs/puppetdb/cli/import_export_roundtrip_test.clj
View file
Expand Up @@ -73,7 +73,8 @@
"--port" (str (:port svc-utils/*base-url*)))))

(svc-utils/call-with-single-quiet-pdb-instance
(assoc-in (svc-utils/create-config) [:command-processing :max-frame-size] 1024)
(assoc-in (svc-utils/create-config-and-clear-db!)
[:command-processing :max-frame-size] 1024)
(fn []
(is (empty? (get-nodes)))

Expand All @@ -90,7 +91,7 @@

(deftest test-max-frame-size
(svc-utils/call-with-single-quiet-pdb-instance
(-> (svc-utils/create-config)
(-> (svc-utils/create-config-and-clear-db!)
(assoc-in [:command-processing :max-frame-size] 1024))
(fn []
(is (empty? (get-nodes)))
Expand Down
2 changes: 1 addition & 1 deletion test/puppetlabs/puppetdb/cli/services_test.clj
View file
Expand Up @@ -133,7 +133,7 @@
cert-config {:ssl-cert "test-resources/localhost.pem"
:ssl-key "test-resources/localhost.key"
:ssl-ca-cert "test-resources/ca.pem"}
config (-> (svc-utils/create-config)
config (-> (svc-utils/create-config-and-clear-db!)
(assoc :jetty (merge cert-config
{:ssl-port 0
:ssl-host "0.0.0.0"
Expand Down
135 changes: 65 additions & 70 deletions test/puppetlabs/puppetdb/http/facts_test.clj
View file
Expand Up @@ -14,7 +14,9 @@
[puppetlabs.puppetdb.http :as http]
[puppetlabs.puppetdb.http.server :as server]
[puppetlabs.puppetdb.jdbc :as jdbc]
[puppetlabs.puppetdb.testutils :refer [get-request
[puppetlabs.puppetdb.testutils :refer [available-postgres-configs
clear-db-for-testing!
get-request
assert-success!
paged-results
paged-results*
Expand Down Expand Up @@ -451,14 +453,6 @@
:product-name "puppetdb"))
nil)))

(defmacro with-shutdown-after
[dbs & body]
`(do ~@body)
`(doseq [db# ~dbs]
(jdbc/with-db-connection db#
(jdbc/do-commands "SHUTDOWN"))
(.close (:datasource db#))))

(deftestseq fact-queries
[[version endpoint] facts-endpoints
method [:get :post]]
Expand Down Expand Up @@ -581,73 +575,74 @@
(is (= body msg))
(is (= status http/status-bad-request)))))))

(deftestseq ^{:postgres false} two-database-fact-query-config
(deftestseq ^{:hsqldb false} two-database-fact-query-config
[[version endpoint] facts-endpoints
method [:get :post]]

(let [read-db-config (create-hsqldb-map)
write-db-config (create-hsqldb-map)
(assert (> (count available-postgres-configs) 1))
(let [read-db-config (available-postgres-configs 0)
write-db-config (available-postgres-configs 1)
config (-> (svc-utils/create-config)
(assoc :read-database read-db-config)
(assoc :database write-db-config))
read-db (-> read-db-config
defaulted-read-db-config
(init-db true))
write-db (-> write-db-config
defaulted-write-db-config
(init-db false))]

(with-shutdown-after [read-db write-db]
(svc-utils/call-with-puppetdb-instance
config
(fn []
(let [pdb (get-service svc-utils/*server* :PuppetDBServer)
shared-globals (cli-svc/shared-globals pdb)
read-db (:scf-read-db shared-globals)
write-db (:scf-write-db shared-globals)
one-db-app (test-app write-db)
two-db-app (test-app read-db write-db)
facts1 {"domain" "testing.com"
"hostname" "foo1"
"kernel" "Linux"
"operatingsystem" "Debian"
"some_version" "1.3.7+build.11.e0f985a"
"uptime_seconds" "4000"}]

(jdbc/with-transacted-connection write-db
(scf-store/add-certname! "foo1")
(scf-store/add-facts! {:certname "foo1"
:values facts1
:timestamp (now)
:environment "DEV"
:producer_timestamp (now)}))

(testing "queries only use the read database"
(let [request (get-request endpoint (json/parse-string nil))
{:keys [status body headers]} (two-db-app request)]
(is (= (headers "Content-Type") c-t))
;; Environments endpoint will return a proper JSON
;; error with a 404, as opposed to an empty array.
(if (= endpoint "/v4/environments/DEV/facts")
(do
(is (= {:error "No information is known about environment DEV"} (json/parse-string body true)))
(is (= status http/status-not-found)))
(do
(is (empty? (json/parse-stream (io/reader body) true)))
(is (= status http/status-ok))))))

(testing "config with only a single database returns results"
(let [request (get-request endpoint (json/parse-string nil))
{:keys [status body headers]} (one-db-app request)]
(is (= status http/status-ok))
(is (= (headers "Content-Type") c-t))
(is (= [{:certname "foo1" :name "domain" :value "testing.com" :environment "DEV"}
{:certname "foo1" :name "hostname" :value "foo1" :environment "DEV"}
{:certname "foo1" :name "kernel" :value "Linux" :environment "DEV"}
{:certname "foo1" :name "operatingsystem" :value "Debian" :environment "DEV"}
{:certname "foo1" :name "some_version" :value "1.3.7+build.11.e0f985a" :environment "DEV"}
{:certname "foo1" :name "uptime_seconds" :value "4000" :environment "DEV"}]
(sort-by :name (json/parse-stream (io/reader body) true))))))))))))
read-config (-> read-db-config defaulted-read-db-config)
write-config (-> write-db-config defaulted-write-db-config)]
(clear-db-for-testing! read-config)
(clear-db-for-testing! write-config)
(init-db read-config true)
(init-db write-config false)
(svc-utils/call-with-puppetdb-instance
config
(fn []
(let [pdb (get-service svc-utils/*server* :PuppetDBServer)
shared-globals (cli-svc/shared-globals pdb)
read-db (:scf-read-db shared-globals)
write-db (:scf-write-db shared-globals)
one-db-app (test-app write-db)
two-db-app (test-app read-db write-db)
facts1 {"domain" "testing.com"
"hostname" "foo1"
"kernel" "Linux"
"operatingsystem" "Debian"
"some_version" "1.3.7+build.11.e0f985a"
"uptime_seconds" "4000"}]

(jdbc/with-transacted-connection write-db
(scf-store/add-certname! "foo1")
(scf-store/add-facts! {:certname "foo1"
:values facts1
:timestamp (now)
:environment "DEV"
:producer_timestamp (now)}))

(testing "queries only use the read database"
(let [request (get-request endpoint)
{:keys [status body headers]} (two-db-app request)]
(is (= (headers "Content-Type") c-t))
;; Environments endpoint will return a proper JSON
;; error with a 404, as opposed to an empty array.
(if (= endpoint "/v4/environments/DEV/facts")
(do
(is (= {:error "No information is known about environment DEV"}
(json/parse-string body true)))
(is (= status http/status-not-found)))
(do
(is (empty? (json/parse-stream (io/reader body) true)))
(is (= status http/status-ok))))))

(testing "config with only a single database returns results"
(let [request (get-request endpoint)
{:keys [status body headers]} (one-db-app request)]
(is (= status http/status-ok))
(is (= (headers "Content-Type") c-t))
(is (= [{:certname "foo1" :name "domain" :value "testing.com" :environment "DEV"}
{:certname "foo1" :name "hostname" :value "foo1" :environment "DEV"}
{:certname "foo1" :name "kernel" :value "Linux" :environment "DEV"}
{:certname "foo1" :name "operatingsystem" :value "Debian" :environment "DEV"}
{:certname "foo1" :name "some_version" :value "1.3.7+build.11.e0f985a" :environment "DEV"}
{:certname "foo1" :name "uptime_seconds" :value "4000" :environment "DEV"}]
(sort-by :name (json/parse-stream (io/reader body)
true)))))))))))

(defn test-paged-results
[endpoint query limit total include_total]
Expand Down
43 changes: 25 additions & 18 deletions test/puppetlabs/puppetdb/testutils.clj
View file
Expand Up @@ -29,20 +29,26 @@
(java.util.UUID/randomUUID)
";hsqldb.tx=mvcc;sql.syntax_pgs=true")})

(def postgres-map
{:classname "org.postgresql.Driver"
:subprotocol "postgresql"
:subname (env :puppetdb-dbsubname "//127.0.0.1:5432/puppetdb_test")
:user (env :puppetdb-dbuser "puppetdb")
:password (env :puppetdb-dbpassword "puppetdb")})
(def available-postgres-configs
[{:classname "org.postgresql.Driver"
:subprotocol "postgresql"
:subname (env :puppetdb-dbsubname "//127.0.0.1:5432/puppetdb_test")
:user (env :puppetdb-dbuser "puppetdb")
:password (env :puppetdb-dbpassword "puppetdb")}
{:classname "org.postgresql.Driver"
:subprotocol "postgresql"
:subname (env :puppetdb2-dbsubname "//127.0.0.1:5432/puppetdb2_test")
:user (env :puppetdb2-dbuser "puppetdb")
:password (env :puppetdb2-dbpassword "puppetdb")}])

(def hsqldb-map (create-hsqldb-map))

(def testing-db-type (env :puppetdb-dbtype "postgres"))

(defn test-db
[]
(if (= testing-db-type "postgres")
postgres-map
(first available-postgres-configs)
hsqldb-map))

(defn drop-table!
Expand All @@ -58,21 +64,22 @@
(jdbc/do-commands (format "DROP SEQUENCE IF EXISTS %s" sequence-name)))

(defn clear-db-for-testing!
"Completely clears the database, dropping all puppetdb tables and other objects
that exist within it. Expects to be called from within a db binding. You
"Completely clears the database specified by config (or the current
database), dropping all puppetdb tables and other objects that exist
within it. Expects to be called from within a db binding. You
Exercise extreme caution when calling this function!"
[]
(jdbc/do-commands "DROP SCHEMA IF EXISTS pdbtestschema CASCADE")
(doseq [table-name (cons "test" (sutils/sql-current-connection-table-names))]
(drop-table! table-name))
(doseq [sequence-name (cons "test" (sutils/sql-current-connection-sequence-names))]
(drop-sequence! sequence-name)))
([config]
(jdbc/with-db-connection config (clear-db-for-testing!)))
([]
(jdbc/do-commands "DROP SCHEMA IF EXISTS pdbtestschema CASCADE")
(doseq [table-name (cons "test" (sutils/sql-current-connection-table-names))]
(drop-table! table-name))
(doseq [sequence-name (cons "test" (sutils/sql-current-connection-sequence-names))]
(drop-sequence! sequence-name))))

(defn clean-db-map
([] (clean-db-map (test-db)))
([db-config]
(jdbc/with-db-connection db-config (clear-db-for-testing!))
db-config))
([db-config] (doto db-config clear-db-for-testing!)))

(defmacro without-jmx
"Disable ActiveMQ's usage of JMX. If you start two AMQ brokers in
Expand Down
39 changes: 25 additions & 14 deletions test/puppetlabs/puppetdb/testutils/services.clj
View file
Expand Up @@ -36,15 +36,23 @@
(def ^:dynamic *base-url* nil) ; Will not have a :version.

(defn create-config
"Creates a default config, populated with a temporary vardir and
a fresh hypersql instance"
"Returns a default config that refers to the default database, and
a temporary vardir."
[]
{:nrepl {}
:global {:vardir (testutils/temp-dir)}
:jetty {:port 0}
:database (testutils/clean-db-map)
:database (testutils/test-db)
:command-processing {}})

(defn create-config-and-clear-db!
"Clears the default database and returns a default config that
refers to that database and a temporary vardir."
[]
(let [cfg (create-config)]
(testutils/clear-db-for-testing! (:database cfg))
cfg))

(defn open-port-num
"Returns a currently open port number"
[]
Expand Down Expand Up @@ -74,18 +82,19 @@
#'config-service])

(defn call-with-puppetdb-instance
"Stands up a puppetdb instance with `config`, tears down once `f` returns.
`services` is a seq of additional services that should be started in
addition to the core PuppetDB services. Binds *server* and
*base-url* to refer to the instance. `attempts` indicates how many
failed attempts should be made to bind to an HTTP port before giving
up, defaults to 10."
([f] (call-with-puppetdb-instance (create-config) f))
"Stands up a puppetdb instance with the specified config, calls f,
and then tears the instance down, binding *server* to the instance
and *base-url* to the instance's URL during the execution of f.
Starts any specified services in addition to the core PuppetDB
services, and tries to bind to an HTTP port up to bind-attempts
times (default 10) before giving up. If a config is not specified,
one will be generated by create-config-and-clear-db!."
([f] (call-with-puppetdb-instance (create-config-and-clear-db!) f))
([config f] (call-with-puppetdb-instance config default-services f))
([config services f]
(call-with-puppetdb-instance config services 10 f))
([config services attempts f]
(when (zero? attempts)
([config services bind-attempts f]
(when (zero? bind-attempts)
(throw (RuntimeException. "Repeated attempts to bind port failed, giving up")))
(let [config (-> config
conf/adjust-and-validate-tk-config
Expand All @@ -106,8 +115,10 @@
*base-url* base-url]
(f)))
(catch java.net.BindException e
(log/errorf e "Error occured when Jetty tried to bind to port %s, attempt #%s" port attempts)
(call-with-puppetdb-instance config services (dec attempts) f))))))
(log/errorf e "Error occured when Jetty tried to bind to port %s, attempt #%s"
port bind-attempts)
(call-with-puppetdb-instance config services (dec bind-attempts)
f))))))

(defn pdb-query-url []
(assoc *base-url* :prefix "/pdb/query" :version :v4))
Expand Down

0 comments on commit 982d0ec

Please sign in to comment.