Merge pull request #249 from Opetushallitus/feature/OPHKIOS-71

OPHKIOS-71: Tutkintolistauksen optimointi

dev/resources/dev.edn

@@ -58,6 +58,7 @@
                                           :access-log #ig/ref :yki.middleware.access-log/with-logging}
 
  :yki.handler/exam-session-public        {:db             #ig/ref :duct.database/sql
+                                          :environment    #ig/ref :yki.env/environment
                                           :payment-config {:paytrail-host   "https://payment.paytrail.com/e2"
                                                            :yki-payment-uri "https://yki.untuvaopintopolku.fi/yki/payment"
                                                            :amount          {:PERUS     153
@@ -93,5 +94,7 @@
  :yki.handler/organizer                  {:db         #ig/ref :duct.database/sql
                                           :url-helper #ig/ref :yki.util/url-helper
                                           :access-log #ig/ref :yki.middleware.access-log/with-logging
-                                          :auth       #ig/ref :yki.middleware.no-auth/with-authentication}}
+                                          :auth       #ig/ref :yki.middleware.no-auth/with-authentication}
+
+ :yki.env/environment {:environment "dev"}}
 

dev/src/profiling.clj

@@ -0,0 +1,17 @@
+(ns profiling
+  (:require [clj-async-profiler.core :as prof]))
+
+(comment
+  ; Profile a single expression.
+  (prof/profile (dotimes [i 10000] (reduce + 0 (range i))))
+  ; Or run profiler for a given time (in seconds)
+  (prof/profile-for 10)
+  ; Or start profiling session manually, do some computation, and then end the session manually.
+  (prof/start)
+  (do '(...))
+  (prof/stop)
+  ; View the results from your filesystem (by default, under /tmp/clj-async-profiler/results/)
+  ; or start a local web UI:
+  (prof/serve-ui 8008)
+
+  )

oph-configuration/config.edn.template

@@ -52,7 +52,8 @@
                                                  :url-helper #ig/ref :yki.util/url-helper
                                                  :retry-duration-in-days 0}
 
-:yki.handler/exam-session-public {:db #ig/ref :duct.database/sql
+:yki.handler/exam-session-public {:db             #ig/ref :duct.database/sql
+                                  :environment    #ig/ref :yki.env/environment
                                   :payment-config {:amount {:PERUS {{yki_exam_session_fee_basic}}
                                                             :KESKI {{yki_exam_session_fee_intermediate}}
                                                             :YLIN {{yki_exam_session_fee_advanced}}
@@ -83,6 +84,7 @@
                       :cookie-attrs {:max-age 3600
                                      :http-only true
                                      :secure true
-                                     :path "/yki"}}}}
-  :duct.profile/prod  {}
-}
+                                     :path "/yki"}}}
+  :yki.env/environment {:environment "{{yki_environment}}"}}
+
+  :duct.profile/prod  {}}

project.clj

@@ -83,13 +83,17 @@
                   :repl-options {:init-ns user}}
    :uberjar      {:aot :all}
    :profiles/dev {}
-   :project/dev  {:source-paths   ["dev/src"]
+   :project/dev  {:jvm-opts ["-Djdk.attach.allowAttachSelf"
+                             "-XX:+UnlockDiagnosticVMOptions"
+                             "-XX:+DebugNonSafepoints"]
+                  :source-paths   ["dev/src"]
                   :resource-paths ["dev/resources"]
                   :dependencies   [[integrant/repl "0.3.3"]
                                    [eftest "0.6.0"]
                                    [peridot "0.5.4"]
                                    [se.haleby/stub-http "0.2.14"]
                                    [com.opentable.components/otj-pg-embedded "1.0.3"]
-                                   [kerodon "0.9.1"]]
+                                   [kerodon "0.9.1"]
+                                   [com.clojure-goes-fast/clj-async-profiler "1.2.0"]]
                   :managed-dependencies [[org.testcontainers/testcontainers "1.19.7"]
                                          [org.testcontainers/postgresql "1.19.7"]]}})

resources/yki/migrations/027-create-index-on-exam_date.0.1.0.up.sql

@@ -0,0 +1,2 @@
+CREATE INDEX IF NOT EXISTS exam_date_exam_date_idx ON exam_date (exam_date ASC);
+

resources/yki/queries.sql

@@ -304,6 +304,58 @@ INNER JOIN organizer o ON es.organizer_id = o.id
 WHERE o.oid = :oid;
 
 -- name: select-exam-sessions
+SELECT
+  e.id,
+  language_code,
+  level_code,
+  ed.exam_date AS session_date,
+  e.max_participants,
+  ed.registration_start_date,
+  ed.registration_end_date,
+  e.post_admission_activated_at,
+  e.post_admission_quota,
+  e.post_admission_active,
+  ed.post_admission_start_date,
+  ed.post_admission_end_date,
+  e.office_oid,
+  e.published_at,
+  (SELECT COUNT(1)
+   FROM exam_session_queue
+   WHERE exam_session_id = e.id) as queue,
+  ((SELECT COUNT(1)
+    FROM exam_session_queue
+    WHERE exam_session_id = e.id) >= 50) as queue_full,
+  (SELECT COUNT(1)
+   FROM registration re
+   WHERE re.exam_session_id = e.id AND re.kind = 'ADMISSION' AND re.state IN ('COMPLETED', 'SUBMITTED', 'STARTED')) as participants,
+  (SELECT COUNT(1)
+   FROM registration re
+   WHERE re.exam_session_id = e.id AND re.kind = 'POST_ADMISSION' AND re.state in ('COMPLETED', 'SUBMITTED', 'STARTED')) as pa_participants,
+  o.oid as organizer_oid,
+  (SELECT array_to_json(array_agg(loc))
+   FROM (SELECT
+           name,
+           street_address,
+           post_office,
+           zip,
+           other_location_info,
+           extra_information,
+           lang
+         FROM exam_session_location
+         WHERE exam_session_id = e.id) loc
+  ) as location,
+  (SELECT post_admission_enabled FROM exam_date WHERE id = e.exam_date_id) AS post_admission_enabled,
+  (within_dt_range(now(), ed.registration_start_date, ed.registration_end_date)
+      OR (within_dt_range(now(), ed.post_admission_start_date, ed.post_admission_end_date) AND e.post_admission_active = TRUE AND ed.post_admission_enabled = TRUE)) as open,
+  (now() AT TIME ZONE 'Europe/Helsinki' < (date_trunc('day', ed.registration_end_date AT TIME ZONE 'Europe/Helsinki') + time '16:00')) AS upcoming_admission,
+  (e.post_admission_active = TRUE AND ed.post_admission_enabled = TRUE AND (now() AT TIME ZONE 'Europe/Helsinki' < (date_trunc('day', ed.post_admission_end_date AT TIME ZONE 'Europe/Helsinki') + time '16:00'))) AS upcoming_post_admission
+FROM exam_session e
+INNER JOIN organizer o ON e.organizer_id = o.id
+INNER JOIN exam_date ed ON e.exam_date_id = ed.id
+WHERE ed.exam_date >= :from
+ORDER BY ed.exam_date ASC;
+
+-- name: select-exam-sessions-for-oid
 SELECT
   e.id,
   language_code,
@@ -368,7 +420,7 @@ FROM exam_session e
 INNER JOIN organizer o ON e.organizer_id = o.id
 INNER JOIN exam_date ed ON e.exam_date_id = ed.id
 WHERE ed.exam_date >= COALESCE(:from, ed.exam_date)
-  AND o.oid = COALESCE(:oid, o.oid)
+  AND o.oid = :oid
 ORDER BY ed.exam_date ASC;
 
 -- name: select-exam-session-by-id

src/yki/boundary/exam_session_db.clj

@@ -69,8 +69,10 @@
   (get-exam-session-participants [db id oid])
   (get-completed-exam-session-participants [db id])
   (get-exam-sessions-to-be-synced [db retry-duration])
-  (get-exam-sessions [db oid from]
-    "Get exam sessions by optional oid and from arguments")
+  (get-exam-sessions [db from]
+    "Get exam sessions with exam date at least 'from'")
+  (get-exam-sessions-for-oid [db oid from]
+    "Get exam sessions by oid and with (optional) exam date at least 'from'")
   (get-exam-sessions-with-queue [db])
   (get-email-added-to-queue? [db email exam-session-id])
   (add-to-exam-session-queue! [db email lang exam-session-id])
@@ -168,9 +170,11 @@
     (q/select-exam-session-participants spec {:id id :oid oid}))
   (get-completed-exam-session-participants [{:keys [spec]} id]
     (q/select-completed-exam-session-participants spec {:id id}))
-  (get-exam-sessions [{:keys [spec]} oid from]
-    (q/select-exam-sessions spec {:oid  oid
-                                  :from from}))
+  (get-exam-sessions [{:keys [spec]} from]
+    (q/select-exam-sessions spec {:from from}))
+  (get-exam-sessions-for-oid [{:keys [spec]} oid from]
+    (q/select-exam-sessions-for-oid spec {:oid  oid
+                                          :from from}))
   (get-email-added-to-queue? [{:keys [spec]} email exam-session-id]
     (int->boolean (:count (first (q/select-email-added-to-queue spec {:email           email
                                                                       :exam_session_id exam-session-id})))))

src/yki/env.clj

@@ -0,0 +1,12 @@
+(ns yki.env
+  (:require
+    [clojure.spec.alpha :as s]
+    [integrant.core :as ig]
+    [yki.spec :as ys]))
+
+(defn- ->environment [environment]
+  {:post [(s/valid? ::ys/environment %)]}
+  (keyword environment))
+
+(defmethod ig/init-key ::environment [_ {:keys [environment]}]
+  (->environment environment))

src/yki/handler/exam_session.clj

@@ -44,7 +44,7 @@
         :query-params [{from :- ::ys/date-type nil}]
         :return ::ys/exam-sessions-response
         (let [from-date (string->date from)]
-          (response {:exam_sessions (exam-session-db/get-exam-sessions db oid from-date)})))
+          (response {:exam_sessions (exam-session-db/get-exam-sessions-for-oid db oid from-date)})))
 
       (POST "/" request
         :body [exam-session ::ys/exam-session]

src/yki/handler/exam_session_public.clj

@@ -1,6 +1,7 @@
 (ns yki.handler.exam-session-public
   (:require
     [clj-time.core :as t]
+    [clojure.spec.alpha :as s]
     [clojure.tools.logging :as log]
     [compojure.api.sweet :refer [context GET POST]]
     [integrant.core :as ig]
@@ -13,18 +14,21 @@
   [payment-config exam-session]
   (get-in payment-config [:amount (keyword (:level_code exam-session))]))
 
-(defmethod ig/init-key :yki.handler/exam-session-public [_ {:keys [db payment-config]}]
-  {:pre [(some? db) (some? payment-config)]}
+(defmethod ig/init-key :yki.handler/exam-session-public [_ {:keys [db environment payment-config]}]
+  {:pre [(some? db) (s/valid? ::ys/environment environment) (some? payment-config)]}
   (context routing/exam-session-public-api-root []
-    :coercion :spec
+    :coercion
+    (when-not (= :prod environment)
+      :spec)
     (GET "/" []
       :return ::ys/exam-sessions-response
       (let [from-date     (t/now)
-            exam-sessions (exam-session-db/get-exam-sessions db nil from-date)
+            exam-sessions (exam-session-db/get-exam-sessions db from-date)
             with-fee      (map #(assoc % :exam_fee (get-exam-fee payment-config %)) exam-sessions)]
         (ok {:exam_sessions with-fee})))
 
     (context "/:id" []
+      :coercion :spec
       (GET "/" []
         :return ::ys/exam-session
         :path-params [id :- ::ys/id]

src/yki/spec.clj

@@ -460,3 +460,5 @@
 (s/def ::open_registrations (s/coll-of ::open-registration) )
 
 (s/def ::user-open-registrations-response (s/keys :req-un [::open_registrations]))
+
+(s/def ::environment #{:dev :qa :prod})

test/yki/handler/base_test.clj

@@ -12,6 +12,7 @@
     [peridot.core :as peridot]
     [yki.boundary.cas]
     [yki.embedded-db :as embedded-db]
+    [yki.env]
     [yki.handler.auth]
     [yki.handler.exam-date]
     [yki.handler.exam-session]
@@ -95,6 +96,9 @@
 (defn db []
   (sql/->Boundary @embedded-db/conn))
 
+(defn environment [env]
+  (ig/init-key :yki.env/environment {:environment env}))
+
 (defn auth [url-helper]
   (ig/init-key :yki.middleware.auth/with-authentication
                {:url-helper     url-helper

test/yki/handler/exam_session_public_test.clj

@@ -15,12 +15,39 @@
 (use-fixtures :once (join-fixtures [embedded-db/with-postgres embedded-db/with-migration]))
 (use-fixtures :each embedded-db/with-transaction)
 
-(defn- send-request [request]
-  (let [handler (api (ig/init-key :yki.handler/exam-session-public {:db             (base/db)
-                                                                    :payment-config {:amount {:PERUS 100
-                                                                                              :KESKI 123
-                                                                                              :YLIN  160}}}))]
-    (handler request)))
+(defn- send-request
+  ([request]
+   (send-request request (base/environment "prod")))
+  ([request env]
+   (let [handler (api (ig/init-key :yki.handler/exam-session-public {:db             (base/db)
+                                                                     :environment    env
+                                                                     :payment-config {:amount {:PERUS 100
+                                                                                               :KESKI 123
+                                                                                               :YLIN  160}}}))]
+     (handler request))))
+
+(deftest get-exam-sessions-test-with-coercions-enabled
+  (base/insert-base-data)
+  ; Exam sessions listing endpoint has coercions turned on only in dev and qa environments due to performance issues.
+  (let [env (base/environment "dev")]
+    (testing "get exam sessions endpoint should return 200 along with a list of upcoming exam sessions"
+      (let [request       (mock/request :get routing/exam-session-public-api-root)
+            response      (send-request request env)
+            response-body (base/body-as-json response)
+            exam-sessions (response-body "exam_sessions")]
+        (is (= (:status response) 200))
+        ; Initially, the single exam session is in the past
+        (is (= (count exam-sessions) 0)))
+      ; Update exam session to future and verify that endpoint now returns exam sessions
+      (let [future-exam-date (base/select-one "SELECT id from exam_date where exam_date >= current_timestamp;")
+            _                (base/execute! (str "UPDATE exam_session SET exam_date_id=" (:id future-exam-date) ";"))
+            request          (mock/request :get routing/exam-session-public-api-root)
+            response         (send-request request env)
+            response-body    (base/body-as-json response)
+            exam-sessions    (response-body "exam_sessions")]
+        (is (= (:status response) 200))
+        (is (= (count exam-sessions) 1))
+        (is (= ((first exam-sessions) "exam_fee") 100))))))
 
 (deftest get-exam-sessions-test
   (base/insert-base-data)