PostgresSQL DB v15 and health check endpoint

Dockerfile

@@ -159,3 +159,4 @@ RUN mkdir -p data share keys logs /tmp/supervisord static
 EXPOSE 8080
 ENTRYPOINT ["/usr/bin/supervisord"]
 CMD ["-c", "supervisord/all.conf"]
+HEALTHCHECK --start-period=30s CMD python3 ${HOME}/manage.py health_check

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "7.2.2",
+  "version": "7.3.0",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "src/api.ts",
   "scripts": {

cvat-core/src/api-implementation.ts

@@ -107,6 +107,16 @@ export default function implementAPI(cvat) {
         return result;
     };
 
+    cvat.server.healthCheck.implementation = async (
+        maxRetries = 1,
+        checkPeriod = 3000,
+        requestTimeout = 5000,
+        progressCallback = undefined,
+    ) => {
+        const result = await serverProxy.server.healthCheck(maxRetries, checkPeriod, requestTimeout, progressCallback);
+        return result;
+    };
+
     cvat.server.request.implementation = async (url, data) => {
         const result = await serverProxy.server.request(url, data);
         return result;

cvat-core/src/api.ts

@@ -257,6 +257,26 @@ function build() {
                 const result = await PluginRegistry.apiWrapper(cvat.server.authorized);
                 return result;
             },
+            /**
+             * Method allows to health check the server
+             * @method healthCheck
+             * @async
+             * @memberof module:API.cvat.server
+             * @param {number} requestTimeout
+             * @returns {Object | undefined} response data if exist
+             * @throws {module:API.cvat.exceptions.PluginError}
+             * @throws {module:API.cvat.exceptions.ServerError}
+             */
+            async healthCheck(maxRetries = 1, checkPeriod = 3000, requestTimeout = 5000, progressCallback = undefined) {
+                const result = await PluginRegistry.apiWrapper(
+                    cvat.server.healthCheck,
+                    maxRetries,
+                    checkPeriod,
+                    requestTimeout,
+                    progressCallback,
+                );
+                return result;
+            },
             /**
              * Method allows to do requests via cvat-core with authorization headers
              * @method request

cvat-core/src/server-proxy.ts

@@ -468,6 +468,29 @@ async function authorized() {
     return true;
 }
 
+async function healthCheck(maxRetries, checkPeriod, requestTimeout, progressCallback, attempt = 0) {
+    const { backendAPI } = config;
+    const url = `${backendAPI}/server/health/?format=json`;
+
+    if (progressCallback) {
+        progressCallback(`${attempt}/${attempt + maxRetries}`);
+    }
+
+    return Axios.get(url, {
+        proxy: config.proxy,
+        timeout: requestTimeout,
+    })
+        .then((response) => response.data)
+        .catch((errorData) => {
+            if (maxRetries > 0) {
+                return new Promise((resolve) => setTimeout(resolve, checkPeriod))
+                    .then(() => healthCheck(maxRetries - 1, checkPeriod,
+                        requestTimeout, progressCallback, attempt + 1));
+            }
+            throw generateError(errorData);
+        });
+}
+
 async function serverRequest(url, data) {
     try {
         return (
@@ -2227,6 +2250,7 @@ export default Object.freeze({
         requestPasswordReset,
         resetPassword,
         authorized,
+        healthCheck,
         register,
         request: serverRequest,
         userAgreements,

cvat-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.44.4",
+  "version": "1.45.0",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

cvat-ui/src/components/cvat-app.tsx

@@ -11,6 +11,8 @@ import Layout from 'antd/lib/layout';
 import Modal from 'antd/lib/modal';
 import notification from 'antd/lib/notification';
 import Spin from 'antd/lib/spin';
+import { DisconnectOutlined } from '@ant-design/icons';
+import Space from 'antd/lib/space';
 import Text from 'antd/lib/typography/Text';
 import 'antd/dist/antd.css';
 
@@ -64,6 +66,7 @@ import showPlatformNotification, {
     showUnsupportedNotification,
 } from 'utils/platform-checker';
 import '../styles.scss';
+import consts from 'consts';
 import EmailConfirmationPage from './email-confirmation-pages/email-confirmed';
 import EmailVerificationSentPage from './email-confirmation-pages/email-verification-sent';
 import IncorrectEmailConfirmationPage from './email-confirmation-pages/incorrect-email-confirmation';
@@ -103,10 +106,23 @@ interface CVATAppProps {
     isModelPluginActive: boolean;
 }
 
-class CVATApplication extends React.PureComponent<CVATAppProps & RouteComponentProps> {
+interface CVATAppState {
+    healthIinitialized: boolean;
+    backendIsHealthy: boolean;
+}
+
+class CVATApplication extends React.PureComponent<CVATAppProps & RouteComponentProps, CVATAppState> {
+    constructor(props: CVATAppProps & RouteComponentProps) {
+        super(props);
+
+        this.state = {
+            healthIinitialized: false,
+            backendIsHealthy: false,
+        };
+    }
     public componentDidMount(): void {
         const core = getCore();
-        const { verifyAuthorized, history, location } = this.props;
+        const { history, location } = this.props;
         // configure({ ignoreRepeatedEventsWhenKeyHeldDown: false });
 
         // Logger configuration
@@ -121,7 +137,46 @@ class CVATApplication extends React.PureComponent<CVATAppProps & RouteComponentP
             customWaViewHit(_location.pathname, _location.search, _location.hash);
         });
 
-        verifyAuthorized();
+        const {
+            HEALH_CHECK_RETRIES, HEALTH_CHECK_PERIOD, HEALTH_CHECK_REQUEST_TIMEOUT, UPGRADE_GUIDE_URL,
+        } = consts;
+        core.server.healthCheck(
+            HEALH_CHECK_RETRIES,
+            HEALTH_CHECK_PERIOD,
+            HEALTH_CHECK_REQUEST_TIMEOUT,
+        ).then(() => {
+            this.setState({
+                healthIinitialized: true,
+                backendIsHealthy: true,
+            });
+        })
+            .catch(() => {
+                this.setState({
+                    healthIinitialized: true,
+                    backendIsHealthy: false,
+                });
+                Modal.error({
+                    title: 'Cannot connect to the server',
+                    className: 'cvat-modal-cannot-connect-server',
+                    closable: false,
+                    content: (
+                        <Text>
+                            Make sure the CVAT backend and all necessary services
+                            (Database, Redis and Open Policy Agent) are running and avaliable.
+                            If you upgraded from version 2.2.0 or earlier, manual actions may be needed, see the&nbsp;
+
+                            <a
+                                target='_blank'
+                                rel='noopener noreferrer'
+                                href={UPGRADE_GUIDE_URL}
+                            >
+                                Upgrade Guide
+                            </a>
+                            .
+                        </Text>
+                    ),
+                });
+            });
 
         const {
             name, version, engine, os,
@@ -198,6 +253,12 @@ class CVATApplication extends React.PureComponent<CVATAppProps & RouteComponentP
             isModelPluginActive,
         } = this.props;
 
+        const { backendIsHealthy } = this.state;
+
+        if (!backendIsHealthy) {
+            return;
+        }
+
         this.showErrors();
         this.showMessages();
 
@@ -332,6 +393,8 @@ class CVATApplication extends React.PureComponent<CVATAppProps & RouteComponentP
             isModelPluginActive,
         } = this.props;
 
+        const { healthIinitialized, backendIsHealthy } = this.state;
+
         const notRegisteredUserInitialized = (userInitialized && (user == null || !user.isVerified));
         let readyForRender = userAgreementsInitialized && authActionsInitialized;
         readyForRender = readyForRender && (notRegisteredUserInitialized ||
@@ -456,7 +519,15 @@ class CVATApplication extends React.PureComponent<CVATAppProps & RouteComponentP
             );
         }
 
-        return <Spin size='large' className='cvat-spinner' />;
+        if (healthIinitialized && !backendIsHealthy) {
+            return (
+                <Space align='center' direction='vertical' className='cvat-spinner'>
+                    <DisconnectOutlined className='cvat-disconnected' />
+                    Cannot connect to the server.
+                </Space>
+            );
+        }
+        return <Spin size='large' className='cvat-spinner' tip='Connecting...' />;
     }
 }
 

cvat-ui/src/consts.ts

@@ -12,6 +12,7 @@ const DISCORD_URL = 'https://discord.gg/fNR3eXfk6C';
 const GITHUB_URL = 'https://github.com/opencv/cvat';
 const GITHUB_IMAGE_URL = 'https://github.com/opencv/cvat/raw/develop/site/content/en/images/cvat.jpg';
 const GUIDE_URL = 'https://opencv.github.io/cvat/docs';
+const UPGRADE_GUIDE_URL = 'https://opencv.github.io/cvat/docs/administration/advanced/upgrade_guide';
 const SHARE_MOUNT_GUIDE_URL =
     'https://opencv.github.io/cvat/docs/administration/basics/installation/#share-path';
 const NUCLIO_GUIDE =
@@ -83,6 +84,10 @@ const DEFAULT_GOOGLE_CLOUD_STORAGE_LOCATIONS: string[][] = [
     ['NAM4', 'US-CENTRAL1 and US-EAST1'],
 ];
 
+const HEALH_CHECK_RETRIES = 10;
+const HEALTH_CHECK_PERIOD = 3000; // ms
+const HEALTH_CHECK_REQUEST_TIMEOUT = 5000; // ms
+
 export default {
     UNDEFINED_ATTRIBUTE_VALUE,
     NO_BREAK_SPACE,
@@ -93,6 +98,7 @@ export default {
     GITHUB_URL,
     GITHUB_IMAGE_URL,
     GUIDE_URL,
+    UPGRADE_GUIDE_URL,
     SHARE_MOUNT_GUIDE_URL,
     CANVAS_BACKGROUND_COLORS,
     NEW_LABEL_COLOR,
@@ -105,4 +111,7 @@ export default {
     DEFAULT_GOOGLE_CLOUD_STORAGE_LOCATIONS,
     OUTSIDE_PIC_URL,
     DATASET_MANIFEST_GUIDE_URL,
+    HEALH_CHECK_RETRIES,
+    HEALTH_CHECK_PERIOD,
+    HEALTH_CHECK_REQUEST_TIMEOUT,
 };

cvat-ui/src/styles.scss

@@ -17,6 +17,10 @@ hr {
     transform: translate(-50%, -50%);
 }
 
+.cvat-disconnected {
+    font-size: 36px;
+}
+
 .cvat-spinner-container {
     position: absolute;
     background: $background-color-1;

cvat/apps/health/__init__.py

@@ -0,0 +1,3 @@
+#  Copyright (C) 2022 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT

cvat/apps/health/apps.py

@@ -0,0 +1,14 @@
+# Copyright (C) 2022 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from django.apps import AppConfig
+
+from health_check.plugins import plugin_dir
+
+class HealthConfig(AppConfig):
+    name = 'cvat.apps.health'
+
+    def ready(self):
+        from .backends import OPAHealthCheck
+        plugin_dir.register(OPAHealthCheck)

cvat/apps/health/backends.py

@@ -0,0 +1,24 @@
+# Copyright (C) 2022 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import requests
+
+from health_check.backends import BaseHealthCheckBackend
+from health_check.exceptions import HealthCheckException
+
+from django.conf import settings
+
+class OPAHealthCheck(BaseHealthCheckBackend):
+    critical_service = True
+
+    def check_status(self):
+        opa_health_url = f'{settings.IAM_OPA_HOST}/health?bundles'
+        try:
+            response = requests.get(opa_health_url)
+            response.raise_for_status()
+        except requests.RequestException as e:
+            raise HealthCheckException(str(e))
+
+    def identifier(self):
+        return self.__class__.__name__

cvat/requirements/base.txt

@@ -51,3 +51,5 @@ natsort==8.0.0
 mistune>=2.0.1 # not directly required, pinned by Snyk to avoid a vulnerability
 dnspython==2.2.0
 setuptools==65.5.1
+django-health-check==3.17.0
+psutil==5.9.4

cvat/settings/base.py

@@ -124,6 +124,11 @@ def add_ssh_keys():
     'allauth.socialaccount.providers.github',
     'allauth.socialaccount.providers.google',
     'dj_rest_auth.registration',
+    'health_check',
+    'health_check.db',
+    'health_check.cache',
+    'health_check.contrib.migrations',
+    'health_check.contrib.psutil',
     'cvat.apps.iam',
     'cvat.apps.dataset_manager',
     'cvat.apps.organizations',
@@ -132,6 +137,7 @@ def add_ssh_keys():
     'cvat.apps.lambda_manager',
     'cvat.apps.opencv',
     'cvat.apps.webhooks',
+    'cvat.apps.health',
 ]
 
 SITE_ID = 1
@@ -246,7 +252,8 @@ def add_ssh_keys():
 IAM_ADMIN_ROLE = 'admin'
 # Index in the list below corresponds to the priority (0 has highest priority)
 IAM_ROLES = [IAM_ADMIN_ROLE, 'business', 'user', 'worker']
-IAM_OPA_DATA_URL = 'http://opa:8181/v1/data'
+IAM_OPA_HOST = 'http://opa:8181'
+IAM_OPA_DATA_URL = f'{IAM_OPA_HOST}/v1/data'
 LOGIN_URL = 'rest_login'
 LOGIN_REDIRECT_URL = '/'
 

cvat/urls.py

@@ -45,3 +45,6 @@
 
 if apps.is_installed('silk'):
     urlpatterns.append(path('profiler/', include('silk.urls')))
+
+if apps.is_installed('health_check'):
+    urlpatterns.append(path('api/server/health/', include('health_check.urls')))

docker-compose.yml

@@ -7,7 +7,7 @@ version: '3.3'
 services:
   cvat_db:
     container_name: cvat_db
-    image: postgres:10-alpine
+    image: postgres:15-alpine
     restart: always
     environment:
       POSTGRES_USER: root
@@ -155,13 +155,14 @@ services:
     labels:
       - traefik.enable=true
       - traefik.http.services.cvat-ui.loadbalancer.server.port=80
-      - traefik.http.routers.cvat-ui.rule=Host(`${CVAT_HOST:-localhost}`)
+      - traefik.http.routers.cvat-ui.rule=Host(`${CVAT_HOST:-localhost}`) &&
+        ! PathPrefix(`/api/`, `/git/`, `/opencv/`, `/static/`, `/admin`, `/documentation/`, `/django-rq`)
       - traefik.http.routers.cvat-ui.entrypoints=web
     networks:
       - cvat
 
   traefik:
-    image: traefik:v2.4
+    image: traefik:v2.9
     container_name: traefik
     restart: always
     command: