diff --git a/cvat-core/package.json b/cvat-core/package.json index c7dfa05d686..1afc44f6436 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "7.2.2", + "version": "7.3.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "src/api.ts", "scripts": { diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts index 6f0d4b14639..dc2dff7c681 100644 --- a/cvat-core/src/api-implementation.ts +++ b/cvat-core/src/api-implementation.ts @@ -107,6 +107,16 @@ export default function implementAPI(cvat) { return result; }; + cvat.server.healthCheck.implementation = async ( + maxRetries = 1, + checkPeriod = 3000, + requestTimeout = 5000, + progressCallback = undefined, + ) => { + const result = await serverProxy.server.healthCheck(maxRetries, checkPeriod, requestTimeout, progressCallback); + return result; + }; + cvat.server.request.implementation = async (url, data) => { const result = await serverProxy.server.request(url, data); return result; diff --git a/cvat-core/src/api.ts b/cvat-core/src/api.ts index 95756786d1a..165e33d7a9a 100644 --- a/cvat-core/src/api.ts +++ b/cvat-core/src/api.ts @@ -257,6 +257,26 @@ function build() { const result = await PluginRegistry.apiWrapper(cvat.server.authorized); return result; }, + /** + * Method allows to health check the server + * @method healthCheck + * @async + * @memberof module:API.cvat.server + * @param {number} requestTimeout + * @returns {Object | undefined} response data if exist + * @throws {module:API.cvat.exceptions.PluginError} + * @throws {module:API.cvat.exceptions.ServerError} + */ + async healthCheck(maxRetries = 1, checkPeriod = 3000, requestTimeout = 5000, progressCallback = undefined) { + const result = await PluginRegistry.apiWrapper( + cvat.server.healthCheck, + maxRetries, + checkPeriod, + requestTimeout, + progressCallback, + ); + return result; + }, /** * Method allows to do requests via cvat-core with authorization headers * @method request diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 6deb7b73e1f..31955c3e4d2 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -468,6 +468,29 @@ async function authorized() { return true; } +async function healthCheck(maxRetries, checkPeriod, requestTimeout, progressCallback, attempt = 0) { + const { backendAPI } = config; + const url = `${backendAPI}/server/health/?format=json`; + + if (progressCallback) { + progressCallback(`${attempt}/${attempt + maxRetries}`); + } + + return Axios.get(url, { + proxy: config.proxy, + timeout: requestTimeout, + }) + .then((response) => response.data) + .catch((errorData) => { + if (maxRetries > 0) { + return new Promise((resolve) => setTimeout(resolve, checkPeriod)) + .then(() => healthCheck(maxRetries - 1, checkPeriod, + requestTimeout, progressCallback, attempt + 1)); + } + throw generateError(errorData); + }); +} + async function serverRequest(url, data) { try { return ( @@ -2227,6 +2250,7 @@ export default Object.freeze({ requestPasswordReset, resetPassword, authorized, + healthCheck, register, request: serverRequest, userAgreements, diff --git a/cvat-ui/package.json b/cvat-ui/package.json index c5200faa5f5..7355bfc8e83 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.44.4", + "version": "1.45.0", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/components/cvat-app.tsx b/cvat-ui/src/components/cvat-app.tsx index fb2e3ed08ed..07f29153d45 100644 --- a/cvat-ui/src/components/cvat-app.tsx +++ b/cvat-ui/src/components/cvat-app.tsx @@ -11,6 +11,8 @@ import Layout from 'antd/lib/layout'; import Modal from 'antd/lib/modal'; import notification from 'antd/lib/notification'; import Spin from 'antd/lib/spin'; +import { DisconnectOutlined } from '@ant-design/icons'; +import Space from 'antd/lib/space'; import Text from 'antd/lib/typography/Text'; import 'antd/dist/antd.css'; @@ -64,6 +66,7 @@ import showPlatformNotification, { showUnsupportedNotification, } from 'utils/platform-checker'; import '../styles.scss'; +import consts from 'consts'; import EmailConfirmationPage from './email-confirmation-pages/email-confirmed'; import EmailVerificationSentPage from './email-confirmation-pages/email-verification-sent'; import IncorrectEmailConfirmationPage from './email-confirmation-pages/incorrect-email-confirmation'; @@ -103,10 +106,23 @@ interface CVATAppProps { isModelPluginActive: boolean; } -class CVATApplication extends React.PureComponent { +interface CVATAppState { + healthIinitialized: boolean; + backendIsHealthy: boolean; +} + +class CVATApplication extends React.PureComponent { + constructor(props: CVATAppProps & RouteComponentProps) { + super(props); + + this.state = { + healthIinitialized: false, + backendIsHealthy: false, + }; + } public componentDidMount(): void { const core = getCore(); - const { verifyAuthorized, history, location } = this.props; + const { history, location } = this.props; // configure({ ignoreRepeatedEventsWhenKeyHeldDown: false }); // Logger configuration @@ -121,7 +137,46 @@ class CVATApplication extends React.PureComponent { + this.setState({ + healthIinitialized: true, + backendIsHealthy: true, + }); + }) + .catch(() => { + this.setState({ + healthIinitialized: true, + backendIsHealthy: false, + }); + Modal.error({ + title: 'Cannot connect to the server', + className: 'cvat-modal-cannot-connect-server', + closable: false, + content: ( + + Make sure the CVAT backend and all necessary services + (Database, Redis and Open Policy Agent) are running and avaliable. + If you upgraded from version 2.2.0 or earlier, manual actions may be needed, see the  + + + Upgrade Guide + + . + + ), + }); + }); const { name, version, engine, os, @@ -198,6 +253,12 @@ class CVATApplication extends React.PureComponent; + if (healthIinitialized && !backendIsHealthy) { + return ( + + + Cannot connect to the server. + + ); + } + return ; } } diff --git a/cvat-ui/src/consts.ts b/cvat-ui/src/consts.ts index be1535c5468..858c335345a 100644 --- a/cvat-ui/src/consts.ts +++ b/cvat-ui/src/consts.ts @@ -12,6 +12,7 @@ const DISCORD_URL = 'https://discord.gg/fNR3eXfk6C'; const GITHUB_URL = 'https://github.com/opencv/cvat'; const GITHUB_IMAGE_URL = 'https://github.com/opencv/cvat/raw/develop/site/content/en/images/cvat.jpg'; const GUIDE_URL = 'https://opencv.github.io/cvat/docs'; +const UPGRADE_GUIDE_URL = 'https://opencv.github.io/cvat/docs/administration/advanced/upgrade_guide'; const SHARE_MOUNT_GUIDE_URL = 'https://opencv.github.io/cvat/docs/administration/basics/installation/#share-path'; const NUCLIO_GUIDE = @@ -83,6 +84,10 @@ const DEFAULT_GOOGLE_CLOUD_STORAGE_LOCATIONS: string[][] = [ ['NAM4', 'US-CENTRAL1 and US-EAST1'], ]; +const HEALH_CHECK_RETRIES = 10; +const HEALTH_CHECK_PERIOD = 3000; // ms +const HEALTH_CHECK_REQUEST_TIMEOUT = 5000; // ms + export default { UNDEFINED_ATTRIBUTE_VALUE, NO_BREAK_SPACE, @@ -93,6 +98,7 @@ export default { GITHUB_URL, GITHUB_IMAGE_URL, GUIDE_URL, + UPGRADE_GUIDE_URL, SHARE_MOUNT_GUIDE_URL, CANVAS_BACKGROUND_COLORS, NEW_LABEL_COLOR, @@ -105,4 +111,7 @@ export default { DEFAULT_GOOGLE_CLOUD_STORAGE_LOCATIONS, OUTSIDE_PIC_URL, DATASET_MANIFEST_GUIDE_URL, + HEALH_CHECK_RETRIES, + HEALTH_CHECK_PERIOD, + HEALTH_CHECK_REQUEST_TIMEOUT, }; diff --git a/cvat-ui/src/styles.scss b/cvat-ui/src/styles.scss index befdaf2547e..cafc0b9f8ba 100644 --- a/cvat-ui/src/styles.scss +++ b/cvat-ui/src/styles.scss @@ -17,6 +17,10 @@ hr { transform: translate(-50%, -50%); } +.cvat-disconnected { + font-size: 36px; +} + .cvat-spinner-container { position: absolute; background: $background-color-1; diff --git a/cvat/apps/health/__init__.py b/cvat/apps/health/__init__.py new file mode 100644 index 00000000000..b5e88414c5f --- /dev/null +++ b/cvat/apps/health/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2022 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT diff --git a/cvat/apps/health/apps.py b/cvat/apps/health/apps.py new file mode 100644 index 00000000000..a457048b87c --- /dev/null +++ b/cvat/apps/health/apps.py @@ -0,0 +1,14 @@ +# Copyright (C) 2022 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from django.apps import AppConfig + +from health_check.plugins import plugin_dir + +class HealthConfig(AppConfig): + name = 'cvat.apps.health' + + def ready(self): + from .backends import OPAHealthCheck + plugin_dir.register(OPAHealthCheck) diff --git a/cvat/apps/health/backends.py b/cvat/apps/health/backends.py new file mode 100644 index 00000000000..e54439639df --- /dev/null +++ b/cvat/apps/health/backends.py @@ -0,0 +1,24 @@ +# Copyright (C) 2022 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import requests + +from health_check.backends import BaseHealthCheckBackend +from health_check.exceptions import HealthCheckException + +from django.conf import settings + +class OPAHealthCheck(BaseHealthCheckBackend): + critical_service = True + + def check_status(self): + opa_health_url = f'{settings.IAM_OPA_HOST}/health?bundles' + try: + response = requests.get(opa_health_url) + response.raise_for_status() + except requests.RequestException as e: + raise HealthCheckException(str(e)) + + def identifier(self): + return self.__class__.__name__ diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 3879dce51c0..03e4eb78081 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -51,3 +51,5 @@ natsort==8.0.0 mistune>=2.0.1 # not directly required, pinned by Snyk to avoid a vulnerability dnspython==2.2.0 setuptools==65.5.1 +django-health-check==3.17.0 +psutil==5.9.4 diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 9cf94d18df0..9e4ccc070a1 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -124,6 +124,11 @@ def add_ssh_keys(): 'allauth.socialaccount.providers.github', 'allauth.socialaccount.providers.google', 'dj_rest_auth.registration', + 'health_check', + 'health_check.db', + 'health_check.cache', + 'health_check.contrib.migrations', + 'health_check.contrib.psutil', 'cvat.apps.iam', 'cvat.apps.dataset_manager', 'cvat.apps.organizations', @@ -132,6 +137,7 @@ def add_ssh_keys(): 'cvat.apps.lambda_manager', 'cvat.apps.opencv', 'cvat.apps.webhooks', + 'cvat.apps.health', ] SITE_ID = 1 @@ -246,7 +252,8 @@ def add_ssh_keys(): IAM_ADMIN_ROLE = 'admin' # Index in the list below corresponds to the priority (0 has highest priority) IAM_ROLES = [IAM_ADMIN_ROLE, 'business', 'user', 'worker'] -IAM_OPA_DATA_URL = 'http://opa:8181/v1/data' +IAM_OPA_HOST = 'http://opa:8181' +IAM_OPA_DATA_URL = f'{IAM_OPA_HOST}/v1/data' LOGIN_URL = 'rest_login' LOGIN_REDIRECT_URL = '/' diff --git a/cvat/urls.py b/cvat/urls.py index 09eed3e9cfe..d084f217f48 100644 --- a/cvat/urls.py +++ b/cvat/urls.py @@ -45,3 +45,6 @@ if apps.is_installed('silk'): urlpatterns.append(path('profiler/', include('silk.urls'))) + +if apps.is_installed('health_check'): + urlpatterns.append(path('api/server/health/', include('health_check.urls'))) diff --git a/docker-compose.yml b/docker-compose.yml index 644db0cbac7..3caad0e0375 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ version: '3.3' services: cvat_db: container_name: cvat_db - image: postgres:10-alpine + image: postgres:15-alpine restart: always environment: POSTGRES_USER: root @@ -161,7 +161,7 @@ services: - cvat traefik: - image: traefik:v2.4 + image: traefik:v2.9 container_name: traefik restart: always command: diff --git a/site/content/en/docs/administration/advanced/upgrade_guide.md b/site/content/en/docs/administration/advanced/upgrade_guide.md index ba7e1746fbd..eb790654dbc 100644 --- a/site/content/en/docs/administration/advanced/upgrade_guide.md +++ b/site/content/en/docs/administration/advanced/upgrade_guide.md @@ -9,6 +9,9 @@ description: 'Instructions for upgrading CVAT deployed with docker compose' ## Upgrade guide +Note: updating CVAT from version 2.2.0 to version 2.3.0 requires additional manual actions with database data due to +upgrading PostgreSQL base image major version. See details [here](#how-to-upgrade-postgresql-database-base-image) + To upgrade CVAT, follow these steps: - It is highly recommended backup all CVAT data before updating, follow the @@ -69,3 +72,43 @@ docker pull cvat/ui:v2.1.0 docker tag cvat/ui:v2.1.0 openvino/cvat_ui:latest docker-compose up -d ``` +## How to upgrade PostgreSQL database base image + +1. It is highly recommended backup all CVAT data before updating, follow the + [backup guide](/docs/administration/advanced/backup_guide/) and backup CVAT database volume. + +1. Run previosly used CVAT version as usual + +1. Backup current database with `pg_dumpall` tool: + ```shell + docker exec -it cvat_db pg_dumpall > cvat.db.dump + ``` + +1. Stop CVAT: + ```shell + docker-compose down + ``` + +1. Delete current PostrgeSQL’s volume, that's why it's important to have a backup: + ```shell + docker volume rm cvat_cvat_db + ``` + +1. Update CVAT source code by any preferable way: clone with git or download zip file from GitHub. + Check the + [installation guide](/docs/administration/basics/installation/#how-to-get-cvat-source-code) for details. + +1. Start database container only: + ```shell + docker-compose up -d cvat_db + ``` + +1. Import PostgreSQL dump into new DB container: + ```shell + docker exec -i cvat_db psql -q -d postgres < cvat.db.dump + ``` + +1. Start CVAT: + ```shell + docker-compose up -d + ``` diff --git a/site/content/en/docs/administration/basics/installation.md b/site/content/en/docs/administration/basics/installation.md index b73a61c38f6..fb586087d9b 100644 --- a/site/content/en/docs/administration/basics/installation.md +++ b/site/content/en/docs/administration/basics/installation.md @@ -345,6 +345,21 @@ unzip v1.7.0.zip && mv cvat-1.7.0 cvat cd cvat ``` +### CVAT healthcheck command +The following command allows to test the CVAT container to make sure it works. +```shell +docker exec -t cvat_server python manage.py health_check +``` +Expected output of a healthy CVAT container: +```shell +Cache backend: default ... working +DatabaseBackend ... working +DiskUsage ... working +MemoryUsage ... working +MigrationsHealthCheck ... working +OPAHealthCheck ... working +``` + ### Deploying CVAT behind a proxy If you deploy CVAT behind a proxy and do not plan to use any of [serverless functions](#semi-automatic-and-automatic-annotation) diff --git a/supervisord/server.conf b/supervisord/server.conf index 9f4cfbe0d3b..4ff9fb6ee9a 100644 --- a/supervisord/server.conf +++ b/supervisord/server.conf @@ -34,4 +34,4 @@ command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_POSTGRES_HOST)s:5432 -t 0 -- bash --limit-request-body 1073741824 --log-level INFO --include-file ~/mod_wsgi.conf \ %(ENV_DJANGO_MODWSGI_EXTRA_ARGS)s --locale %(ENV_LC_ALL)s \ --server-root /tmp/cvat-server" -numprocs=%(ENV_NUMPROCS)s \ No newline at end of file +numprocs=%(ENV_NUMPROCS)s diff --git a/tests/python/shared/assets/cvat_db/restore.sql b/tests/python/shared/assets/cvat_db/restore.sql index 84f6db388aa..a8af5949cc7 100644 --- a/tests/python/shared/assets/cvat_db/restore.sql +++ b/tests/python/shared/assets/cvat_db/restore.sql @@ -1,8 +1,11 @@ -SELECT pg_terminate_backend(pg_stat_activity.pid) -FROM pg_stat_activity -WHERE pg_stat_activity.datname = 'cvat' AND pid <> pg_backend_pid(); +SELECT :'from' = 'cvat' AS fromcvat \gset -DROP DATABASE IF EXISTS :to; +\if :fromcvat + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE pg_stat_activity.datname = 'cvat' AND pid <> pg_backend_pid(); +\endif -CREATE DATABASE :to WITH TEMPLATE :from; +DROP DATABASE IF EXISTS :to WITH (FORCE); +CREATE DATABASE :to WITH TEMPLATE :from;