Merge branch 'develop' into bs/refactored_root

cvat-ai · Jun 7, 2024 · 363eb48 · 363eb48
2 parents 4f975fd + 729322f
commit 363eb48
Show file tree

Hide file tree

Showing 11 changed files with 88 additions and 22 deletions.
diff --git a/changelog.d/20240606_134734_boris_bitmap_memoryleak.md b/changelog.d/20240606_134734_boris_bitmap_memoryleak.md
@@ -0,0 +1,4 @@
+### Fixed
+
+- Significant memory leak related to the frames, which did not memory after become unused
+  (<https://github.com/cvat-ai/cvat/pull/7995>)
diff --git a/cvat-canvas/package.json b/cvat-canvas/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cvat-canvas",
-  "version": "2.20.2",
+  "version": "2.20.3",
   "type": "module",
   "description": "Part of Computer Vision Annotation Tool which presents its canvas library",
   "main": "src/canvas.ts",

diff --git a/cvat-canvas/src/typescript/canvasView.ts b/cvat-canvas/src/typescript/canvasView.ts
@@ -451,6 +451,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
                     results.forEach((bitmap, idx) => {
                         const [curLeft, curTop] = objects[idx].points.slice(-4, -2);
                         canvas.getContext('2d').drawImage(bitmap, curLeft - left, curTop - top);
+                        bitmap.close();
                     });
 
                     const imageData = canvas.getContext('2d')

diff --git a/cvat-core/package.json b/cvat-core/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "15.0.5",
+  "version": "15.0.6",
   "type": "module",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "src/api.ts",

diff --git a/cvat-core/src/frames.ts b/cvat-core/src/frames.ts
@@ -295,8 +295,8 @@ Object.defineProperty(FrameData.prototype.data, 'implementation', {
                     const nextChunkNumber = findTheNextNotDecodedChunk(this.number);
                     const predecodeChunksMax = Math.floor(decodedBlocksCacheSize / 2);
                     if (nextChunkNumber * chunkSize <= stopFrame &&
-                        nextChunkNumber <= chunkNumber + predecodeChunksMax) {
-                        provider.cleanup(1);
+                        nextChunkNumber <= chunkNumber + predecodeChunksMax
+                    ) {
                         frameDataCache[this.jobID].activeChunkRequest = new Promise((resolveForward) => {
                             const releasePromise = (): void => {
                                 resolveForward();
@@ -306,6 +306,14 @@ Object.defineProperty(FrameData.prototype.data, 'implementation', {
                             frameDataCache[this.jobID].getChunk(
                                 nextChunkNumber, ChunkQuality.COMPRESSED,
                             ).then((chunk: ArrayBuffer) => {
+                                if (!(this.jobID in frameDataCache)) {
+                                    // check if frameDataCache still exist
+                                    // as it may be released during chunk request
+                                    resolveForward();
+                                    return;
+                                }
+
+                                provider.cleanup(1);
                                 provider.requestDecodeBlock(
                                     chunk,
                                     nextChunkNumber * chunkSize,
@@ -333,13 +341,13 @@ Object.defineProperty(FrameData.prototype.data, 'implementation', {
             onServerRequest();
             frameDataCache[this.jobID].latestFrameDecodeRequest = requestId;
             (frameDataCache[this.jobID].activeChunkRequest || Promise.resolve()).finally(() => {
-                if (frameDataCache[this.jobID].latestFrameDecodeRequest !== requestId) {
+                if (frameDataCache[this.jobID]?.latestFrameDecodeRequest !== requestId) {
                     // not relevant request anymore
                     reject(this.number);
                     return;
                 }
 
-                // it might appear during decoding, so, check again
+                // it might appear during previous decoding, so, check again
                 const currentFrame = provider.frame(this.number);
                 if (currentFrame) {
                     resolve({
@@ -359,6 +367,14 @@ Object.defineProperty(FrameData.prototype.data, 'implementation', {
                         chunkNumber, ChunkQuality.COMPRESSED,
                     ).then((chunk: ArrayBuffer) => {
                         try {
+                            if (!(this.jobID in frameDataCache)) {
+                                // check if frameDataCache still exist
+                                // as it may be released during chunk request
+                                resolveLoadAndDecode();
+                                reject(this.number);
+                                return;
+                            }
+
                             provider
                                 .requestDecodeBlock(
                                     chunk,
@@ -703,6 +719,13 @@ export function getCachedChunks(jobID): number[] {
 
 export function clear(jobID: number): void {
     if (jobID in frameDataCache) {
+        frameDataCache[jobID].provider.close();
+        for (const contextImagesByFrame of Object.values(frameDataCache[jobID].contextCache)) {
+            for (const image of Object.values(contextImagesByFrame.data)) {
+                image.close();
+            }
+        }
+
         delete frameDataCache[jobID];
         delete frameMetaCache[jobID];
     }

diff --git a/cvat-data/package.json b/cvat-data/package.json
@@ -1,7 +1,7 @@
 {
   "name": "cvat-data",
   "type": "module",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "",
   "main": "src/ts/cvat-data.ts",
   "scripts": {

diff --git a/cvat-data/src/ts/cvat-data.ts b/cvat-data/src/ts/cvat-data.ts
@@ -98,7 +98,8 @@ export class FrameDecoder {
     // used for video chunks to get correct side after decoding
     private renderWidth: number;
     private renderHeight: number;
-    private zipWorker: Worker;
+    private zipWorker: Worker | null;
+    private videoWorker: Worker | null;
 
     constructor(
         blockType: BlockType,
@@ -108,6 +109,8 @@ export class FrameDecoder {
     ) {
         this.mutex = new Mutex();
         this.orderedStack = [];
+        this.zipWorker = null;
+        this.videoWorker = null;
 
         this.cachedChunksLimit = Math.max(1, cachedBlockCount);
         this.dimension = dimension;
@@ -139,6 +142,14 @@ export class FrameDecoder {
             if (typeof lastChunk === 'undefined') {
                 return;
             }
+
+            for (const frame of Object.keys(this.decodedChunks[lastChunk])) {
+                const data = this.decodedChunks[lastChunk][frame];
+                if (data instanceof ImageBitmap) {
+                    data.close();
+                }
+            }
+
             delete this.decodedChunks[lastChunk];
             length--;
         }
@@ -232,7 +243,15 @@ export class FrameDecoder {
 
     async startDecode(): Promise<void> {
         const blockToDecode = { ...this.requestedChunkToDecode };
-        const release = await this.mutex.acquire();
+        const releaseMutex = await this.mutex.acquire();
+        const release = (): void => {
+            if (this.videoWorker) {
+                this.videoWorker.terminate();
+                this.videoWorker = null;
+            }
+
+            releaseMutex();
+        };
         try {
             const { start, end, block } = this.requestedChunkToDecode;
             if (start !== blockToDecode.start) {
@@ -251,12 +270,12 @@ export class FrameDecoder {
             this.requestedChunkToDecode = null;
 
             if (this.blockType === BlockType.MP4VIDEO) {
-                const worker = new Worker(
+                this.videoWorker = new Worker(
                     new URL('./3rdparty/Decoder.worker', import.meta.url),
                 );
                 let index = start;
 
-                worker.onmessage = (e) => {
+                this.videoWorker.onmessage = (e) => {
                     if (e.data.consoleLog) {
                         // ignore initialization message
                         return;
@@ -283,22 +302,20 @@ export class FrameDecoder {
                             this.decodedChunks[chunkNumber] = decodedFrames;
                             this.chunkIsBeingDecoded.onDecodeAll();
                             this.chunkIsBeingDecoded = null;
-                            worker.terminate();
                             release();
                         }
                     });
 
                     index++;
                 };
 
-                worker.onerror = (event: ErrorEvent) => {
+                this.videoWorker.onerror = (event: ErrorEvent) => {
                     release();
-                    worker.terminate();
                     this.chunkIsBeingDecoded.onReject(event.error);
                     this.chunkIsBeingDecoded = null;
                 };
 
-                worker.postMessage({
+                this.videoWorker.postMessage({
                     type: 'Broadway.js - Worker init',
                     options: {
                         rgb: true,
@@ -314,12 +331,12 @@ export class FrameDecoder {
                 const sps = avc.sps[0];
                 const pps = avc.pps[0];
 
-                worker.postMessage({ buf: sps, offset: 0, length: sps.length });
-                worker.postMessage({ buf: pps, offset: 0, length: pps.length });
+                this.videoWorker.postMessage({ buf: sps, offset: 0, length: sps.length });
+                this.videoWorker.postMessage({ buf: pps, offset: 0, length: pps.length });
 
                 for (let sample = 0; sample < video.getSampleCount(); sample++) {
                     video.getSampleNALUnits(sample).forEach((nal) => {
-                        worker.postMessage({ buf: nal, offset: 0, length: nal.length });
+                        this.videoWorker.postMessage({ buf: nal, offset: 0, length: nal.length });
                     });
                 }
             } else {
@@ -368,6 +385,20 @@ export class FrameDecoder {
         }
     }
 
+    public close(): void {
+        if (this.zipWorker) {
+            this.zipWorker.terminate();
+            this.zipWorker = null;
+        }
+
+        if (this.videoWorker) {
+            this.videoWorker.terminate();
+            this.videoWorker = null;
+        }
+
+        this.cleanup(Number.MAX_SAFE_INTEGER);
+    }
+
     public cachedChunks(includeInProgress = false): number[] {
         const chunkIsBeingDecoded = includeInProgress && this.chunkIsBeingDecoded ?
             Math.floor(this.chunkIsBeingDecoded.start / this.chunkSize) : null;

diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts
@@ -872,7 +872,11 @@ export function closeJob(): ThunkAction {
     return async (dispatch: ActionCreator<Dispatch>, getState): Promise<void> => {
         const state = getState();
         const { instance: canvasInstance } = state.annotation.canvas;
-        const { jobInstance } = receiveAnnotationsParameters();
+        const { jobInstance, groundTruthInstance } = receiveAnnotationsParameters();
+
+        if (groundTruthInstance) {
+            await groundTruthInstance.close();
+        }
 
         if (jobInstance) {
             await jobInstance.close();

diff --git a/site/content/en/docs/administration/advanced/analytics.md b/site/content/en/docs/administration/advanced/analytics.md
@@ -176,6 +176,7 @@ Here is a short overview of how CVAT deals with the user's working time:
   - A user clicks the "Save" button on the annotation view
   - A user opens the annotation view
   - A user closes the annotation view (but not the tab/browser)
+  - A user clicks **Logout** button
 
 - When events reach the server, it calculates working time based on timestamps of the events.
 

diff --git a/site/content/en/docs/manual/advanced/formats/_index.md b/site/content/en/docs/manual/advanced/formats/_index.md
@@ -31,7 +31,7 @@ The table below outlines the available formats for data export in CVAT.
 | [COCO Keypoints 1.0](coco-keypoints)                                                                                                | .xml          | Keypoints                                                   | OpenPose, PoseNet, AlphaPose, <br> SPM (Single Person Model), <br>Mask R-CNN with Keypoint Detection:, and others.                                                                      | Skeletons                                                                              | Specific attributes  | Not supported |
 | {{< ilink "/docs/manual/advanced/formats/format-cvat#cvat-for-videos-export" "CVAT for images 1.1" >}}                                            | .xml          | Universal format<br> for all types of <br>annotations.      | Universal format<br> for all types of <br>models.                                                                                                                                       | Bounding Boxes, Polygons, <br>Polylines, Points, Cuboids, <br>Skeletons, Tags.         | All attributes       | Not supported |
 | {{< ilink "/docs/manual/advanced/formats/format-cvat#cvat-for-videos-export" "CVAT for video 1.1" >}}                                             | .xml          | Universal format<br> for all types of <br>annotations.      | Universal format<br> for all types of <br>annotations.                                                                                                                                  | Bounding Boxes, Polygons, <br>Polylines, Points, Cuboids, <br>Skeletons, Tags, Tracks. | All attributes       | Supported     |
-| [Datumaro 1.0](format-datumaro)                                                                                                     | JSON          | Universal format<br> for all types of <br>annotations.      | Universal format<br> for all types of <br>models.                                                                                                                                       | Bounding Boxes, Polygons, <br>Polylines, Points, Cuboids, <br>Skeletons, Tags, Tracks. | All attributes       | Supported     |
+| [Datumaro 1.0](format-datumaro)                                                                                                     | JSON          | Universal format<br> for all types of <br>annotations.      | Universal format<br> for all types of <br>models.                                                                                                                                       | Bounding Boxes, Polygons, <br>Polylines, Points, Cuboids, <br>Tags, Tracks. | All attributes       | Supported     |
 | [ICDAR](format-icdar)<br> Includes ICDAR Recognition 1.0, <br>ICDAR Detection 1.0, <br>and ICDAR Segmentation 1.0 <br>descriptions. | .txt          | Text recognition, <br>Text detection, <br>Text segmentation | EAST: Efficient and Accurate <br>Scene Text Detector, CRNN, Mask TextSpotter, TextSnake, <br>and others.                                                                                | Tag, Bounding Boxes, Polygons                                                          | Specific attributes  | Not supported |
 | [ImageNet 1.0](format-imagenet)                                                                                                     | .jpg <br>.txt | Semantic Segmentation, <br>Classification, <br>Detection    | VGG (VGG16, VGG19), Inception, YOLO, Faster R-CNN , U-Net, and others                                                                                                                   | Tags                                                                                   | No attributes        | Not supported |
 | [KITTI 1.0](format-kitti)                                                                                                           | .txt <br>.png | Semantic Segmentation, Detection, 3D                        | PointPillars, SECOND, AVOD, YOLO, DeepSORT, PWC-Net, ORB-SLAM, and others.                                                                                                              | Bounding Boxes, Polygons                                                               | Specific attributes  | Not supported |

diff --git a/site/content/en/docs/manual/advanced/formats/format-datumaro.md b/site/content/en/docs/manual/advanced/formats/format-datumaro.md
@@ -20,7 +20,8 @@ For more information, see:
 
 For export of images: any 2D shapes, tags
 
-- Supported annotations: Bounding Boxes, Polygons.
+- Supported annotations: Bounding Boxes, Polygons, Polylines,
+  Masks, Points, Cuboids, Tags
 - Attributes: Supported.
 - Tracks: Supported.
 
@@ -39,7 +40,8 @@ taskname.zip/
 
 # Import annotations in Datumaro format
 
-- supported annotations: any 2D shapes, labels
+- supported annotations: Bounding Boxes, Polygons, Polylines,
+  Masks, Points, Cuboids, Labels
 - supported attributes: any
 
 Uploaded file: a zip archive of the following structure: