Port Django get_valid_filename utility to front-end code to open files with unicode characters and space

jsapp/js/components/submissions/submissionDataTable.es6

@@ -9,6 +9,7 @@ import {renderQuestionTypeIcon} from 'js/assetUtils';
 import {
   DISPLAY_GROUP_TYPES,
   getSubmissionDisplayData,
+  getMediaAttachment,
 } from 'js/components/submissions/submissionUtils';
 import {
   META_QUESTION_TYPES,
@@ -203,18 +204,6 @@ class SubmissionDataTable extends React.Component {
     });
   }
 
-  /**
-   * @prop {string} filename
-   * @returns {object|undefined}
-   */
-  findAttachmentData(targetFilename) {
-    // Match filename with full filename in attachment list
-    // BUG: this works but is possible to find bad attachment as `includes` can match multiple
-    return this.props.submissionData._attachments.find((attachment) => {
-      return attachment.filename.endsWith(`/${targetFilename}`);
-    });
-  }
-
   /**
    * @prop {string} data
    */
@@ -265,10 +254,8 @@ class SubmissionDataTable extends React.Component {
    * @prop {string} filename
    */
   renderAttachment(type, filename) {
-    const fileNameNoSpaces = filename.replace(/ /g, '_');
-    const attachment = this.findAttachmentData(fileNameNoSpaces);
-
-    if (attachment) {
+    const attachment = getMediaAttachment(this.props.submissionData, filename);
+    if (attachment && attachment instanceof Object) {
       if (type === QUESTION_TYPES.image.id) {
         return (
           <a href={attachment.download_url} target='_blank'>
@@ -280,7 +267,7 @@ class SubmissionDataTable extends React.Component {
       }
     // In the case that an attachment is missing, don't crash the page
     } else {
-      return(t('Could not retrieve ##filename##').replace('##filename##', filename));
+      return attachment;
     }
   }
 

jsapp/js/components/submissions/submissionUtils.mocks.es6

@@ -1679,3 +1679,37 @@ export const matrixRepeatSurveyDisplayData = [
     ],
   },
 ];
+
+export const submissionWithAttachmentsWithUnicode = {
+  '_id': 18,
+  'A_picture': 'Un été au Québec (Canada)-19_41_32.jpg',
+  'meta/instanceID': 'uuid:4cfa16e8-f29b-41a9-984c-2bf7fe05064b',
+  'meta/deprecatedID': 'uuid:f79e88d3-2329-40c7-ab7a-66dde871480c',
+  'formhub/uuid': '45748fd461814880bd9545c8c8827d78',
+  '__version__': 'vUdsH7ovQn4eCdBtPJyBag',
+  '_xform_id_string': 'azCy24QgjprZGrdvbHQXr3',
+  '_uuid': '4cfa16e8-f29b-41a9-984c-2bf7fe05064b',
+  '_attachments': [
+    {
+      download_url: 'http://kc.kobo.local/media/original?media_file=kobo%2Fattachments%2F45748fd461814880bd9545c8c8827d78%2F4cfa16e8-f29b-41a9-984c-2bf7fe05064b%2FUn_ete_au_Quebec_Canada-19_41_32.jpg',
+      download_large_url: 'http://kc.kobo.local/media/large?media_file=kobo%2Fattachments%2F45748fd461814880bd9545c8c8827d78%2F4cfa16e8-f29b-41a9-984c-2bf7fe05064b%2FUn_ete_au_Quebec_Canada-19_41_32.jpg',
+      download_medium_url: 'http://kc.kobo.local/media/medium?media_file=kobo%2Fattachments%2F45748fd461814880bd9545c8c8827d78%2F4cfa16e8-f29b-41a9-984c-2bf7fe05064b%2FUn_ete_au_Quebec_Canada-19_41_32.jpg',
+      download_small_url: 'http://kc.kobo.local/media/small?media_file=kobo%2Fattachments%2F45748fd461814880bd9545c8c8827d78%2F4cfa16e8-f29b-41a9-984c-2bf7fe05064b%2FUn_ete_au_Quebec_Canada-19_41_32.jpgg',
+      mimetype: 'image/jpeg',
+      filename: 'kobo/attachments/45748fd461814880bd9545c8c8827d78/4cfa16e8-f29b-41a9-984c-2bf7fe05064b/Un_ete_au_Quebec_Canada-19_41_32.jpg',
+      instance: 18,
+      xform: 4,
+      id: 13
+    },
+  ],
+  _geolocation: [
+    null,
+    null
+  ],
+  _notes: [],
+  _tags: [],
+  _status: 'submitted_via_web',
+  _submission_time: '2022-01-26T19:40:11',
+  _submitted_by: null,
+  _validation_status: {}
+}

jsapp/js/components/submissions/submissionUtils.tests.es6

@@ -27,8 +27,9 @@ import {
   matrixRepeatSurveyChoices,
   matrixRepeatSurveySubmission,
   matrixRepeatSurveyDisplayData,
+  submissionWithAttachmentsWithUnicode,
 } from './submissionUtils.mocks';
-import {getSubmissionDisplayData} from './submissionUtils';
+import {getValidFilename, getMediaAttachment, getSubmissionDisplayData} from './submissionUtils';
 
 describe('getSubmissionDisplayData', () => {
   it('should return a valid data for a survey with a group', () => {
@@ -79,3 +80,21 @@ describe('getSubmissionDisplayData', () => {
       expect(test).to.deep.equal(target);
   });
 });
+
+describe('getValidFilename', () => {
+  it('should return a file name which matches Django renaming', () => {
+    const fileName = submissionWithAttachmentsWithUnicode.A_picture;
+    const test = getValidFilename(fileName);
+    const target = 'Un_ete_au_Quebec_Canada-19_41_32.jpg';
+    expect(test).to.equal(target);
+  });
+});
+
+describe('getMediaAttachment', () => {
+  it('should return an attachment object', () => {
+    const fileName = submissionWithAttachmentsWithUnicode.A_picture;
+    const test = getMediaAttachment(submissionWithAttachmentsWithUnicode, fileName);
+    const target = submissionWithAttachmentsWithUnicode._attachments[0];
+    expect(test).to.deep.equal(target);
+  });
+});

jsapp/js/components/submissions/submissionUtils.ts

@@ -477,21 +477,34 @@ export function getMediaAttachment(
   submission: SubmissionResponse,
   fileName: string
 ): string | SubmissionAttachment {
-  const fileNameNoSpaces = fileName.replace(/ /g, '_');
+  const validFileName = getValidFilename(fileName);
   let mediaAttachment: string | SubmissionAttachment = t('Could not find ##fileName##').replace(
     '##fileName##',
     fileName,
   );
 
   submission._attachments.forEach((attachment) => {
-    if (attachment.filename.includes(fileNameNoSpaces)) {
+    if (attachment.filename.includes(validFileName)) {
       mediaAttachment = attachment;
     }
   });
 
   return mediaAttachment;
 }
 
+/**
+  Mimics Django get_valid_filename() to match back-end renaming when an
+  attachment is saved in storage.
+  See https://github.com/django/django/blob/832adb31f27cfc18ad7542c7eda5a1b6ed5f1669/django/utils/text.py#L224
+ */
+export function getValidFilename(
+  fileName: string
+): string {
+  return fileName.normalize('NFD').replace(/\p{Diacritic}/gu, '')
+    .replace(/ /g, '_')
+    .replace(/[^\p{L}\p{M}\.\d_-]/gu, '');
+}
+
 export default {
   DISPLAY_GROUP_TYPES,
   getSubmissionDisplayData,

kpi/deployment_backends/kc_access/shadow_models.py

@@ -622,8 +622,12 @@ def protected_path(self, format_: Optional[str] = None):
             attachment_file_path = self.absolute_path
 
         if not isinstance(get_kobocat_storage(), KobocatS3Boto3Storage):
-            protected_url = attachment_file_path.replace(
-                settings.KOBOCAT_MEDIA_PATH, '/protected'
+            # Django normally sanitizes accented characters in file names during
+            # save on disk but some languages have extra letters
+            # (out of ASCII character set) and must be encoded to let NGINX serve
+            # them
+            protected_url = urlquote(attachment_file_path.replace(
+                settings.KOBOCAT_MEDIA_PATH, '/protected')
             )
         else:
             # Double-encode the S3 URL to take advantage of NGINX's