Skip to content

Commit

Permalink
Merge pull request #26141 from vector-im/actions/localazy-download
Browse files Browse the repository at this point in the history
Localazy Download
  • Loading branch information
RiotRobot committed Sep 8, 2023
2 parents 8c83307 + e84e161 commit 6dc6c80
Show file tree
Hide file tree
Showing 56 changed files with 129 additions and 1,290 deletions.
2 changes: 1 addition & 1 deletion .eslintrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ module.exports = {
},
overrides: [
{
files: ["src/**/*.{ts,tsx}", "test/**/*.{ts,tsx}"],
files: ["src/**/*.{ts,tsx}", "test/**/*.{ts,tsx}", "scripts/*.ts"],
extends: ["plugin:matrix-org/typescript", "plugin:matrix-org/react"],
// NOTE: These rules are frozen and new rules should not be added here.
// New changes belong in https://github.com/matrix-org/eslint-plugin-matrix-org/
Expand Down
2 changes: 1 addition & 1 deletion localazy.json
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
],
"includeSourceLang": "${includeSourceLang|false}",
"langAliases": {
"en-GB": "en-EN"
"en": "en-EN"
}
}
}
7 changes: 4 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
"build": "yarn clean && yarn build:genfiles && yarn build:bundle",
"build-stats": "yarn clean && yarn build:genfiles && yarn build:bundle-stats",
"build:jitsi": "ts-node scripts/build-jitsi.ts",
"build:res": "node scripts/copy-res.js",
"build:res": "ts-node scripts/copy-res.ts",
"build:genfiles": "yarn build:res && yarn build:jitsi && yarn build:module_system",
"build:modernizr": "modernizr -c .modernizr.json -d src/vector/modernizr.js",
"build:bundle": "webpack --progress --bail --mode production",
Expand All @@ -47,7 +47,7 @@
"dist": "scripts/package.sh",
"start": "yarn build:module_system && concurrently --kill-others-on-fail --prefix \"{time} [{name}]\" -n res,element-js \"yarn start:res\" \"yarn start:js\"",
"start:https": "concurrently --kill-others-on-fail --prefix \"{time} [{name}]\" -n res,element-js \"yarn start:res\" \"yarn start:js --https\"",
"start:res": "yarn build:jitsi && node scripts/copy-res.js -w",
"start:res": "yarn build:jitsi && ts-node scripts/copy-res.ts -w",
"start:js": "webpack-dev-server --output-filename=bundles/_dev_/[name].js --output-chunk-filename=bundles/_dev_/[name].js -w --mode development --disable-host-check --hot",
"lint": "yarn lint:types && yarn lint:js && yarn lint:style",
"lint:js": "yarn lint:js:src && yarn lint:js:module_system",
Expand Down Expand Up @@ -109,6 +109,7 @@
"@types/jest": "^29.0.0",
"@types/jitsi-meet": "^2.0.2",
"@types/jsrsasign": "^10.5.4",
"@types/loader-utils": "^2.0.4",
"@types/lodash": "^4.14.197",
"@types/modernizr": "^3.5.3",
"@types/node": "^16",
Expand All @@ -123,7 +124,7 @@
"babel-loader": "^8.2.2",
"chokidar": "^3.5.1",
"concurrently": "^8.0.0",
"cpx": "^1.5.0",
"cpx": "1.5.0",
"css-loader": "^4",
"dotenv": "^16.0.2",
"eslint": "8.48.0",
Expand Down
120 changes: 35 additions & 85 deletions scripts/copy-res.js → scripts/copy-res.ts
Original file line number Diff line number Diff line change
@@ -1,66 +1,27 @@
#!/usr/bin/env node

const loaderUtils = require("loader-utils");

// copies the resources into the webapp directory.
//

// Languages are listed manually, so we can choose when to include a translation in the app
// (because having a translation with only 3 strings translated is just frustrating)
// This could readily be automated, but it's nice to explicitly control when new languages are available.
const INCLUDE_LANGS = [
"bg",
"ca",
"cs",
"da",
"de_DE",
"el",
"en_EN",
"en_US",
"eo",
"es",
"et",
"eu",
"fi",
"fr",
"gl",
"he",
"hi",
"hu",
"id",
"is",
"it",
"ja",
"kab",
"ko",
"lo",
"lt",
"lv",
"nb_NO",
"nl",
"nn",
"pl",
"pt",
"pt_BR",
"ru",
"sk",
"sq",
"sr",
"sv",
"te",
"th",
"tr",
"uk",
"vi",
"vls",
"zh_Hans",
"zh_Hant",
];

import parseArgs from "minimist";
import * as chokidar from "chokidar";
import * as fs from "node:fs";
import _ from "lodash";
import { Cpx } from "cpx";
import * as loaderUtils from "loader-utils";

const I18N_BASE_PATH = "src/i18n/strings/";
const INCLUDE_LANGS = fs.readdirSync(I18N_BASE_PATH).filter((fn) => fn.endsWith(".json"));

// cpx includes globbed parts of the filename in the destination, but excludes
// common parents. Hence, "res/{a,b}/**": the output will be "dest/a/..." and
// "dest/b/...".
const COPY_LIST = [
const COPY_LIST: [
sourceGlob: string,
outputPath: string,
opts?: {
directwatch?: 1;
},
][] = [
["res/apple-app-site-association", "webapp"],
["res/manifest.json", "webapp"],
["res/sw.js", "webapp"],
Expand All @@ -74,19 +35,12 @@ const COPY_LIST = [
["./config.json", "webapp", { directwatch: 1 }],
["contribute.json", "webapp"],
];

const parseArgs = require("minimist");
const Cpx = require("cpx");
const chokidar = require("chokidar");
const fs = require("fs");
const _ = require("lodash");

const argv = parseArgs(process.argv.slice(2), {});

const watch = argv.w;
const verbose = argv.v;

function errCheck(err) {
function errCheck(err?: Error): void {
if (err) {
console.error(err.message);
process.exit(1);
Expand All @@ -102,7 +56,7 @@ if (!fs.existsSync("webapp/i18n/")) {
fs.mkdirSync("webapp/i18n/");
}

function next(i, err) {
function next(i: number, err?: Error): void {
errCheck(err);

if (i >= COPY_LIST.length) {
Expand All @@ -113,13 +67,9 @@ function next(i, err) {
const source = ent[0];
const dest = ent[1];
const opts = ent[2] || {};
let cpx = undefined;

if (!opts.lang) {
cpx = new Cpx.Cpx(source, dest);
}
const cpx = new Cpx(source, dest);

if (verbose && cpx) {
if (verbose) {
cpx.on("copy", (event) => {
console.log(`Copied: ${event.srcPath} --> ${event.dstPath}`);
});
Expand All @@ -128,7 +78,7 @@ function next(i, err) {
});
}

const cb = (err) => {
const cb = (err?: Error): void => {
next(i + 1, err);
};

Expand All @@ -138,7 +88,7 @@ function next(i, err) {
// which in the case of config.json is '.', which inevitably takes
// ages to crawl. So we create our own watcher on the files
// instead.
const copy = () => {
const copy = (): void => {
cpx.copy(errCheck);
};
chokidar.watch(source).on("add", copy).on("change", copy).on("ready", cb).on("error", errCheck);
Expand All @@ -152,9 +102,9 @@ function next(i, err) {
}
}

function genLangFile(lang, dest) {
function genLangFile(lang: string, dest: string): string {
const reactSdkFile = "node_modules/matrix-react-sdk/src/i18n/strings/" + lang + ".json";
const riotWebFile = "src/i18n/strings/" + lang + ".json";
const riotWebFile = I18N_BASE_PATH + lang + ".json";

let translations = {};
[reactSdkFile, riotWebFile].forEach(function (f) {
Expand All @@ -170,7 +120,7 @@ function genLangFile(lang, dest) {

const json = JSON.stringify(translations, null, 4);
const jsonBuffer = Buffer.from(json);
const digest = loaderUtils.getHashDigest(jsonBuffer, null, null, 7);
const digest = loaderUtils.getHashDigest(jsonBuffer, null, "hex", 7);
const filename = `${lang}.${digest}.json`;

fs.writeFileSync(dest + filename, json);
Expand All @@ -181,8 +131,8 @@ function genLangFile(lang, dest) {
return filename;
}

function genLangList(langFileMap) {
const languages = {};
function genLangList(langFileMap: Record<string, string>): void {
const languages: Record<string, string> = {};
INCLUDE_LANGS.forEach(function (lang) {
const normalizedLanguage = lang.toLowerCase().replace("_", "-");
const languageParts = normalizedLanguage.split("-");
Expand All @@ -194,7 +144,7 @@ function genLangList(langFileMap) {
});
fs.writeFile("webapp/i18n/languages.json", JSON.stringify(languages, null, 4), function (err) {
if (err) {
console.error("Copy Error occured: " + err);
console.error("Copy Error occured: " + err.message);
throw new Error("Failed to generate languages.json");
}
});
Expand All @@ -208,15 +158,15 @@ function genLangList(langFileMap) {
* regenerate the file, adding its content-hashed filename to langFileMap
* and regenerating languages.json with the new filename
*/
function watchLanguage(lang, dest, langFileMap) {
function watchLanguage(lang: string, dest: string, langFileMap: Record<string, string>): void {
const reactSdkFile = "node_modules/matrix-react-sdk/src/i18n/strings/" + lang + ".json";
const riotWebFile = "src/i18n/strings/" + lang + ".json";
const riotWebFile = I18N_BASE_PATH + lang + ".json";

// XXX: Use a debounce because for some reason if we read the language
// file immediately after the FS event is received, the file contents
// appears empty. Possibly https://github.com/nodejs/node/issues/6112
let makeLangDebouncer;
const makeLang = () => {
let makeLangDebouncer: ReturnType<typeof setTimeout>;
const makeLang = (): void => {
if (makeLangDebouncer) {
clearTimeout(makeLangDebouncer);
}
Expand All @@ -234,15 +184,15 @@ function watchLanguage(lang, dest, langFileMap) {

// language resources
const I18N_DEST = "webapp/i18n/";
const I18N_FILENAME_MAP = INCLUDE_LANGS.reduce((m, l) => {
const I18N_FILENAME_MAP = INCLUDE_LANGS.reduce<Record<string, string>>((m, l) => {
const filename = genLangFile(l, I18N_DEST);
m[l] = filename;
return m;
}, {});
genLangList(I18N_FILENAME_MAP);

if (watch) {
INCLUDE_LANGS.forEach((l) => watchLanguage(l.value, I18N_DEST, I18N_FILENAME_MAP));
INCLUDE_LANGS.forEach((l) => watchLanguage(l, I18N_DEST, I18N_FILENAME_MAP));
}

// non-language resources
Expand Down
43 changes: 43 additions & 0 deletions src/@types/cpx.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
Copyright 2023 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

declare module "cpx" {
export class Cpx {
public constructor(source: string, outDir: string, options?: object);

public on(eventName: "copy", fn: (event: { srcPath: string; dstPath: string }) => void): void;
public on(eventName: "remove", fn: (event: { path: string }) => void): void;
public on(eventName: "watch-ready", fn: () => void): void;
public on(eventName: "watch-error", fn: (error: Error) => void): void;

/**
* Copy all files that matches `this.source` pattern to `this.outDir`.
*
* @param {function} [cb = null] - A callback function.
* @returns {void}
*/
public copy(cb: Function | null): void;

/**
* Copy all files that matches `this.source` pattern to `this.outDir`.
* And watch changes in `this.base`, and copy only the file every time.
*
* @returns {void}
* @throws {Error} This had been watching already.
*/
public watch(): void;
}
}
28 changes: 28 additions & 0 deletions src/@types/loader-utils.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
Copyright 2023 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import * as LoaderUtils from "loader-utils";

declare module "loader-utils" {
export function getHashDigest(
buffer: Buffer,
hashType: null,
digestType: LoaderUtils.DigestType,
maxLength: number,
): string;
}

export as namespace Cpx;
39 changes: 0 additions & 39 deletions src/i18n/strings/ar.json

This file was deleted.

0 comments on commit 6dc6c80

Please sign in to comment.