diff --git a/.gitignore b/.gitignore index 417a17773b9c..e072e5d7a0e6 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,4 @@ third-party/libcrafter/ prime/ stage/ parts/ +hooks/.enabled diff --git a/Makefile.in b/Makefile.in index 5b326e59dcb5..d0bcd09b9da0 100755 --- a/Makefile.in +++ b/Makefile.in @@ -157,7 +157,7 @@ endif .NOTPARALLEL: default all -default: $(NDPI_LIB_DEP) $(LIB_TARGETS) $(TARGET) +default: hooks/.enabled $(NDPI_LIB_DEP) $(LIB_TARGETS) $(TARGET) all: default @@ -188,6 +188,10 @@ $(LIBRRDTOOL_LIB): $(JSON_LIB): cd $(JSON_HOME); ./autogen.sh; ./configure; @GMAKE@ +hooks/.enabled: + git config core.hooksPath hooks || true + touch hooks/.enabled + clean: -rm -f src/*.o src/*~ include/*~ *~ #config.h -rm -f $(TARGET) diff --git a/hooks/pre-commit b/hooks/pre-commit new file mode 100755 index 000000000000..047d42be05a0 --- /dev/null +++ b/hooks/pre-commit @@ -0,0 +1,23 @@ +#!/bin/sh + +if which lua 2>&1 >/dev/null; then + # Sort locales + for fname in `ls scripts/locales`; do + full_path=scripts/locales/${fname} + + if ! git diff --staged --quiet $full_path; then + locale=${fname%%.lua} + tools/localization/localize.sh sort $locale + git add $full_path + fi + done +fi + +if which uglifyjs 2>&1 >/dev/null; then + # Minify scripts + if ! git diff --staged --quiet httpdocs/js; then + make minify + git add -u httpdocs/js/*.min.js + git add -u httpdocs/js/*.min.js.map + fi +fi diff --git a/tools/localization/README.md b/tools/localization/README.md new file mode 100644 index 000000000000..c95bc439a3a3 --- /dev/null +++ b/tools/localization/README.md @@ -0,0 +1,54 @@ +# Localization tools + +The tool `localize.sh` can be used to perform many localization checks and +operations to complement the actual file localization tasks. + +Here are some examples of what to do in order to fulfil the localization +requirements. + +## After a manual modification of a localization file the file must be sorted +``` +tools/localization/localize.sh sort en +``` + +## Verify the current localization status of a language +``` +tools/localization/localize.sh status de +``` + +## Send the new strings to localize to a third party +``` +tools/localization/localize.sh missing de > to_localize_de.txt +``` + +This will generate the `to_localize_de.txt` which can be sent to the third party +translator. It has the following format: + +``` +lang.manage_users.manage = "Manage" +lang.manage_users.manage_user_x = "Manage User %{user}" +... +``` + +*Important*: the third party translator should only modify the strings located at the +*right* of the `=` sign while trying to maintain the same punctation and format used. +Strings enclosed in curly braces `%{user}` *should not* be localized. + +Moreover, the translator should not modify *the structure* of the txt file. After +translating the file, the translator should send back the modified text file. + +Here is an example of the localized version of `to_localize_de.txt`: + +``` +lang.manage_users.manage = "Verwalten" +lang.manage_users.manage_user_x = "Verwalte User %{user}" +... +``` + +By keeping the file format consistent, the localized strings can be easily +integrated as explained below. + +## Integrate the third party localized strings +``` +tools/localization/localize.sh extend de path_to_localized_strings.txt +``` diff --git a/tools/localization/localize.sh b/tools/localization/localize.sh new file mode 100755 index 000000000000..357d16039b21 --- /dev/null +++ b/tools/localization/localize.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +function usage { + echo -e "Usage: `basename $0` action parameters" + echo + echo -e "Actions:" + echo -e " sort [lang]: sorts the specified localization file after modification" + echo -e " status [lang]: verifies the localization status of lang" + echo -e " missing [lang]: get a report of missing strings to localize for the lang" + echo -e " all [lang]: get a report of all the localized strings for lang" + echo -e " extend [lang] [extension_txt]: extends the localization with a localized report" + echo + echo -e "A report is in the format:" + echo -e " lang.manage_users.manage = \"Verwalten\"" + echo -e " lang.manage_users.manage_user_x = \"Verwalte User %{user}\"" + echo -e " ..." + exit 1 +} + +# pro root +base_path="../tools/localization" +root_path=".." + +if [[ -d src ]]; then + # ntopng root + base_path="tools/localization" + root_path="." +elif [[ ! -d tools ]]; then + # inside localization folder + base_path="." + root_path="../../.." +fi + +if [[ $# -lt 1 ]]; then + usage +fi + +case $1 in +sort) + lang=$2 + if [[ -z $lang ]]; then usage; fi + + lua "$base_path/sort_localization_file.lua" "$lang" + ;; +status) + lang=$2 + if [[ -z $lang ]]; then usage; fi + + "$base_path/missing_localization.py" "$root_path/scripts/locales/en.lua" "$root_path/pro/scripts/locales/${lang}.lua" | grep -v ".nedge." + ;; +missing) + lang=$2 + if [[ -z $lang ]]; then usage; fi + + "$base_path/missing_localization.py" "$root_path/scripts/locales/en.lua" "$root_path/pro/scripts/locales/${lang}.lua" | grep -v ".nedge." | awk '{ $1=""; $2 = ""; print $0; }' + ;; +all) + lang=$2 + if [[ -z $lang ]]; then usage; fi + loc_path= + + if [[ $lang == "en" ]]; then + loc_path="$root_path/scripts/locales" + else + loc_path="$root_path/pro/scripts/locales" + fi + "$base_path/missing_localization.py" /dev/null "${loc_path}/${lang}.lua" | grep -v ".nedge." | awk '{ $1=""; $2 = ""; print $0; }' + ;; +extend) + lang=$2 + extension_file=$3 + if [[ -z $lang ]]; then usage; fi + if [[ -z $extension_file ]]; then usage; fi + + lua "$base_path/sort_localization_file.lua" "$lang" "$extension_file" + ;; +*) usage +esac diff --git a/tools/localization/missing_localization.py b/tools/localization/missing_localization.py new file mode 100755 index 000000000000..50882f6b898a --- /dev/null +++ b/tools/localization/missing_localization.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +# +# missing_localization.py Emanuele Faranda +# A tool to find missing localization strings +# +# Sample invocation: +# tools/localization/missing_localization.py scripts/locales/en.lua pro/scripts/locales/de.lua | grep -v ".nedge." | awk '{ $2 = ""; print $0; }' + +import sys +import difflib + +def extract_table_key(x): + if x.startswith('["') and x.endswith('"]'): + return x[2:-2] + return x + +class LocalizationFile(object): + def __init__(self, name): + self.f = open(name, "r", encoding='UTF-8') + self.next_skip = False + self.cur_section = [] + self.line_no = 0 + + def __iter__(self): + return self + + # Iterates on localization ids + def __next__(self): + while True: + line = next(self.f).strip() + self.line_no += 1 + is_section_start = line.endswith("{") and line.find(" = ") != None + is_section_end = line.startswith("}") + + if is_section_start: + self.cur_section.append(extract_table_key(line.split(" = ", 1)[0].split()[-1])) + elif is_section_end: + self.cur_section.pop() + elif not line.startswith("--") and self.cur_section: + if not self.next_skip: + value = line.split("=", 1) + + if len(value) == 2: + localized_id, localized_str = value + localized_id = extract_table_key(localized_id.strip()) + localized_str = localized_str.strip().strip(",").strip('"') + + if localized_str.endswith(".."): + # String continues on next line + self.next_skip = True + + return ".".join(self.cur_section) + "." + localized_id, self.line_no, localized_str + else: + self.next_skip = line.endswith("..") + +# Wrapper to provide len and indexing on the LocalizationFile +class LocalizationReaderWrapper(object): + def __init__(self, localization_file_obj): + self.localiz_obj = localization_file_obj + self.lines = [] + self.line_id_to_line = {} + self.populateLines() + + def __len__(self): + return len(self.lines) + + def __getitem__(self, idx): + return self.lines[idx] + + def __iter__(self): + return self.lines.__iter__() + + def populateLines(self): + for line in self.localiz_obj: + self.lines.append(line[0]) + self.line_id_to_line[line[0]] = line + + def getLineInfo(self, line_id): + return self.line_id_to_line[line_id] + +if __name__ == "__main__": + def usage(): + print("Usage: " + sys.argv[0] + " base_file cmp_file") + exit(1) + + if len(sys.argv) != 3: + usage() + + base_file = LocalizationReaderWrapper(LocalizationFile(sys.argv[1])) + cmp_file = LocalizationReaderWrapper(LocalizationFile(sys.argv[2])) + difftool = difflib.Differ() + diff = difftool.compare(base_file, cmp_file) + + for line in diff: + if not line.startswith(" "): + if line.startswith("-"): + wrapper = base_file + elif line.startswith("+"): + wrapper = cmp_file + else: + print(line) + continue + + line_info = wrapper.getLineInfo(line.split()[-1]) + + print("%d) %s = \"%s\"" % ( + line_info[1], + line, + line_info[2], + )) diff --git a/tools/localization/persistence.lua b/tools/localization/persistence.lua new file mode 100644 index 000000000000..5b0a9a4e39d6 --- /dev/null +++ b/tools/localization/persistence.lua @@ -0,0 +1,218 @@ +-- Adapted from http://lua-users.org/wiki/TablePersistence to preserve order and +-- put single values at the end of the stream + +local write, writeIndent, writers, refCount; + +function pairsByKeyVals(t, f) + local a = {} + + -- io.write(debug.traceback().."\n") + for n in pairs(t) do table.insert(a, n) end + table.sort(a, function(x, y) return f(x, y, t[x], t[y]) end) + local i = 0 -- iterator variable + local iter = function () -- iterator function + i = i + 1 + if a[i] == nil then return nil + else return a[i], t[a[i]] + end + end + return iter +end +function asc_table_after(a,b,t_a,t_b) + local a_is_table = (type(t_a) == "table") + local b_is_table = (type(t_b) == "table") + + if a_is_table and not b_is_table then + return false + elseif not a_is_table and b_is_table then + return true + else + return (a < b) + end +end + +persistence = +{ + store = function (path, ...) + local file, e = io.open(path, "w"); + if not file then + return error(e); + end + local n = select("#", ...); + -- Count references + local objRefCount = {}; -- Stores reference that will be exported + for i = 1, n do + refCount(objRefCount, (select(i,...))); + end; + -- Export Objects with more than one ref and assign name + -- First, create empty tables for each + local objRefNames = {}; + local objRefIdx = 0; + + local has_objRefCount = false + for obj, count in pairs(objRefCount) do + if count > 1 then + has_objRefCount = true + break + end + end + + if has_objRefCount then + file:write("-- Persistent Data\n"); + file:write("local multiRefObjects = {\n"); + for obj, count in pairs(objRefCount) do + if count > 1 then + objRefIdx = objRefIdx + 1; + objRefNames[obj] = objRefIdx; + file:write("{};"); -- table objRefIdx + end; + end; + file:write("\n} -- multiRefObjects\n"); + + -- Then fill them (this requires all empty multiRefObjects to exist) + for obj, idx in pairs(objRefNames) do + for k, v in pairs(obj) do + file:write("multiRefObjects["..idx.."]["); + write(file, k, 0, objRefNames); + file:write("] = "); + write(file, v, 0, objRefNames); + file:write("\n"); + end; + end; + end + -- Create the remaining objects + for i = 1, n do + local suffix = "" + if i > 1 then + suffix = (i-1) .. "" + end + file:write("local ".."lang"..suffix.." = "); + write(file, (select(i,...)), 0, objRefNames); + file:write("\n"); + end + -- Return them + if n > 0 then + file:write("\nreturn lang"); + for i = 2, n do + file:write(" ,lang"..(i-1)); + end; + file:write("\n"); + else + file:write("return\n"); + end; + if type(path) == "string" then + file:close(); + end; + end; + + load = function (path) + local f, e; + if type(path) == "string" then + f, e = loadfile(path); + else + f, e = path:read('*a') + end + if f then + return f(); + else + return nil, e; + end; + end; +} + +-- Private methods + +-- write thing (dispatcher) +write = function (file, item, level, objRefNames) + writers[type(item)](file, item, level, objRefNames); +end; + +-- write indent +writeIndent = function (file, level) + for i = 1, level do + file:write(" "); + end; +end; + +-- recursively count references +refCount = function (objRefCount, item) + -- only count reference types (tables) + if type(item) == "table" then + -- Increase ref count + if objRefCount[item] then + objRefCount[item] = objRefCount[item] + 1; + else + objRefCount[item] = 1; + -- If first encounter, traverse + for k, v in pairs(item) do + refCount(objRefCount, k); + refCount(objRefCount, v); + end; + end; + end; +end; + +-- Format items for the purpose of restoring +writers = { + ["nil"] = function (file, item) + file:write("nil"); + end; + ["number"] = function (file, item) + file:write(tostring(item)); + end; + ["string"] = function (file, item) + file:write(string.format("%q", item)); + end; + ["boolean"] = function (file, item) + if item then + file:write("true"); + else + file:write("false"); + end + end; + ["table"] = function (file, item, level, objRefNames) + local refIdx = objRefNames[item]; + if refIdx then + -- Table with multiple references + file:write("multiRefObjects["..refIdx.."]"); + else + -- Single use table + file:write("{\n"); + for k, v in pairsByKeyVals(item, asc_table_after) do + writeIndent(file, level+1); + file:write("["); + write(file, k, level+1, objRefNames); + file:write("] = "); + write(file, v, level+1, objRefNames); + file:write(",\n"); + end + writeIndent(file, level); + file:write("}"); + end; + end; + ["function"] = function (file, item) + -- Does only work for "normal" functions, not those + -- with upvalues or c functions + local dInfo = debug.getinfo(item, "uS"); + if dInfo.nups > 0 then + file:write("nil --[[functions with upvalue not supported]]"); + elseif dInfo.what ~= "Lua" then + file:write("nil --[[non-lua function not supported]]"); + else + local r, s = pcall(string.dump,item); + if r then + file:write(string.format("loadstring(%q)", s)); + else + file:write("nil --[[function could not be dumped]]"); + end + end + end; + ["thread"] = function (file, item) + file:write("nil --[[thread]]\n"); + end; + ["userdata"] = function (file, item) + file:write("nil --[[userdata]]\n"); + end; +} + +return persistence diff --git a/tools/localization/persistence.luar b/tools/localization/persistence.luar new file mode 100644 index 000000000000..fda590cf4dbf Binary files /dev/null and b/tools/localization/persistence.luar differ diff --git a/tools/localization/sort_localization_file.lua b/tools/localization/sort_localization_file.lua new file mode 100644 index 000000000000..089833d7c2b5 --- /dev/null +++ b/tools/localization/sort_localization_file.lua @@ -0,0 +1,99 @@ +-- A script to sort a localization file + +local community_base_path = "scripts/locales" +local pro_base_path = "pro/scripts/locales" + +if (#arg ~= 1) and (#arg ~= 2) then + print([[Usage: lua ]] .. arg[0] .. [[ localization_code [merge_strings] + +The merge_strings parameter can be used to load additional strings to merge into +the localization. The strings should be a list similar to this: + + lang.manage_users.add_new_user = "Add New User" + lang.manage_users.expires_after = "Expires after" + +Example: ]] .. arg[0] .. [[ en"]]) + + os.exit(1) +end + +local lang_code = arg[1] +local merge_strings_file = arg[2] + +local root_path +local pkgpath_path +local base_path + +local function file_exists(name) + local f=io.open(name,"r") + if f~=nil then io.close(f) return true else return false end +end + +if file_exists("persistence.lua") then + -- localization folder + root_path = "../../.." + pkgpath_path = "." +elseif file_exists("src/Ntop.cpp") then + -- ntopng root + root_path = "." + pkgpath_path = "tools/localization" +else + -- pro root + root_path = ".." + pkgpath_path = "../tools/localization" +end + +package.path = pkgpath_path .. "/?.lua;" .. package.path + +if file_exists(root_path.."/"..community_base_path.."/"..lang_code..".lua") then + base_path = root_path.."/"..community_base_path +else + base_path = root_path.."/"..pro_base_path +end + +package.path = base_path .. "/?.lua;" .. package.path +local persistance = require("persistence") +local lang_file = base_path .. "/" .. lang_code .. ".lua" + +local lang = require(lang_code) + +if merge_strings_file then + local f = assert(io.open(merge_strings_file, "r")) + local lines = f:read("*all") + f:close() + + for line in lines:gmatch("[^\r\n]+") do + line = line:gsub("\\\"", "\"") + local k, v = line:gmatch("%s*([^%s]+)%s*=%s*\"(.+)\"$")() + + -- merge + if (k ~= nil) and (v ~= nil) then + print(k .. " = " .. v) + local t = lang + local prev_t = t + local prev_k = nil + + for part in k:gmatch("[^%.]+") do + part = part:gmatch("%[\"([^\"]+)\"%]")() or part + + if not ((t == lang) and (part == "lang")) then + if t[part] == nil then + t[part] = {} + end + + prev_t = t + t = t[part] + prev_k = part + end + end + + if prev_k ~= nil then + prev_t[prev_k] = v + end + end + end + + +end + +persistence.store(lang_file, lang) diff --git a/tools/localization/sort_localization_file.luar b/tools/localization/sort_localization_file.luar new file mode 100644 index 000000000000..4e1d2880be4f Binary files /dev/null and b/tools/localization/sort_localization_file.luar differ