Add pre-commit hooks to sort localization and minify

.gitignore

@@ -90,3 +90,4 @@ third-party/libcrafter/
 prime/
 stage/
 parts/
+hooks/.enabled

Makefile.in

@@ -157,7 +157,7 @@ endif
 
 .NOTPARALLEL: default all
 
-default: $(NDPI_LIB_DEP) $(LIB_TARGETS) $(TARGET)
+default: hooks/.enabled $(NDPI_LIB_DEP) $(LIB_TARGETS) $(TARGET)
 
 all: default
 
@@ -188,6 +188,10 @@ $(LIBRRDTOOL_LIB):
 $(JSON_LIB):
 	cd $(JSON_HOME); ./autogen.sh; ./configure; @GMAKE@
 
+hooks/.enabled:
+	git config core.hooksPath hooks || true
+	touch hooks/.enabled
+
 clean:
 	-rm -f src/*.o src/*~ include/*~ *~ #config.h
 	-rm -f $(TARGET)

hooks/pre-commit

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+if which lua 2>&1 >/dev/null; then
+	# Sort locales
+	for fname in `ls scripts/locales`; do
+		full_path=scripts/locales/${fname}
+
+		if ! git diff --staged --quiet $full_path; then
+			locale=${fname%%.lua}
+			tools/localization/localize.sh sort $locale
+			git add $full_path
+		fi
+	done
+fi
+
+if which uglifyjs 2>&1 >/dev/null; then
+	# Minify scripts
+	if ! git diff --staged --quiet httpdocs/js; then
+		make minify
+		git add -u httpdocs/js/*.min.js
+		git add -u httpdocs/js/*.min.js.map
+	fi
+fi

tools/localization/README.md

@@ -0,0 +1,54 @@
+# Localization tools
+
+The tool `localize.sh` can be used to perform many localization checks and
+operations to complement the actual file localization tasks.
+
+Here are some examples of what to do in order to fulfil the localization
+requirements.
+
+## After a manual modification of a localization file the file must be sorted
+```
+tools/localization/localize.sh sort en
+```
+
+## Verify the current localization status of a language
+```
+tools/localization/localize.sh status de
+```
+
+## Send the new strings to localize to a third party
+```
+tools/localization/localize.sh missing de > to_localize_de.txt
+```
+
+This will generate the `to_localize_de.txt` which can be sent to the third party
+translator. It has the following format:
+
+```
+lang.manage_users.manage = "Manage"
+lang.manage_users.manage_user_x = "Manage User %{user}"
+...
+```
+
+*Important*: the third party translator should only modify the strings located at the
+*right* of the `=` sign while trying to maintain the same punctation and format used.
+Strings enclosed in curly braces `%{user}` *should not* be localized.
+
+Moreover, the translator should not modify *the structure* of the txt file. After
+translating the file, the translator should send back the modified text file.
+
+Here is an example of the localized version of `to_localize_de.txt`:
+
+```
+lang.manage_users.manage = "Verwalten"
+lang.manage_users.manage_user_x = "Verwalte User %{user}"
+...
+```
+
+By keeping the file format consistent, the localized strings can be easily
+integrated as explained below.
+
+## Integrate the third party localized strings
+```
+tools/localization/localize.sh extend de path_to_localized_strings.txt
+```

tools/localization/localize.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+function usage {
+  echo -e "Usage: `basename $0` action parameters"
+  echo
+  echo -e "Actions:"
+  echo -e " sort [lang]: sorts the specified localization file after modification"
+  echo -e " status [lang]: verifies the localization status of lang"
+  echo -e " missing [lang]: get a report of missing strings to localize for the lang"
+  echo -e " all [lang]: get a report of all the localized strings for lang"
+  echo -e " extend [lang] [extension_txt]: extends the localization with a localized report"
+  echo
+  echo -e "A report is in the format:"
+  echo -e " lang.manage_users.manage = \"Verwalten\""
+  echo -e " lang.manage_users.manage_user_x = \"Verwalte User %{user}\""
+  echo -e " ..."
+  exit 1
+}
+
+# pro root
+base_path="../tools/localization"
+root_path=".."
+
+if [[ -d src ]]; then
+  # ntopng root
+  base_path="tools/localization"
+  root_path="."
+elif [[ ! -d tools ]]; then
+  # inside localization folder
+  base_path="."
+  root_path="../../.."
+fi
+
+if [[ $# -lt 1 ]]; then
+  usage
+fi
+
+case $1 in
+sort)
+  lang=$2
+  if [[ -z $lang ]]; then usage; fi
+
+  lua "$base_path/sort_localization_file.lua" "$lang"
+  ;;
+status)
+  lang=$2
+  if [[ -z $lang ]]; then usage; fi
+
+  "$base_path/missing_localization.py" "$root_path/scripts/locales/en.lua" "$root_path/pro/scripts/locales/${lang}.lua" | grep -v ".nedge."
+  ;;
+missing)
+  lang=$2
+  if [[ -z $lang ]]; then usage; fi
+
+  "$base_path/missing_localization.py" "$root_path/scripts/locales/en.lua" "$root_path/pro/scripts/locales/${lang}.lua" | grep -v ".nedge." | awk '{ $1=""; $2 = ""; print $0; }'
+  ;;
+all)
+  lang=$2
+  if [[ -z $lang ]]; then usage; fi
+  loc_path=
+
+  if [[ $lang == "en" ]]; then
+    loc_path="$root_path/scripts/locales"
+  else
+    loc_path="$root_path/pro/scripts/locales"
+  fi
+  "$base_path/missing_localization.py" /dev/null "${loc_path}/${lang}.lua" | grep -v ".nedge." | awk '{ $1=""; $2 = ""; print $0; }'
+  ;;
+extend)
+  lang=$2
+  extension_file=$3
+  if [[ -z $lang ]]; then usage; fi
+  if [[ -z $extension_file ]]; then usage; fi
+
+  lua "$base_path/sort_localization_file.lua" "$lang" "$extension_file"
+  ;;
+*)  usage
+esac

tools/localization/missing_localization.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+#
+# missing_localization.py                                     Emanuele Faranda
+# A tool to find missing localization strings
+#
+# Sample invocation:
+#   tools/localization/missing_localization.py scripts/locales/en.lua pro/scripts/locales/de.lua | grep -v ".nedge." | awk '{ $2 = ""; print $0; }'
+
+import sys
+import difflib
+
+def extract_table_key(x):
+  if x.startswith('["') and x.endswith('"]'):
+    return x[2:-2]
+  return x
+
+class LocalizationFile(object):
+  def __init__(self, name):
+    self.f = open(name, "r", encoding='UTF-8')
+    self.next_skip = False
+    self.cur_section = []
+    self.line_no = 0
+
+  def __iter__(self):
+    return self
+
+  # Iterates on localization ids
+  def __next__(self):
+    while True:
+      line = next(self.f).strip()
+      self.line_no += 1
+      is_section_start = line.endswith("{") and line.find(" = ") != None
+      is_section_end = line.startswith("}")
+
+      if is_section_start:
+        self.cur_section.append(extract_table_key(line.split(" = ", 1)[0].split()[-1]))
+      elif is_section_end:
+        self.cur_section.pop()
+      elif not line.startswith("--") and self.cur_section:
+        if not self.next_skip:
+          value = line.split("=", 1)
+
+          if len(value) == 2:
+            localized_id, localized_str = value
+            localized_id = extract_table_key(localized_id.strip())
+            localized_str = localized_str.strip().strip(",").strip('"')
+
+            if localized_str.endswith(".."):
+              # String continues on next line
+              self.next_skip = True
+
+            return ".".join(self.cur_section) + "." + localized_id, self.line_no, localized_str
+        else:
+          self.next_skip = line.endswith("..")
+
+# Wrapper to provide len and indexing on the LocalizationFile
+class LocalizationReaderWrapper(object):
+  def __init__(self, localization_file_obj):
+    self.localiz_obj = localization_file_obj
+    self.lines = []
+    self.line_id_to_line = {}
+    self.populateLines()
+
+  def __len__(self):
+    return len(self.lines)
+
+  def __getitem__(self, idx):
+      return self.lines[idx]
+
+  def __iter__(self):
+    return self.lines.__iter__()
+
+  def populateLines(self):
+    for line in self.localiz_obj:
+      self.lines.append(line[0])
+      self.line_id_to_line[line[0]] = line
+
+  def getLineInfo(self, line_id):
+    return self.line_id_to_line[line_id]
+
+if __name__ == "__main__":
+  def usage():
+    print("Usage: " + sys.argv[0] + " base_file cmp_file")
+    exit(1)
+
+  if len(sys.argv) != 3:
+    usage()
+
+  base_file = LocalizationReaderWrapper(LocalizationFile(sys.argv[1]))
+  cmp_file = LocalizationReaderWrapper(LocalizationFile(sys.argv[2]))
+  difftool = difflib.Differ()
+  diff = difftool.compare(base_file, cmp_file)
+
+  for line in diff:
+    if not line.startswith(" "):
+      if line.startswith("-"):
+        wrapper = base_file
+      elif line.startswith("+"):
+        wrapper = cmp_file
+      else:
+        print(line)
+        continue
+
+      line_info = wrapper.getLineInfo(line.split()[-1])
+
+      print("%d) %s = \"%s\"" % (
+        line_info[1],
+        line,
+        line_info[2],
+      ))