Convert python scripts to python3 (#808)

* Convert python scripts to python3
* Enable python3-pip in travis
* run_linter uses shutil.which instead of 'which'
* Reimplement extract_strings in python
* Cleanup pylint
* Remove unused script
* Fix string-length checker

.travis.yml

@@ -46,6 +46,8 @@ addons:
       - mingw32-runtime
       - libfltk1.3-dev
       - gcovr
+      - python3
+      - python3-pip
 
 language: minimal
 #Do NOT set these.  They set 'CC' which breaks the build
@@ -60,7 +62,7 @@ before_install:
   - export PYTHONWARNINGS="ignore:A true SSLContext object is not available,ignore:An HTTPS request has been made"
   - export GITHUB_TOKEN='789c538d758e0a718c8ae537b7f6656776e20d768de18d546014e20876e1898933cd94648f75e06117e20000b71f080d'
   # TRAVIS does not override the TRAVIS_COMMIT_MSG with the 'message' value passed via the API (travis issue #10184)
-  - source <(curl -s -L "https://api.travis-ci.org/repos/$TRAVIS_REPO_SLUG/builds/$TRAVIS_BUILD_ID" | python -c 'import sys, json; a = json.load(sys.stdin); print "export TRAVIS_COMMIT_MSG=\"" + a.get("message","").replace("\"", "\\\"") + "\""')
+  - source <(curl -s -L "https://api.travis-ci.org/repos/$TRAVIS_REPO_SLUG/builds/$TRAVIS_BUILD_ID" | python3 -c 'import sys, json; a = json.load(sys.stdin); print("export TRAVIS_COMMIT_MSG=\"" + a.get("message","").replace("\"", "\\\"") + "\"")')
 
 install:
    - |
@@ -74,8 +76,8 @@ install:
      esac
      echo "Requirements: $reqs"
    # These are separated to make it easier to visualize the build in the travis logs
-   - if [[ $reqs == *"test"* ]]; then pip install --user cpp-coveralls; fi
-   - if [[ $reqs == *"lint"* ]]; then pip install --user cpplint==1.3.0; fi
+   - if [[ $reqs == *"test"* ]]; then pip3 install --user cpp-coveralls; fi
+   - if [[ $reqs == *"lint"* ]]; then pip3 install --user cpplint==1.3.0; fi
    - |
      if [[ $reqs == *"arm"* ]]; then
        if [[ ! -d "$HOME/gcc-arm-none-eabi-8-2018-q4-major/bin" ]]; then
@@ -134,7 +136,7 @@ install:
      fi
    - |
      if [[ $TRAVIS_BUILD_STAGE_NAME == "Nightly" || $TRAVIS_BUILD_STAGE_NAME == "Release" ]]; then
-         pip install --user urllib3==1.23 transifex-client
+         pip3 install --user urllib3==1.23 transifex-client
          openssl aes-256-cbc -K $encrypted_54cd4d6ff016_key -iv $encrypted_54cd4d6ff016_iv -in install_nightlies.tar.enc -out install_nightlies.tar -d
          tar -xf install_nightlies.tar;
          rm -f install_nightlies.tar

src/Makefile

@@ -222,7 +222,7 @@ distclean:
 	rm -rf deviation*.zip
 
 language:
-	../utils/extract_strings.pl -po > fs/language/deviation.po
+	../utils/extract_strings.py -po > fs/language/deviation.po
 
 
 zip: $(TARGET).zip
@@ -309,7 +309,7 @@ $(LAST_MODEL): model_template.ini tx_template.ini $(FONTS) $(PRE_FS)
 	cp model_template.ini filesystem/$(FILESYSTEM)/models/default.ini
 ifdef LANGUAGE
 	mkdir filesystem/$(FILESYSTEM)/language 2> /dev/null; \
-               ../utils/extract_strings.pl -po -fs filesystem/$(FILESYSTEM)/language -targets $(LANGUAGE) -update -objdir $(ODIR)
+               ../utils/extract_strings.py -po -fs filesystem/$(FILESYSTEM)/language -targets $(LANGUAGE) -update -objdir $(ODIR)
 endif
 	export tx=$(FILESYSTEM); \
 	number=2 ; while [ $$number -le $(NUM_MODELS) ] ; do \
@@ -318,9 +318,9 @@ endif
 		done
 	@echo " + Checking string list length for $(FILESYSTEM)"
 ifeq "$(TYPE)" "dev"
-	../utils/check_string_size.pl -target $(FILESYSTEM) -objdir $(ODIR)
+	../utils/check_string_size.pl -target $(FILESYSTEM) -objdir $(ODIR) -language $(LANGUAGE)
 else
-	../utils/check_string_size.pl -target $(FILESYSTEM) -objdir $(ODIR) -quiet
+	../utils/check_string_size.pl -target $(FILESYSTEM) -objdir $(ODIR) -quiet -language $(LANGUAGE)
 endif
 	../utils/run_linter.py --diff --skip-github --no-fail
 

src/target/tx/devo/devo12/Makefile.inc

@@ -4,7 +4,7 @@ FILESYSTEMS := common base_fonts 320x240x16 480x272x16
 DFU_ARGS    := -c 12 -b 0x08004000
 FONTS        = filesystem/$(FILESYSTEM)/media/15normal.fon \
                filesystem/$(FILESYSTEM)/media/23bold.fon
-LANGUAGE    := devo8,devo12
+LANGUAGE    := devo8 devo12
 
 OPTIMIZE_DFU := 1
 

utils/archive_build_size.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """Track changes in the size of an image using Travis-CI and GitHub statuses.
 
 This code is adapted from https://github.com/danvk/travis-weigh-in
@@ -18,7 +18,7 @@
 import re
 import sys
 import json
-import urllib2
+import urllib.request, urllib.error, urllib.parse
 import zlib
 from collections import namedtuple
 
@@ -64,18 +64,18 @@ def post_status(url, state, context, description):
     }
     headers = {'Authorization': 'token ' + get_token()}
 
-    request = urllib2.Request(url, json.dumps(data), headers)
-    res = urllib2.urlopen(request)
+    request = urllib.request.Request(url, json.dumps(data), headers)
+    res = urllib.request.urlopen(request)
     raise_for_status(url, res)
 
-    print 'Posted %s' % json.dumps(data)
+    print('Posted %s' % json.dumps(data))
 
 
 def get_status(url, context):
     """Request status for previous build from GitHub"""
     headers = {'Authorization': 'token ' + get_token()}
-    request = urllib2.Request(url, None, headers)
-    res = urllib2.urlopen(request)
+    request = urllib.request.Request(url, None, headers)
+    res = urllib.request.urlopen(request)
     raise_for_status(url, res)
 
     data = json.loads(res.read())
@@ -93,8 +93,8 @@ def get_pr_info(slug, pull_number):
     """Retireve pull-request info"""
     url = 'https://api.github.com/repos/%s/pulls/%s' % (slug, pull_number)
     headers = {'Authorization': 'token ' + get_token()}
-    request = urllib2.Request(url, None, headers)
-    res = urllib2.urlopen(request)
+    request = urllib.request.Request(url, None, headers)
+    res = urllib.request.urlopen(request)
     raise_for_status(url, res)
     return json.loads(res.read())
 
@@ -127,8 +127,8 @@ def format_description(current_size, previous_size):
         rom_delta = current_size.rom - previous_size.rom
         ram_delta = current_size.ram - previous_size.ram
         return (
-            u'ROM: {:,.0f} \u0394: {:+,.0f}b/{:+0.2f}% '
-            u'RAM: {:,.0f} \u0394: {:+,.0f}b/{:+0.2f}%'.format(
+            'ROM: {:,.0f} \u0394: {:+,.0f}b/{:+0.2f}% '
+            'RAM: {:,.0f} \u0394: {:+,.0f}b/{:+0.2f}%'.format(
                 current_size.rom, rom_delta, 100.0 * rom_delta /  current_size.rom,
                 current_size.ram, ram_delta, 100.0 * ram_delta /  current_size.ram))
 
@@ -204,24 +204,24 @@ def main():
 
     filename = sys.argv[1].replace("zip_", "").replace("win_", "") + ".map"
     if not os.path.exists(filename):
-        print "No map file generated ({}): {}".format(sys.argv[1], filename)
+        print("No map file generated ({}): {}".format(sys.argv[1], filename))
         return
 
     check_environment()
 
     current_size = get_image_size(filename)
     previous_size = get_base_size(filename)
 
-    print '%s Current:  ROM: %s RAM: %s' % (filename, current_size.rom, current_size.ram)
+    print('%s Current:  ROM: %s RAM: %s' % (filename, current_size.rom, current_size.ram))
     if previous_size:
-        print '%s Previous: ROM: %s RAM: %s' % (filename, previous_size.rom, previous_size.ram)
+        print('%s Previous: ROM: %s RAM: %s' % (filename, previous_size.rom, previous_size.ram))
 
     if TRAVIS_STATUS_URL:
         url = TRAVIS_STATUS_URL
     else:
         url = 'https://api.github.com/repos/%s/statuses/%s' % (TRAVIS_REPO_SLUG, TRAVIS_COMMIT)
 
-    print 'POSTing to %s' % url
+    print('POSTing to %s' % url)
     post_status(url, 'success', get_context(filename),
                 format_description(current_size, previous_size))
 

utils/check_string_size.pl

@@ -9,8 +9,9 @@
 my $quiet;
 my $objdir;
 my $elffile;
+my @language;
 my $log = "";
-GetOptions("target=s" => \$target, "quiet" => \$quiet, "objdir=s" => \$objdir, "elffile=s" => \$elffile);
+GetOptions("target=s" => \$target, "quiet" => \$quiet, "objdir=s" => \$objdir, "elffile=s" => \$elffile, "language=s{1,}" => \@language);
 
 my @dirs = grep {$_ !~ /common/ && (! $target || $_ =~ /$target/)} glob("filesystem/*");
 my $max_line_length = 0;
@@ -69,7 +70,7 @@
         $target_count = $count if($count > $target_count);
         $target_line_length = $line_length if($line_length > $target_line_length);
     }
-    my $cmd = "../utils/extract_strings.pl -target $target";
+    my $cmd = "../utils/extract_strings.py -target @language";
     $cmd .= " -objdir $objdir" if($objdir);
     $cmd .= " -elffile $elffile" if($elffile);
     print $cmd;

utils/clean_transifex_merge.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 import subprocess
@@ -9,7 +9,7 @@
     cmd += sys.argv[1:]
 
 res = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-lines = [line.rstrip() for line in res.stdout]
+lines = [line.decode('utf-8').rstrip() for line in res.stdout]
 lines.append("diff") # Crude way to ensure final file is handled
 
 filename = None

utils/datalog2csv.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 import sys
@@ -33,7 +33,7 @@ def add_elem(self, data):
             item.append(self.format_data(i, data[idx:]))
             idx += size
         if idx != self.capture_size:
-            print "Size mismatch"
+            print("Size mismatch")
             sys.exit(1)
         self.data.append(item)
     def to_model(self, value):
@@ -44,7 +44,7 @@ def to_model(self, value):
         telem_extra_items = 49
         telem_extra = []
         for i in range(telem_extra_items):
-            telem_extra.append("TELEM_" + `i`)
+            telem_extra.append("TELEM_" + repr(i))
         inp    = []
         outch  = ["Channel1", "Channel2", "Channel3", "Channel4",
                   "Channel5", "Channel6", "Channel7", "Channel8",
@@ -227,10 +227,10 @@ def main():
                       help="write outcpature # <value>")
     (opt, args) = parser.parse_args()
     if not opt.bin:
-        print "Must specify --bin"
+        print("Must specify --bin")
         return
     if not os.path.exists(opt.bin):
-        print "Could not locate bin file: " + opt.bin
+        print("Could not locate bin file: " + opt.bin)
         return
     if opt.clear:
         stat = os.stat(opt.bin)
@@ -269,9 +269,7 @@ def main():
         for o in out:
             printf("%s", o)
 def parse_file(bin):
-    data = []
-    for i in open(bin,'rb').read() :
-        data.append(ord(i))
+    data = open(bin,'rb').read()
     idx = 0
     size = 1
     info = []