ON-13632: Update scripts folder to python 3

ON-13632: Update onload packaging to remove python2 references

ON-13632: Update onload spec file with appropriate python dev dependencies

scripts/anonymise_stackdump.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python2
+#! /usr/bin/env python3
 #
 # SPDX-License-Identifier: BSD-2-Clause
 # X-SPDX-Copyright-Text: (c) Copyright 2015-2020 Xilinx, Inc.

scripts/debian/debian-templ/control

@@ -2,13 +2,13 @@ Source: #TYPE#
 Section: net
 Priority: extra
 Maintainer: Support <support@solarflare.com>
-Build-Depends: debhelper (>= 8.0.0), bzip2, gawk, glibc | libc6, python2-dev | python-dev, libpcap-dev, libcap-dev
+Build-Depends: debhelper (>= 8.0.0), bzip2, gawk, glibc | libc6, python3-dev, libpcap-dev, libcap-dev
 Standards-Version: 3.9.3
 Homepage: http://www.openonload.org
 
 Package: #TYPE#-user
 Architecture: any
-Depends: adduser, psmisc, ${shlibs:Depends}, ${misc:Depends}, python2 | python
+Depends: adduser, psmisc, ${shlibs:Depends}, ${misc:Depends}, python3
 Suggests: #TYPE#-modules-${source:Version}
 Description: Onload TCP/IP stack - library
 

scripts/license-validate.py

@@ -1,13 +1,19 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # SPDX-License-Identifier: Solarflare-Binary
 # X-SPDX-Copyright-Text: (c) Copyright 2019-2020 Xilinx, Inc.
 from __future__ import print_function
-import sys, os, os.path, re, argparse, urlparse, collections, fnmatch
+import sys, os, os.path, re, argparse, collections, fnmatch
+
+# Python 2 backwards compatibility
+if sys.version_info < (3,0):
+    from urlparse import urlparse
+else:
+    from urllib.parse import urlparse
 
 
 def is_binary_header(header):
-    return (header.startswith('!<arch>./       ')
-            or header.startswith('\x7fELF'))
+    return (header.startswith(b'!<arch>./       ')
+            or header.startswith(b'\x7fELF'))
 
 
 _error_count = 0
@@ -46,7 +52,7 @@ def is_known_licence(value):
 
 def is_url(value):
     try:
-        urlparse.urlparse(value)
+        urlparse(value)
         return True
     except ValueError:
         return False
@@ -73,7 +79,7 @@ def check_tagset(path, tags):
         counts[k] += 1
         if not kt.validator(v):
             err(path, 0, '', "Malformed SPDX value %s: '%s'" % (k, v))
-    for k,t in _known_tags.iteritems():
+    for k,t in _known_tags.items():
         if not (t.min <= counts[k] <= t.max):
             if not counts[k]:
                 err(path, 0, '', "Missing required SPDX tag '%s'" % k)
@@ -130,6 +136,7 @@ def validate(path):
     maybe_continuation = False
     linenum = 0
     for line in f:
+        line = line.decode(errors='ignore')
         linenum += 1
         m = _strict_re.match(line)
         if m:

scripts/makelinuxinstall

@@ -1,4 +1,4 @@
-#! /usr/bin/env python2
+#! /usr/bin/env python3
 #
 # SPDX-License-Identifier: BSD-2-Clause
 # X-SPDX-Copyright-Text: (c) Copyright 2009-2020 Xilinx, Inc.
@@ -10,11 +10,6 @@
 #
 # David Russell, July 2010
 
-
-# THIS SCRIPT MUST RUN ON PYTHON 2.2 - 2.6
-#
-# Be careful with imports and syntax
-
 import os, sys, optparse, shutil
 
 # Each file is specified either as a string, in which case it is copied to the
@@ -86,7 +81,7 @@ def choose_files(onload):
         other = onload_files
     else:
         other = kernel_files
-    for tree,files in (common_files.items() + other.items()):
+    for tree,files in (list(common_files.items()) + list(other.items())):
         for f in files: yield tree,f
 
 def parse_args():
@@ -120,11 +115,11 @@ def parse_args():
         else:
             options.target_dir = 'kernel_install_package'
         if not options.ultree:
-            print "ERROR: -u/--ultree is required\n"
+            print("ERROR: -u/--ultree is required\n")
             parser.print_help()
             sys.exit(1)
         if (not options.onload) and (not options.ktree):
-            print "ERROR: -k/--ktree is required for kernel packages\n"
+            print("ERROR: -k/--ktree is required for kernel packages\n")
             parser.print_help()
             sys.exit(1)
     return options
@@ -134,24 +129,24 @@ def main(onload, ktree, ultree, target_dir=None, allow_missing=None,
          print_kernel_build_dirs=False):
     if print_ul_build_dirs:
         for directory in ul_build_dirs:
-            print directory
+            print(directory)
     if print_kernel_build_dirs:
         for directory in kernel_build_dirs:
-            print directory
+            print(directory)
     if print_ul_build_dirs or print_kernel_build_dirs:
         sys.exit(0)
 
     if allow_missing is None:
         allow_missing = []
     for path in [ktree, ultree, ultree32]:
         if path and not os.path.isdir(path):
-            print "ERROR: Cannot find",path
+            print("ERROR: Cannot find",path)
             sys.exit(2)
 
     if os.path.isdir(target_dir):
         shutil.rmtree(target_dir)
 
-    print "TARGET=" + target_dir
+    print("TARGET=" + target_dir)
 
     for tree,f in choose_files(onload):
         if tree == 'ktree':
@@ -168,20 +163,20 @@ def main(onload, ktree, ultree, target_dir=None, allow_missing=None,
             destdir = os.path.join(target_dir, subdir)
             if not os.path.isfile(source):
                 if os.path.basename(source) in allow_missing:
-                    print "WARNING: Cannot find", source
+                    print("WARNING: Cannot find", source)
                     continue
                 else:
-                    print "ERROR: Cannot find",source
+                    print("ERROR: Cannot find",source)
                     sys.exit(3)
-            print "Copy '%s' to '$TARGET/%s'" % (source, subdir)
+            print("Copy '%s' to '$TARGET/%s'" % (source, subdir))
             if not os.path.isdir(destdir):
                 os.makedirs(destdir)
             if os.path.islink(source):
                 dest = os.path.realpath(source)
-                print "treating '%s' as a link to '%s'"%(source, dest)
+                print("treating '%s' as a link to '%s'"%(source, dest))
                 source = dest
             if os.system("cp '%s' '%s'" % (source, destdir)) != 0:
-                print "ERROR: Failed to copy",source,"to",destdir
+                print("ERROR: Failed to copy",source,"to",destdir)
                 sys.exit(4)
 
 if __name__ == '__main__':

scripts/onload_install

@@ -171,7 +171,7 @@ install_solar_clusterd() {
 
   python_cmd="from distutils.sysconfig import get_python_lib; import sys; "
   python_cmd="$python_cmd sys.stdout.write(get_python_lib(plat_specific=1))"
-  python_sitelib=$(python2 -c "$python_cmd" 2> /dev/null)
+  python_sitelib=$(python3 -c "$python_cmd" 2> /dev/null)
 
   if [ $? != 0 ]; then
     log "WARNING: solar_clusterd will not be installed as python" \
@@ -285,9 +285,9 @@ install_userland() {
   [ -f setup.py ] || try $do cp "$TOP/scripts/onload_misc/setup.py" .
   if [ "$i_usr" = "/usr" ]; then
     # Debian and Ubuntu mess up if --prefix=/usr is set.
-    $do python2 setup.py install --record=$python_manifest
+    $do python3 setup.py install --record=$python_manifest
   else
-    $do python2 setup.py install --prefix="$i_usr"\
+    $do python3 setup.py install --prefix="$i_usr"\
                                  --record=$python_manifest
   fi
 

scripts/onload_iptables

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0
 # X-SPDX-Copyright-Text: (c) Copyright 2012-2019 Xilinx, Inc.
 
@@ -39,13 +39,8 @@ def GenerateIptables():
 		cmdline.append( '-v' )
 
 	p = subprocess.Popen( cmdline, stdout=subprocess.PIPE )
-	while p.returncode is not None:
-		# While it's alive, keep reading
-		for line in p.stdout:
-			yield data
-	# And grab any data that it spat out before it died that we didn't read yet
 	for line in p.stdout:
-		yield line
+		yield line.decode()
 
 def StripEntry(x):
 	''' Just strip whitespace from each line of x '''
@@ -163,7 +158,7 @@ def GenerateBits( line ):
 		rval['rule'] = [ bits[6] ]
 
 	if len(bits) > 7:
-		for i in xrange( 7, len(bits) ):
+		for i in range( 7, len(bits) ):
 			rval['rule'].append( bits[i] )
 
 	rval['action'] 		= bits[0]
@@ -298,11 +293,6 @@ def SysFsAction( action ):
 
 def GenerateInterfaces():
 	p = subprocess.Popen( ['ls', '/proc/sys/net/ipv4/conf/'], stdout=subprocess.PIPE )
-	while p.returncode is not None:
-		# While it's alive, keep reading
-		for line in p.stdout:
-			yield data
-	# And grab any data that it spat out before it died that we didn't read yet
 	for line in p.stdout:
 		yield line
 
@@ -393,7 +383,7 @@ def PrintInstalledRules(filename=None):
 			rules = fp.read()
 			fp.close()
 		except IOError:
-			print "Cannot find the file", filename
+			print("Cannot find the file", filename)
 			sys.exit(1)
 		print(rules)
 
@@ -454,7 +444,7 @@ def ParseCommandLineOptions():
 	parser.add_option("--use-extended", action="store_true", dest="use_verbose_iptables", default=False,
 				help="Use extended IPtables (-L -n -v, rather than just -L -n) input, which includes which interface each rule is applied to.  Disabled by default, as the format may vary." )
 	parser.add_option("--treat-as-outgoing", action="store_true", dest="treat_as_outgoing", default=False,
-				help="Treat the chain as an OUTPUT chain, and so treat its dest rules as remote hosts.\Note: Onload can ONLY block incoming packets; but this is often suffcieint to prevent the connection." )
+				help="Treat the chain as an OUTPUT chain, and so treat its dest rules as remote hosts. Note: Onload can ONLY block incoming packets; but this is often suffcieint to prevent the connection." )
 	(options, args) = parser.parse_args()
 	unparsable_is_fatal = options.unparsable_is_fatal and not options.verbose
 	use_verbose_iptables = options.use_verbose_iptables

scripts/onload_misc/openonload.spec

@@ -115,7 +115,7 @@ ExclusiveArch	: i386 i586 i686 x86_64 ppc64
 BuildRequires	: gawk gcc sed make bash libpcap libpcap-devel automake libtool autoconf libcap-devel
 # The glibc, python-devel, and libcap packages we need depend on distro and platform
 %if %{redhat}
-BuildRequires	: glibc-common python2-devel libcap
+BuildRequires	: glibc-common python3-devel libcap
 %else
 BuildRequires	: glibc-devel glibc python-devel libcap2
 %ifarch x86_64

scripts/sfcaffinity_config

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0
 # X-SPDX-Copyright-Text: (c) Copyright 2009-2020 Xilinx, Inc.
 
@@ -132,7 +132,7 @@ class EthInterface(object):
 
 def top_and_tail(x, pre="", post=""):
     if type(x) is list:
-        return map(lambda a: pre + str(a) + post, x)
+        return [pre + str(a) + post for a in x]
     else:
         return pre + str(x) + post
 
@@ -185,7 +185,8 @@ def fail(x, exit_code=1):
 
 
 def isint(x):
-    return type(x) is int or type(x) is long
+    # Python 2 backwards compatibility
+    return type(x) is int or (sys.version_info < (3,0) and type(x) is long)
 
 
 def do_system(command, loglevel=LL_SYSTEM):
@@ -559,19 +560,6 @@ def interface_show(ethx):
         out("%s not registered with sfc_resource" % ethx)
 
 
-def my_min(seq, key=None):
-    # Yuk -- older python versions don't support "key" arg to min(), so we
-    # need our own implementation for the time being.
-    if key is None:
-        key = lambda x: x
-    min_s = seq[0]
-    if len(seq) > 1:
-        for s in seq[1:]:
-            if cmp(key(s), key(min_s)) < 0:
-                min_s = s
-    return min_s
-
-
 def generate_core_to_rxq_map(topology, rxq_to_core_id):
     """Generate a mapping from core_id to rxq.  If core has its own rxq,
     use it.  Otherwise map to an rxq on a nearby core.  If there are
@@ -596,7 +584,7 @@ def generate_core_to_rxq_map(topology, rxq_to_core_id):
             if not cwq:
                 continue
             # Of those, find the core with the fewest other cores mapped on.
-            core = my_min(cwq, key=(lambda c: c.n_mapped))
+            core = min(cwq, key=(lambda c: c.n_mapped))
             core.n_mapped += 1
             # Which rxq?  Pick the first.  (Ultimately it would be better
             # to spread evenly over them if there are multiple).
@@ -627,7 +615,7 @@ def cores_spread_over(topology, core_groups):
         core_groups = core_groups[1:] + [g]
         g_cores = [topology.id2core[core_id] for core_id in g.core_ids_left]
         if g_cores:
-            core_w_fewest_irqs = my_min(g_cores, key=(lambda c: c.n_irqs))
+            core_w_fewest_irqs = min(g_cores, key=(lambda c: c.n_irqs))
             g.core_ids_left.discard(core_w_fewest_irqs.id)
             core_ids.append(core_w_fewest_irqs.id)
     return core_ids

scripts/sfcirqaffinity

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0
 # X-SPDX-Copyright-Text: (c) Copyright 2010-2019 Xilinx, Inc.
 
@@ -18,7 +18,7 @@ options = None
 
 def top_and_tail(x, pre="", post=""):
     if type(x) is list:
-        return map(lambda a: pre + str(a) + post, x)
+        return [pre + str(a) + post for a in x]
     else:
         return pre + str(x) + post