python 2-->3 / configd

First (functional) attempt, this needs to stay on devel for some time there might be dragons ;)

src/etc/rc.d/configd --> command_interpreter could cause restart issues after an upgrade, the rc system doesn't like command changes it seems. Maybe not a real world problem, just haven't tried it yet.
unit tests are somewhat functional, although generating all templates will likely fail, since the test config doesn't include all data involved.

src/etc/rc.d/configd

@@ -26,7 +26,7 @@ configd_load_rc_config()
 	required_files=""
 	command_args="${required_args}"
 	command=/usr/local/opnsense/service/configd.py
-	command_interpreter=/usr/local/bin/python2.7
+	command_interpreter=/usr/local/bin/python3.6
 }
 
 #

src/opnsense/service/configd.py

@@ -1,8 +1,8 @@
-#!/usr/local/bin/python2.7
+#!/usr/local/bin/python3.6
 # -*- coding: utf-8 -*-
 
 """
-    Copyright (c) 2014-2016 Ad Schellevis <ad@opnsense.org>
+    Copyright (c) 2014-2019 Ad Schellevis <ad@opnsense.org>
     All rights reserved.
 
     Redistribution and use in source and binary forms, with or without

src/opnsense/service/configd_ctl.py

@@ -1,7 +1,7 @@
-#!/usr/local/bin/python2.7
+#!/usr/local/bin/python3.6
 
 """
-    Copyright (c) 2015 Ad Schellevis <ad@opnsense.org>
+    Copyright (c) 2015-2019 Ad Schellevis <ad@opnsense.org>
     All rights reserved.
 
     Redistribution and use in source and binary forms, with or without
@@ -59,10 +59,10 @@ def exec_config_cmd(exec_command):
         return None
 
     try:
-        sock.send(exec_command)
+        sock.send(exec_command.encode())
         data = []
         while True:
-            line = sock.recv(65536)
+            line = sock.recv(65536).decode()
             if line:
                 data.append(line)
             else:
@@ -82,7 +82,7 @@ def exec_config_cmd(exec_command):
 
 # validate parameters
 if len(sys.argv) <= 1:
-    print ('usage : %s [-m] <command>'%sys.argv[0])
+    print('usage : %s [-m] <command>'%sys.argv[0])
     sys.exit(0)
 
 # check if configd socket exists
@@ -95,7 +95,7 @@ def exec_config_cmd(exec_command):
     i += 1
 
 if not os.path.exists(configd_socket_name):
-    print ('configd socket missing (@%s)'%configd_socket_name)
+    print('configd socket missing (@%s)'%configd_socket_name)
     sys.exit(-1)
 
 if sys.argv[1] == '-m':

src/opnsense/service/modules/addons/template_helpers.py

@@ -1,5 +1,5 @@
 """
-    Copyright (c) 2015 Ad Schellevis <ad@opnsense.org>
+    Copyright (c) 2015-2019 Ad Schellevis <ad@opnsense.org>
     All rights reserved.
 
     Redistribution and use in source and binary forms, with or without

src/opnsense/service/modules/config.py

@@ -1,5 +1,5 @@
 """
-    Copyright (c) 2015 Ad Schellevis <ad@opnsense.org>
+    Copyright (c) 2015-2019 Ad Schellevis <ad@opnsense.org>
     All rights reserved.
 
     Redistribution and use in source and binary forms, with or without

src/opnsense/service/modules/csconfigparser.py

@@ -1,5 +1,5 @@
 """
-    Copyright (c) 2015 Ad Schellevis <ad@opnsense.org>
+    Copyright (c) 2015-2019 Ad Schellevis <ad@opnsense.org>
     All rights reserved.
 
     Redistribution and use in source and binary forms, with or without
@@ -29,7 +29,7 @@
     function: make standard config parser case sensitive
 """
 
-from ConfigParser import ConfigParser
+from configparser import ConfigParser
 
 
 class CSConfigParser(ConfigParser):

src/opnsense/service/modules/ph_inline_actions.py

@@ -1,5 +1,5 @@
 """
-    Copyright (c) 2014 Ad Schellevis <ad@opnsense.org>
+    Copyright (c) 2014-2019 Ad Schellevis <ad@opnsense.org>
     All rights reserved.
 
     Redistribution and use in source and binary forms, with or without
@@ -30,8 +30,8 @@
 """
 
 import syslog
-import template
-import config
+from . import template
+from . import config
 
 __author__ = 'Ad Schellevis'
 
@@ -86,11 +86,11 @@ def execute(action, parameters):
             return 'ERR'
     elif action.command == 'configd.actions':
         # list all available configd actions
-        from processhandler import ActionHandler
+        from .processhandler import ActionHandler
         act_handler = ActionHandler()
         actions = act_handler.list_actions(['message', 'description'])
 
-        if unicode(parameters).lower() == 'json':
+        if str(parameters).lower() == 'json':
             import json
             return json.dumps(actions)
         else:

src/opnsense/service/modules/processhandler.py

@@ -1,5 +1,5 @@
 """
-    Copyright (c) 2014 Ad Schellevis <ad@opnsense.org>
+    Copyright (c) 2014-2019 Ad Schellevis <ad@opnsense.org>
     All rights reserved.
 
     Redistribution and use in source and binary forms, with or without
@@ -34,13 +34,13 @@
 import traceback
 import syslog
 import threading
-import ConfigParser
+import configparser
 import glob
 import time
 import uuid
 import shlex
 import tempfile
-import ph_inline_actions
+from . import ph_inline_actions
 from modules import singleton
 
 __author__ = 'Ad Schellevis'
@@ -126,7 +126,7 @@ def run(self):
                 return
             except Exception:
                 # something went wrong... send traceback to syslog, restart listener (wait for a short time)
-                print (traceback.format_exc())
+                print(traceback.format_exc())
                 syslog.syslog(syslog.LOG_ERR, 'Handler died on %s' % traceback.format_exc())
                 time.sleep(1)
 
@@ -163,12 +163,12 @@ def run(self):
         # noinspection PyBroadException
         try:
             # receive command, maximum data length is 4k... longer messages will be truncated
-            data = self.connection.recv(4096)
+            data = self.connection.recv(4096).decode()
             # map command to action
             data_parts = shlex.split(data)
             if len(data_parts) == 0 or len(data_parts[0]) == 0:
                 # no data found
-                self.connection.sendall('no data\n')
+                self.connection.sendall(('no data\n').encode())
             else:
                 exec_command = data_parts[0]
                 if exec_command[0] == "&":
@@ -187,7 +187,7 @@ def run(self):
                 # when running in background, return this message uuid and detach socket
                 if exec_in_background:
                     result = self.message_uuid
-                    self.connection.sendall('%s\n%c%c%c' % (result, chr(0), chr(0), chr(0)))
+                    self.connection.sendall(('%s\n%c%c%c' % (result, chr(0), chr(0), chr(0))).encode())
                     self.connection.shutdown(socket.SHUT_RDWR)
                     self.connection.close()
 
@@ -200,22 +200,22 @@ def run(self):
 
                 if not exec_in_background:
                     # send response back to client( including trailing enter )
-                    self.connection.sendall('%s\n' % result)
+                    self.connection.sendall(('%s\n' % result).encode())
                 else:
                     # log response
                     syslog.syslog(syslog.LOG_INFO, "message %s [%s.%s] returned %s " % (self.message_uuid,
                                                                                         exec_command,
                                                                                         exec_action,
-                                                                                        unicode(result)[:100]))
+                                                                                        result[:100]))
 
             # send end of stream characters
             if not exec_in_background:
-                self.connection.sendall("%c%c%c" % (chr(0), chr(0), chr(0)))
+                self.connection.sendall(("%c%c%c" % (chr(0), chr(0), chr(0))).encode())
         except SystemExit:
             # ignore system exit related errors
             pass
         except Exception:
-            print (traceback.format_exc())
+            print(traceback.format_exc())
             syslog.syslog(
                 syslog.LOG_ERR,
                 'unable to sendback response [%s] for [%s][%s][%s] {%s}, message was %s' % (result,
@@ -268,7 +268,7 @@ def load_config(self):
                 self.action_map[topic_name] = {}
 
             # traverse config directory and open all filenames starting with actions_
-            cnf = ConfigParser.RawConfigParser()
+            cnf = configparser.RawConfigParser()
             cnf.read(config_filename)
             for section in cnf.sections():
                 # map configuration data on object
@@ -370,10 +370,10 @@ def show_action(self, command, action, parameters, message_uuid):
         :return: None
         """
         action_obj = self.find_action(command, action, parameters)
-        print ('---------------------------------------------------------------------')
-        print ('execute %s.%s with parameters : %s ' % (command, action, parameters))
-        print ('action object %s (%s) %s' % (action_obj, action_obj.command, message_uuid))
-        print ('---------------------------------------------------------------------')
+        print('---------------------------------------------------------------------')
+        print('execute %s.%s with parameters : %s ' % (command, action, parameters))
+        print('action object %s (%s) %s' % (action_obj, action_obj.command, message_uuid))
+        print('---------------------------------------------------------------------')
 
 
 class Action(object):
@@ -488,7 +488,7 @@ def execute(self, parameters, message_uuid):
                                               '[%s] Script action stderr returned "%s"' %
                                               (message_uuid, script_error_output.strip()[:255])
                                               )
-                            return script_output
+                            return script_output.decode()
                 except Exception as script_exception:
                     syslog.syslog(syslog.LOG_ERR, '[%s] Script action failed with %s at %s' % (message_uuid,
                                                                                                script_exception,

src/opnsense/service/modules/template.py

@@ -1,5 +1,5 @@
 """
-    Copyright (c) 2015 Ad Schellevis <ad@opnsense.org>
+    Copyright (c) 2015-2019 Ad Schellevis <ad@opnsense.org>
     All rights reserved.
 
     Redistribution and use in source and binary forms, with or without
@@ -39,7 +39,7 @@
 import copy
 import codecs
 import jinja2
-import addons.template_helpers
+from .addons import template_helpers
 
 __author__ = 'Ad Schellevis'
 
@@ -59,7 +59,6 @@ def __init__(self, target_root_directory="/"):
         self._template_dir = os.path.dirname(os.path.abspath(__file__)) + '/../templates/'
         self._j2_env = jinja2.Environment(loader=jinja2.FileSystemLoader(self._template_dir), trim_blocks=True,
                                           extensions=["jinja2.ext.do", "jinja2.ext.loopcontrols"])
-
         # register additional filters
         self._j2_env.filters['decode_idna'] = lambda x:x.decode('idna')
         self._j2_env.filters['encode_idna'] = self._encode_idna
@@ -68,7 +67,7 @@ def __init__(self, target_root_directory="/"):
     def _encode_idna(x):
         """ encode string to idna, preserve leading dots
         """
-        return ''.join(map(lambda x:'.', range(len(x) - len(x.lstrip('.'))))) + x.lstrip('.').encode('idna')
+        return b''.join([b''.join([b'.' for x in range(len(x) - len(x.lstrip('.')))]), x.lstrip('.').encode('idna')])
 
     def list_module(self, module_name):
         """ list single module content
@@ -84,20 +83,21 @@ def list_module(self, module_name):
 
         for target_source in target_sources:
             if os.path.exists(target_source):
-                for line in open(target_source, 'r').read().split('\n'):
-                    parts = line.split(':')
-                    if len(parts) > 1 and parts[0].strip()[0] != '#':
-                        source_file = parts[0].strip()
-                        target_name = parts[1].strip()
-                        if target_name in result['+TARGETS'].values():
-                            syslog.syslog(syslog.LOG_NOTICE, "template overlay %s with %s" % (
-                                target_name, os.path.basename(target_source)
-                            ))
-                        result['+TARGETS'][source_file] = target_name
-                        if len(parts) == 2:
-                            result['+CLEANUP_TARGETS'][source_file] = target_name
-                        elif parts[2].strip() != "":
-                            result['+CLEANUP_TARGETS'][source_file] = parts[2].strip()
+                with open(target_source, 'r') as fhandle:
+                    for line in fhandle.read().split('\n'):
+                        parts = line.split(':')
+                        if len(parts) > 1 and parts[0].strip()[0] != '#':
+                            source_file = parts[0].strip()
+                            target_name = parts[1].strip()
+                            if target_name in list(result['+TARGETS'].values()):
+                                syslog.syslog(syslog.LOG_NOTICE, "template overlay %s with %s" % (
+                                    target_name, os.path.basename(target_source)
+                                ))
+                            result['+TARGETS'][source_file] = target_name
+                            if len(parts) == 2:
+                                result['+CLEANUP_TARGETS'][source_file] = target_name
+                            elif parts[2].strip() != "":
+                                result['+CLEANUP_TARGETS'][source_file] = parts[2].strip()
         return result
 
     def list_modules(self):
@@ -155,9 +155,9 @@ def __find_filters(self, tags):
                     config_ptr = config_ptr[xmlNodeName]
                 elif xmlNodeName == '%':
                     if type(config_ptr) in (collections.OrderedDict, dict):
-                        target_keys = config_ptr.keys()
+                        target_keys = list(config_ptr)
                     else:
-                        target_keys = map(lambda x: str(x), range(len(config_ptr)))
+                        target_keys = [str(x) for x in range(len(config_ptr))]
                 else:
                     # config pointer is reused when the match is exact, so we need to reset it here
                     # if the tag was not found.
@@ -217,15 +217,15 @@ def _generate(self, module_name, create_directory=True):
         """
         result = []
         module_data = self.list_module(module_name)
-        for src_template in module_data['+TARGETS'].keys():
+        for src_template in list(module_data['+TARGETS']):
             target = module_data['+TARGETS'][src_template]
 
             target_filename_tags = self.__find_string_tags(target)
             target_filters = self.__find_filters(target_filename_tags)
             result_filenames = {target: {}}
-            for target_filter in target_filters.keys():
-                for key in target_filters[target_filter].keys():
-                    for filename in result_filenames.keys():
+            for target_filter in list(target_filters):
+                for key in list(target_filters[target_filter]):
+                    for filename in list(result_filenames):
                         if target_filters[target_filter][key] is not None \
                                 and filename.find('[%s]' % target_filter) > -1:
                             new_filename = filename.replace('[%s]' % target_filter, target_filters[target_filter][key])
@@ -240,14 +240,14 @@ def _generate(self, module_name, create_directory=True):
             except jinja2.exceptions.TemplateSyntaxError as templExc:
                 raise Exception("%s %s %s" % (module_name, template_filename, templExc))
 
-            for filename in result_filenames.keys():
+            for filename in list(result_filenames):
                 if not (filename.find('[') != -1 and filename.find(']') != -1):
                     # copy config data
                     cnf_data = copy.deepcopy(self._config)
                     cnf_data['TARGET_FILTERS'] = result_filenames[filename]
 
                     # link template helpers
-                    self._j2_env.globals['helpers'] = addons.template_helpers.Helpers(cnf_data)
+                    self._j2_env.globals['helpers'] = template_helpers.Helpers(cnf_data)
 
                     # make sure we're only rendering output once
                     if filename not in result:
@@ -271,7 +271,7 @@ def _generate(self, module_name, create_directory=True):
                         # It looks like Jinja sometimes isn't consistent on placing this last end-of-line in.
                         if len(content) > 1 and content[-1] != '\n':
                             src_file = '%s%s' % (self._template_dir, template_filename)
-                            src_file_handle = open(src_file, 'r')
+                            src_file_handle = open(src_file, 'rb')
                             src_file_handle.seek(-1, os.SEEK_END)
                             last_bytes_template = src_file_handle.read()
                             src_file_handle.close()
@@ -342,7 +342,7 @@ def cleanup(self, module_name):
         for template_name in self.iter_modules(module_name):
             syslog.syslog(syslog.LOG_NOTICE, "cleanup template container %s" % template_name)
             module_data = self.list_module(module_name)
-            for src_template in module_data['+CLEANUP_TARGETS'].keys():
+            for src_template in list(module_data['+CLEANUP_TARGETS']):
                 target = module_data['+CLEANUP_TARGETS'][src_template]
                 for filename in glob.glob(target):
                     os.remove(filename)

src/opnsense/service/run_unittests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3.6
 """
     Copyright (c) 2016 Ad Schellevis <ad@opnsense.org>
     All rights reserved.

src/opnsense/service/tests/config/config.xml

@@ -1,5 +1,6 @@
 <?xml version="1.0"?>
 <opnsense>
+  <version>1</version>
   <interfaces>
     <wan>
       <enable/>