diff --git a/.github/workflows/FMITest.yml b/.github/workflows/FMITest.yml index 60d4e3788..633b79cbc 100644 --- a/.github/workflows/FMITest.yml +++ b/.github/workflows/FMITest.yml @@ -38,7 +38,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install future pyparsing numpy psutil pyzmq pytest pytest-md pytest-emoji + pip install . pytest pytest-md pytest-emoji - name: Set timezone uses: szenius/set-timezone@v2.0 diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index 66f404e53..0490b77df 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -38,13 +38,17 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install future pyparsing numpy psutil pyzmq pytest pytest-md pytest-emoji + pip install . pytest pytest-md pytest-emoji flake8 - name: Set timezone uses: szenius/set-timezone@v2.0 with: timezoneLinux: 'Europe/Berlin' + - name: Lint with flake8 + run: | + flake8 . --count --statistics + - name: Run pytest uses: pavelzw/pytest-action@v2 with: diff --git a/OMPython/OMParser/__init__.py b/OMPython/OMParser/__init__.py index 9bfa7688a..f1708947d 100755 --- a/OMPython/OMParser/__init__.py +++ b/OMPython/OMParser/__init__.py @@ -32,11 +32,6 @@ Version: 1.0 """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from builtins import int, range - import sys result = dict() @@ -207,8 +202,6 @@ def delete_elements(strings): def make_subset_sets(strings, name): - index = 0 - anchor = 0 main_set_name = "SET1" subset_name = "Subset1" set_name = "Set1" @@ -285,8 +278,6 @@ def make_subset_sets(strings, name): def make_sets(strings, name): if strings == "{}": return - index = 0 - anchor = 0 main_set_name = "SET1" set_name = "Set1" @@ -415,7 +406,6 @@ def get_inner_sets(strings, for_this, name): def make_elements(strings): - original_string = strings index = 0 main_set_name = "SET1" @@ -563,7 +553,6 @@ def skip_all_inner_sets(position): max_count = main_count last_set = 0 last_subset = 0 - last_brace = 0 pos = position while pos < len(string): @@ -616,7 +605,6 @@ def skip_all_inner_sets(position): break elif ch == "(": brace_count += 1 - brace_start = position position += 1 while position < end_of_main_set: s = string[position] @@ -625,7 +613,6 @@ def skip_all_inner_sets(position): elif s == ")": brace_count -= 1 if brace_count == 0: - last_brace = position break elif s == "=" and string[position + 1] == "{": indx = position + 2 @@ -672,7 +659,6 @@ def skip_all_inner_sets(position): break elif ch == "(": brace_count += 1 - brace_start = position position += 1 while position < end_of_main_set: s = string[position] @@ -681,13 +667,11 @@ def skip_all_inner_sets(position): elif s == ")": brace_count -= 1 if brace_count == 0: - last_brace = position break position += 1 position += 1 elif char == "(": brace_count += 1 - brace_start = position position += 1 while position < end_of_main_set: s = string[position] @@ -696,7 +680,6 @@ def skip_all_inner_sets(position): elif s == ")": brace_count -= 1 if brace_count == 0: - last_brace = position break position += 1 diff --git a/OMPython/OMTypedParser.py b/OMPython/OMTypedParser.py index b4f0d249f..28807a920 100644 --- a/OMPython/OMTypedParser.py +++ b/OMPython/OMTypedParser.py @@ -1,10 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from builtins import int, range - __author__ = "Anand Kalaiarasi Ganeson, ganan642@student.liu.se, 2012-03-19, and Martin Sjölund" __license__ = """ This file is part of OpenModelica. @@ -59,6 +54,7 @@ import sys + def convertNumbers(s, l, toks): n = toks[0] try: @@ -75,7 +71,8 @@ def convertString2(s, s2): tmp = tmp.replace("\n", "\\n") tmp = tmp.replace("\r", "\\r") tmp = tmp.replace("\t", "\\t") - return "'"+tmp+"'"; + return "'"+tmp+"'" + def convertString(s, s2): return s2[0].replace("\\\"", '"') @@ -89,7 +86,6 @@ def convertTuple(t): return tuple(t[0]) - def evaluateExpression(s, loc, toks): # Convert the tokens (ParseResults) into a string expression flat_list = [item for sublist in toks[0] for item in sublist] @@ -100,6 +96,7 @@ def evaluateExpression(s, loc, toks): except Exception: return expr + # Number parsing (supports arithmetic expressions in dimensions) (e.g., {1 + 1, 1}) arrayDimension = infixNotation( Word(alphas + "_", alphanums + "_") | Word(nums), @@ -123,7 +120,7 @@ def evaluateExpression(s, loc, toks): Optional('.' + Word(nums)) + Optional(Word('eE', exact=1) + Word(nums + '+-', nums))) -#ident = Word(alphas + "_", alphanums + "_") | Combine("'" + Word(alphanums + "!#$%&()*+,-./:;<>=?@[]^{}|~ ") + "'") +# ident = Word(alphas + "_", alphanums + "_") | Combine("'" + Word(alphanums + "!#$%&()*+,-./:;<>=?@[]^{}|~ ") + "'") ident = Word(alphas + "_", alphanums + "_") | QuotedString(quoteChar='\'', escChar='\\').setParseAction(convertString2) fqident = Forward() fqident << ((ident + "." + fqident) | ident) @@ -143,7 +140,7 @@ def evaluateExpression(s, loc, toks): def parseString(string): res = omcGrammar.parseString(string) if len(res) == 0: - return + return return res[0] diff --git a/OMPython/__init__.py b/OMPython/__init__.py index c817657c5..f4a59334b 100644 --- a/OMPython/__init__.py +++ b/OMPython/__init__.py @@ -7,14 +7,7 @@ omc.sendExpression("command") """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from future.utils import with_metaclass -from builtins import int, range -from copy import deepcopy import shutil - import abc import csv import getpass @@ -32,12 +25,11 @@ import time import uuid import xml.etree.ElementTree as ET -from collections import OrderedDict import numpy as np import pyparsing import importlib import zmq -import warnings + if sys.platform == 'darwin': # On Mac let's assume omc is installed here and there might be a broken omniORB installed in a bad place @@ -91,17 +83,21 @@ logger.addHandler(logger_console_handler) logger.setLevel(logging.WARNING) + class DummyPopen(): - def __init__(self, pid): - self.pid = pid - self.process = psutil.Process(pid) - self.returncode = 0 - def poll(self): - return None if self.process.is_running() else True - def kill(self): - return os.kill(self.pid, signal.SIGKILL) - def wait(self, timeout): - return self.process.wait(timeout=timeout) + def __init__(self, pid): + self.pid = pid + self.process = psutil.Process(pid) + self.returncode = 0 + + def poll(self): + return None if self.process.is_running() else True + + def kill(self): + return os.kill(self.pid, signal.SIGKILL) + + def wait(self, timeout): + return self.process.wait(timeout=timeout) class OMCSessionHelper: @@ -136,7 +132,7 @@ def _get_omc_path(self): raise -class OMCSessionBase(with_metaclass(abc.ABCMeta, object)): +class OMCSessionBase(metaclass=abc.ABCMeta): def __init__(self, readonly=False): self.readonly = readonly @@ -163,25 +159,25 @@ def __init__(self, readonly=False): def __del__(self): try: - self.sendExpression("quit()") - except: - pass + self.sendExpression("quit()") + except Exception: + pass self._omc_log_file.close() if sys.version_info.major >= 3: - try: - self._omc_process.wait(timeout=2.0) - except: - if self._omc_process: - self._omc_process.kill() + try: + self._omc_process.wait(timeout=2.0) + except Exception: + if self._omc_process: + self._omc_process.kill() else: - for i in range(0,100): - time.sleep(0.02) - if self._omc_process and (self._omc_process.poll() is not None): - break + for i in range(0, 100): + time.sleep(0.02) + if self._omc_process and (self._omc_process.poll() is not None): + break # kill self._omc_process process if it is still running/exists if self._omc_process is not None and self._omc_process.returncode is None: print("OMC did not exit after being sent the quit() command; killing the process with pid=%s" % str(self._omc_process.pid)) - if sys.platform=="win32": + if sys.platform == "win32": self._omc_process.kill() self._omc_process.wait() else: @@ -209,52 +205,52 @@ def _start_omc_process(self, timeout): # Because we spawned a shell, and we need to be able to kill OMC, create a new process group for this self._omc_process = subprocess.Popen(self._omc_command, shell=True, stdout=self._omc_log_file, stderr=self._omc_log_file, env=my_env, preexec_fn=os.setsid) if self._docker: - for i in range(0,40): + for i in range(0, 40): + try: + with open(self._dockerCidFile, "r") as fin: + self._dockerCid = fin.read().strip() + except Exception: + pass + if self._dockerCid: + break + time.sleep(timeout / 40.0) try: - with open(self._dockerCidFile, "r") as fin: - self._dockerCid = fin.read().strip() - except: - pass - if self._dockerCid: - break - time.sleep(timeout / 40.0) - try: - os.remove(self._dockerCidFile) - except: - pass - if self._dockerCid is None: - logger.error("Docker did not start. Log-file says:\n%s" % (open(self._omc_log_file.name).read())) - raise Exception("Docker did not start (timeout=%f might be too short especially if you did not docker pull the image before this command)." % timeout) + os.remove(self._dockerCidFile) + except Exception: + pass + if self._dockerCid is None: + logger.error("Docker did not start. Log-file says:\n%s" % (open(self._omc_log_file.name).read())) + raise Exception("Docker did not start (timeout=%f might be too short especially if you did not docker pull the image before this command)." % timeout) if self._docker or self._dockerContainer: - if self._dockerNetwork == "separate": - self._serverIPAddress = json.loads(subprocess.check_output(["docker", "inspect", self._dockerCid]).decode().strip())[0]["NetworkSettings"]["IPAddress"] - for i in range(0,40): - if sys.platform == 'win32': - break - dockerTop = subprocess.check_output(["docker", "top", self._dockerCid]).decode().strip() - self._omc_process = None - for line in dockerTop.split("\n"): - columns = line.split() - if self._random_string in line: - try: - self._omc_process = DummyPopen(int(columns[1])) - except psutil.NoSuchProcess: - raise Exception("Could not find PID %d - is this a docker instance spawned without --pid=host?\nLog-file says:\n%s" % (self._random_string, dockerTop, open(self._omc_log_file.name).read())) - break - if self._omc_process is not None: - break - time.sleep(timeout / 40.0) - if self._omc_process is None: - raise Exception("Docker top did not contain omc process %s:\n%s\nLog-file says:\n%s" % (self._random_string, dockerTop, open(self._omc_log_file.name).read())) + if self._dockerNetwork == "separate": + self._serverIPAddress = json.loads(subprocess.check_output(["docker", "inspect", self._dockerCid]).decode().strip())[0]["NetworkSettings"]["IPAddress"] + for i in range(0, 40): + if sys.platform == 'win32': + break + dockerTop = subprocess.check_output(["docker", "top", self._dockerCid]).decode().strip() + self._omc_process = None + for line in dockerTop.split("\n"): + columns = line.split() + if self._random_string in line: + try: + self._omc_process = DummyPopen(int(columns[1])) + except psutil.NoSuchProcess: + raise Exception(f"Could not find PID {dockerTop} - is this a docker instance spawned without --pid=host?\nLog-file says:\n{open(self._omc_log_file.name).read()}") + break + if self._omc_process is not None: + break + time.sleep(timeout / 40.0) + if self._omc_process is None: + raise Exception("Docker top did not contain omc process %s:\n%s\nLog-file says:\n%s" % (self._random_string, dockerTop, open(self._omc_log_file.name).read())) return self._omc_process def _getuid(self): - """ - The uid to give to docker. - On Windows, volumes are mapped with all files are chmod ugo+rwx, - so uid does not matter as long as it is not the root user. - """ - return 1000 if sys.platform == 'win32' else os.getuid() + """ + The uid to give to docker. + On Windows, volumes are mapped with all files are chmod ugo+rwx, + so uid does not matter as long as it is not the root user. + """ + return 1000 if sys.platform == 'win32' else os.getuid() def _set_omc_command(self, omc_path_and_args_list): """Define the command that will be called by the subprocess module. @@ -272,7 +268,7 @@ def _set_omc_command(self, omc_path_and_args_list): if self._docker: if sys.platform == "win32": p = int(self._interactivePort) - dockerNetworkStr = ["-p", "127.0.0.1:%d:%d" % (p,p)] + dockerNetworkStr = ["-p", "127.0.0.1:%d:%d" % (p, p)] elif self._dockerNetwork == "host" or self._dockerNetwork is None: dockerNetworkStr = ["--network=host"] elif self._dockerNetwork == "separate": @@ -533,11 +529,12 @@ def getClassNames(self, className=None, recursive=False, qualified=False, sort=F str(builtin).lower(), str(showProtected).lower())) return value + class OMCSessionZMQ(OMCSessionHelper, OMCSessionBase): - def __init__(self, readonly=False, timeout = 10.00, - docker = None, dockerContainer = None, dockerExtraArgs = None, dockerOpenModelicaPath = "omc", - dockerNetwork = None, port = None, omhome: str = None): + def __init__(self, readonly=False, timeout=10.00, + docker=None, dockerContainer=None, dockerExtraArgs=None, dockerOpenModelicaPath="omc", + dockerNetwork=None, port=None, omhome: str = None): if dockerExtraArgs is None: dockerExtraArgs = [] @@ -581,7 +578,7 @@ def _connect_to_omc(self, timeout): try: self._port = subprocess.check_output(["docker", "exec", self._dockerCid, "cat", self._port_file], stderr=subprocess.DEVNULL if (sys.version_info > (3, 0)) else subprocess.STDOUT).decode().strip() break - except: + except Exception: pass else: if os.path.isfile(self._port_file): @@ -596,7 +593,7 @@ def _connect_to_omc(self, timeout): name = self._omc_log_file.name self._omc_log_file.close() logger.error("OMC Server did not start. Please start it! Log-file says:\n%s" % open(name).read()) - raise Exception("OMC Server did not start (timeout=%f). Could not open file %s" % (timeout,self._port_file)) + raise Exception("OMC Server did not start (timeout=%f). Could not open file %s" % (timeout, self._port_file)) time.sleep(timeout / 80.0) self._port = self._port.replace("0.0.0.0", self._serverIPAddress) @@ -605,18 +602,18 @@ def _connect_to_omc(self, timeout): # Create the ZeroMQ socket and connect to OMC server context = zmq.Context.instance() self._omc = context.socket(zmq.REQ) - self._omc.setsockopt(zmq.LINGER, 0) # Dismisses pending messages if closed - self._omc.setsockopt(zmq.IMMEDIATE, True) # Queue messages only to completed connections + self._omc.setsockopt(zmq.LINGER, 0) # Dismisses pending messages if closed + self._omc.setsockopt(zmq.IMMEDIATE, True) # Queue messages only to completed connections self._omc.connect(self._port) def execute(self, command): - ## check for process is running + # check for process is running return self.sendExpression(command, parsed=False) def sendExpression(self, command, parsed=True): - ## check for process is running - p=self._omc_process.poll() - if (p == None): + # check for process is running + p = self._omc_process.poll() + if p is None: attempts = 0 while True: try: @@ -644,10 +641,12 @@ def sendExpression(self, command, parsed=True): else: raise Exception("Process Exited, No connection with OMC. Create a new instance of OMCSessionZMQ") + class ModelicaSystemError(Exception): pass -class ModelicaSystem(object): + +class ModelicaSystem: def __init__(self, fileName=None, modelName=None, lmodel=None, commandLineOptions=None, variableFilter=None, customBuildDirectory=None, verbose=True, raiseerrors=False, omhome: str = None, session: OMCSessionBase = None): # 1 @@ -688,11 +687,11 @@ def __init__(self, fileName=None, modelName=None, lmodel=None, commandLineOption else: self.getconn = OMCSessionZMQ(omhome=omhome) - ## needed for properly deleting the session + # needed for properly deleting the session self._omc_log_file = self.getconn._omc_log_file self._omc_process = self.getconn._omc_process - ## set commandLineOptions if provided by users + # set commandLineOptions if provided by users self.setCommandLineOptions(commandLineOptions=commandLineOptions) if lmodel is None: @@ -714,9 +713,9 @@ def __init__(self, fileName=None, modelName=None, lmodel=None, commandLineOption if fileName is not None and not os.path.exists(self.fileName): # if file does not exist raise IOError("File Error:" + os.path.abspath(self.fileName) + " does not exist!!!") - ## set default command Line Options for linearization as - ## linearize() will use the simulation executable and runtime - ## flag -l to perform linearization + # set default command Line Options for linearization as + # linearize() will use the simulation executable and runtime + # flag -l to perform linearization self.sendExpression("setCommandLineOptions(\"--linearizationDumpLanguage=python\")") self.sendExpression("setCommandLineOptions(\"--generateSymbolicLinearization\")") @@ -726,14 +725,14 @@ def __init__(self, fileName=None, modelName=None, lmodel=None, commandLineOption self.loadLibrary() self.loadFile() - ## allow directly loading models from MSL without fileName + # allow directly loading models from MSL without fileName if fileName is None and modelName is not None: self.loadLibrary() self.buildModel(variableFilter) def setCommandLineOptions(self, commandLineOptions: str): - ## set commandLineOptions if provided by users + # set commandLineOptions if provided by users if commandLineOptions is not None: exp = "".join(["setCommandLineOptions(", "\"", commandLineOptions, "\"", ")"]) cmdexp = self.sendExpression(exp) @@ -744,7 +743,7 @@ def loadFile(self): # load file loadFileExp = "".join(["loadFile(", "\"", self.fileName, "\"", ")"]).replace("\\", "/") loadMsg = self.sendExpression(loadFileExp) - ## Show notification or warnings to the user when verbose=True OR if some error occurred i.e., not result + # Show notification or warnings to the user when verbose=True OR if some error occurred i.e., not result if self._verbose or not loadMsg: self._check_error() @@ -771,7 +770,7 @@ def loadLibrary(self): "The following patterns are supported:\n" + "1)[\"Modelica\"]\n" + "2)[(\"Modelica\",\"3.2.3\"), \"PowerSystems\"]\n") - ## Show notification or warnings to the user when verbose=True OR if some error occurred i.e., not result + # Show notification or warnings to the user when verbose=True OR if some error occurred i.e., not result if self._verbose or not result: self._check_error() @@ -799,7 +798,7 @@ def _run_cmd(self, cmd: list): if platform.system() == "Windows": dllPath = "" - ## set the process environment from the generated .bat file in windows which should have all the dependencies + # set the process environment from the generated .bat file in windows which should have all the dependencies batFilePath = os.path.join(self.tempdir, '{}.{}'.format(self.modelName, "bat")).replace("\\", "/") if (not os.path.exists(batFilePath)): print("Error: bat does not exist " + batFilePath) @@ -950,7 +949,7 @@ def getQuantities(self, names=None): # 3 >>> getQuantities("Name1") >>> getQuantities(["Name1","Name2"]) """ - if (names == None): + if names is None: return self.quantitiesList elif (isinstance(names, str)): return [x for x in self.quantitiesList if x["name"] == names] @@ -1010,7 +1009,7 @@ def getParameters(self, names=None): # 5 >>> getParameters("Name1") >>> getParameters(["Name1","Name2"]) """ - if (names == None): + if names is None: return self.paramlist elif (isinstance(names, str)): return [self.paramlist.get(names, "NotExist")] @@ -1036,7 +1035,7 @@ def getInputs(self, names=None): # 6 If *name is None then the function will return dict which contain all input names as key and value as corresponding values. eg., getInputs() Otherwise variable number of arguments can be passed as input name in string format separated by commas. eg., getInputs('iName1', 'iName2') """ - if (names == None): + if names is None: return self.inputlist elif (isinstance(names, str)): return [self.inputlist.get(names, "NotExist")] @@ -1053,14 +1052,14 @@ def getOutputs(self, names=None): # 7 >>> getOutputs(["Name1","Name2"]) """ if not self.simulationFlag: - if (names == None): + if names is None: return self.outputlist elif (isinstance(names, str)): return [self.outputlist.get(names, "NotExist")] else: return ([self.outputlist.get(x, "NotExist") for x in names]) else: - if (names == None): + if names is None: for i in self.outputlist: value = self.getSolutions(i) self.outputlist[i] = value[0][-1] @@ -1092,7 +1091,7 @@ def getSimulationOptions(self, names=None): # 8 >>> getSimulationOptions("Name1") >>> getSimulationOptions(["Name1","Name2"]) """ - if (names == None): + if names is None: return self.simulateOptions elif (isinstance(names, str)): return [self.simulateOptions.get(names, "NotExist")] @@ -1108,7 +1107,7 @@ def getLinearizationOptions(self, names=None): # 9 >>> getLinearizationOptions("Name1") >>> getLinearizationOptions(["Name1","Name2"]) """ - if (names == None): + if names is None: return self.linearOptions elif (isinstance(names, str)): return [self.linearOptions.get(names, "NotExist")] @@ -1122,7 +1121,7 @@ def getOptimizationOptions(self, names=None): # 10 >>> getOptimizationOptions("Name1") >>> getOptimizationOptions(["Name1","Name2"]) """ - if (names == None): + if names is None: return self.optimizeOptions elif (isinstance(names, str)): return [self.optimizeOptions.get(names, "NotExist")] @@ -1173,7 +1172,7 @@ def simulate(self, resultfile=None, simflags=None): # 11 if (self.inputFlag): # if model has input quantities for i in self.inputlist: val = self.inputlist[i] - if (val == None): + if val is None: val = [(float(self.simulateOptions["startTime"]), 0.0), (float(self.simulateOptions["stopTime"]), 0.0)] self.inputlist[i] = [(float(self.simulateOptions["startTime"]), 0.0), @@ -1222,7 +1221,7 @@ def getSolutions(self, varList=None, resultfile=None): # 12 >>> getSolutions("Name1",resultfile=""c:/a.mat"") >>> getSolutions(["Name1","Name2"],resultfile=""c:/a.mat"") """ - if (resultfile == None): + if resultfile is None: resFile = self.resultfile else: resFile = resultfile @@ -1236,7 +1235,7 @@ def getSolutions(self, varList=None, resultfile=None): # 12 else: resultVars = self.sendExpression("readSimulationResultVars(\"" + resFile + "\")") self.sendExpression("closeSimulationResultFile()") - if (varList == None): + if varList is None: return resultVars elif (isinstance(varList, str)): if (varList not in resultVars and varList != "time"): @@ -1284,13 +1283,13 @@ def apply_single(args1): args1 = self.strip_space(args1) value = args1.split("=") if value[0] in args2: - if (args3 == "parameter" and self.isParameterChangeable(value[0], value[1])): + if args3 == "parameter" and self.isParameterChangeable(value[0], value[1]): args2[value[0]] = value[1] - if (args4 != None): + if args4 is not None: args4[value[0]] = value[1] - elif (args3 != "parameter"): + elif args3 != "parameter": args2[value[0]] = value[1] - if (args4 != None): + if args4 is not None: args4[value[0]] = value[1] return True @@ -1431,7 +1430,7 @@ def createCSVData(self): sl = list() # Actual timestamps skip = False - ## check for NONE in input list and replace with proper data (e.g) [(startTime, 0.0), (stopTime, 0.0)] + # check for NONE in input list and replace with proper data (e.g) [(startTime, 0.0), (stopTime, 0.0)] tmpinputlist = {} for (key, value) in self.inputlist.items(): if (value is None): @@ -1526,7 +1525,7 @@ def createCSVData(self): l = [] l.append(name) for i in range(0, len(sl)): - a = ("%s,%s" % (str(float(sl[i])), ",".join(list(str(float(inppp[i])) \ + a = ("%s,%s" % (str(float(sl[i])), ",".join(list(str(float(inppp[i])) for inppp in interpolated_inputs_all)))) + ',0' l.append(a) @@ -1558,7 +1557,7 @@ def convertMo2Fmu(self, version="2.0", fmuType="me_cs", fileNamePrefix="