From e69d4737c1f3e647e395420eb994843ce5a408f2 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Fri, 29 Sep 2023 17:04:37 +0300 Subject: [PATCH 01/17] First version of instruction coverage --- .../main/python/utbot_executor/pyproject.toml | 2 +- .../utbot_executor/utbot_executor/executor.py | 50 ++++++------ .../utbot_executor/utbot_executor/listener.py | 6 +- .../utbot_executor/utbot_executor/parser.py | 6 +- .../utbot_executor/ut_tracer.py | 77 +++++++++++++++++-- .../utbot_executor/utbot_executor/utils.py | 13 ++++ .../src/main/resources/utbot_executor_version | 2 +- .../samples/controlflow/multi_conditions.py | 14 ++++ .../kotlin/org/utbot/python/PythonEngine.kt | 12 +-- .../evaluation/PythonCodeSocketExecutor.kt | 16 ++-- .../evaluation/PythonCoverageReceiver.kt | 12 ++- .../ExecutionResultDeserializer.kt | 5 +- .../utbot/python/evaluation/utils/Utils.kt | 33 ++++++++ 13 files changed, 192 insertions(+), 56 deletions(-) create mode 100644 utbot-python/samples/samples/controlflow/multi_conditions.py diff --git a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml index 13b3063e09..90717a8f6d 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml +++ b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "utbot-executor" -version = "1.7.0" +version = "1.7.0.dev7" description = "" authors = ["Vyacheslav Tamarin "] readme = "README.md" diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py index a1e10ec57e..13b64a1060 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py @@ -1,7 +1,7 @@ """Python code executor for UnitTestBot""" import copy +import dis import importlib -import inspect import logging import pathlib import socket @@ -17,8 +17,8 @@ from utbot_executor.deep_serialization.utils import PythonId, getattr_by_path from utbot_executor.memory_compressor import compress_memory from utbot_executor.parser import ExecutionRequest, ExecutionResponse, ExecutionFailResponse, ExecutionSuccessResponse -from utbot_executor.ut_tracer import UtTracer -from utbot_executor.utils import suppress_stdout as __suppress_stdout +from utbot_executor.ut_tracer import UtTracer, UtCoverageSender +from utbot_executor.utils import suppress_stdout as __suppress_stdout, get_instructions __all__ = ['PythonExecutor'] @@ -111,14 +111,15 @@ def run_function(self, request: ExecutionRequest) -> ExecutionResponse: state_init = _update_states(loader.reload_id(), state_init_memory) serialized_state_init = serialize_memory_dump(state_init) - def _coverage_sender(info: typing.Tuple[str, int]): - if pathlib.Path(info[0]) == pathlib.Path(request.filepath): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - logging.debug("Coverage message: %s:%d", request.coverage_id, info[1]) - logging.debug("Port: %d", self.coverage_port) - message = bytes(f'{request.coverage_id}:{info[1]}', encoding='utf-8') - sock.sendto(message, (self.coverage_hostname, self.coverage_port)) - logging.debug("ID: %s, Coverage: %s", request.coverage_id, info) + # def _coverage_sender(info: typing.Tuple[str, int]): + # if pathlib.Path(info[0]) == pathlib.Path(request.filepath): + # sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + # logging.debug("Coverage message: %s:%d", request.coverage_id, info[1]) + # logging.debug("Port: %d", self.coverage_port) + # message = bytes(f'{request.coverage_id}:{info[1]}', encoding='utf-8') + # sock.sendto(message, (self.coverage_hostname, self.coverage_port)) + # logging.debug("ID: %s, Coverage: %s", request.coverage_id, info) + _coverage_sender = UtCoverageSender(request.coverage_id, self.coverage_hostname, self.coverage_port) value = _run_calculate_function_value( function, @@ -126,7 +127,7 @@ def _coverage_sender(info: typing.Tuple[str, int]): kwargs, request.filepath, serialized_state_init, - tracer=UtTracer(_coverage_sender) + tracer=UtTracer(pathlib.Path(request.filepath), [sys.prefix, sys.exec_prefix], _coverage_sender) ) except Exception as _: logging.debug("Error \n%s", traceback.format_exc()) @@ -172,10 +173,10 @@ def _run_calculate_function_value( __is_exception = False - (__sources, __start, ) = inspect.getsourcelines(function) - __not_empty_lines = [i for i, line in enumerate(__sources, __start) if len(line.strip()) != 0] - logging.debug("Not empty lines %s", __not_empty_lines) - __end = __start + len(__sources) + # bytecode = dis.get_instructions(function) + # __all_code_lines = [instr.starts_line for instr in bytecode if instr.starts_line is not None] + __all_code_lines = list(get_instructions(function)) + __start = min([op[0] for op in __all_code_lines]) __tracer = tracer @@ -189,14 +190,15 @@ def _run_calculate_function_value( logging.debug("Coverage: %s", __tracer.counts) logging.debug("Fullpath: %s", fullpath) - module_path = pathlib.Path(fullpath) - __stmts = [x[1] for x in __tracer.counts if pathlib.Path(x[0]) == module_path] - __stmts_filtered = [x for x in __not_empty_lines if x in __stmts] - __stmts_filtered_with_def = [__start] + __stmts_filtered - __missed_filtered = [x for x in __not_empty_lines if x not in __stmts_filtered_with_def] - logging.debug("Covered lines: %s", __stmts_filtered_with_def) + __stmts = [x for x in __tracer.counts] + __stmts_with_def = [(1, 0)] + __stmts + __missed_filtered = [x for x in __all_code_lines if x not in __stmts_with_def] + logging.debug("Covered lines: %s", __stmts_with_def) logging.debug("Missed lines: %s", __missed_filtered) + __str_statements = [":".join(map(str, x)) for x in __stmts_with_def] + __str_missed_statements = [":".join(map(str, x)) for x in __missed_filtered] + args_ids, kwargs_ids, result_id, state_after, serialized_state_after = _serialize_state(args, kwargs, __result) ids = args_ids + list(kwargs_ids.values()) # state_before, state_after = compress_memory(ids, state_before, state_after) @@ -205,8 +207,8 @@ def _run_calculate_function_value( return ExecutionSuccessResponse( status="success", is_exception=__is_exception, - statements=__stmts_filtered_with_def, - missed_statements=__missed_filtered, + statements=__str_statements, + missed_statements=__str_missed_statements, state_init=state_init, state_before=serialized_state_before, state_after=serialized_state_after, diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py index 78df1a507c..5d9d55a2b3 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py @@ -80,9 +80,9 @@ def handler(self) -> None: response_size = str(len(bytes_data)) self.clientsocket.send((response_size + os.linesep).encode()) - sended_size = 0 - while len(bytes_data) > sended_size: - sended_size += self.clientsocket.send(bytes_data[sended_size:]) + sent_size = 0 + while len(bytes_data) > sent_size: + sent_size += self.clientsocket.send(bytes_data[sent_size:]) logging.debug('Sent all data') logging.info('All done...') diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py index 838711d305..d247c28290 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py @@ -1,6 +1,6 @@ import dataclasses import json -from typing import Dict, List, Union +from typing import Dict, List, Union, Tuple @dataclasses.dataclass @@ -24,8 +24,8 @@ class ExecutionResponse: class ExecutionSuccessResponse(ExecutionResponse): status: str is_exception: bool - statements: List[int] - missed_statements: List[int] + statements: List[str] + missed_statements: List[str] state_init: str state_before: str state_after: str diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py index d761c02d74..7d88e3cbb9 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py @@ -1,5 +1,12 @@ +import dis +import inspect +import logging import os +import pathlib +import queue +import socket import sys +import threading import typing @@ -9,12 +16,48 @@ def _modname(path): return filename +class UtCoverageSender: + def __init__(self, coverage_id: str, host: str, port: int, use_thread: bool = False): + self.coverage_id = coverage_id + self.host = host + self.port = port + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.message_queue = queue.Queue() + + self.use_thread = use_thread + if use_thread: + self.thread = threading.Thread(target=self.send_loop, daemon=True) + self.thread.start() + + def send_loop(self): + try: + while True: + self.send_message_thread() + except Exception as _: + self.send_loop() + + def send_message(self, message: bytes): + self.sock.sendto(*message) + + def send_message_thread(self): + message = self.message_queue.get() + self.send_message(message) + + def put_message(self, key: str): + message = bytes(f"{self.coverage_id}:{key}", encoding="utf-8") + if self.use_thread: + self.message_queue.put((message, (self.host, self.port))) + else: + self.send_message(message) + + class UtTracer: - def __init__(self, sender: typing.Callable[[typing.Tuple[str, int]], None]): - self.globaltrace = self.globaltrace_lt + def __init__(self, tested_file: pathlib.Path, ignore_dirs: typing.List[str], sender: UtCoverageSender): + self.tested_file = tested_file self.counts = {} self.localtrace = self.localtrace_count self.globaltrace = self.globaltrace_lt + self.ignore_dirs = ignore_dirs self.sender = sender def runfunc(self, func, /, *args, **kw): @@ -31,13 +74,16 @@ def coverage(self, filename: str) -> typing.List[int]: return [line for file, line in self.counts.keys() if file == filename] def localtrace_count(self, frame, why, arg): - if why == "line": - filename = frame.f_code.co_filename + filename = frame.f_code.co_filename + if pathlib.Path(filename) == self.tested_file: lineno = frame.f_lineno - key = filename, lineno + offset = 0 + if why == "opcode": + offset = frame.f_lasti + key = (lineno, offset) if key not in self.counts: try: - self.sender(key) + self.sender.put_message(":".join(map(str, key))) except Exception: pass self.counts[key] = self.counts.get(key, 0) + 1 @@ -45,8 +91,10 @@ def localtrace_count(self, frame, why, arg): def globaltrace_lt(self, frame, why, arg): if why == 'call': + frame.f_trace_opcodes = True + frame.f_trace_lines = False filename = frame.f_globals.get('__file__', None) - if filename: + if filename and all(not filename.startswith(d + os.sep) for d in self.ignore_dirs): modulename = _modname(filename) if modulename is not None: return self.localtrace @@ -60,3 +108,18 @@ def __init__(self): def runfunc(self, func, /, *args, **kw): return func(*args, **kw) + + +def f(x): + if 0 < x < 10 and x % 2 == 0: + return 1 + else: + return [100, + x**2, + x + 1 + ] + + +if __name__ in "__main__": + tracer = UtTracer(pathlib.Path(__file__), [], UtCoverageSender("1", "localhost", 0)) + tracer.runfunc(f, 70) \ No newline at end of file diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py index 35c10d0f83..577b76399f 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py @@ -1,5 +1,7 @@ +import dis import os import sys +import typing from contextlib import contextmanager @@ -12,3 +14,14 @@ def suppress_stdout(): yield finally: sys.stdout = old_stdout + + +def get_instructions(obj: object) -> typing.Iterator[tuple[int, int]]: + def inner_get_instructions(x, current_line): + for i, el in enumerate(dis.get_instructions(x)): + if el.starts_line is not None: + current_line = el.starts_line + yield current_line, el.offset + if "" in str(type(el.argval)): + inner_get_instructions(el.argval, current_line) + return inner_get_instructions(obj, None) diff --git a/utbot-python-executor/src/main/resources/utbot_executor_version b/utbot-python-executor/src/main/resources/utbot_executor_version index 9dbb0c0052..b04d552932 100644 --- a/utbot-python-executor/src/main/resources/utbot_executor_version +++ b/utbot-python-executor/src/main/resources/utbot_executor_version @@ -1 +1 @@ -1.7.0 \ No newline at end of file +1.7.0.dev7 \ No newline at end of file diff --git a/utbot-python/samples/samples/controlflow/multi_conditions.py b/utbot-python/samples/samples/controlflow/multi_conditions.py new file mode 100644 index 0000000000..b7ba4bce58 --- /dev/null +++ b/utbot-python/samples/samples/controlflow/multi_conditions.py @@ -0,0 +1,14 @@ +def check_interval(x: float, left: float, right: float) -> str: + if left < x < right or right < x < left: + return "between" + elif x < left and x < right: + return "less" + elif x > left and x > right: + return "more" + elif left == right: + return "all equals" + elif x == left: + return "left" + elif x == right: + return "right" + return "what?" diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index cc0730bffe..2dc0c7f547 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -12,7 +12,9 @@ import org.utbot.python.evaluation.* import org.utbot.python.evaluation.serialization.MemoryDump import org.utbot.python.evaluation.serialization.toPythonTree import org.utbot.python.evaluation.utils.CoverageIdGenerator +import org.utbot.python.evaluation.utils.PyInstruction import org.utbot.python.evaluation.utils.coveredLinesToInstructions +import org.utbot.python.evaluation.utils.makeInstructions import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.PythonTreeModel import org.utbot.python.framework.api.python.PythonTreeWrapper @@ -92,7 +94,7 @@ class PythonEngine( private fun handleTimeoutResult( arguments: List, methodUnderTestDescription: PythonMethodDescription, - coveredLines: Collection, + coveredInstructions: List, ): FuzzingExecutionFeedback { val summary = arguments .zip(methodUnderTest.arguments) @@ -109,7 +111,6 @@ class PythonEngine( val beforeThisObject = beforeThisObjectTree?.let { PythonTreeModel(it.tree) } val beforeModelList = beforeModelListTree.map { PythonTreeModel(it.tree) } - val coveredInstructions = coveredLinesToInstructions(coveredLines, methodUnderTest) val coverage = Coverage(coveredInstructions) val utFuzzedExecution = PythonUtExecution( stateInit = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), @@ -134,7 +135,8 @@ class PythonEngine( ): FuzzingExecutionFeedback { val prohibitedExceptions = listOf( "builtins.AttributeError", - "builtins.TypeError" + "builtins.TypeError", + "builtins.NotImplementedError", ) val summary = arguments @@ -237,8 +239,8 @@ class PythonEngine( is PythonEvaluationTimeout -> { val coveredLines = manager.coverageReceiver.coverageStorage.getOrDefault(coverageId, mutableSetOf()) - val utTimeoutException = handleTimeoutResult(arguments, description, coveredLines) - val coveredInstructions = coveredLinesToInstructions(coveredLines, methodUnderTest) + val coveredInstructions = makeInstructions(coveredLines, methodUnderTest) + val utTimeoutException = handleTimeoutResult(arguments, description, coveredInstructions) val trieNode: Trie.Node = if (coveredInstructions.isEmpty()) Trie.emptyNode() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index e4c53f02d6..7a77a4c080 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -13,6 +13,8 @@ import org.utbot.python.evaluation.serialization.PythonExecutionResult import org.utbot.python.evaluation.serialization.SuccessExecution import org.utbot.python.evaluation.serialization.serializeObjects import org.utbot.python.evaluation.utils.CoverageIdGenerator +import org.utbot.python.evaluation.utils.PyInstruction +import org.utbot.python.evaluation.utils.toPyInstruction import org.utbot.python.framework.api.python.util.pythonAnyClassId import org.utbot.python.newtyping.PythonCallableTypeDescription import org.utbot.python.newtyping.pythonDescription @@ -132,9 +134,11 @@ class PythonCodeSocketExecutor( val stateBefore = ExecutionResultDeserializer.parseMemoryDump(executionResult.stateBefore) ?: return parsingException val stateAfter = ExecutionResultDeserializer.parseMemoryDump(executionResult.stateAfter) ?: return parsingException val diffIds = executionResult.diffIds.map {it.toLong()} + val statements = executionResult.statements.mapNotNull { it.toPyInstruction() } + val missedStatements = executionResult.missedStatements.mapNotNull { it.toPyInstruction() } PythonEvaluationSuccess( executionResult.isException, - calculateCoverage(executionResult.statements, executionResult.missedStatements), + calculateCoverage(statements, missedStatements), stateInit, stateBefore, stateAfter, @@ -151,15 +155,15 @@ class PythonCodeSocketExecutor( } } - private fun calculateCoverage(statements: List, missedStatements: List): Coverage { + private fun calculateCoverage(statements: List, missedStatements: List): Coverage { val covered = statements.filter { it !in missedStatements } return Coverage( coveredInstructions=covered.map { Instruction( method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, method.methodSignature(), - it, - it.toLong() + it.lineNumber, + it.offset ) }, instructionsCount = statements.size.toLong(), @@ -167,8 +171,8 @@ class PythonCodeSocketExecutor( Instruction( method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, method.methodSignature(), - it, - it.toLong() + it.lineNumber, + it.offset ) } ) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt index 413ff85aa4..4972cba89e 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt @@ -1,6 +1,8 @@ package org.utbot.python.evaluation import mu.KotlinLogging +import org.utbot.python.evaluation.utils.PyInstruction +import org.utbot.python.evaluation.utils.toPyInstruction import java.io.IOException import java.net.DatagramPacket import java.net.DatagramSocket @@ -11,7 +13,7 @@ import kotlin.math.max class PythonCoverageReceiver( val until: Long, ) : Thread() { - val coverageStorage = mutableMapOf>() + val coverageStorage = mutableMapOf>() private val socket = DatagramSocket() private val logger = KotlinLogging.logger {} @@ -39,11 +41,13 @@ class PythonCoverageReceiver( val buf = ByteArray(256) val request = DatagramPacket(buf, buf.size) socket.receive(request) - val requestData = request.data.decodeToString().take(request.length).split(":") + val requestData = request.data.decodeToString().take(request.length).split(":", limit=2) if (requestData.size == 2) { val (id, line) = requestData - val lineNumber = line.toInt() - coverageStorage.getOrPut(id) { mutableSetOf() }.add(lineNumber) + val instruction = line.toPyInstruction() + if (instruction != null) { + coverageStorage.getOrPut(id) { mutableSetOf() }.add(instruction) + } } } } catch (ex: SocketException) { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt index 2909143096..74c577b3ed 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt @@ -4,6 +4,7 @@ import com.squareup.moshi.JsonEncodingException import com.squareup.moshi.Moshi import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory +import org.utbot.python.evaluation.utils.PyInstruction object ExecutionResultDeserializer { private val moshi = Moshi.Builder() @@ -43,8 +44,8 @@ sealed class PythonExecutionResult data class SuccessExecution( val isException: Boolean, - val statements: List, - val missedStatements: List, + val statements: List, + val missedStatements: List, val stateInit: String, val stateBefore: String, val stateAfter: String, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt index 51d8a11e31..91eef93736 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt @@ -15,3 +15,36 @@ fun coveredLinesToInstructions(coveredLines: Collection, method: PythonMeth ) } } + +data class PyInstruction( + val lineNumber: Int, + val offset: Long +) { + constructor(lineNumber: Int) : this(lineNumber, lineNumber.toLong()) +} + +fun String.toPyInstruction(): PyInstruction? { + val data = this.split(":") + if (data.size == 2) { + val line = data[0].toInt() + val offset = data[1].toLong() + return PyInstruction(line, offset) + } else if (data.size == 1) { + val line = data[0].toInt() + return PyInstruction(line) + } + return null +} + +fun makeInstructions(coveredInstructions: Collection, method: PythonMethod): List { + return coveredInstructions.map { + val line = it.lineNumber + val offset = it.offset + Instruction( + method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, + method.methodSignature(), + line, + offset + ) + } +} From f28deff8c82b03edb2a5063f9bf945f6c5fba8e3 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 4 Oct 2023 15:31:22 +0300 Subject: [PATCH 02/17] instruction coverage --- gradle.properties | 2 +- .../main/python/utbot_executor/pyproject.toml | 2 +- .../utbot_executor/utbot_executor/executor.py | 5 ----- .../utbot_executor/utbot_executor/ut_tracer.py | 18 ++++++++++++------ .../src/main/resources/utbot_executor_version | 2 +- .../samples/controlflow/multi_conditions.py | 17 +++++++++++++++++ .../kotlin/org/utbot/python/PythonEngine.kt | 2 +- .../evaluation/PythonCoverageReceiver.kt | 4 ++-- 8 files changed, 35 insertions(+), 17 deletions(-) diff --git a/gradle.properties b/gradle.properties index 00719b83c7..1e41b8971b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -10,7 +10,7 @@ ultimateEdition=Ultimate # IU, IC, PC, PY # IC for AndroidStudio -ideType=IC +ideType=IU ideVersion=232.8660.185 # ALL, NOJS buildType=NOJS diff --git a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml index 90717a8f6d..a7581f1ef3 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml +++ b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "utbot-executor" -version = "1.7.0.dev7" +version = "1.7.0.dev13" description = "" authors = ["Vyacheslav Tamarin "] readme = "README.md" diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py index 13b64a1060..4d16e5d618 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py @@ -1,13 +1,10 @@ """Python code executor for UnitTestBot""" import copy -import dis import importlib import logging import pathlib -import socket import sys import traceback -import typing from typing import Any, Callable, Dict, Iterable, List, Tuple from utbot_executor.deep_serialization.deep_serialization import serialize_memory_dump, \ @@ -173,8 +170,6 @@ def _run_calculate_function_value( __is_exception = False - # bytecode = dis.get_instructions(function) - # __all_code_lines = [instr.starts_line for instr in bytecode if instr.starts_line is not None] __all_code_lines = list(get_instructions(function)) __start = min([op[0] for op in __all_code_lines]) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py index 7d88e3cbb9..8fbe0392c5 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py @@ -8,6 +8,7 @@ import sys import threading import typing +from concurrent.futures import ThreadPoolExecutor def _modname(path): @@ -26,8 +27,7 @@ def __init__(self, coverage_id: str, host: str, port: int, use_thread: bool = Fa self.use_thread = use_thread if use_thread: - self.thread = threading.Thread(target=self.send_loop, daemon=True) - self.thread.start() + self.thread = ThreadPoolExecutor(max_workers=4) def send_loop(self): try: @@ -37,7 +37,8 @@ def send_loop(self): self.send_loop() def send_message(self, message: bytes): - self.sock.sendto(*message) + logging.debug(f"SEND {message}") + self.sock.sendto(message, (self.host, self.port)) def send_message_thread(self): message = self.message_queue.get() @@ -45,8 +46,10 @@ def send_message_thread(self): def put_message(self, key: str): message = bytes(f"{self.coverage_id}:{key}", encoding="utf-8") + logging.debug(f"PUT {message}") if self.use_thread: self.message_queue.put((message, (self.host, self.port))) + self.thread.submit(self.send_message_thread) else: self.send_message(message) @@ -81,9 +84,12 @@ def localtrace_count(self, frame, why, arg): if why == "opcode": offset = frame.f_lasti key = (lineno, offset) + logging.debug(filename, key) if key not in self.counts: + message = ":".join(map(str, key)) try: - self.sender.put_message(":".join(map(str, key))) + # self.sender.send_message(message) + self.sender.put_message(message) except Exception: pass self.counts[key] = self.counts.get(key, 0) + 1 @@ -121,5 +127,5 @@ def f(x): if __name__ in "__main__": - tracer = UtTracer(pathlib.Path(__file__), [], UtCoverageSender("1", "localhost", 0)) - tracer.runfunc(f, 70) \ No newline at end of file + tracer = UtTracer(pathlib.Path(__file__), [], UtCoverageSender("1", "localhost", 0, use_thread=False)) + tracer.runfunc(f, 6) \ No newline at end of file diff --git a/utbot-python-executor/src/main/resources/utbot_executor_version b/utbot-python-executor/src/main/resources/utbot_executor_version index b04d552932..0c0a3dedc4 100644 --- a/utbot-python-executor/src/main/resources/utbot_executor_version +++ b/utbot-python-executor/src/main/resources/utbot_executor_version @@ -1 +1 @@ -1.7.0.dev7 \ No newline at end of file +1.7.0.dev13 \ No newline at end of file diff --git a/utbot-python/samples/samples/controlflow/multi_conditions.py b/utbot-python/samples/samples/controlflow/multi_conditions.py index b7ba4bce58..7ee91a9fc8 100644 --- a/utbot-python/samples/samples/controlflow/multi_conditions.py +++ b/utbot-python/samples/samples/controlflow/multi_conditions.py @@ -1,14 +1,31 @@ +import time + + def check_interval(x: float, left: float, right: float) -> str: if left < x < right or right < x < left: + print(1) + time.sleep(4) return "between" elif x < left and x < right: + print(1) + time.sleep(4) return "less" elif x > left and x > right: + print(1) + time.sleep(4) return "more" elif left == right: + print(1) + time.sleep(4) return "all equals" elif x == left: + print(1) + time.sleep(4) return "left" elif x == right: + print(1) + time.sleep(4) return "right" + print(1) + time.sleep(4) return "what?" diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 2dc0c7f547..7a7472b9ba 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -238,7 +238,7 @@ class PythonEngine( is PythonEvaluationTimeout -> { val coveredLines = - manager.coverageReceiver.coverageStorage.getOrDefault(coverageId, mutableSetOf()) + manager.coverageReceiver.coverageStorage.getOrDefault(coverageId, mutableListOf()) val coveredInstructions = makeInstructions(coveredLines, methodUnderTest) val utTimeoutException = handleTimeoutResult(arguments, description, coveredInstructions) val trieNode: Trie.Node = diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt index 4972cba89e..7e21ab744b 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt @@ -13,7 +13,7 @@ import kotlin.math.max class PythonCoverageReceiver( val until: Long, ) : Thread() { - val coverageStorage = mutableMapOf>() + val coverageStorage = mutableMapOf>() private val socket = DatagramSocket() private val logger = KotlinLogging.logger {} @@ -46,7 +46,7 @@ class PythonCoverageReceiver( val (id, line) = requestData val instruction = line.toPyInstruction() if (instruction != null) { - coverageStorage.getOrPut(id) { mutableSetOf() }.add(instruction) + coverageStorage.getOrPut(id) { mutableListOf() }.add(instruction) } } } From dc62e03b6ceb6bbd9bb2adc1232752869acda943 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Thu, 5 Oct 2023 17:04:41 +0300 Subject: [PATCH 03/17] instruction coverage update --- .../main/python/utbot_executor/pyproject.toml | 2 +- .../utbot_executor/utbot_executor/__main__.py | 54 +++++++++++-------- .../utbot_executor/utbot_executor/executor.py | 40 +++++++------- .../utbot_executor/utbot_executor/listener.py | 7 +-- .../utbot_executor/ut_tracer.py | 25 +++++---- .../utbot_executor/utbot_executor/utils.py | 17 ++++++ .../src/main/resources/utbot_executor_version | 2 +- .../samples/controlflow/multi_conditions.py | 17 ------ utbot-python/samples/test_configuration.json | 11 ++++ .../evaluation/PythonCodeSocketExecutor.kt | 2 +- .../python/evaluation/PythonWorkerManager.kt | 1 + 11 files changed, 104 insertions(+), 74 deletions(-) diff --git a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml index a7581f1ef3..a1b7b52fd2 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml +++ b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "utbot-executor" -version = "1.7.0.dev13" +version = "1.8.0" description = "" authors = ["Vyacheslav Tamarin "] readme = "README.md" diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py index ab1ce3fb60..89439a6c84 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py @@ -2,35 +2,45 @@ import logging from utbot_executor.listener import PythonExecuteServer +from utbot_executor.utils import TraceMode -def main(hostname: str, port: int, coverage_hostname: str, coverage_port: str): - server = PythonExecuteServer(hostname, port, coverage_hostname, coverage_port) +def main(hostname: str, port: int, coverage_hostname: str, coverage_port: int, trace_mode: TraceMode): + server = PythonExecuteServer(hostname, port, coverage_hostname, coverage_port, trace_mode) server.run() -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( - prog='UtBot Python Executor', - description='Listen socket stream and execute function value', - ) - parser.add_argument('hostname') - parser.add_argument('port', type=int) - parser.add_argument('--logfile', default=None) + prog="UtBot Python Executor", + description="Listen socket stream and execute function value", + ) + parser.add_argument("hostname") + parser.add_argument("port", type=int) + parser.add_argument("--logfile", default=None) parser.add_argument( - '--loglevel', - choices=["DEBUG", "INFO", "ERROR"], - default="ERROR", - ) - parser.add_argument('coverage_hostname') - parser.add_argument('coverage_port', type=int) + "--loglevel", + choices=["DEBUG", "INFO", "WARNING", "ERROR"], + default="ERROR", + ) + parser.add_argument("coverage_hostname") + parser.add_argument("coverage_port", type=int) + parser.add_argument( + "--coverage_type", choices=["lines", "instructions"], default="instructions" + ) args = parser.parse_args() - loglevel = {"DEBUG": logging.DEBUG, "INFO": logging.INFO, "ERROR": logging.ERROR}[args.loglevel] + loglevel = { + "DEBUG": logging.DEBUG, + "INFO": logging.INFO, + "WARNING": logging.WARNING, + "ERROR": logging.ERROR, + }[args.loglevel] logging.basicConfig( - filename=args.logfile, - format='%(asctime)s | %(levelname)s | %(funcName)s - %(message)s', - datefmt='%m/%d/%Y %H:%M:%S', - level=loglevel, - ) - main(args.hostname, args.port, args.coverage_hostname, args.coverage_port) + filename=args.logfile, + format="%(asctime)s | %(levelname)s | %(funcName)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=loglevel, + ) + trace_mode = TraceMode.Lines if args.coverage_type == "lines" else TraceMode.Instructions + main(args.hostname, args.port, args.coverage_hostname, args.coverage_port, trace_mode) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py index 4d16e5d618..c6865eb4ea 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py @@ -15,7 +15,12 @@ from utbot_executor.memory_compressor import compress_memory from utbot_executor.parser import ExecutionRequest, ExecutionResponse, ExecutionFailResponse, ExecutionSuccessResponse from utbot_executor.ut_tracer import UtTracer, UtCoverageSender -from utbot_executor.utils import suppress_stdout as __suppress_stdout, get_instructions +from utbot_executor.utils import ( + suppress_stdout as __suppress_stdout, + get_instructions, + filter_instructions, + TraceMode, +) __all__ = ['PythonExecutor'] @@ -38,9 +43,10 @@ def _load_objects(objs: List[Any]) -> MemoryDump: class PythonExecutor: - def __init__(self, coverage_hostname: str, coverage_port: int): + def __init__(self, coverage_hostname: str, coverage_port: int, trace_mode: TraceMode): self.coverage_hostname = coverage_hostname self.coverage_port = coverage_port + self.trace_mode = trace_mode @staticmethod def add_syspaths(syspaths: Iterable[str]): @@ -108,24 +114,21 @@ def run_function(self, request: ExecutionRequest) -> ExecutionResponse: state_init = _update_states(loader.reload_id(), state_init_memory) serialized_state_init = serialize_memory_dump(state_init) - # def _coverage_sender(info: typing.Tuple[str, int]): - # if pathlib.Path(info[0]) == pathlib.Path(request.filepath): - # sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - # logging.debug("Coverage message: %s:%d", request.coverage_id, info[1]) - # logging.debug("Port: %d", self.coverage_port) - # message = bytes(f'{request.coverage_id}:{info[1]}', encoding='utf-8') - # sock.sendto(message, (self.coverage_hostname, self.coverage_port)) - # logging.debug("ID: %s, Coverage: %s", request.coverage_id, info) _coverage_sender = UtCoverageSender(request.coverage_id, self.coverage_hostname, self.coverage_port) value = _run_calculate_function_value( - function, - args, - kwargs, - request.filepath, - serialized_state_init, - tracer=UtTracer(pathlib.Path(request.filepath), [sys.prefix, sys.exec_prefix], _coverage_sender) - ) + function, + args, + kwargs, + request.filepath, + serialized_state_init, + tracer=UtTracer( + pathlib.Path(request.filepath), + [sys.prefix, sys.exec_prefix], + _coverage_sender, + self.trace_mode, + ), + ) except Exception as _: logging.debug("Error \n%s", traceback.format_exc()) return ExecutionFailResponse("fail", traceback.format_exc()) @@ -170,7 +173,7 @@ def _run_calculate_function_value( __is_exception = False - __all_code_lines = list(get_instructions(function)) + __all_code_lines = filter_instructions(get_instructions(function), tracer.mode) __start = min([op[0] for op in __all_code_lines]) __tracer = tracer @@ -196,7 +199,6 @@ def _run_calculate_function_value( args_ids, kwargs_ids, result_id, state_after, serialized_state_after = _serialize_state(args, kwargs, __result) ids = args_ids + list(kwargs_ids.values()) - # state_before, state_after = compress_memory(ids, state_before, state_after) diff_ids = compress_memory(ids, state_before, state_after) return ExecutionSuccessResponse( diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py index 5d9d55a2b3..7985026930 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py @@ -6,7 +6,7 @@ from utbot_executor.deep_serialization.memory_objects import PythonSerializer from utbot_executor.parser import parse_request, serialize_response, ExecutionFailResponse from utbot_executor.executor import PythonExecutor - +from utbot_executor.utils import TraceMode RECV_SIZE = 2**15 @@ -17,12 +17,13 @@ def __init__( hostname: str, port: int, coverage_hostname: str, - coverage_port: str, + coverage_port: int, + trace_mode: TraceMode, ): logging.info('PythonExecutor is creating...') self.clientsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.clientsocket.connect((hostname, port)) - self.executor = PythonExecutor(coverage_hostname, coverage_port) + self.executor = PythonExecutor(coverage_hostname, coverage_port, trace_mode) def run(self) -> None: logging.info('PythonExecutor is ready...') diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py index 8fbe0392c5..01ad78d33b 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py @@ -1,15 +1,14 @@ -import dis -import inspect import logging import os import pathlib import queue import socket import sys -import threading import typing from concurrent.futures import ThreadPoolExecutor +from utbot_executor.utils import TraceMode + def _modname(path): base = os.path.basename(path) @@ -55,13 +54,20 @@ def put_message(self, key: str): class UtTracer: - def __init__(self, tested_file: pathlib.Path, ignore_dirs: typing.List[str], sender: UtCoverageSender): + def __init__( + self, + tested_file: pathlib.Path, + ignore_dirs: typing.List[str], + sender: UtCoverageSender, + mode: TraceMode = TraceMode.Instructions, + ): self.tested_file = tested_file self.counts = {} self.localtrace = self.localtrace_count self.globaltrace = self.globaltrace_lt self.ignore_dirs = ignore_dirs self.sender = sender + self.mode = mode def runfunc(self, func, /, *args, **kw): result = None @@ -80,15 +86,13 @@ def localtrace_count(self, frame, why, arg): filename = frame.f_code.co_filename if pathlib.Path(filename) == self.tested_file: lineno = frame.f_lineno - offset = 0 + offset = lineno * 2 if why == "opcode": offset = frame.f_lasti key = (lineno, offset) - logging.debug(filename, key) if key not in self.counts: message = ":".join(map(str, key)) try: - # self.sender.send_message(message) self.sender.put_message(message) except Exception: pass @@ -97,8 +101,9 @@ def localtrace_count(self, frame, why, arg): def globaltrace_lt(self, frame, why, arg): if why == 'call': - frame.f_trace_opcodes = True - frame.f_trace_lines = False + if self.mode == TraceMode.Instructions: + frame.f_trace_opcodes = True + frame.f_trace_lines = False filename = frame.f_globals.get('__file__', None) if filename and all(not filename.startswith(d + os.sep) for d in self.ignore_dirs): modulename = _modname(filename) @@ -127,5 +132,5 @@ def f(x): if __name__ in "__main__": - tracer = UtTracer(pathlib.Path(__file__), [], UtCoverageSender("1", "localhost", 0, use_thread=False)) + tracer = UtTracer(pathlib.Path(__file__), [], UtCoverageSender("1", "localhost", 0, use_thread=False), mode=TraceMode.Lines) tracer.runfunc(f, 6) \ No newline at end of file diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py index 577b76399f..23b6fbe69b 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py @@ -1,10 +1,17 @@ import dis +import enum +import inspect import os import sys import typing from contextlib import contextmanager +class TraceMode(enum.Enum): + Lines = 1 + Instructions = 2 + + @contextmanager def suppress_stdout(): with open(os.devnull, "w") as devnull: @@ -25,3 +32,13 @@ def inner_get_instructions(x, current_line): if "" in str(type(el.argval)): inner_get_instructions(el.argval, current_line) return inner_get_instructions(obj, None) + + +def filter_instructions( + instructions: typing.Iterable[tuple[int, int]], + mode: TraceMode = TraceMode.Instructions, +) -> list[tuple[int, int]]: + if mode == TraceMode.Lines: + return [(it, it) for it in {line for line, op in instructions}] + return list(instructions) + diff --git a/utbot-python-executor/src/main/resources/utbot_executor_version b/utbot-python-executor/src/main/resources/utbot_executor_version index 0c0a3dedc4..afa2b3515e 100644 --- a/utbot-python-executor/src/main/resources/utbot_executor_version +++ b/utbot-python-executor/src/main/resources/utbot_executor_version @@ -1 +1 @@ -1.7.0.dev13 \ No newline at end of file +1.8.0 \ No newline at end of file diff --git a/utbot-python/samples/samples/controlflow/multi_conditions.py b/utbot-python/samples/samples/controlflow/multi_conditions.py index 7ee91a9fc8..b7ba4bce58 100644 --- a/utbot-python/samples/samples/controlflow/multi_conditions.py +++ b/utbot-python/samples/samples/controlflow/multi_conditions.py @@ -1,31 +1,14 @@ -import time - - def check_interval(x: float, left: float, right: float) -> str: if left < x < right or right < x < left: - print(1) - time.sleep(4) return "between" elif x < left and x < right: - print(1) - time.sleep(4) return "less" elif x > left and x > right: - print(1) - time.sleep(4) return "more" elif left == right: - print(1) - time.sleep(4) return "all equals" elif x == left: - print(1) - time.sleep(4) return "left" elif x == right: - print(1) - time.sleep(4) return "right" - print(1) - time.sleep(4) return "what?" diff --git a/utbot-python/samples/test_configuration.json b/utbot-python/samples/test_configuration.json index 37f4d5645c..1f5bb02851 100644 --- a/utbot-python/samples/test_configuration.json +++ b/utbot-python/samples/test_configuration.json @@ -248,6 +248,17 @@ "coverage": 75 } ] + }, + { + "name": "multi_conditions", + "groups": [ + { + "classes": null, + "methods": null, + "timeout": 15, + "coverage": 100 + } + ] } ] }, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index 7a77a4c080..d404bd10c4 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -166,7 +166,7 @@ class PythonCodeSocketExecutor( it.offset ) }, - instructionsCount = statements.size.toLong(), + instructionsCount = (covered.size + missedStatements.size).toLong(), missedInstructions = missedStatements.map { Instruction( method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index 2ef2e2eeaf..eb58a1edba 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -47,6 +47,7 @@ class PythonWorkerManager( coverageReceiver.address().second, "--logfile", logfile.absolutePath, "--loglevel", logLevel, // "DEBUG", "INFO", "WARNING", "ERROR" + "--coverage_type", "instructions", )) timeout = max(until - processStartTime, 0) if (this::workerSocket.isInitialized && !workerSocket.isClosed) { From e62a18078f1481af4a4c5d9e839c19a35d8cd41d Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Mon, 9 Oct 2023 18:19:54 +0300 Subject: [PATCH 04/17] instruction coverage update and fix bugs --- .../main/python/utbot_executor/pyproject.toml | 2 +- .../utbot_executor/utbot_executor/executor.py | 7 +- .../utbot_executor/ut_tracer.py | 19 +---- .../utbot_executor/utbot_executor/utils.py | 8 +- utbot-python/samples/run_tests.py | 78 ++++++++++++++----- utbot-python/samples/test_configuration.json | 10 +-- .../inference/baseline/BaselineAlgorithm.kt | 8 +- 7 files changed, 79 insertions(+), 53 deletions(-) diff --git a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml index a1b7b52fd2..7cb3da0082 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml +++ b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "utbot-executor" -version = "1.8.0" +version = "1.8.0.dev4" description = "" authors = ["Vyacheslav Tamarin "] readme = "README.md" diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py index c6865eb4ea..fd403919b1 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py @@ -1,6 +1,7 @@ """Python code executor for UnitTestBot""" import copy import importlib +import inspect import logging import pathlib import sys @@ -173,8 +174,8 @@ def _run_calculate_function_value( __is_exception = False - __all_code_lines = filter_instructions(get_instructions(function), tracer.mode) - __start = min([op[0] for op in __all_code_lines]) + _, __start = inspect.getsourcelines(function) + __all_code_lines = filter_instructions(get_instructions(function, __start), tracer.mode) __tracer = tracer @@ -189,7 +190,7 @@ def _run_calculate_function_value( logging.debug("Coverage: %s", __tracer.counts) logging.debug("Fullpath: %s", fullpath) __stmts = [x for x in __tracer.counts] - __stmts_with_def = [(1, 0)] + __stmts + __stmts_with_def = [(__start, 0)] + __stmts __missed_filtered = [x for x in __all_code_lines if x not in __stmts_with_def] logging.debug("Covered lines: %s", __stmts_with_def) logging.debug("Missed lines: %s", __missed_filtered) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py index 01ad78d33b..62e60e3f5a 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py @@ -84,8 +84,8 @@ def coverage(self, filename: str) -> typing.List[int]: def localtrace_count(self, frame, why, arg): filename = frame.f_code.co_filename - if pathlib.Path(filename) == self.tested_file: - lineno = frame.f_lineno + lineno = frame.f_lineno + if pathlib.Path(filename) == self.tested_file and lineno is not None: offset = lineno * 2 if why == "opcode": offset = frame.f_lasti @@ -119,18 +119,3 @@ def __init__(self): def runfunc(self, func, /, *args, **kw): return func(*args, **kw) - - -def f(x): - if 0 < x < 10 and x % 2 == 0: - return 1 - else: - return [100, - x**2, - x + 1 - ] - - -if __name__ in "__main__": - tracer = UtTracer(pathlib.Path(__file__), [], UtCoverageSender("1", "localhost", 0, use_thread=False), mode=TraceMode.Lines) - tracer.runfunc(f, 6) \ No newline at end of file diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py index 23b6fbe69b..22543c98b1 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py @@ -1,6 +1,5 @@ import dis import enum -import inspect import os import sys import typing @@ -23,15 +22,15 @@ def suppress_stdout(): sys.stdout = old_stdout -def get_instructions(obj: object) -> typing.Iterator[tuple[int, int]]: +def get_instructions(obj: object, start_line: int) -> typing.Iterator[tuple[int, int]]: def inner_get_instructions(x, current_line): for i, el in enumerate(dis.get_instructions(x)): if el.starts_line is not None: current_line = el.starts_line yield current_line, el.offset - if "" in str(type(el.argval)): + if any(t in str(type(el.argval)) for t in [""]): inner_get_instructions(el.argval, current_line) - return inner_get_instructions(obj, None) + return inner_get_instructions(obj, start_line) def filter_instructions( @@ -41,4 +40,3 @@ def filter_instructions( if mode == TraceMode.Lines: return [(it, it) for it in {line for line, op in instructions}] return list(instructions) - diff --git a/utbot-python/samples/run_tests.py b/utbot-python/samples/run_tests.py index fd4064d1f7..9a353584c7 100644 --- a/utbot-python/samples/run_tests.py +++ b/utbot-python/samples/run_tests.py @@ -7,10 +7,14 @@ -c """ import argparse +import contextlib import json import os import shutil +import sys import typing +import tqdm +from tqdm.contrib import DummyTqdmFile import pathlib @@ -38,11 +42,41 @@ def parse_arguments(): return parser.parse_args() +def inner_zip(collection: dict[str, typing.Iterable], keys: list[str]) -> typing.Iterator[list[typing.Any]]: + key, inner_keys = keys[0], keys[1:] + if len(inner_keys) == 0: + yield [collection[key]] + return + for inner_collection in collection[key]: + for group in inner_zip(inner_collection, inner_keys): + yield [collection[key]] + group + + +def test_inner_zip(): + data = {"1": [{"2": [1, 2, 3]}, {"2": [4, 5, 6]}]} + actual = inner_zip(data, ["1", "2"]) + assert list(actual) == [[data["1"], data["1"][0]["2"]], [data["1"], data["1"][1]["2"]]] + + def parse_config(config_path: str): with open(config_path, "r") as fin: return json.loads(fin.read()) +@contextlib.contextmanager +def std_out_err_redirect_tqdm(): + orig_out_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = map(DummyTqdmFile, orig_out_err) + yield orig_out_err[0] + # Relay exceptions + except Exception as exc: + raise exc + # Always restore sys.stdout/err if necessary + finally: + sys.stdout, sys.stderr = orig_out_err + + def generate_tests( java: str, jar_path: str, @@ -60,7 +94,7 @@ def generate_tests( command += f" -c {','.join(class_names)}" if method_names is not None: command += f" -m {','.join(method_names)}" - print(command) + tqdm.tqdm.write(command) code = os.system(command) return code @@ -71,7 +105,7 @@ def run_tests( samples_dir: str, ): command = f'{python_path} -m coverage run --source={samples_dir} -m unittest discover -p "utbot_*" {tests_dir}' - print(command) + tqdm.tqdm.write(command) code = os.system(command) return code @@ -119,25 +153,27 @@ def main_test_generation(args): config = parse_config(args.config_file) if pathlib.Path(args.coverage_output_dir).exists(): shutil.rmtree(args.coverage_output_dir) - for part in config['parts']: - for file in part['files']: - for group in file['groups']: - full_name = pathlib.PurePath(args.path_to_test_dir, part['path'], file['name']) - output_file = pathlib.PurePath(args.output_dir, f"utbot_tests_{part['path'].replace('/', '_')}_{file['name']}.py") - coverage_output_file = pathlib.PurePath(args.coverage_output_dir, f"coverage_{part['path'].replace('/', '_')}_{file['name']}.json") - generate_tests( - args.java, - args.jar, - [args.path_to_test_dir], - args.python_path, - str(full_name), - group['timeout'], - str(output_file), - str(coverage_output_file), - group['classes'], - group['methods'] - ) - + with std_out_err_redirect_tqdm() as orig_stdout: + # for (part, file, group) in tqdm.tqdm(inner_zip(config, ["parts", "files", "groups"]), file=orig_stdout, dynamic_ncols=True): + for part in tqdm.tqdm(config["parts"], file=orig_stdout, dynamic_ncols=True): + for file in tqdm.tqdm(part["files"], file=orig_stdout, dynamic_ncols=True): + for group in tqdm.tqdm(file["groups"], file=orig_stdout, dynamic_ncols=True): + full_name = pathlib.PurePath(args.path_to_test_dir, part['path'], file['name']) + output_file = pathlib.PurePath(args.output_dir, f"utbot_tests_{part['path'].replace('/', '_')}_{file['name']}.py") + coverage_output_file = pathlib.PurePath(args.coverage_output_dir, f"coverage_{part['path'].replace('/', '_')}_{file['name']}.json") + generate_tests( + args.java, + args.jar, + [args.path_to_test_dir], + args.python_path, + str(full_name), + group['timeout'], + str(output_file), + str(coverage_output_file), + group['classes'], + group['methods'] + ) + if __name__ == '__main__': arguments = parse_arguments() diff --git a/utbot-python/samples/test_configuration.json b/utbot-python/samples/test_configuration.json index 1f5bb02851..41a64e6f28 100644 --- a/utbot-python/samples/test_configuration.json +++ b/utbot-python/samples/test_configuration.json @@ -10,7 +10,7 @@ "classes": null, "methods": null, "timeout": 10, - "coverage": 100 + "coverage": 93 } ] }, @@ -233,8 +233,8 @@ { "classes": null, "methods": null, - "timeout": 30, - "coverage": 72 + "timeout": 180, + "coverage": 94 } ] }, @@ -256,7 +256,7 @@ "classes": null, "methods": null, "timeout": 15, - "coverage": 100 + "coverage": 93 } ] } @@ -531,7 +531,7 @@ { "classes": ["Matrix"], "methods": null, - "timeout": 240, + "timeout": 180, "coverage": 100 } ] diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt index 5272f0ed40..fd1173ce17 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt @@ -52,6 +52,8 @@ class BaselineAlgorithm( private val openedStates: MutableMap> = mutableMapOf() private val statistic: MutableMap = mutableMapOf() + private val checkedSignatures: MutableSet = mutableSetOf() + private fun getRandomType(): UtType? { val weights = states.map { 1.0 / (it.anyNodes.size * it.anyNodes.size + 1) } val state = weightedRandom(states, weights, random) @@ -92,15 +94,19 @@ class BaselineAlgorithm( val state = chooseState(states) val newState = expandState(state, storage) if (newState != null) { - logger.info("Checking ${newState.signature.pythonTypeRepresentation()}") + logger.info("Checking new state ${newState.signature.pythonTypeRepresentation()}") if (checkSignature(newState.signature as FunctionType, fileForMypyRuns, configFile)) { logger.debug("Found new state!") openedStates[newState.signature] = newState to state return newState.signature } } else if (state.anyNodes.isEmpty()) { + if (state.signature in checkedSignatures) { + return state.signature + } logger.info("Checking ${state.signature.pythonTypeRepresentation()}") if (checkSignature(state.signature as FunctionType, fileForMypyRuns, configFile)) { + checkedSignatures.add(state.signature) return state.signature } else { states.remove(state) From 85b9971dd875f83d63e048dae3274136d70c2df5 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Mon, 9 Oct 2023 18:30:00 +0300 Subject: [PATCH 05/17] version update --- .../src/main/python/utbot_executor/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml index 7cb3da0082..a1b7b52fd2 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml +++ b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "utbot-executor" -version = "1.8.0.dev4" +version = "1.8.0" description = "" authors = ["Vyacheslav Tamarin "] readme = "README.md" From e0f3332782b40203b0d9e270c32fcd8a4c085750 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Tue, 10 Oct 2023 15:41:18 +0300 Subject: [PATCH 06/17] update coverage --- gradle.properties | 2 +- .../python/utbot_executor/utbot_executor/__main__.py | 10 +++++++--- .../python/utbot_executor/utbot_executor/executor.py | 10 ++++++++-- .../python/utbot_executor/utbot_executor/listener.py | 3 ++- .../python/utbot_executor/utbot_executor/ut_tracer.py | 8 +++++--- .../main/python/utbot_executor/utbot_executor/utils.py | 2 +- .../org/utbot/python/evaluation/PythonWorkerManager.kt | 3 ++- 7 files changed, 26 insertions(+), 12 deletions(-) diff --git a/gradle.properties b/gradle.properties index 1e41b8971b..00719b83c7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -10,7 +10,7 @@ ultimateEdition=Ultimate # IU, IC, PC, PY # IC for AndroidStudio -ideType=IU +ideType=IC ideVersion=232.8660.185 # ALL, NOJS buildType=NOJS diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py index 89439a6c84..c9356cdcdb 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py @@ -5,8 +5,8 @@ from utbot_executor.utils import TraceMode -def main(hostname: str, port: int, coverage_hostname: str, coverage_port: int, trace_mode: TraceMode): - server = PythonExecuteServer(hostname, port, coverage_hostname, coverage_port, trace_mode) +def main(hostname: str, port: int, coverage_hostname: str, coverage_port: int, trace_mode: TraceMode, send_coverage: bool): + server = PythonExecuteServer(hostname, port, coverage_hostname, coverage_port, trace_mode, send_coverage) server.run() @@ -28,6 +28,9 @@ def main(hostname: str, port: int, coverage_hostname: str, coverage_port: int, t parser.add_argument( "--coverage_type", choices=["lines", "instructions"], default="instructions" ) + parser.add_argument( + "--send_coverage", action=argparse.BooleanOptionalAction + ) args = parser.parse_args() loglevel = { @@ -43,4 +46,5 @@ def main(hostname: str, port: int, coverage_hostname: str, coverage_port: int, t level=loglevel, ) trace_mode = TraceMode.Lines if args.coverage_type == "lines" else TraceMode.Instructions - main(args.hostname, args.port, args.coverage_hostname, args.coverage_port, trace_mode) + send_coverage = args.send_coverage + main(args.hostname, args.port, args.coverage_hostname, args.coverage_port, trace_mode, send_coverage) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py index fd403919b1..5586d67be2 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py @@ -44,10 +44,11 @@ def _load_objects(objs: List[Any]) -> MemoryDump: class PythonExecutor: - def __init__(self, coverage_hostname: str, coverage_port: int, trace_mode: TraceMode): + def __init__(self, coverage_hostname: str, coverage_port: int, trace_mode: TraceMode, send_coverage: bool): self.coverage_hostname = coverage_hostname self.coverage_port = coverage_port self.trace_mode = trace_mode + self.send_coverage = send_coverage @staticmethod def add_syspaths(syspaths: Iterable[str]): @@ -115,7 +116,12 @@ def run_function(self, request: ExecutionRequest) -> ExecutionResponse: state_init = _update_states(loader.reload_id(), state_init_memory) serialized_state_init = serialize_memory_dump(state_init) - _coverage_sender = UtCoverageSender(request.coverage_id, self.coverage_hostname, self.coverage_port) + _coverage_sender = UtCoverageSender( + request.coverage_id, + self.coverage_hostname, + self.coverage_port, + send_coverage=self.send_coverage, + ) value = _run_calculate_function_value( function, diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py index 7985026930..a1c91e8c00 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py @@ -19,11 +19,12 @@ def __init__( coverage_hostname: str, coverage_port: int, trace_mode: TraceMode, + send_coverage: bool ): logging.info('PythonExecutor is creating...') self.clientsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.clientsocket.connect((hostname, port)) - self.executor = PythonExecutor(coverage_hostname, coverage_port, trace_mode) + self.executor = PythonExecutor(coverage_hostname, coverage_port, trace_mode, send_coverage) def run(self) -> None: logging.info('PythonExecutor is ready...') diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py index 62e60e3f5a..454826a682 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py @@ -17,12 +17,13 @@ def _modname(path): class UtCoverageSender: - def __init__(self, coverage_id: str, host: str, port: int, use_thread: bool = False): + def __init__(self, coverage_id: str, host: str, port: int, use_thread: bool = False, send_coverage: bool = True): self.coverage_id = coverage_id self.host = host self.port = port self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.message_queue = queue.Queue() + self.send_coverage = send_coverage self.use_thread = use_thread if use_thread: @@ -36,8 +37,9 @@ def send_loop(self): self.send_loop() def send_message(self, message: bytes): - logging.debug(f"SEND {message}") - self.sock.sendto(message, (self.host, self.port)) + if self.send_coverage: + logging.debug(f"SEND {message}") + self.sock.sendto(message, (self.host, self.port)) def send_message_thread(self): message = self.message_queue.get() diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py index 22543c98b1..29ad7d0f85 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py @@ -38,5 +38,5 @@ def filter_instructions( mode: TraceMode = TraceMode.Instructions, ) -> list[tuple[int, int]]: if mode == TraceMode.Lines: - return [(it, it) for it in {line for line, op in instructions}] + return [(it, 2 * it) for it in {line for line, op in instructions}] return list(instructions) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index eb58a1edba..143718842f 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -47,7 +47,8 @@ class PythonWorkerManager( coverageReceiver.address().second, "--logfile", logfile.absolutePath, "--loglevel", logLevel, // "DEBUG", "INFO", "WARNING", "ERROR" - "--coverage_type", "instructions", + "--coverage_type", "instructions", // "lines", "instructions" + "--send_coverage", // "--send_coverage", "--no-send_coverage" )) timeout = max(until - processStartTime, 0) if (this::workerSocket.isInitialized && !workerSocket.isClosed) { From 6dd5f5ed8855a93e2b5fe2ceead6b8c434e4b83e Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Tue, 10 Oct 2023 16:37:46 +0300 Subject: [PATCH 07/17] Add cli arguments --- .../python/PythonGenerateTestsCommand.kt | 13 +++++++++- .../kotlin/org/utbot/python/PythonEngine.kt | 7 ++++-- .../utbot/python/PythonTestCaseGenerator.kt | 9 +++++-- .../python/PythonTestGenerationConfig.kt | 3 +++ .../python/PythonTestGenerationProcessor.kt | 4 ++- .../python/evaluation/PythonWorkerManager.kt | 15 +++++++++-- .../utbot/python/evaluation/utils/Utils.kt | 25 ++++++++++++------- 7 files changed, 59 insertions(+), 17 deletions(-) diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt index 89448aebb9..cdb5565ab7 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt @@ -16,6 +16,8 @@ import org.utbot.python.PythonTestSet import org.utbot.python.utils.RequirementsInstaller import org.utbot.python.TestFileInformation import org.utbot.python.code.PythonCode +import org.utbot.python.evaluation.utils.PythonCoverageMode +import org.utbot.python.evaluation.utils.toPythonCoverageMode import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.codegen.model.Pytest import org.utbot.python.framework.codegen.model.Unittest @@ -117,6 +119,13 @@ class PythonGenerateTestsCommand : CliktCommand( private val doNotGenerateRegressionSuite by option("--do-not-generate-regression-suite", help = "Do not generate regression test suite") .flag(default = false) + private val coverageMeasureMode by option("--coverage-measure-mode", help = "Use LINES or INSTRUCTIONS for coverage measurement") + .choice("INSTRUCTIONS", "LINES") + .default("INSTRUCTIONS") + + private val doNotSendCoverageContinuously by option("--do-not-send-coverage-continuously", help = "") + .flag(default = false) + private val testFramework: TestFramework get() = when (testFrameworkAsString) { @@ -252,7 +261,9 @@ class PythonGenerateTestsCommand : CliktCommand( testSourceRootPath = Paths.get(output.toAbsolutePath()).parent.toAbsolutePath(), withMinimization = !doNotMinimize, isCanceled = { false }, - runtimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.valueOf(runtimeExceptionTestsBehaviour) + runtimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.valueOf(runtimeExceptionTestsBehaviour), + coverageMeasureMode = coverageMeasureMode.toPythonCoverageMode() ?: PythonCoverageMode.Instructions, + sendCoverageContinuously = !doNotSendCoverageContinuously ) val processor = PythonCliProcessor( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 7a7472b9ba..28bd6d501c 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -12,8 +12,7 @@ import org.utbot.python.evaluation.* import org.utbot.python.evaluation.serialization.MemoryDump import org.utbot.python.evaluation.serialization.toPythonTree import org.utbot.python.evaluation.utils.CoverageIdGenerator -import org.utbot.python.evaluation.utils.PyInstruction -import org.utbot.python.evaluation.utils.coveredLinesToInstructions +import org.utbot.python.evaluation.utils.PythonCoverageMode import org.utbot.python.evaluation.utils.makeInstructions import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.PythonTreeModel @@ -44,6 +43,8 @@ class PythonEngine( private val fuzzedConcreteValues: List, private val timeoutForRun: Long, private val pythonTypeStorage: PythonTypeHintsStorage, + private val coverageMode: PythonCoverageMode = PythonCoverageMode.Instructions, + private val sendCoverageContinuously: Boolean = true, ) { private val cache = EvaluationCache() @@ -302,6 +303,8 @@ class PythonEngine( serverSocket, pythonPath, until, + coverageMode, + sendCoverageContinuously, ) { constructEvaluationInput(it) } } catch (_: TimeoutException) { return@flow diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index f6ad5c1419..1d69149adf 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -6,6 +6,7 @@ import org.utbot.framework.minimization.minimizeExecutions import org.utbot.framework.plugin.api.UtError import org.utbot.framework.plugin.api.UtExecution import org.utbot.framework.plugin.api.UtExecutionSuccess +import org.utbot.python.evaluation.utils.PythonCoverageMode import org.utbot.python.framework.api.python.PythonUtExecution import org.utbot.python.framework.api.python.util.pythonStrClassId import org.utbot.python.fuzzing.* @@ -40,7 +41,9 @@ class PythonTestCaseGenerator( private val timeoutForRun: Long = 0, private val sourceFileContent: String, private val mypyStorage: MypyInfoBuild, - private val mypyReportLine: List + private val mypyReportLine: List, + private val coverageMode: PythonCoverageMode = PythonCoverageMode.Instructions, + private val sendCoverageContinuously: Boolean = true, ) { private val storageForMypyMessages: MutableList = mutableListOf() @@ -153,7 +156,9 @@ class PythonTestCaseGenerator( pythonPath, constants, timeoutForRun, - PythonTypeHintsStorage.get(mypyStorage) + PythonTypeHintsStorage.get(mypyStorage), + coverageMode, + sendCoverageContinuously, ) val namesInModule = mypyStorage.names .getOrDefault(curModule, emptyList()) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt index 405a1abf0d..cd62ee8c89 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt @@ -2,6 +2,7 @@ package org.utbot.python import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour import org.utbot.framework.codegen.domain.TestFramework +import org.utbot.python.evaluation.utils.PythonCoverageMode import java.nio.file.Path data class TestFileInformation( @@ -22,4 +23,6 @@ class PythonTestGenerationConfig( val withMinimization: Boolean, val isCanceled: () -> Boolean, val runtimeExceptionTestsBehaviour: RuntimeExceptionTestsBehaviour, + val coverageMeasureMode: PythonCoverageMode = PythonCoverageMode.Instructions, + val sendCoverageContinuously: Boolean = true, ) \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt index 59821778f5..bec976059d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt @@ -67,7 +67,9 @@ abstract class PythonTestGenerationProcessor { timeoutForRun = configuration.timeoutForRun, sourceFileContent = configuration.testFileInformation.testedFileContent, mypyStorage = mypyStorage, - mypyReportLine = emptyList() + mypyReportLine = emptyList(), + coverageMode = configuration.coverageMeasureMode, + sendCoverageContinuously = configuration.sendCoverageContinuously, ) val until = startTime + configuration.timeout diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index 143718842f..26ada603a3 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -11,6 +11,7 @@ import java.net.ServerSocket import java.net.Socket import java.net.SocketTimeoutException import org.apache.logging.log4j.LogManager +import org.utbot.python.evaluation.utils.PythonCoverageMode private val logger = KotlinLogging.logger {} @@ -18,6 +19,8 @@ class PythonWorkerManager( private val serverSocket: ServerSocket, val pythonPath: String, val until: Long, + private val coverageMeasureMode: PythonCoverageMode = PythonCoverageMode.Instructions, + private val sendCoverageContinuously: Boolean = true, val pythonCodeExecutorConstructor: (PythonWorker) -> PythonCodeExecutor, ) { var timeout: Long = 0 @@ -47,8 +50,8 @@ class PythonWorkerManager( coverageReceiver.address().second, "--logfile", logfile.absolutePath, "--loglevel", logLevel, // "DEBUG", "INFO", "WARNING", "ERROR" - "--coverage_type", "instructions", // "lines", "instructions" - "--send_coverage", // "--send_coverage", "--no-send_coverage" + "--coverage_type", coverageMeasureMode.toString(), // "lines", "instructions" + sendCoverageContinuously.toSendCoverageContinuouslyString(), // "--send_coverage", "--no-send_coverage" )) timeout = max(until - processStartTime, 0) if (this::workerSocket.isInitialized && !workerSocket.isClosed) { @@ -122,5 +125,13 @@ class PythonWorkerManager( companion object { val logfile = TemporaryFileManager.createTemporaryFile("", "utbot_executor.log", "log", true) + + fun Boolean.toSendCoverageContinuouslyString(): String { + return if (this) { + "--send_coverage" + } else { + "--no-send_coverage" + } + } } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt index 91eef93736..c26e17cae4 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt @@ -5,14 +5,21 @@ import org.utbot.python.PythonMethod import org.utbot.python.framework.api.python.util.pythonAnyClassId import org.utbot.python.newtyping.pythonTypeRepresentation -fun coveredLinesToInstructions(coveredLines: Collection, method: PythonMethod): List { - return coveredLines.map { - Instruction( - method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, - method.methodSignature(), - it, - it.toLong() - ) +enum class PythonCoverageMode { + Lines { + override fun toString() = "lines" + }, + + Instructions { + override fun toString() = "instructions" + } +} + +fun String.toPythonCoverageMode(): PythonCoverageMode? { + return when (this.lowercase()) { + "lines" -> PythonCoverageMode.Lines + "instructions" -> PythonCoverageMode.Instructions + else -> null } } @@ -47,4 +54,4 @@ fun makeInstructions(coveredInstructions: Collection, method: Pyt offset ) } -} +} \ No newline at end of file From c933a792891d9258a07a46c9e26e2e1b47c02b57 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Tue, 10 Oct 2023 17:10:02 +0300 Subject: [PATCH 08/17] Update run_tests.py --- utbot-python/samples/run_tests.py | 166 ++++++++++++++++-------------- 1 file changed, 90 insertions(+), 76 deletions(-) diff --git a/utbot-python/samples/run_tests.py b/utbot-python/samples/run_tests.py index 9a353584c7..f0a7900957 100644 --- a/utbot-python/samples/run_tests.py +++ b/utbot-python/samples/run_tests.py @@ -11,7 +11,9 @@ import json import os import shutil +from subprocess import Popen, PIPE import sys +import threading import typing import tqdm from tqdm.contrib import DummyTqdmFile @@ -20,44 +22,27 @@ def parse_arguments(): parser = argparse.ArgumentParser( - prog='UtBot Python test', - description='Generate tests for example files' + prog="UtBot Python test", description="Generate tests for example files" ) subparsers = parser.add_subparsers(dest="command") - parser_generate = subparsers.add_parser('generate', help='Generate tests') - parser_generate.add_argument('java') - parser_generate.add_argument('jar') - parser_generate.add_argument('path_to_test_dir') - parser_generate.add_argument('-c', '--config_file', required=True) - parser_generate.add_argument('-p', '--python_path', required=True) - parser_generate.add_argument('-o', '--output_dir', required=True) - parser_generate.add_argument('-i', '--coverage_output_dir', required=True) - parser_run = subparsers.add_parser('run', help='Run tests') - parser_run.add_argument('-p', '--python_path', required=True) - parser_run.add_argument('-t', '--test_directory', required=True) - parser_run.add_argument('-c', '--code_directory', required=True) - parser_coverage = subparsers.add_parser('check_coverage', help='Check coverage') - parser_coverage.add_argument('-i', '--coverage_output_dir', required=True) - parser_coverage.add_argument('-c', '--config_file', required=True) + parser_generate = subparsers.add_parser("generate", help="Generate tests") + parser_generate.add_argument("java") + parser_generate.add_argument("jar") + parser_generate.add_argument("path_to_test_dir") + parser_generate.add_argument("-c", "--config_file", required=True) + parser_generate.add_argument("-p", "--python_path", required=True) + parser_generate.add_argument("-o", "--output_dir", required=True) + parser_generate.add_argument("-i", "--coverage_output_dir", required=True) + parser_run = subparsers.add_parser("run", help="Run tests") + parser_run.add_argument("-p", "--python_path", required=True) + parser_run.add_argument("-t", "--test_directory", required=True) + parser_run.add_argument("-c", "--code_directory", required=True) + parser_coverage = subparsers.add_parser("check_coverage", help="Check coverage") + parser_coverage.add_argument("-i", "--coverage_output_dir", required=True) + parser_coverage.add_argument("-c", "--config_file", required=True) return parser.parse_args() -def inner_zip(collection: dict[str, typing.Iterable], keys: list[str]) -> typing.Iterator[list[typing.Any]]: - key, inner_keys = keys[0], keys[1:] - if len(inner_keys) == 0: - yield [collection[key]] - return - for inner_collection in collection[key]: - for group in inner_zip(inner_collection, inner_keys): - yield [collection[key]] + group - - -def test_inner_zip(): - data = {"1": [{"2": [1, 2, 3]}, {"2": [4, 5, 6]}]} - actual = inner_zip(data, ["1", "2"]) - assert list(actual) == [[data["1"], data["1"][0]["2"]], [data["1"], data["1"][1]["2"]]] - - def parse_config(config_path: str): with open(config_path, "r") as fin: return json.loads(fin.read()) @@ -78,31 +63,37 @@ def std_out_err_redirect_tqdm(): def generate_tests( - java: str, - jar_path: str, - sys_paths: list[str], - python_path: str, - file_under_test: str, - timeout: int, - output: str, - coverage_output: str, - class_names: typing.Optional[list[str]] = None, - method_names: typing.Optional[list[str]] = None - ): + java: str, + jar_path: str, + sys_paths: list[str], + python_path: str, + file_under_test: str, + timeout: int, + output: str, + coverage_output: str, + class_names: typing.Optional[list[str]] = None, + method_names: typing.Optional[list[str]] = None, +): command = f"{java} -jar {jar_path} generate_python {file_under_test}.py -p {python_path} -o {output} -s {' '.join(sys_paths)} --timeout {timeout * 1000} --install-requirements --runtime-exception-behaviour PASS --coverage={coverage_output}" if class_names is not None: command += f" -c {','.join(class_names)}" if method_names is not None: command += f" -m {','.join(method_names)}" - tqdm.tqdm.write(command) - code = os.system(command) - return code + tqdm.tqdm.write("\n" + command) + + def stdout_printer(p): + for line in p.stdout: + tqdm.tqdm.write(line.rstrip().decode()) + + p = Popen(command.split(), stdout=PIPE) + t = threading.Thread(target=stdout_printer, args=(p,)) + t.run() def run_tests( - python_path: str, - tests_dir: str, - samples_dir: str, + python_path: str, + tests_dir: str, + samples_dir: str, ): command = f'{python_path} -m coverage run --source={samples_dir} -m unittest discover -p "utbot_*" {tests_dir}' tqdm.tqdm.write(command) @@ -111,28 +102,38 @@ def run_tests( def check_coverage( - config_file: str, - coverage_output_dir: str, + config_file: str, + coverage_output_dir: str, ): config = parse_config(config_file) report: typing.Dict[str, bool] = {} coverage: typing.Dict[str, typing.Tuple[float, float]] = {} - for part in config['parts']: - for file in part['files']: - for group in file['groups']: - expected_coverage = group.get('coverage', 0) + for part in config["parts"]: + for file in part["files"]: + for group in file["groups"]: + expected_coverage = group.get("coverage", 0) file_suffix = f"{part['path'].replace('/', '_')}_{file['name']}" - coverage_output_file = pathlib.Path(coverage_output_dir, f"coverage_{file_suffix}.json") + coverage_output_file = pathlib.Path( + coverage_output_dir, f"coverage_{file_suffix}.json" + ) if coverage_output_file.exists(): with open(coverage_output_file, "rt") as fin: actual_coverage_json = json.loads(fin.readline()) - actual_covered = sum(lines['end'] - lines['start'] + 1 for lines in actual_coverage_json['covered']) - actual_not_covered = sum(lines['end'] - lines['start'] + 1 for lines in actual_coverage_json['notCovered']) + actual_covered = sum( + lines["end"] - lines["start"] + 1 + for lines in actual_coverage_json["covered"] + ) + actual_not_covered = sum( + lines["end"] - lines["start"] + 1 + for lines in actual_coverage_json["notCovered"] + ) if actual_covered + actual_not_covered == 0: actual_coverage = 0 else: - actual_coverage = round(actual_covered / (actual_not_covered + actual_covered) * 100) + actual_coverage = round( + actual_covered / (actual_not_covered + actual_covered) * 100 + ) else: actual_coverage = 0 @@ -154,32 +155,45 @@ def main_test_generation(args): if pathlib.Path(args.coverage_output_dir).exists(): shutil.rmtree(args.coverage_output_dir) with std_out_err_redirect_tqdm() as orig_stdout: - # for (part, file, group) in tqdm.tqdm(inner_zip(config, ["parts", "files", "groups"]), file=orig_stdout, dynamic_ncols=True): - for part in tqdm.tqdm(config["parts"], file=orig_stdout, dynamic_ncols=True): - for file in tqdm.tqdm(part["files"], file=orig_stdout, dynamic_ncols=True): - for group in tqdm.tqdm(file["groups"], file=orig_stdout, dynamic_ncols=True): - full_name = pathlib.PurePath(args.path_to_test_dir, part['path'], file['name']) - output_file = pathlib.PurePath(args.output_dir, f"utbot_tests_{part['path'].replace('/', '_')}_{file['name']}.py") - coverage_output_file = pathlib.PurePath(args.coverage_output_dir, f"coverage_{part['path'].replace('/', '_')}_{file['name']}.json") + for part in tqdm.tqdm( + config["parts"], file=orig_stdout, dynamic_ncols=True, desc="Progress" + ): + for file in tqdm.tqdm( + part["files"], file=orig_stdout, dynamic_ncols=True, desc=part["path"] + ): + for group in file["groups"]: + full_name = pathlib.PurePath( + args.path_to_test_dir, part["path"], file["name"] + ) + output_file = pathlib.PurePath( + args.output_dir, + f"utbot_tests_{part['path'].replace('/', '_')}_{file['name']}.py", + ) + coverage_output_file = pathlib.PurePath( + args.coverage_output_dir, + f"coverage_{part['path'].replace('/', '_')}_{file['name']}.json", + ) generate_tests( args.java, args.jar, [args.path_to_test_dir], args.python_path, str(full_name), - group['timeout'], + group["timeout"], str(output_file), str(coverage_output_file), - group['classes'], - group['methods'] + group["classes"], + group["methods"], ) - -if __name__ == '__main__': + +if __name__ == "__main__": arguments = parse_arguments() - if arguments.command == 'generate': + if arguments.command == "generate": main_test_generation(arguments) - elif arguments.command == 'run': - run_tests(arguments.python_path, arguments.test_directory, arguments.code_directory) - elif arguments.command == 'check_coverage': + elif arguments.command == "run": + run_tests( + arguments.python_path, arguments.test_directory, arguments.code_directory + ) + elif arguments.command == "check_coverage": check_coverage(arguments.config_file, arguments.coverage_output_dir) From cf67b46bf73e4f0d3501e8eabc343c2ad8fc8550 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Tue, 17 Oct 2023 10:47:12 +0300 Subject: [PATCH 09/17] Add instructions/lines trace mode, update coverage output --- .../python/PythonGenerateTestsCommand.kt | 20 ++- .../utbot_executor/utbot_executor/executor.py | 13 +- .../utbot_executor/ut_tracer.py | 33 +++- .../utbot_executor/utbot_executor/utils.py | 33 +++- .../kotlin/org/utbot/python/PythonEngine.kt | 34 ++-- .../utbot/python/PythonTestCaseGenerator.kt | 2 +- .../python/PythonTestGenerationConfig.kt | 4 +- .../python/PythonTestGenerationProcessor.kt | 157 +++++++++++++----- .../python/evaluation/CodeEvaluationApi.kt | 4 +- .../evaluation/PythonCodeSocketExecutor.kt | 35 +--- .../evaluation/PythonCoverageReceiver.kt | 4 +- .../python/evaluation/PythonWorkerManager.kt | 2 +- .../python/evaluation/coverage/CoverageApi.kt | 109 ++++++++++++ .../CoverageIdGenerator.kt | 6 +- .../ExecutionResultDeserializer.kt | 1 - .../utbot/python/evaluation/utils/Utils.kt | 57 ------- .../org/utbot/python/fuzzing/PythonApi.kt | 6 +- 17 files changed, 337 insertions(+), 183 deletions(-) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt rename utbot-python/src/main/kotlin/org/utbot/python/evaluation/{utils => coverage}/CoverageIdGenerator.kt (52%) delete mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt index cdb5565ab7..126f811e1b 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt @@ -10,14 +10,15 @@ import org.parsers.python.PythonParser import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour import org.utbot.framework.codegen.domain.TestFramework import org.utbot.framework.plugin.api.UtExecutionSuccess +import org.utbot.python.evaluation.coverage.CoverageOutputFormat import org.utbot.python.PythonMethodHeader import org.utbot.python.PythonTestGenerationConfig import org.utbot.python.PythonTestSet -import org.utbot.python.utils.RequirementsInstaller import org.utbot.python.TestFileInformation +import org.utbot.python.utils.RequirementsInstaller import org.utbot.python.code.PythonCode -import org.utbot.python.evaluation.utils.PythonCoverageMode -import org.utbot.python.evaluation.utils.toPythonCoverageMode +import org.utbot.python.evaluation.coverage.PythonCoverageMode +import org.utbot.python.evaluation.coverage.toPythonCoverageMode import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.codegen.model.Pytest import org.utbot.python.framework.codegen.model.Unittest @@ -116,16 +117,20 @@ class PythonGenerateTestsCommand : CliktCommand( .choice("PASS", "FAIL") .default("FAIL") - private val doNotGenerateRegressionSuite by option("--do-not-generate-regression-suite", help = "Do not generate regression test suite") + private val doNotGenerateRegressionSuite by option("--do-not-generate-regression-suite", help = "Do not generate regression test suite.") .flag(default = false) - private val coverageMeasureMode by option("--coverage-measure-mode", help = "Use LINES or INSTRUCTIONS for coverage measurement") + private val coverageMeasureMode by option("--coverage-measure-mode", help = "Use LINES or INSTRUCTIONS for coverage measurement.") .choice("INSTRUCTIONS", "LINES") .default("INSTRUCTIONS") - private val doNotSendCoverageContinuously by option("--do-not-send-coverage-continuously", help = "") + private val doNotSendCoverageContinuously by option("--do-not-send-coverage-continuously", help = "Do not send coverage during execution.") .flag(default = false) + private val coverageOutputFormat by option("--coverage-output-format", help = "Use Lines, Instructions or TopFrameInstructions.") + .choice("Instructions", "Lines", "TopFrameInstructions") + .default("Instructions") + private val testFramework: TestFramework get() = when (testFrameworkAsString) { @@ -263,7 +268,8 @@ class PythonGenerateTestsCommand : CliktCommand( isCanceled = { false }, runtimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.valueOf(runtimeExceptionTestsBehaviour), coverageMeasureMode = coverageMeasureMode.toPythonCoverageMode() ?: PythonCoverageMode.Instructions, - sendCoverageContinuously = !doNotSendCoverageContinuously + sendCoverageContinuously = !doNotSendCoverageContinuously, + coverageOutputFormat = CoverageOutputFormat.valueOf(coverageOutputFormat), ) val processor = PythonCliProcessor( diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py index 5586d67be2..94c6009302 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py @@ -20,7 +20,7 @@ suppress_stdout as __suppress_stdout, get_instructions, filter_instructions, - TraceMode, + TraceMode, UtInstruction, ) __all__ = ['PythonExecutor'] @@ -181,7 +181,7 @@ def _run_calculate_function_value( __is_exception = False _, __start = inspect.getsourcelines(function) - __all_code_lines = filter_instructions(get_instructions(function, __start), tracer.mode) + __all_code_stmts = filter_instructions(get_instructions(function, __start), tracer.mode) __tracer = tracer @@ -195,14 +195,13 @@ def _run_calculate_function_value( logging.debug("Coverage: %s", __tracer.counts) logging.debug("Fullpath: %s", fullpath) - __stmts = [x for x in __tracer.counts] - __stmts_with_def = [(__start, 0)] + __stmts - __missed_filtered = [x for x in __all_code_lines if x not in __stmts_with_def] + __stmts_with_def = [UtInstruction(__start, 0, 0)] + list(__tracer.counts.keys()) + __missed_filtered = [x for x in __all_code_stmts if x not in __stmts_with_def] logging.debug("Covered lines: %s", __stmts_with_def) logging.debug("Missed lines: %s", __missed_filtered) - __str_statements = [":".join(map(str, x)) for x in __stmts_with_def] - __str_missed_statements = [":".join(map(str, x)) for x in __missed_filtered] + __str_statements = [x.serialize() for x in __stmts_with_def] + __str_missed_statements = [x.serialize() for x in __missed_filtered] args_ids, kwargs_ids, result_id, state_after, serialized_state_after = _serialize_state(args, kwargs, __result) ids = args_ids + list(kwargs_ids.values()) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py index 454826a682..c62ed2278e 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py @@ -1,3 +1,5 @@ +import dis +import inspect import logging import os import pathlib @@ -7,7 +9,7 @@ import typing from concurrent.futures import ThreadPoolExecutor -from utbot_executor.utils import TraceMode +from utbot_executor.utils import TraceMode, UtInstruction def _modname(path): @@ -55,6 +57,11 @@ def put_message(self, key: str): self.send_message(message) +class PureSender(UtCoverageSender): + def __init__(self): + super().__init__("000000", "localhost", 0, use_thread=False, send_coverage=False) + + class UtTracer: def __init__( self, @@ -64,20 +71,23 @@ def __init__( mode: TraceMode = TraceMode.Instructions, ): self.tested_file = tested_file - self.counts = {} + self.counts: dict[UtInstruction, int] = {} self.localtrace = self.localtrace_count self.globaltrace = self.globaltrace_lt self.ignore_dirs = ignore_dirs self.sender = sender self.mode = mode + self.depth = 0 def runfunc(self, func, /, *args, **kw): result = None + self.depth = 0 sys.settrace(self.globaltrace) try: result = func(*args, **kw) finally: sys.settrace(None) + self.depth = 0 return result def coverage(self, filename: str) -> typing.List[int]: @@ -91,9 +101,9 @@ def localtrace_count(self, frame, why, arg): offset = lineno * 2 if why == "opcode": offset = frame.f_lasti - key = (lineno, offset) + key = UtInstruction(lineno, offset, self.depth) if key not in self.counts: - message = ":".join(map(str, key)) + message = key.serialize() try: self.sender.put_message(message) except Exception: @@ -102,11 +112,12 @@ def localtrace_count(self, frame, why, arg): return self.localtrace def globaltrace_lt(self, frame, why, arg): + self.depth += 1 if why == 'call': if self.mode == TraceMode.Instructions: frame.f_trace_opcodes = True frame.f_trace_lines = False - filename = frame.f_globals.get('__file__', None) + filename = frame.f_code.co_filename if filename and all(not filename.startswith(d + os.sep) for d in self.ignore_dirs): modulename = _modname(filename) if modulename is not None: @@ -121,3 +132,15 @@ def __init__(self): def runfunc(self, func, /, *args, **kw): return func(*args, **kw) + + +def f(x): + def g(x): + xs = [[j for j in range(i)] for i in range(10)] + return x * 2 + return x * g(x) + 2 + + +if __name__ == "__main__": + tracer = UtTracer(pathlib.Path(__file__), [], PureSender()) + tracer.runfunc(f, 2) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py index 29ad7d0f85..b815e45350 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py @@ -1,3 +1,4 @@ +import dataclasses import dis import enum import os @@ -11,6 +12,19 @@ class TraceMode(enum.Enum): Instructions = 2 +@dataclasses.dataclass +class UtInstruction: + line: int + offset: int + depth: int + + def serialize(self) -> str: + return ":".join(map(str, [self.line, self.offset, self.depth])) + + def __hash__(self): + return hash((self.line, self.offset, self.depth)) + + @contextmanager def suppress_stdout(): with open(os.devnull, "w") as devnull: @@ -22,21 +36,24 @@ def suppress_stdout(): sys.stdout = old_stdout -def get_instructions(obj: object, start_line: int) -> typing.Iterator[tuple[int, int]]: - def inner_get_instructions(x, current_line): +def get_instructions(obj: object, start_line: int) -> typing.Iterator[UtInstruction]: + def inner_get_instructions(x, current_line, depth): for i, el in enumerate(dis.get_instructions(x)): if el.starts_line is not None: current_line = el.starts_line - yield current_line, el.offset + yield UtInstruction(current_line, el.offset, depth) if any(t in str(type(el.argval)) for t in [""]): - inner_get_instructions(el.argval, current_line) - return inner_get_instructions(obj, start_line) + inner_get_instructions(el.argval, current_line, depth + 1) + return inner_get_instructions(obj, start_line, 0) def filter_instructions( - instructions: typing.Iterable[tuple[int, int]], + instructions: typing.Iterable[UtInstruction], mode: TraceMode = TraceMode.Instructions, -) -> list[tuple[int, int]]: +) -> list[UtInstruction]: if mode == TraceMode.Lines: - return [(it, 2 * it) for it in {line for line, op in instructions}] + unique_line_instructions: set[UtInstruction] = set() + for it in instructions: + unique_line_instructions.add(UtInstruction(it.line, it.line * 2, it.depth)) + return list({UtInstruction(it.line, 2 * it.line, it.depth) for it in instructions}) return list(instructions) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 28bd6d501c..095e4761f7 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -8,12 +8,21 @@ import org.utbot.fuzzing.Control import org.utbot.fuzzing.NoSeedValueException import org.utbot.fuzzing.fuzz import org.utbot.fuzzing.utils.Trie -import org.utbot.python.evaluation.* +import org.utbot.python.evaluation.EvaluationCache +import org.utbot.python.evaluation.PythonCodeExecutor +import org.utbot.python.evaluation.PythonCodeSocketExecutor +import org.utbot.python.evaluation.PythonEvaluationError +import org.utbot.python.evaluation.PythonEvaluationSuccess +import org.utbot.python.evaluation.PythonEvaluationTimeout +import org.utbot.python.evaluation.PythonWorker +import org.utbot.python.evaluation.PythonWorkerManager +import org.utbot.python.evaluation.coverage.CoverageIdGenerator +import org.utbot.python.evaluation.coverage.PyInstruction +import org.utbot.python.evaluation.coverage.PythonCoverageMode +import org.utbot.python.evaluation.coverage.calculateCoverage +import org.utbot.python.evaluation.coverage.makeInstructions import org.utbot.python.evaluation.serialization.MemoryDump import org.utbot.python.evaluation.serialization.toPythonTree -import org.utbot.python.evaluation.utils.CoverageIdGenerator -import org.utbot.python.evaluation.utils.PythonCoverageMode -import org.utbot.python.evaluation.utils.makeInstructions import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.PythonTreeModel import org.utbot.python.framework.api.python.PythonTreeWrapper @@ -95,7 +104,7 @@ class PythonEngine( private fun handleTimeoutResult( arguments: List, methodUnderTestDescription: PythonMethodDescription, - coveredInstructions: List, + coveredInstructions: List, ): FuzzingExecutionFeedback { val summary = arguments .zip(methodUnderTest.arguments) @@ -112,7 +121,7 @@ class PythonEngine( val beforeThisObject = beforeThisObjectTree?.let { PythonTreeModel(it.tree) } val beforeModelList = beforeModelListTree.map { PythonTreeModel(it.tree) } - val coverage = Coverage(coveredInstructions) + val coverage = Coverage(makeInstructions(coveredInstructions, methodUnderTest)) val utFuzzedExecution = PythonUtExecution( stateInit = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), stateBefore = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), @@ -176,7 +185,7 @@ class PythonEngine( stateAfter = EnvironmentModels(afterThisObject, afterModelList, emptyMap(), executableToCall = null), diffIds = evaluationResult.diffIds, result = executionResult, - coverage = evaluationResult.coverage, + coverage = calculateCoverage(evaluationResult.coverage, methodUnderTest), testMethodName = testMethodName.testName?.camelToSnakeCase(), displayName = testMethodName.displayName, summary = summary.map { DocRegularStmt(it) }, @@ -238,11 +247,10 @@ class PythonEngine( } is PythonEvaluationTimeout -> { - val coveredLines = + val coveredInstructions = manager.coverageReceiver.coverageStorage.getOrDefault(coverageId, mutableListOf()) - val coveredInstructions = makeInstructions(coveredLines, methodUnderTest) val utTimeoutException = handleTimeoutResult(arguments, description, coveredInstructions) - val trieNode: Trie.Node = + val trieNode: Trie.Node = if (coveredInstructions.isEmpty()) Trie.emptyNode() else @@ -266,7 +274,7 @@ class PythonEngine( val typeInferenceFeedback = if (result is ValidExecution) SuccessFeedback else InvalidTypeFeedback when (result) { is ValidExecution -> { - val trieNode: Trie.Node = description.tracer.add(coveredInstructions) + val trieNode: Trie.Node = description.tracer.add(coveredInstructions) description.limitManager.addSuccessExecution() PythonExecutionResult( result, @@ -316,7 +324,7 @@ class PythonEngine( parameters, fuzzedConcreteValues, pythonTypeStorage, - Trie(Instruction::id), + Trie(PyInstruction::id), Random(0), TestGenerationLimitManager(ExecutionWithTimoutMode, until, isRootManager = true), methodUnderTest.definition.type, @@ -352,7 +360,7 @@ class PythonEngine( val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) }) val mem = cache.get(pair) if (mem != null) { - logger.debug("Repeat in fuzzing ${arguments.map {it.tree}}") + logger.debug { "Repeat in fuzzing ${arguments.map {it.tree}}" } description.limitManager.addSuccessExecution() emit(CachedExecutionFeedback(mem.fuzzingExecutionFeedback)) return@PythonFuzzing mem.fuzzingPlatformFeedback.fromCache() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 1d69149adf..6549b8f454 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -6,7 +6,7 @@ import org.utbot.framework.minimization.minimizeExecutions import org.utbot.framework.plugin.api.UtError import org.utbot.framework.plugin.api.UtExecution import org.utbot.framework.plugin.api.UtExecutionSuccess -import org.utbot.python.evaluation.utils.PythonCoverageMode +import org.utbot.python.evaluation.coverage.PythonCoverageMode import org.utbot.python.framework.api.python.PythonUtExecution import org.utbot.python.framework.api.python.util.pythonStrClassId import org.utbot.python.fuzzing.* diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt index cd62ee8c89..909e8c8f28 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt @@ -2,7 +2,8 @@ package org.utbot.python import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour import org.utbot.framework.codegen.domain.TestFramework -import org.utbot.python.evaluation.utils.PythonCoverageMode +import org.utbot.python.evaluation.coverage.CoverageOutputFormat +import org.utbot.python.evaluation.coverage.PythonCoverageMode import java.nio.file.Path data class TestFileInformation( @@ -25,4 +26,5 @@ class PythonTestGenerationConfig( val runtimeExceptionTestsBehaviour: RuntimeExceptionTestsBehaviour, val coverageMeasureMode: PythonCoverageMode = PythonCoverageMode.Instructions, val sendCoverageContinuously: Boolean = true, + val coverageOutputFormat: CoverageOutputFormat = CoverageOutputFormat.Lines, ) \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt index bec976059d..777dadae17 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt @@ -1,7 +1,5 @@ package org.utbot.python -import com.squareup.moshi.Moshi -import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory import mu.KotlinLogging import org.parsers.python.PythonParser import org.utbot.framework.codegen.domain.HangingTestsTimeout @@ -12,6 +10,9 @@ import org.utbot.framework.plugin.api.UtExecutionSuccess import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.withUtContext import org.utbot.python.code.PythonCode +import org.utbot.python.evaluation.coverage.CoverageOutputFormat +import org.utbot.python.evaluation.coverage.PyInstruction +import org.utbot.python.evaluation.coverage.toPair import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.api.python.PythonMethodId import org.utbot.python.framework.api.python.PythonModel @@ -54,6 +55,8 @@ abstract class PythonTestGenerationProcessor { ) } + + fun testGenerate(mypyStorage: MypyInfoBuild): List { val startTime = System.currentTimeMillis() @@ -255,56 +258,128 @@ abstract class PythonTestGenerationProcessor { paths } - data class InstructionSet( - val start: Int, - val end: Int - ) + sealed class CoverageFormat + data class LineCoverage(val start: Int, val end: Int) : CoverageFormat() { + override fun equals(other: Any?): Boolean { + if (other is LineCoverage) { + return start == other.start && end == other.end + } + return false + } - data class CoverageInfo( - val covered: List, - val notCovered: List - ) + override fun hashCode(): Int { + var result = start + result = 31 * result + end + return result + } + } + data class InstructionCoverage(val line: Int, val offset: Long) : CoverageFormat() { + override fun equals(other: Any?): Boolean { + if (other is InstructionCoverage) { + return line == other.line && offset == other.offset + } + return false + } - private val moshi: Moshi = Moshi.Builder().addLast(KotlinJsonAdapterFactory()).build() - private val jsonAdapter = moshi.adapter(CoverageInfo::class.java) - - private fun getInstructionSetList(instructions: Collection): List = - instructions.sorted().fold(emptyList()) { acc, lineNumber -> - if (acc.isEmpty()) - return@fold listOf(InstructionSet(lineNumber, lineNumber)) - val elem = acc.last() - if (elem.end + 1 == lineNumber) - acc.dropLast(1) + listOf(InstructionSet(elem.start, lineNumber)) - else - acc + listOf(InstructionSet(lineNumber, lineNumber)) + override fun hashCode(): Int { + var result = line + result = 31 * result + offset.hashCode() + return result } - protected fun getCoverageInfo(testSets: List): CoverageInfo { - val covered = mutableSetOf() - val missed = mutableSetOf>() + } + + data class InstructionIdCoverage(val line: Int, val id: Long) : CoverageFormat() { + override fun equals(other: Any?): Boolean { + if (other is InstructionCoverage) { + return line == other.line && id == other.offset + } + return false + } + + override fun hashCode(): Int { + var result = line + result = 31 * result + id.hashCode() + return result + } + } + + data class CoverageInfo( + val covered: List, + val notCovered: List, + ) + + private fun getLinesList(instructions: Collection): List = + instructions + .map { it.lineNumber } + .sorted() + .fold(emptyList()) { acc, lineNumber -> + if (acc.isEmpty()) + return@fold listOf(LineCoverage(lineNumber, lineNumber)) + val elem = acc.last() + if (elem.end + 1 == lineNumber || elem.end == lineNumber ) + acc.dropLast(1) + listOf(LineCoverage(elem.start, lineNumber)) + else + acc + listOf(LineCoverage(lineNumber, lineNumber)) + } + + private fun filterMissedLines(covered: Collection, missed: Collection): List = + missed.filterNot { missedInstruction -> covered.any { it.start <= missedInstruction.lineNumber && missedInstruction.lineNumber <= it.end } } + + private fun getInstructionsListWithId(instructions: Collection): List = + instructions.map { InstructionIdCoverage(it.lineNumber, it.id) }.toSet().toList() + + private fun getInstructionsListWithOffset(instructions: Collection): List = + instructions.map { InstructionCoverage(it.lineNumber, it.offset) }.toSet().toList() + + private fun getCoverageInfo(testSets: List): CoverageInfo { + val covered = mutableSetOf() + val missed = mutableSetOf() testSets.forEach { testSet -> testSet.executions.forEach inner@{ execution -> val coverage = execution.coverage ?: return@inner - coverage.coveredInstructions.forEach { covered.add(it.lineNumber) } - missed.add(coverage.missedInstructions.map { it.lineNumber }.toSet()) + covered.addAll(coverage.coveredInstructions.map { PyInstruction(it.lineNumber, it.id) }) + missed.addAll(coverage.missedInstructions.map { PyInstruction(it.lineNumber, it.id) }) } } - val coveredInstructionSets = getInstructionSetList(covered) - val missedInstructionSets = - if (missed.isEmpty()) - emptyList() - else - getInstructionSetList(missed.reduce { a, b -> a intersect b }) - - return CoverageInfo( - coveredInstructionSets, - missedInstructionSets - ) + missed -= covered + val info = when (this.configuration.coverageOutputFormat) { + CoverageOutputFormat.Lines -> { + val coveredLines = getLinesList(covered) + val filteredMissed = filterMissedLines(coveredLines, missed) + val missedLines = getLinesList(filteredMissed) + CoverageInfo(coveredLines, missedLines) + } + CoverageOutputFormat.Instructions -> CoverageInfo(getInstructionsListWithId(covered), getInstructionsListWithId(missed)) + CoverageOutputFormat.TopFrameInstructions -> { + val filteredCovered = covered.filter { it.id.toPair().second == 0L } + val filteredMissed = missed.filter { it.id.toPair().second == 0L } + + val coveredInstructions = getInstructionsListWithOffset(filteredCovered) + val missedInstructions = getInstructionsListWithOffset(filteredMissed) + + CoverageInfo(coveredInstructions, (missedInstructions.toSet() - coveredInstructions.toSet()).toList()) + } + } + return CoverageInfo(info.covered.toSet().toList(), info.notCovered.toSet().toList()) + } + + private fun toJson(coverageInfo: CoverageInfo): String { + val covered = coverageInfo.covered.map { toJson(it) } + val notCovered = coverageInfo.notCovered.map { toJson(it) } + return "{\"covered\": [${covered.joinToString(", ")}], \"notCovered\": [${notCovered.joinToString(", ")}]}" + } + + private fun toJson(coverageFormat: CoverageFormat): String { + return when (coverageFormat) { + is LineCoverage -> "{\"start\": ${coverageFormat.start}, \"end\": ${coverageFormat.end}}" + is InstructionCoverage -> "{\"line\": ${coverageFormat.line}, \"offset\": ${coverageFormat.offset}}" + is InstructionIdCoverage -> "{\"line\": ${coverageFormat.line}, \"id\": ${coverageFormat.id}}" + } } protected fun getStringCoverageInfo(testSets: List): String { - return jsonAdapter.toJson( - getCoverageInfo(testSets) - ) + val value = getCoverageInfo(testSets) + return toJson(value) } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt index 3030ed6870..41154c2baa 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt @@ -1,9 +1,9 @@ package org.utbot.python.evaluation -import org.utbot.framework.plugin.api.Coverage import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod import org.utbot.python.evaluation.serialization.MemoryDump +import org.utbot.python.evaluation.coverage.PyCoverage interface PythonCodeExecutor { val method: PythonMethod @@ -40,7 +40,7 @@ data class PythonEvaluationTimeout( data class PythonEvaluationSuccess( val isException: Boolean, - val coverage: Coverage, + val coverage: PyCoverage, val stateInit: MemoryDump, val stateBefore: MemoryDump, val stateAfter: MemoryDump, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index d404bd10c4..5b8fba4501 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -1,8 +1,6 @@ package org.utbot.python.evaluation import mu.KotlinLogging -import org.utbot.framework.plugin.api.Coverage -import org.utbot.framework.plugin.api.Instruction import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod import org.utbot.python.evaluation.serialization.ExecutionRequest @@ -12,14 +10,12 @@ import org.utbot.python.evaluation.serialization.FailExecution import org.utbot.python.evaluation.serialization.PythonExecutionResult import org.utbot.python.evaluation.serialization.SuccessExecution import org.utbot.python.evaluation.serialization.serializeObjects -import org.utbot.python.evaluation.utils.CoverageIdGenerator -import org.utbot.python.evaluation.utils.PyInstruction -import org.utbot.python.evaluation.utils.toPyInstruction -import org.utbot.python.framework.api.python.util.pythonAnyClassId +import org.utbot.python.evaluation.coverage.CoverageIdGenerator +import org.utbot.python.evaluation.coverage.PyCoverage +import org.utbot.python.evaluation.coverage.toPyInstruction import org.utbot.python.newtyping.PythonCallableTypeDescription import org.utbot.python.newtyping.pythonDescription import org.utbot.python.newtyping.pythonTypeName -import org.utbot.python.newtyping.pythonTypeRepresentation import org.utbot.python.newtyping.utils.isNamed import java.net.SocketException @@ -138,7 +134,7 @@ class PythonCodeSocketExecutor( val missedStatements = executionResult.missedStatements.mapNotNull { it.toPyInstruction() } PythonEvaluationSuccess( executionResult.isException, - calculateCoverage(statements, missedStatements), + PyCoverage(statements, missedStatements), stateInit, stateBefore, stateAfter, @@ -155,29 +151,6 @@ class PythonCodeSocketExecutor( } } - private fun calculateCoverage(statements: List, missedStatements: List): Coverage { - val covered = statements.filter { it !in missedStatements } - return Coverage( - coveredInstructions=covered.map { - Instruction( - method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, - method.methodSignature(), - it.lineNumber, - it.offset - ) - }, - instructionsCount = (covered.size + missedStatements.size).toLong(), - missedInstructions = missedStatements.map { - Instruction( - method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, - method.methodSignature(), - it.lineNumber, - it.offset - ) - } - ) - } - override fun stop() { pythonWorker.stopServer() } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt index 7e21ab744b..63a5e52944 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt @@ -1,8 +1,8 @@ package org.utbot.python.evaluation import mu.KotlinLogging -import org.utbot.python.evaluation.utils.PyInstruction -import org.utbot.python.evaluation.utils.toPyInstruction +import org.utbot.python.evaluation.coverage.PyInstruction +import org.utbot.python.evaluation.coverage.toPyInstruction import java.io.IOException import java.net.DatagramPacket import java.net.DatagramSocket diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index 26ada603a3..264299816c 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -11,7 +11,7 @@ import java.net.ServerSocket import java.net.Socket import java.net.SocketTimeoutException import org.apache.logging.log4j.LogManager -import org.utbot.python.evaluation.utils.PythonCoverageMode +import org.utbot.python.evaluation.coverage.PythonCoverageMode private val logger = KotlinLogging.logger {} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt new file mode 100644 index 0000000000..2f69428bd2 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt @@ -0,0 +1,109 @@ +package org.utbot.python.evaluation.coverage + +import org.utbot.framework.plugin.api.Coverage +import org.utbot.framework.plugin.api.Instruction +import org.utbot.python.PythonMethod +import org.utbot.python.framework.api.python.util.pythonAnyClassId +import org.utbot.python.newtyping.pythonTypeRepresentation + +enum class PythonCoverageMode { + Lines { + override fun toString() = "lines" + }, + + Instructions { + override fun toString() = "instructions" + } +} + +fun String.toPythonCoverageMode(): PythonCoverageMode? { + return when (this.lowercase()) { + "lines" -> PythonCoverageMode.Lines + "instructions" -> PythonCoverageMode.Instructions + else -> null + } +} + +data class PyInstruction( + val lineNumber: Int, + val offset: Long, + val depth: Int = 0, +) { + override fun toString(): String = listOf(lineNumber, offset, depth).joinToString(":") + + val id: Long = (offset to depth.toLong()).toCoverageId() + + constructor(lineNumber: Int) : this(lineNumber, lineNumber.toLong()) + constructor(lineNumber: Int, id: Long) : this(lineNumber, id.toPair().first, id.toPair().second.toInt()) +} + +fun String.toPyInstruction(): PyInstruction? { + val data = this.split(":") + when (data.size) { + 3 -> { + val line = data[0].toInt() + val offset = data[1].toLong() + val depth = data[2].toInt() + return PyInstruction(line, offset, depth) + } + 2 -> { + val line = data[0].toInt() + val offset = data[1].toLong() + return PyInstruction(line, offset) + } + 1 -> { + val line = data[0].toInt() + return PyInstruction(line) + } + else -> return null + } +} + +fun makeInstructions(coveredInstructions: Collection, method: PythonMethod): List { + return coveredInstructions.map { + Instruction( + method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, + method.methodSignature(), + it.lineNumber, + it.id + ) + } +} + +data class PyCoverage( + val coveredInstructions: List, + val missedInstructions: List +) + +fun calculateCoverage(coverage: PyCoverage, method: PythonMethod): Coverage { + return calculateCoverage(coverage.coveredInstructions, coverage.missedInstructions, method) +} + +fun calculateCoverage(statements: List, missedStatements: List, method: PythonMethod): Coverage { + val covered = statements.filter { it !in missedStatements } + return Coverage( + coveredInstructions=covered.map { + Instruction( + method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, + method.methodSignature(), + it.lineNumber, + it.id + ) + }, + instructionsCount = (covered.size + missedStatements.size).toLong(), + missedInstructions = missedStatements.map { + Instruction( + method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, + method.methodSignature(), + it.lineNumber, + it.id + ) + } + ) +} + +enum class CoverageOutputFormat { + Lines, + Instructions, + TopFrameInstructions; +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageIdGenerator.kt similarity index 52% rename from utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt rename to utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageIdGenerator.kt index 0e13ce16b7..b9248d03f1 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageIdGenerator.kt @@ -1,11 +1,11 @@ -package org.utbot.python.evaluation.utils +package org.utbot.python.evaluation.coverage import java.util.concurrent.atomic.AtomicLong object CoverageIdGenerator { - private const val lower_bound: Long = 1500_000_000 + private const val LOWER_BOUND: Long = 1500_000_000 - private val lastId: AtomicLong = AtomicLong(lower_bound) + private val lastId: AtomicLong = AtomicLong(LOWER_BOUND) fun createId(): String { return lastId.incrementAndGet().toString(radix = 16) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt index 74c577b3ed..9cc140a12a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt @@ -4,7 +4,6 @@ import com.squareup.moshi.JsonEncodingException import com.squareup.moshi.Moshi import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory -import org.utbot.python.evaluation.utils.PyInstruction object ExecutionResultDeserializer { private val moshi = Moshi.Builder() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt deleted file mode 100644 index c26e17cae4..0000000000 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt +++ /dev/null @@ -1,57 +0,0 @@ -package org.utbot.python.evaluation.utils - -import org.utbot.framework.plugin.api.Instruction -import org.utbot.python.PythonMethod -import org.utbot.python.framework.api.python.util.pythonAnyClassId -import org.utbot.python.newtyping.pythonTypeRepresentation - -enum class PythonCoverageMode { - Lines { - override fun toString() = "lines" - }, - - Instructions { - override fun toString() = "instructions" - } -} - -fun String.toPythonCoverageMode(): PythonCoverageMode? { - return when (this.lowercase()) { - "lines" -> PythonCoverageMode.Lines - "instructions" -> PythonCoverageMode.Instructions - else -> null - } -} - -data class PyInstruction( - val lineNumber: Int, - val offset: Long -) { - constructor(lineNumber: Int) : this(lineNumber, lineNumber.toLong()) -} - -fun String.toPyInstruction(): PyInstruction? { - val data = this.split(":") - if (data.size == 2) { - val line = data[0].toInt() - val offset = data[1].toLong() - return PyInstruction(line, offset) - } else if (data.size == 1) { - val line = data[0].toInt() - return PyInstruction(line) - } - return null -} - -fun makeInstructions(coveredInstructions: Collection, method: PythonMethod): List { - return coveredInstructions.map { - val line = it.lineNumber - val offset = it.offset - Instruction( - method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, - method.methodSignature(), - line, - offset - ) - } -} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt index 00c3fb48e2..6f0a88dea2 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt @@ -1,11 +1,11 @@ package org.utbot.python.fuzzing import mu.KotlinLogging -import org.utbot.framework.plugin.api.Instruction import org.utbot.framework.plugin.api.UtError import org.utbot.fuzzer.FuzzedContext import org.utbot.fuzzing.* import org.utbot.fuzzing.utils.Trie +import org.utbot.python.evaluation.coverage.PyInstruction import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.PythonUtExecution import org.utbot.python.fuzzing.provider.* @@ -35,7 +35,7 @@ class PythonMethodDescription( parameters: List, val concreteValues: Collection = emptyList(), val pythonTypeStorage: PythonTypeHintsStorage, - val tracer: Trie, + val tracer: Trie, val random: Random, val limitManager: TestGenerationLimitManager, val type: FunctionType, @@ -56,7 +56,7 @@ data class PythonExecutionResult( data class PythonFeedback( override val control: Control = Control.CONTINUE, - val result: Trie.Node = Trie.emptyNode(), + val result: Trie.Node = Trie.emptyNode(), val typeInferenceFeedback: InferredTypeFeedback = InvalidTypeFeedback, val fromCache: Boolean = false, ) : Feedback { From c9c1a0e3434190b6353f29070dc8b68ebf6bfdd3 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Tue, 17 Oct 2023 10:54:57 +0300 Subject: [PATCH 10/17] Fix code style --- .../org/utbot/python/PythonTestGenerationProcessor.kt | 2 -- .../org/utbot/python/evaluation/coverage/CoverageApi.kt | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt index 777dadae17..3e0edda17e 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt @@ -55,8 +55,6 @@ abstract class PythonTestGenerationProcessor { ) } - - fun testGenerate(mypyStorage: MypyInfoBuild): List { val startTime = System.currentTimeMillis() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt index 2f69428bd2..880390a246 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt @@ -27,13 +27,13 @@ fun String.toPythonCoverageMode(): PythonCoverageMode? { data class PyInstruction( val lineNumber: Int, val offset: Long, - val depth: Int = 0, + val depth: Int, ) { override fun toString(): String = listOf(lineNumber, offset, depth).joinToString(":") val id: Long = (offset to depth.toLong()).toCoverageId() - constructor(lineNumber: Int) : this(lineNumber, lineNumber.toLong()) + constructor(lineNumber: Int) : this(lineNumber, lineNumber.toLong(), 0) constructor(lineNumber: Int, id: Long) : this(lineNumber, id.toPair().first, id.toPair().second.toInt()) } @@ -49,7 +49,7 @@ fun String.toPyInstruction(): PyInstruction? { 2 -> { val line = data[0].toInt() val offset = data[1].toLong() - return PyInstruction(line, offset) + return PyInstruction(line, offset, 0) } 1 -> { val line = data[0].toInt() From 99c66ea381bbb490219d823168f0250699ddd322 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Tue, 17 Oct 2023 13:18:28 +0300 Subject: [PATCH 11/17] Update cli arguments and coverage format --- .../python/PythonGenerateTestsCommand.kt | 11 +++--- .../utbot_executor/ut_tracer.py | 20 +++++++---- .../utbot_executor/utbot_executor/utils.py | 17 ++++----- .../python/PythonTestGenerationProcessor.kt | 18 +++++----- .../python/evaluation/coverage/CoverageApi.kt | 36 +++++++++++-------- .../utbot/python/evaluation/coverage/Utils.kt | 22 ++++++++++++ 6 files changed, 80 insertions(+), 44 deletions(-) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/Utils.kt diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt index 126f811e1b..c5b917bd7b 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt @@ -18,7 +18,6 @@ import org.utbot.python.TestFileInformation import org.utbot.python.utils.RequirementsInstaller import org.utbot.python.code.PythonCode import org.utbot.python.evaluation.coverage.PythonCoverageMode -import org.utbot.python.evaluation.coverage.toPythonCoverageMode import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.codegen.model.Pytest import org.utbot.python.framework.codegen.model.Unittest @@ -127,9 +126,9 @@ class PythonGenerateTestsCommand : CliktCommand( private val doNotSendCoverageContinuously by option("--do-not-send-coverage-continuously", help = "Do not send coverage during execution.") .flag(default = false) - private val coverageOutputFormat by option("--coverage-output-format", help = "Use Lines, Instructions or TopFrameInstructions.") - .choice("Instructions", "Lines", "TopFrameInstructions") - .default("Instructions") + private val coverageOutputFormat by option("--coverage-output-format", help = "Use LINES, INSTRUCTIONS or TOPFRAMEINSTRUCTIONS.") + .choice("INSTRUCTIONS", "LINES", "TOPFRAMEINSTRUCTIONS") + .default("LINES") private val testFramework: TestFramework get() = @@ -267,9 +266,9 @@ class PythonGenerateTestsCommand : CliktCommand( withMinimization = !doNotMinimize, isCanceled = { false }, runtimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.valueOf(runtimeExceptionTestsBehaviour), - coverageMeasureMode = coverageMeasureMode.toPythonCoverageMode() ?: PythonCoverageMode.Instructions, + coverageMeasureMode = PythonCoverageMode.parse(coverageMeasureMode), sendCoverageContinuously = !doNotSendCoverageContinuously, - coverageOutputFormat = CoverageOutputFormat.valueOf(coverageOutputFormat), + coverageOutputFormat = CoverageOutputFormat.parse(coverageOutputFormat), ) val processor = PythonCliProcessor( diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py index c62ed2278e..2e66524e5c 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py @@ -77,17 +77,19 @@ def __init__( self.ignore_dirs = ignore_dirs self.sender = sender self.mode = mode - self.depth = 0 + self.global_offset = 0 + self.local_offset = 0 + self.offsets = {} def runfunc(self, func, /, *args, **kw): result = None - self.depth = 0 + self.global_offset = 0 sys.settrace(self.globaltrace) try: result = func(*args, **kw) finally: sys.settrace(None) - self.depth = 0 + self.global_offset = 0 return result def coverage(self, filename: str) -> typing.List[int]: @@ -98,10 +100,12 @@ def localtrace_count(self, frame, why, arg): filename = frame.f_code.co_filename lineno = frame.f_lineno if pathlib.Path(filename) == self.tested_file and lineno is not None: - offset = lineno * 2 + offset = 0 if why == "opcode": offset = frame.f_lasti - key = UtInstruction(lineno, offset, self.depth) + self.local_offset = offset + key = UtInstruction(lineno, offset, self.global_offset) + print(key) if key not in self.counts: message = key.serialize() try: @@ -112,7 +116,11 @@ def localtrace_count(self, frame, why, arg): return self.localtrace def globaltrace_lt(self, frame, why, arg): - self.depth += 1 + print("Global", frame, id(frame), frame.f_lasti, self.global_offset) + if frame not in self.offsets: + self.offsets[frame] = self.global_offset + self.local_offset + self.global_offset = self.offsets[frame] + if why == 'call': if self.mode == TraceMode.Instructions: frame.f_trace_opcodes = True diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py index b815e45350..d19f940e87 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py @@ -16,13 +16,13 @@ class TraceMode(enum.Enum): class UtInstruction: line: int offset: int - depth: int + global_offset: int def serialize(self) -> str: - return ":".join(map(str, [self.line, self.offset, self.depth])) + return ":".join(map(str, [self.line, self.offset, self.global_offset])) def __hash__(self): - return hash((self.line, self.offset, self.depth)) + return hash((self.line, self.offset, self.global_offset)) @contextmanager @@ -37,13 +37,13 @@ def suppress_stdout(): def get_instructions(obj: object, start_line: int) -> typing.Iterator[UtInstruction]: - def inner_get_instructions(x, current_line, depth): + def inner_get_instructions(x, current_line, offset): for i, el in enumerate(dis.get_instructions(x)): if el.starts_line is not None: current_line = el.starts_line - yield UtInstruction(current_line, el.offset, depth) + yield UtInstruction(current_line, el.offset, el.offset + offset) if any(t in str(type(el.argval)) for t in [""]): - inner_get_instructions(el.argval, current_line, depth + 1) + inner_get_instructions(el.argval, current_line, el.offset + offset) return inner_get_instructions(obj, start_line, 0) @@ -52,8 +52,5 @@ def filter_instructions( mode: TraceMode = TraceMode.Instructions, ) -> list[UtInstruction]: if mode == TraceMode.Lines: - unique_line_instructions: set[UtInstruction] = set() - for it in instructions: - unique_line_instructions.add(UtInstruction(it.line, it.line * 2, it.depth)) - return list({UtInstruction(it.line, 2 * it.line, it.depth) for it in instructions}) + return list({UtInstruction(it.line, 0, 0) for it in instructions}) return list(instructions) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt index 3e0edda17e..b6d8331c67 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt @@ -286,19 +286,21 @@ abstract class PythonTestGenerationProcessor { } } - data class InstructionIdCoverage(val line: Int, val id: Long) : CoverageFormat() { + data class InstructionIdCoverage(val line: Int, val offset: Long, val globalOffset: Long) : CoverageFormat() { override fun equals(other: Any?): Boolean { - if (other is InstructionCoverage) { - return line == other.line && id == other.offset + if (other is InstructionIdCoverage) { + return line == other.line && offset == other.offset && globalOffset == other.globalOffset } return false } override fun hashCode(): Int { var result = line - result = 31 * result + id.hashCode() + result = 31 * result + offset.hashCode() + result = 31 * result + globalOffset.hashCode() return result } + } data class CoverageInfo( @@ -324,7 +326,7 @@ abstract class PythonTestGenerationProcessor { missed.filterNot { missedInstruction -> covered.any { it.start <= missedInstruction.lineNumber && missedInstruction.lineNumber <= it.end } } private fun getInstructionsListWithId(instructions: Collection): List = - instructions.map { InstructionIdCoverage(it.lineNumber, it.id) }.toSet().toList() + instructions.map { InstructionIdCoverage(it.lineNumber, it.offset, it.globalOffset) }.toSet().toList() private fun getInstructionsListWithOffset(instructions: Collection): List = instructions.map { InstructionCoverage(it.lineNumber, it.offset) }.toSet().toList() @@ -349,8 +351,8 @@ abstract class PythonTestGenerationProcessor { } CoverageOutputFormat.Instructions -> CoverageInfo(getInstructionsListWithId(covered), getInstructionsListWithId(missed)) CoverageOutputFormat.TopFrameInstructions -> { - val filteredCovered = covered.filter { it.id.toPair().second == 0L } - val filteredMissed = missed.filter { it.id.toPair().second == 0L } + val filteredCovered = covered.filter { it.id.toPair().first == it.id.toPair().second } + val filteredMissed = missed.filter { it.id.toPair().first == it.id.toPair().second } val coveredInstructions = getInstructionsListWithOffset(filteredCovered) val missedInstructions = getInstructionsListWithOffset(filteredMissed) @@ -371,7 +373,7 @@ abstract class PythonTestGenerationProcessor { return when (coverageFormat) { is LineCoverage -> "{\"start\": ${coverageFormat.start}, \"end\": ${coverageFormat.end}}" is InstructionCoverage -> "{\"line\": ${coverageFormat.line}, \"offset\": ${coverageFormat.offset}}" - is InstructionIdCoverage -> "{\"line\": ${coverageFormat.line}, \"id\": ${coverageFormat.id}}" + is InstructionIdCoverage -> "{\"line\": ${coverageFormat.line}, \"offset\": ${coverageFormat.offset}, \"globalOffset\": ${coverageFormat.globalOffset}}" } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt index 880390a246..f6b840e662 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt @@ -13,28 +13,28 @@ enum class PythonCoverageMode { Instructions { override fun toString() = "instructions" - } -} + }; -fun String.toPythonCoverageMode(): PythonCoverageMode? { - return when (this.lowercase()) { - "lines" -> PythonCoverageMode.Lines - "instructions" -> PythonCoverageMode.Instructions - else -> null + companion object { + fun parse(name: String): PythonCoverageMode { + return PythonCoverageMode.values().first { + it.name.lowercase() == name.lowercase() + } + } } } data class PyInstruction( val lineNumber: Int, val offset: Long, - val depth: Int, + val globalOffset: Long, ) { - override fun toString(): String = listOf(lineNumber, offset, depth).joinToString(":") + override fun toString(): String = listOf(lineNumber, offset, globalOffset).joinToString(":") - val id: Long = (offset to depth.toLong()).toCoverageId() + val id: Long = (offset to globalOffset).toCoverageId() constructor(lineNumber: Int) : this(lineNumber, lineNumber.toLong(), 0) - constructor(lineNumber: Int, id: Long) : this(lineNumber, id.toPair().first, id.toPair().second.toInt()) + constructor(lineNumber: Int, id: Long) : this(lineNumber, id.toPair().first, id.toPair().second) } fun String.toPyInstruction(): PyInstruction? { @@ -43,8 +43,8 @@ fun String.toPyInstruction(): PyInstruction? { 3 -> { val line = data[0].toInt() val offset = data[1].toLong() - val depth = data[2].toInt() - return PyInstruction(line, offset, depth) + val globalOffset = data[2].toLong() + return PyInstruction(line, offset, globalOffset) } 2 -> { val line = data[0].toInt() @@ -106,4 +106,12 @@ enum class CoverageOutputFormat { Lines, Instructions, TopFrameInstructions; -} \ No newline at end of file + + companion object { + fun parse(name: String): CoverageOutputFormat { + return CoverageOutputFormat.values().first { + it.name.lowercase() == name.lowercase() + } + } + } +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/Utils.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/Utils.kt new file mode 100644 index 0000000000..62b1d19afb --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/Utils.kt @@ -0,0 +1,22 @@ +package org.utbot.python.evaluation.coverage + +import kotlin.math.ceil +import kotlin.math.max +import kotlin.math.min +import kotlin.math.sqrt + +fun Long.toPair(): Pair { + val n = ceil(sqrt(this + 2.0)).toLong() - 1 + val k = this - (n * n - 1) + return if (k <= n + 1) { + n + 1 to k + } else { + k to n + 1 + } +} + +fun Pair.toCoverageId(): Long { + val n = max(this.first, this.second) - 1 + val k = min(this.first, this.second) + return (n * n - 1) + k +} \ No newline at end of file From 09c730e522e9dcc7322ac8fcc4ec1cce4f2c48d5 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Thu, 19 Oct 2023 16:43:27 +0300 Subject: [PATCH 12/17] Remove inner frames logic --- .../python/PythonGenerateTestsCommand.kt | 4 +-- .../utbot_executor/utbot_executor/executor.py | 9 ++--- .../utbot_executor/ut_tracer.py | 29 +++++++-------- .../utbot_executor/utbot_executor/utils.py | 26 +++++++------- .../python/PythonTestGenerationProcessor.kt | 35 ++----------------- .../python/evaluation/coverage/CoverageApi.kt | 21 ++++++----- .../fuzzing/provider/DictValueProvider.kt | 5 ++- 7 files changed, 45 insertions(+), 84 deletions(-) diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt index c5b917bd7b..e8dbee86a3 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt @@ -126,8 +126,8 @@ class PythonGenerateTestsCommand : CliktCommand( private val doNotSendCoverageContinuously by option("--do-not-send-coverage-continuously", help = "Do not send coverage during execution.") .flag(default = false) - private val coverageOutputFormat by option("--coverage-output-format", help = "Use LINES, INSTRUCTIONS or TOPFRAMEINSTRUCTIONS.") - .choice("INSTRUCTIONS", "LINES", "TOPFRAMEINSTRUCTIONS") + private val coverageOutputFormat by option("--coverage-output-format", help = "Use LINES, INSTRUCTIONS (only from function frame).") + .choice("INSTRUCTIONS", "LINES") .default("LINES") private val testFramework: TestFramework diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py index 94c6009302..01fdb290d4 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py @@ -6,6 +6,7 @@ import pathlib import sys import traceback +import types from typing import Any, Callable, Dict, Iterable, List, Tuple from utbot_executor.deep_serialization.deep_serialization import serialize_memory_dump, \ @@ -96,7 +97,7 @@ def run_function(self, request: ExecutionRequest) -> ExecutionResponse: importlib.import_module(request.function_module), request.function_name ) - if not callable(function): + if not isinstance(function, types.FunctionType): return ExecutionFailResponse( "fail", f"Invalid function path {request.function_module}.{request.function_name}" @@ -165,7 +166,7 @@ def _serialize_state( def _run_calculate_function_value( - function: Callable, + function: types.FunctionType, args: List[Any], kwargs: Dict[str, Any], fullpath: str, @@ -181,7 +182,7 @@ def _run_calculate_function_value( __is_exception = False _, __start = inspect.getsourcelines(function) - __all_code_stmts = filter_instructions(get_instructions(function, __start), tracer.mode) + __all_code_stmts = filter_instructions(get_instructions(function.__code__), tracer.mode) __tracer = tracer @@ -195,7 +196,7 @@ def _run_calculate_function_value( logging.debug("Coverage: %s", __tracer.counts) logging.debug("Fullpath: %s", fullpath) - __stmts_with_def = [UtInstruction(__start, 0, 0)] + list(__tracer.counts.keys()) + __stmts_with_def = [UtInstruction(__start, 0, True)] + list(__tracer.counts.keys()) __missed_filtered = [x for x in __all_code_stmts if x not in __stmts_with_def] logging.debug("Covered lines: %s", __stmts_with_def) logging.debug("Missed lines: %s", __missed_filtered) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py index 2e66524e5c..eb0d6d5b41 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py @@ -77,19 +77,15 @@ def __init__( self.ignore_dirs = ignore_dirs self.sender = sender self.mode = mode - self.global_offset = 0 - self.local_offset = 0 - self.offsets = {} def runfunc(self, func, /, *args, **kw): result = None - self.global_offset = 0 sys.settrace(self.globaltrace) + self.f_code = func.__code__ try: result = func(*args, **kw) finally: sys.settrace(None) - self.global_offset = 0 return result def coverage(self, filename: str) -> typing.List[int]: @@ -100,12 +96,11 @@ def localtrace_count(self, frame, why, arg): filename = frame.f_code.co_filename lineno = frame.f_lineno if pathlib.Path(filename) == self.tested_file and lineno is not None: - offset = 0 - if why == "opcode": + if self.mode == TraceMode.Instructions and frame.f_lasti is not None: offset = frame.f_lasti - self.local_offset = offset - key = UtInstruction(lineno, offset, self.global_offset) - print(key) + else: + offset = 0 + key = UtInstruction(lineno, offset, frame.f_code == self.f_code) if key not in self.counts: message = key.serialize() try: @@ -116,16 +111,11 @@ def localtrace_count(self, frame, why, arg): return self.localtrace def globaltrace_lt(self, frame, why, arg): - print("Global", frame, id(frame), frame.f_lasti, self.global_offset) - if frame not in self.offsets: - self.offsets[frame] = self.global_offset + self.local_offset - self.global_offset = self.offsets[frame] - if why == 'call': if self.mode == TraceMode.Instructions: frame.f_trace_opcodes = True frame.f_trace_lines = False - filename = frame.f_code.co_filename + filename = frame.f_globals.get('__file__', None) if filename and all(not filename.startswith(d + os.sep) for d in self.ignore_dirs): modulename = _modname(filename) if modulename is not None: @@ -142,13 +132,18 @@ def runfunc(self, func, /, *args, **kw): return func(*args, **kw) +def g1(x): + return x * 2 + + def f(x): def g(x): xs = [[j for j in range(i)] for i in range(10)] return x * 2 - return x * g(x) + 2 + return g1(x) * g(x) + 2 if __name__ == "__main__": tracer = UtTracer(pathlib.Path(__file__), [], PureSender()) tracer.runfunc(f, 2) + print(tracer.counts) \ No newline at end of file diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py index d19f940e87..d3f6d14700 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py @@ -1,10 +1,11 @@ +from __future__ import annotations import dataclasses -import dis import enum import os import sys import typing from contextlib import contextmanager +from types import CodeType class TraceMode(enum.Enum): @@ -16,13 +17,13 @@ class TraceMode(enum.Enum): class UtInstruction: line: int offset: int - global_offset: int + from_main_frame: bool def serialize(self) -> str: - return ":".join(map(str, [self.line, self.offset, self.global_offset])) + return ":".join(map(str, [self.line, self.offset, int(self.from_main_frame)])) def __hash__(self): - return hash((self.line, self.offset, self.global_offset)) + return hash((self.line, self.offset, self.from_main_frame)) @contextmanager @@ -36,15 +37,8 @@ def suppress_stdout(): sys.stdout = old_stdout -def get_instructions(obj: object, start_line: int) -> typing.Iterator[UtInstruction]: - def inner_get_instructions(x, current_line, offset): - for i, el in enumerate(dis.get_instructions(x)): - if el.starts_line is not None: - current_line = el.starts_line - yield UtInstruction(current_line, el.offset, el.offset + offset) - if any(t in str(type(el.argval)) for t in [""]): - inner_get_instructions(el.argval, current_line, el.offset + offset) - return inner_get_instructions(obj, start_line, 0) +def get_instructions(obj: CodeType) -> list[UtInstruction]: + return [UtInstruction(line, start_offset, True) for start_offset, _, line in obj.co_lines() if None not in {start_offset, line}] def filter_instructions( @@ -52,5 +46,9 @@ def filter_instructions( mode: TraceMode = TraceMode.Instructions, ) -> list[UtInstruction]: if mode == TraceMode.Lines: - return list({UtInstruction(it.line, 0, 0) for it in instructions}) + return list({UtInstruction(it.line, 0, True) for it in instructions}) return list(instructions) + + +def get_lines(instructions: typing.Iterable[UtInstruction]) -> list[int]: + return [instruction.line for instruction in filter_instructions(instructions)] diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt index b6d8331c67..243ecd7111 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt @@ -12,7 +12,6 @@ import org.utbot.framework.plugin.api.util.withUtContext import org.utbot.python.code.PythonCode import org.utbot.python.evaluation.coverage.CoverageOutputFormat import org.utbot.python.evaluation.coverage.PyInstruction -import org.utbot.python.evaluation.coverage.toPair import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.api.python.PythonMethodId import org.utbot.python.framework.api.python.PythonModel @@ -286,23 +285,6 @@ abstract class PythonTestGenerationProcessor { } } - data class InstructionIdCoverage(val line: Int, val offset: Long, val globalOffset: Long) : CoverageFormat() { - override fun equals(other: Any?): Boolean { - if (other is InstructionIdCoverage) { - return line == other.line && offset == other.offset && globalOffset == other.globalOffset - } - return false - } - - override fun hashCode(): Int { - var result = line - result = 31 * result + offset.hashCode() - result = 31 * result + globalOffset.hashCode() - return result - } - - } - data class CoverageInfo( val covered: List, val notCovered: List, @@ -325,10 +307,7 @@ abstract class PythonTestGenerationProcessor { private fun filterMissedLines(covered: Collection, missed: Collection): List = missed.filterNot { missedInstruction -> covered.any { it.start <= missedInstruction.lineNumber && missedInstruction.lineNumber <= it.end } } - private fun getInstructionsListWithId(instructions: Collection): List = - instructions.map { InstructionIdCoverage(it.lineNumber, it.offset, it.globalOffset) }.toSet().toList() - - private fun getInstructionsListWithOffset(instructions: Collection): List = + private fun getInstructionsList(instructions: Collection): List = instructions.map { InstructionCoverage(it.lineNumber, it.offset) }.toSet().toList() private fun getCoverageInfo(testSets: List): CoverageInfo { @@ -349,16 +328,7 @@ abstract class PythonTestGenerationProcessor { val missedLines = getLinesList(filteredMissed) CoverageInfo(coveredLines, missedLines) } - CoverageOutputFormat.Instructions -> CoverageInfo(getInstructionsListWithId(covered), getInstructionsListWithId(missed)) - CoverageOutputFormat.TopFrameInstructions -> { - val filteredCovered = covered.filter { it.id.toPair().first == it.id.toPair().second } - val filteredMissed = missed.filter { it.id.toPair().first == it.id.toPair().second } - - val coveredInstructions = getInstructionsListWithOffset(filteredCovered) - val missedInstructions = getInstructionsListWithOffset(filteredMissed) - - CoverageInfo(coveredInstructions, (missedInstructions.toSet() - coveredInstructions.toSet()).toList()) - } + CoverageOutputFormat.Instructions -> CoverageInfo(getInstructionsList(covered), getInstructionsList(missed)) } return CoverageInfo(info.covered.toSet().toList(), info.notCovered.toSet().toList()) } @@ -373,7 +343,6 @@ abstract class PythonTestGenerationProcessor { return when (coverageFormat) { is LineCoverage -> "{\"start\": ${coverageFormat.start}, \"end\": ${coverageFormat.end}}" is InstructionCoverage -> "{\"line\": ${coverageFormat.line}, \"offset\": ${coverageFormat.offset}}" - is InstructionIdCoverage -> "{\"line\": ${coverageFormat.line}, \"offset\": ${coverageFormat.offset}, \"globalOffset\": ${coverageFormat.globalOffset}}" } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt index f6b840e662..abc3e30b7a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt @@ -27,14 +27,14 @@ enum class PythonCoverageMode { data class PyInstruction( val lineNumber: Int, val offset: Long, - val globalOffset: Long, + val fromMainFrame: Boolean, ) { - override fun toString(): String = listOf(lineNumber, offset, globalOffset).joinToString(":") + override fun toString(): String = listOf(lineNumber, offset, fromMainFrame).joinToString(":") - val id: Long = (offset to globalOffset).toCoverageId() + val id: Long = (lineNumber.toLong() to offset).toCoverageId() - constructor(lineNumber: Int) : this(lineNumber, lineNumber.toLong(), 0) - constructor(lineNumber: Int, id: Long) : this(lineNumber, id.toPair().first, id.toPair().second) + constructor(lineNumber: Int) : this(lineNumber, lineNumber.toLong(), true) + constructor(lineNumber: Int, id: Long) : this(lineNumber, id.toPair().second, true) } fun String.toPyInstruction(): PyInstruction? { @@ -43,13 +43,13 @@ fun String.toPyInstruction(): PyInstruction? { 3 -> { val line = data[0].toInt() val offset = data[1].toLong() - val globalOffset = data[2].toLong() - return PyInstruction(line, offset, globalOffset) + val fromMainFrame = data[2].toInt() != 0 + return PyInstruction(line, offset, fromMainFrame) } 2 -> { val line = data[0].toInt() val offset = data[1].toLong() - return PyInstruction(line, offset, 0) + return PyInstruction(line, offset, true) } 1 -> { val line = data[0].toInt() @@ -80,7 +80,7 @@ fun calculateCoverage(coverage: PyCoverage, method: PythonMethod): Coverage { } fun calculateCoverage(statements: List, missedStatements: List, method: PythonMethod): Coverage { - val covered = statements.filter { it !in missedStatements } + val covered = statements.filter { it !in missedStatements && it.fromMainFrame } return Coverage( coveredInstructions=covered.map { Instruction( @@ -104,8 +104,7 @@ fun calculateCoverage(statements: List, missedStatements: List - val items = mapOf(v[0].tree to v[1].tree).toMutableMap() + construct = Routine.Create(emptyList()) { v -> PythonFuzzedValue( - PythonTree.DictNode(items), + PythonTree.DictNode(mutableMapOf()), "%var% = ${type.pythonTypeRepresentation()}" ) }, From 5316143fc6ac8519f4717fc29966a26094ada833 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Fri, 20 Oct 2023 11:45:26 +0300 Subject: [PATCH 13/17] Update test configuration and matrix class --- .../samples/samples/structures/matrix.py | 19 ++++--- utbot-python/samples/test_configuration.json | 51 ++++++++++++++----- 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/utbot-python/samples/samples/structures/matrix.py b/utbot-python/samples/samples/structures/matrix.py index 300851f49c..eda69179b9 100644 --- a/utbot-python/samples/samples/structures/matrix.py +++ b/utbot-python/samples/samples/structures/matrix.py @@ -10,15 +10,16 @@ def __init__(self, description): class Matrix: def __init__(self, elements: List[List[float]]): - self.dim = ( - len(elements), - max(len(elements[i]) for i in range(len(elements))) - if len(elements) > 0 else 0 + assert all(len(elements[i-1]) == len(row) for i, row in enumerate(elements)) + self.elements = elements + + @property + def dim(self) -> tuple[int, int]: + return ( + len(self.elements), + max(len(self.elements[i]) for i in range(len(self.elements))) + if len(self.elements) > 0 else 0 ) - self.elements = [ - row + [0] * (self.dim[1] - len(row)) - for row in elements - ] def __repr__(self): return str(self.elements) @@ -59,6 +60,8 @@ def __matmul__(self, other): for k in range(self.dim[1]) ) return Matrix(result) + else: + MatrixException("Wrong dimensions") else: raise MatrixException("Wrong Type") diff --git a/utbot-python/samples/test_configuration.json b/utbot-python/samples/test_configuration.json index 41a64e6f28..41af37e97b 100644 --- a/utbot-python/samples/test_configuration.json +++ b/utbot-python/samples/test_configuration.json @@ -70,7 +70,7 @@ "classes": ["Dictionary"], "methods": ["translate"], "timeout": 10, - "coverage": 89 + "coverage": 88 } ] }, @@ -149,10 +149,16 @@ "name": "dicts", "groups": [ { - "classes": null, - "methods": null, + "classes": ["Dictionary"], + "methods": ["__eq__"], "timeout": 10, "coverage": 100 + }, + { + "classes": ["Dictionary"], + "methods": ["translate"], + "timeout": 20, + "coverage": 100 } ] }, @@ -196,7 +202,7 @@ "classes": null, "methods": null, "timeout": 10, - "coverage": 100 + "coverage": 75 } ] }, @@ -234,7 +240,7 @@ "classes": null, "methods": null, "timeout": 180, - "coverage": 94 + "coverage": 83 } ] }, @@ -343,7 +349,7 @@ "classes": null, "methods": null, "timeout": 180, - "coverage": 100 + "coverage": 97 } ] } @@ -465,9 +471,24 @@ "groups": [ { "classes": null, - "methods": null, + "methods": [ + "concat", + "concat_pair", + "string_constants", + "contains", + "const_contains", + "to_str", + "starts_with", + "join_str" + ], "timeout": 160, "coverage": 100 + }, + { + "classes": null, + "methods": ["separated_str"], + "timeout": 60, + "coverage": 100 } ] } @@ -499,7 +520,7 @@ "classes": ["Graph"], "methods": null, "timeout": 150, - "coverage": 100 + "coverage": 64 } ] }, @@ -521,7 +542,7 @@ "classes": null, "methods": null, "timeout": 120, - "coverage": 100 + "coverage": 96 } ] }, @@ -530,8 +551,14 @@ "groups": [ { "classes": ["Matrix"], - "methods": null, - "timeout": 180, + "methods": ["__repr__", "__eq__", "__add__", "__mul__", "is_diagonal"], + "timeout": 120, + "coverage": 100 + }, + { + "classes": ["Matrix"], + "methods": ["__matmul__"], + "timeout": 120, "coverage": 100 } ] @@ -543,7 +570,7 @@ "classes": null, "methods": null, "timeout": 180, - "coverage": 100 + "coverage": 62 } ] } From f308bd29a5edbdedd294fc1688579bb92ad68810 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Fri, 20 Oct 2023 15:59:19 +0300 Subject: [PATCH 14/17] Update test configuration and matrix class --- utbot-python/samples/run_tests.py | 20 +++++++++--- utbot-python/samples/samples/classes/field.py | 10 ------ .../samples/samples/structures/matrix.py | 2 +- utbot-python/samples/test_configuration.json | 31 +++++-------------- 4 files changed, 23 insertions(+), 40 deletions(-) delete mode 100644 utbot-python/samples/samples/classes/field.py diff --git a/utbot-python/samples/run_tests.py b/utbot-python/samples/run_tests.py index f0a7900957..c426fed41c 100644 --- a/utbot-python/samples/run_tests.py +++ b/utbot-python/samples/run_tests.py @@ -110,10 +110,15 @@ def check_coverage( coverage: typing.Dict[str, typing.Tuple[float, float]] = {} for part in config["parts"]: for file in part["files"]: - for group in file["groups"]: + for i, group in enumerate(file["groups"]): + if i > 0: + suffix = f"_{i}" + else: + suffix = "" + expected_coverage = group.get("coverage", 0) - file_suffix = f"{part['path'].replace('/', '_')}_{file['name']}" + file_suffix = f"{part['path'].replace('/', '_')}_{file['name']}{suffix}" coverage_output_file = pathlib.Path( coverage_output_dir, f"coverage_{file_suffix}.json" ) @@ -161,17 +166,22 @@ def main_test_generation(args): for file in tqdm.tqdm( part["files"], file=orig_stdout, dynamic_ncols=True, desc=part["path"] ): - for group in file["groups"]: + for i, group in enumerate(file["groups"]): + if i > 0: + suffix = f"_{i}" + else: + suffix = "" + full_name = pathlib.PurePath( args.path_to_test_dir, part["path"], file["name"] ) output_file = pathlib.PurePath( args.output_dir, - f"utbot_tests_{part['path'].replace('/', '_')}_{file['name']}.py", + f"utbot_tests_{part['path'].replace('/', '_')}_{file['name']}{suffix}.py", ) coverage_output_file = pathlib.PurePath( args.coverage_output_dir, - f"coverage_{part['path'].replace('/', '_')}_{file['name']}.json", + f"coverage_{part['path'].replace('/', '_')}_{file['name']}{suffix}.json", ) generate_tests( args.java, diff --git a/utbot-python/samples/samples/classes/field.py b/utbot-python/samples/samples/classes/field.py deleted file mode 100644 index 55a4676041..0000000000 --- a/utbot-python/samples/samples/classes/field.py +++ /dev/null @@ -1,10 +0,0 @@ -class NoTestsProblem: - def __init__(self): - self.board = [] - - def set_position(self, row, col, symbol): - self.board[row][col] = symbol - return symbol - - def start(self): - self.set_position(1, 2, "O") diff --git a/utbot-python/samples/samples/structures/matrix.py b/utbot-python/samples/samples/structures/matrix.py index eda69179b9..b9b284d334 100644 --- a/utbot-python/samples/samples/structures/matrix.py +++ b/utbot-python/samples/samples/structures/matrix.py @@ -50,7 +50,7 @@ def __mul__(self, other): else: raise MatrixException("Wrong Type") - def __matmul__(self, other): + def __matmul__(self, other: Matrix): if isinstance(other, Matrix): if self.dim[1] == other.dim[0]: result = [[0 for _ in range(self.dim[0])] * other.dim[1]] diff --git a/utbot-python/samples/test_configuration.json b/utbot-python/samples/test_configuration.json index 41af37e97b..2051d5515a 100644 --- a/utbot-python/samples/test_configuration.json +++ b/utbot-python/samples/test_configuration.json @@ -10,7 +10,7 @@ "classes": null, "methods": null, "timeout": 10, - "coverage": 93 + "coverage": 92 } ] }, @@ -69,8 +69,8 @@ { "classes": ["Dictionary"], "methods": ["translate"], - "timeout": 10, - "coverage": 88 + "timeout": 30, + "coverage": 100 } ] }, @@ -96,17 +96,6 @@ } ] }, - { - "name": "field", - "groups": [ - { - "classes": ["NoTestsProblem"], - "methods": null, - "timeout": 10, - "coverage": 100 - } - ] - }, { "name": "inner_class", "groups": [ @@ -149,14 +138,8 @@ "name": "dicts", "groups": [ { - "classes": ["Dictionary"], - "methods": ["__eq__"], - "timeout": 10, - "coverage": 100 - }, - { - "classes": ["Dictionary"], - "methods": ["translate"], + "classes": null, + "methods": null, "timeout": 20, "coverage": 100 } @@ -558,8 +541,8 @@ { "classes": ["Matrix"], "methods": ["__matmul__"], - "timeout": 120, - "coverage": 100 + "timeout": 80, + "coverage": 90 } ] }, From d9b299bb03592f19ab0a5cc3b21e5ad31798b16a Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 25 Oct 2023 15:08:35 +0300 Subject: [PATCH 15/17] Refactor python coverage --- .../python/PythonGenerateTestsCommand.kt | 4 +- .../utbot/framework/plugin/api/CoverageApi.kt | 6 +- .../kotlin/org/utbot/python/PythonEngine.kt | 15 ++- .../utbot/python/PythonTestCaseGenerator.kt | 2 +- .../python/PythonTestGenerationConfig.kt | 4 +- .../python/PythonTestGenerationProcessor.kt | 93 +++----------- .../org/utbot/python/coverage/CoverageApi.kt | 80 ++++++++++++ .../coverage/CoverageIdGenerator.kt | 2 +- .../utbot/python/coverage/CoverageOutput.kt | 41 +++++++ .../python/{evaluation => }/coverage/Utils.kt | 2 +- .../python/evaluation/CodeEvaluationApi.kt | 5 +- .../evaluation/PythonCodeSocketExecutor.kt | 10 +- .../evaluation/PythonCoverageReceiver.kt | 4 +- .../python/evaluation/PythonWorkerManager.kt | 2 +- .../python/evaluation/coverage/CoverageApi.kt | 116 ------------------ .../org/utbot/python/fuzzing/PythonApi.kt | 2 +- 16 files changed, 166 insertions(+), 222 deletions(-) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt rename utbot-python/src/main/kotlin/org/utbot/python/{evaluation => }/coverage/CoverageIdGenerator.kt (86%) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageOutput.kt rename utbot-python/src/main/kotlin/org/utbot/python/{evaluation => }/coverage/Utils.kt (91%) delete mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt index e8dbee86a3..43b081d4a7 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt @@ -10,14 +10,14 @@ import org.parsers.python.PythonParser import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour import org.utbot.framework.codegen.domain.TestFramework import org.utbot.framework.plugin.api.UtExecutionSuccess -import org.utbot.python.evaluation.coverage.CoverageOutputFormat +import org.utbot.python.coverage.CoverageOutputFormat import org.utbot.python.PythonMethodHeader import org.utbot.python.PythonTestGenerationConfig import org.utbot.python.PythonTestSet import org.utbot.python.TestFileInformation import org.utbot.python.utils.RequirementsInstaller import org.utbot.python.code.PythonCode -import org.utbot.python.evaluation.coverage.PythonCoverageMode +import org.utbot.python.coverage.PythonCoverageMode import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.codegen.model.Pytest import org.utbot.python.framework.codegen.model.Unittest diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt index 06c4c9350f..923fd99b7e 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt @@ -10,11 +10,11 @@ package org.utbot.framework.plugin.api * * @see Test minimization */ -data class Instruction( +open class Instruction( val internalName: String, val methodSignature: String, - val lineNumber: Int, - val id: Long + open val lineNumber: Int, + open val id: Long ) { val className: String get() = internalName.replace('/', '.') } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 095e4761f7..076194a87e 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -16,11 +16,10 @@ import org.utbot.python.evaluation.PythonEvaluationSuccess import org.utbot.python.evaluation.PythonEvaluationTimeout import org.utbot.python.evaluation.PythonWorker import org.utbot.python.evaluation.PythonWorkerManager -import org.utbot.python.evaluation.coverage.CoverageIdGenerator -import org.utbot.python.evaluation.coverage.PyInstruction -import org.utbot.python.evaluation.coverage.PythonCoverageMode -import org.utbot.python.evaluation.coverage.calculateCoverage -import org.utbot.python.evaluation.coverage.makeInstructions +import org.utbot.python.coverage.CoverageIdGenerator +import org.utbot.python.coverage.PyInstruction +import org.utbot.python.coverage.PythonCoverageMode +import org.utbot.python.coverage.buildCoverage import org.utbot.python.evaluation.serialization.MemoryDump import org.utbot.python.evaluation.serialization.toPythonTree import org.utbot.python.framework.api.python.PythonTree @@ -121,7 +120,7 @@ class PythonEngine( val beforeThisObject = beforeThisObjectTree?.let { PythonTreeModel(it.tree) } val beforeModelList = beforeModelListTree.map { PythonTreeModel(it.tree) } - val coverage = Coverage(makeInstructions(coveredInstructions, methodUnderTest)) + val coverage = Coverage(coveredInstructions) val utFuzzedExecution = PythonUtExecution( stateInit = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), stateBefore = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), @@ -185,7 +184,7 @@ class PythonEngine( stateAfter = EnvironmentModels(afterThisObject, afterModelList, emptyMap(), executableToCall = null), diffIds = evaluationResult.diffIds, result = executionResult, - coverage = calculateCoverage(evaluationResult.coverage, methodUnderTest), + coverage = buildCoverage(evaluationResult.coveredStatements, evaluationResult.missedStatements), testMethodName = testMethodName.testName?.camelToSnakeCase(), displayName = testMethodName.displayName, summary = summary.map { DocRegularStmt(it) }, @@ -263,7 +262,7 @@ class PythonEngine( } is PythonEvaluationSuccess -> { - val coveredInstructions = evaluationResult.coverage.coveredInstructions + val coveredInstructions = evaluationResult.coveredStatements val result = handleSuccessResult( arguments, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 6549b8f454..cd43648659 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -6,7 +6,7 @@ import org.utbot.framework.minimization.minimizeExecutions import org.utbot.framework.plugin.api.UtError import org.utbot.framework.plugin.api.UtExecution import org.utbot.framework.plugin.api.UtExecutionSuccess -import org.utbot.python.evaluation.coverage.PythonCoverageMode +import org.utbot.python.coverage.PythonCoverageMode import org.utbot.python.framework.api.python.PythonUtExecution import org.utbot.python.framework.api.python.util.pythonStrClassId import org.utbot.python.fuzzing.* diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt index 909e8c8f28..99a9500e18 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt @@ -2,8 +2,8 @@ package org.utbot.python import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour import org.utbot.framework.codegen.domain.TestFramework -import org.utbot.python.evaluation.coverage.CoverageOutputFormat -import org.utbot.python.evaluation.coverage.PythonCoverageMode +import org.utbot.python.coverage.CoverageOutputFormat +import org.utbot.python.coverage.PythonCoverageMode import java.nio.file.Path data class TestFileInformation( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt index 243ecd7111..cd8c835e45 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt @@ -10,8 +10,13 @@ import org.utbot.framework.plugin.api.UtExecutionSuccess import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.withUtContext import org.utbot.python.code.PythonCode -import org.utbot.python.evaluation.coverage.CoverageOutputFormat -import org.utbot.python.evaluation.coverage.PyInstruction +import org.utbot.python.coverage.CoverageFormat +import org.utbot.python.coverage.CoverageInfo +import org.utbot.python.coverage.CoverageOutputFormat +import org.utbot.python.coverage.PyInstruction +import org.utbot.python.coverage.filterMissedLines +import org.utbot.python.coverage.getInstructionsList +import org.utbot.python.coverage.getLinesList import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.api.python.PythonMethodId import org.utbot.python.framework.api.python.PythonModel @@ -255,69 +260,14 @@ abstract class PythonTestGenerationProcessor { paths } - sealed class CoverageFormat - data class LineCoverage(val start: Int, val end: Int) : CoverageFormat() { - override fun equals(other: Any?): Boolean { - if (other is LineCoverage) { - return start == other.start && end == other.end - } - return false - } - - override fun hashCode(): Int { - var result = start - result = 31 * result + end - return result - } - } - data class InstructionCoverage(val line: Int, val offset: Long) : CoverageFormat() { - override fun equals(other: Any?): Boolean { - if (other is InstructionCoverage) { - return line == other.line && offset == other.offset - } - return false - } - - override fun hashCode(): Int { - var result = line - result = 31 * result + offset.hashCode() - return result - } - } - - data class CoverageInfo( - val covered: List, - val notCovered: List, - ) - - private fun getLinesList(instructions: Collection): List = - instructions - .map { it.lineNumber } - .sorted() - .fold(emptyList()) { acc, lineNumber -> - if (acc.isEmpty()) - return@fold listOf(LineCoverage(lineNumber, lineNumber)) - val elem = acc.last() - if (elem.end + 1 == lineNumber || elem.end == lineNumber ) - acc.dropLast(1) + listOf(LineCoverage(elem.start, lineNumber)) - else - acc + listOf(LineCoverage(lineNumber, lineNumber)) - } - - private fun filterMissedLines(covered: Collection, missed: Collection): List = - missed.filterNot { missedInstruction -> covered.any { it.start <= missedInstruction.lineNumber && missedInstruction.lineNumber <= it.end } } - - private fun getInstructionsList(instructions: Collection): List = - instructions.map { InstructionCoverage(it.lineNumber, it.offset) }.toSet().toList() - private fun getCoverageInfo(testSets: List): CoverageInfo { val covered = mutableSetOf() val missed = mutableSetOf() testSets.forEach { testSet -> testSet.executions.forEach inner@{ execution -> val coverage = execution.coverage ?: return@inner - covered.addAll(coverage.coveredInstructions.map { PyInstruction(it.lineNumber, it.id) }) - missed.addAll(coverage.missedInstructions.map { PyInstruction(it.lineNumber, it.id) }) + covered.addAll(coverage.coveredInstructions.filterIsInstance()) + missed.addAll(coverage.missedInstructions.filterIsInstance()) } } missed -= covered @@ -328,29 +278,20 @@ abstract class PythonTestGenerationProcessor { val missedLines = getLinesList(filteredMissed) CoverageInfo(coveredLines, missedLines) } - CoverageOutputFormat.Instructions -> CoverageInfo(getInstructionsList(covered), getInstructionsList(missed)) + CoverageOutputFormat.Instructions -> CoverageInfo( + getInstructionsList(covered), + getInstructionsList(missed) + ) } return CoverageInfo(info.covered.toSet().toList(), info.notCovered.toSet().toList()) } - private fun toJson(coverageInfo: CoverageInfo): String { - val covered = coverageInfo.covered.map { toJson(it) } - val notCovered = coverageInfo.notCovered.map { toJson(it) } - return "{\"covered\": [${covered.joinToString(", ")}], \"notCovered\": [${notCovered.joinToString(", ")}]}" - } - - private fun toJson(coverageFormat: CoverageFormat): String { - return when (coverageFormat) { - is LineCoverage -> "{\"start\": ${coverageFormat.start}, \"end\": ${coverageFormat.end}}" - is InstructionCoverage -> "{\"line\": ${coverageFormat.line}, \"offset\": ${coverageFormat.offset}}" - } - } - protected fun getStringCoverageInfo(testSets: List): String { - val value = getCoverageInfo(testSets) - return toJson(value) + val coverageInfo = getCoverageInfo(testSets) + val covered = coverageInfo.covered.map { it.toJson() } + val notCovered = coverageInfo.notCovered.map { it.toJson() } + return "{\"covered\": [${covered.joinToString(", ")}], \"notCovered\": [${notCovered.joinToString(", ")}]}" } - } data class SelectedMethodIsNotAFunctionDefinition(val methodName: String): Exception() \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt new file mode 100644 index 0000000000..0effe8b010 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt @@ -0,0 +1,80 @@ +package org.utbot.python.coverage + +import org.utbot.framework.plugin.api.Coverage +import org.utbot.framework.plugin.api.Instruction + +enum class PythonCoverageMode { + Lines { + override fun toString() = "lines" + }, + + Instructions { + override fun toString() = "instructions" + }; + + companion object { + fun parse(name: String): PythonCoverageMode { + return PythonCoverageMode.values().first { + it.name.lowercase() == name.lowercase() + } + } + } +} + +data class PyInstruction( + override val lineNumber: Int, + val offset: Long, + val fromMainFrame: Boolean, +): Instruction("", "", lineNumber, 0) { + override fun toString(): String = listOf(lineNumber, offset, fromMainFrame).joinToString(":") + + override val id: Long = (lineNumber.toLong() to offset).toCoverageId() * 2 + fromMainFrame.toLong() + + constructor(lineNumber: Int) : this(lineNumber, lineNumber.toLong(), true) + constructor(lineNumber: Int, id: Long) : this(lineNumber, id.floorDiv(2).toPair().second, id % 2 == 1L) +} + +fun Boolean.toLong() = if (this) 1L else 0L + +fun String.toPyInstruction(): PyInstruction? { + val data = this.split(":") + when (data.size) { + 3 -> { + val line = data[0].toInt() + val offset = data[1].toLong() + val fromMainFrame = data[2].toInt() != 0 + return PyInstruction(line, offset, fromMainFrame) + } + 2 -> { + val line = data[0].toInt() + val offset = data[1].toLong() + return PyInstruction(line, offset, true) + } + 1 -> { + val line = data[0].toInt() + return PyInstruction(line) + } + else -> return null + } +} + +fun buildCoverage(coveredStatements: List, missedStatements: List): Coverage { + return Coverage( + coveredInstructions = coveredStatements, + instructionsCount = (coveredStatements.size + missedStatements.size).toLong(), + missedInstructions = missedStatements + ) +} + +enum class CoverageOutputFormat { + Lines, + Instructions; + + companion object { + fun parse(name: String): CoverageOutputFormat { + return CoverageOutputFormat.values().first { + it.name.lowercase() == name.lowercase() + } + } + } +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageIdGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageIdGenerator.kt similarity index 86% rename from utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageIdGenerator.kt rename to utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageIdGenerator.kt index b9248d03f1..15aafacb54 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageIdGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageIdGenerator.kt @@ -1,4 +1,4 @@ -package org.utbot.python.evaluation.coverage +package org.utbot.python.coverage import java.util.concurrent.atomic.AtomicLong diff --git a/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageOutput.kt b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageOutput.kt new file mode 100644 index 0000000000..0bd4055da4 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageOutput.kt @@ -0,0 +1,41 @@ +package org.utbot.python.coverage + +sealed interface CoverageFormat { + fun toJson(): String +} +data class LineCoverage(val start: Int, val end: Int) : CoverageFormat { + override fun toJson(): String = "{\"start\": ${start}, \"end\": ${end}}" +} + +data class InstructionCoverage( + val line: Int, + val offset: Long, + val fromMainFrame: Boolean +) : CoverageFormat { + override fun toJson(): String = "{\"line\": ${line}, \"offset\": ${offset}, \"fromMainFrame\": ${fromMainFrame}}" +} + +data class CoverageInfo( + val covered: List, + val notCovered: List, +) + +fun getLinesList(instructions: Collection): List = + instructions + .map { it.lineNumber } + .sorted() + .fold(emptyList()) { acc, lineNumber -> + if (acc.isEmpty()) + return@fold listOf(LineCoverage(lineNumber, lineNumber)) + val elem = acc.last() + if (elem.end + 1 == lineNumber || elem.end == lineNumber ) + acc.dropLast(1) + listOf(LineCoverage(elem.start, lineNumber)) + else + acc + listOf(LineCoverage(lineNumber, lineNumber)) + } + +fun filterMissedLines(covered: Collection, missed: Collection): List = + missed.filterNot { missedInstruction -> covered.any { it.start <= missedInstruction.lineNumber && missedInstruction.lineNumber <= it.end } } + +fun getInstructionsList(instructions: Collection): List = + instructions.map { InstructionCoverage(it.lineNumber, it.offset, it.fromMainFrame) }.toSet().toList() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/Utils.kt b/utbot-python/src/main/kotlin/org/utbot/python/coverage/Utils.kt similarity index 91% rename from utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/Utils.kt rename to utbot-python/src/main/kotlin/org/utbot/python/coverage/Utils.kt index 62b1d19afb..37e04a747d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/Utils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/coverage/Utils.kt @@ -1,4 +1,4 @@ -package org.utbot.python.evaluation.coverage +package org.utbot.python.coverage import kotlin.math.ceil import kotlin.math.max diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt index 41154c2baa..7e6eed438a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt @@ -3,7 +3,7 @@ package org.utbot.python.evaluation import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod import org.utbot.python.evaluation.serialization.MemoryDump -import org.utbot.python.evaluation.coverage.PyCoverage +import org.utbot.python.coverage.PyInstruction interface PythonCodeExecutor { val method: PythonMethod @@ -40,7 +40,8 @@ data class PythonEvaluationTimeout( data class PythonEvaluationSuccess( val isException: Boolean, - val coverage: PyCoverage, + val coveredStatements: List, + val missedStatements: List, val stateInit: MemoryDump, val stateBefore: MemoryDump, val stateAfter: MemoryDump, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index 5b8fba4501..f30872204a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -10,17 +10,14 @@ import org.utbot.python.evaluation.serialization.FailExecution import org.utbot.python.evaluation.serialization.PythonExecutionResult import org.utbot.python.evaluation.serialization.SuccessExecution import org.utbot.python.evaluation.serialization.serializeObjects -import org.utbot.python.evaluation.coverage.CoverageIdGenerator -import org.utbot.python.evaluation.coverage.PyCoverage -import org.utbot.python.evaluation.coverage.toPyInstruction +import org.utbot.python.coverage.CoverageIdGenerator +import org.utbot.python.coverage.toPyInstruction import org.utbot.python.newtyping.PythonCallableTypeDescription import org.utbot.python.newtyping.pythonDescription import org.utbot.python.newtyping.pythonTypeName import org.utbot.python.newtyping.utils.isNamed import java.net.SocketException -private val logger = KotlinLogging.logger {} - class PythonCodeSocketExecutor( override val method: PythonMethod, override val moduleToImport: String, @@ -134,7 +131,8 @@ class PythonCodeSocketExecutor( val missedStatements = executionResult.missedStatements.mapNotNull { it.toPyInstruction() } PythonEvaluationSuccess( executionResult.isException, - PyCoverage(statements, missedStatements), + statements, + missedStatements, stateInit, stateBefore, stateAfter, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt index 63a5e52944..e28cc2654b 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt @@ -1,8 +1,8 @@ package org.utbot.python.evaluation import mu.KotlinLogging -import org.utbot.python.evaluation.coverage.PyInstruction -import org.utbot.python.evaluation.coverage.toPyInstruction +import org.utbot.python.coverage.PyInstruction +import org.utbot.python.coverage.toPyInstruction import java.io.IOException import java.net.DatagramPacket import java.net.DatagramSocket diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index 264299816c..0841e1c36a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -11,7 +11,7 @@ import java.net.ServerSocket import java.net.Socket import java.net.SocketTimeoutException import org.apache.logging.log4j.LogManager -import org.utbot.python.evaluation.coverage.PythonCoverageMode +import org.utbot.python.coverage.PythonCoverageMode private val logger = KotlinLogging.logger {} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt deleted file mode 100644 index abc3e30b7a..0000000000 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/coverage/CoverageApi.kt +++ /dev/null @@ -1,116 +0,0 @@ -package org.utbot.python.evaluation.coverage - -import org.utbot.framework.plugin.api.Coverage -import org.utbot.framework.plugin.api.Instruction -import org.utbot.python.PythonMethod -import org.utbot.python.framework.api.python.util.pythonAnyClassId -import org.utbot.python.newtyping.pythonTypeRepresentation - -enum class PythonCoverageMode { - Lines { - override fun toString() = "lines" - }, - - Instructions { - override fun toString() = "instructions" - }; - - companion object { - fun parse(name: String): PythonCoverageMode { - return PythonCoverageMode.values().first { - it.name.lowercase() == name.lowercase() - } - } - } -} - -data class PyInstruction( - val lineNumber: Int, - val offset: Long, - val fromMainFrame: Boolean, -) { - override fun toString(): String = listOf(lineNumber, offset, fromMainFrame).joinToString(":") - - val id: Long = (lineNumber.toLong() to offset).toCoverageId() - - constructor(lineNumber: Int) : this(lineNumber, lineNumber.toLong(), true) - constructor(lineNumber: Int, id: Long) : this(lineNumber, id.toPair().second, true) -} - -fun String.toPyInstruction(): PyInstruction? { - val data = this.split(":") - when (data.size) { - 3 -> { - val line = data[0].toInt() - val offset = data[1].toLong() - val fromMainFrame = data[2].toInt() != 0 - return PyInstruction(line, offset, fromMainFrame) - } - 2 -> { - val line = data[0].toInt() - val offset = data[1].toLong() - return PyInstruction(line, offset, true) - } - 1 -> { - val line = data[0].toInt() - return PyInstruction(line) - } - else -> return null - } -} - -fun makeInstructions(coveredInstructions: Collection, method: PythonMethod): List { - return coveredInstructions.map { - Instruction( - method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, - method.methodSignature(), - it.lineNumber, - it.id - ) - } -} - -data class PyCoverage( - val coveredInstructions: List, - val missedInstructions: List -) - -fun calculateCoverage(coverage: PyCoverage, method: PythonMethod): Coverage { - return calculateCoverage(coverage.coveredInstructions, coverage.missedInstructions, method) -} - -fun calculateCoverage(statements: List, missedStatements: List, method: PythonMethod): Coverage { - val covered = statements.filter { it !in missedStatements && it.fromMainFrame } - return Coverage( - coveredInstructions=covered.map { - Instruction( - method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, - method.methodSignature(), - it.lineNumber, - it.id - ) - }, - instructionsCount = (covered.size + missedStatements.size).toLong(), - missedInstructions = missedStatements.map { - Instruction( - method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, - method.methodSignature(), - it.lineNumber, - it.id - ) - } - ) -} - -enum class CoverageOutputFormat { - Lines, - Instructions; - - companion object { - fun parse(name: String): CoverageOutputFormat { - return CoverageOutputFormat.values().first { - it.name.lowercase() == name.lowercase() - } - } - } -} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt index 6f0a88dea2..2d7f63ac35 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt @@ -5,7 +5,7 @@ import org.utbot.framework.plugin.api.UtError import org.utbot.fuzzer.FuzzedContext import org.utbot.fuzzing.* import org.utbot.fuzzing.utils.Trie -import org.utbot.python.evaluation.coverage.PyInstruction +import org.utbot.python.coverage.PyInstruction import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.PythonUtExecution import org.utbot.python.fuzzing.provider.* From d578c90d419a73a9326e4b2b5afe76d89366a3bb Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Thu, 26 Oct 2023 09:27:04 +0300 Subject: [PATCH 16/17] Move instruction id calculation to constructor --- .../main/kotlin/org/utbot/python/coverage/CoverageApi.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt index 0effe8b010..214011ed74 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt @@ -25,11 +25,13 @@ data class PyInstruction( override val lineNumber: Int, val offset: Long, val fromMainFrame: Boolean, -): Instruction("", "", lineNumber, 0) { +) : Instruction( + "", + "", + lineNumber, + (lineNumber.toLong() to offset).toCoverageId() * 2 + fromMainFrame.toLong()) { override fun toString(): String = listOf(lineNumber, offset, fromMainFrame).joinToString(":") - override val id: Long = (lineNumber.toLong() to offset).toCoverageId() * 2 + fromMainFrame.toLong() - constructor(lineNumber: Int) : this(lineNumber, lineNumber.toLong(), true) constructor(lineNumber: Int, id: Long) : this(lineNumber, id.floorDiv(2).toPair().second, id % 2 == 1L) } From 47662e115329cf338bd8990448d18f245821d5d0 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Thu, 26 Oct 2023 09:50:38 +0300 Subject: [PATCH 17/17] Make coverage line and id not open --- .../kotlin/org/utbot/framework/plugin/api/CoverageApi.kt | 4 ++-- .../main/kotlin/org/utbot/python/coverage/CoverageApi.kt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt index 923fd99b7e..3c3f31cb08 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt @@ -13,8 +13,8 @@ package org.utbot.framework.plugin.api open class Instruction( val internalName: String, val methodSignature: String, - open val lineNumber: Int, - open val id: Long + val lineNumber: Int, + val id: Long ) { val className: String get() = internalName.replace('/', '.') } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt index 214011ed74..34403ad42a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt @@ -22,14 +22,14 @@ enum class PythonCoverageMode { } data class PyInstruction( - override val lineNumber: Int, + val pyLineNumber: Int, val offset: Long, val fromMainFrame: Boolean, ) : Instruction( "", "", - lineNumber, - (lineNumber.toLong() to offset).toCoverageId() * 2 + fromMainFrame.toLong()) { + pyLineNumber, + (pyLineNumber.toLong() to offset).toCoverageId() * 2 + fromMainFrame.toLong()) { override fun toString(): String = listOf(lineNumber, offset, fromMainFrame).joinToString(":") constructor(lineNumber: Int) : this(lineNumber, lineNumber.toLong(), true)