diff --git a/.github/workflows/IntegrationTests.yaml b/.github/workflows/IntegrationTests.yaml new file mode 100644 index 000000000..83d027e40 --- /dev/null +++ b/.github/workflows/IntegrationTests.yaml @@ -0,0 +1,22 @@ +name: Integration Tests + +on: + schedule: + - cron: '30 18 * * 1' + +jobs: + Verify: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 14 + uses: actions/setup-java@v2 + with: + java-version: '14' + distribution: 'adopt' + - name: Set up Python 3.8.10 + uses: actions/setup-python@v2 + with: + python-version: '3.8.10' + - name: Run integration tests + run: mvn clean verify -B \ No newline at end of file diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 1b11dfff8..54f1b1b0c 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -16,7 +16,7 @@ jobs: java-version: '14' distribution: 'adopt' - name: Build with Maven - run: mvn clean compile -B -q + run: mvn clean compile -B test: strategy: @@ -35,7 +35,7 @@ jobs: with: python-version: '3.8.10' - name: Build with Maven - run: mvn clean verify -B -DskipITs + run: mvn clean verify -B -DskipITs - name: archive artifacts uses: actions/upload-artifact@v2 with: @@ -109,7 +109,7 @@ jobs: name: artifacts path: ./ - name: Build with Maven - run: cp -a transpiler/src/test/resources/org/polystat/py2eo/transpiler/runEO . && cp -a transpiler/target/results/* ./runEO && cp transpiler/src/main/eo/xmodules/*.eo ./runEO/xmodules/ && cp -a transpiler/src/main/eo/preface ./runEO && cd ./runEO && mvn clean test + run: cp -a transpiler/src/test/resources/org/polystat/py2eo/transpiler/runEO . && cp -a transpiler/target/results/* ./runEO && cp transpiler/src/main/eo/xmodules/*.eo ./runEO/xmodules/ && cp -a transpiler/src/main/eo/preface ./runEO && cd ./runEO && rm exceptions-finally-3.eo && mvn clean test && rm *.eo && cp transpiler/target/results/exceptions-finally-3.eo . && mvn clean test - name: upload artifacts uses: actions/upload-artifact@v2 with: diff --git a/README.md b/README.md index 3911f0308..fe50ee3ee 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,14 @@ 2. [Quick Start](#quick-start) 3. [How to contribute](#how-to-contribute) 4. [How to transpile Py to EO](#how-to-transpile-py-to-eo) -5. [Python syntax and tests coverage](#python-syntax-and-tests-coverage) -6. [Syntax support and passed tests info](#for-now-we-support-529-of-python-syntax-and-572-are-passed-successefully) -7. [Big project transpilation results](#py2eo-is-capable-of-transpiling-more-than-hundreds-of-thousands-lines-of-python-code) +5. [A list of not supported language features](#a-list-of-not-supported-language-features) +6. [Python syntax and tests coverage](#python-syntax-and-tests-coverage) +7. [Syntax support and passed tests info](#for-now-we-support-529-of-python-syntax-and-572-are-passed-successefully) +8. [Big project transpilation results](#py2eo-is-capable-of-transpiling-more-than-hundreds-of-thousands-lines-of-python-code) - [Django](#django) - [CPython](#cpython) -8. [Architecture and design](#architecture-and-design) -9. [How we translate Python to EOLang](#how-do-we-project-python-to-eolang) +9. [Architecture and design](#architecture-and-design) +10. [How we translate Python to EOLang](#how-do-we-project-python-to-eolang) - [while](#while) - [while-try-break](#while-try-break) - [function definition](#function-definition) @@ -143,6 +144,16 @@ $ docker run -v $(pwd):/eo yegor256/py2eo hello.py -o hello.eo This command will translate `hello.py` in the current directory, saving the output to the `hello.eo` file. +## A list of not supported language features ## +1. Any kind of yield, also coroutines and generators (incl generator expressions) -- no support in EO +1. Threads, async, futures, await -- no support in EO +1. Dynamic features of python (dynamic creation/change/lookup/deletion of variables, creation of classes with metaclasses etc., dynamic features of import) -- using completely dynamic features would make the output EO not statically analyzable +1. Multiple inheritance -- not obvious how to do that for a general case, but with the EO delegation principle in mind +1. The majority of standard library -- it is mostly written in C, so even if we support all of the python syntax, it is still a problem to support the library without rewriting it manually. +1. Star expressions are mostly not supported -- possible, but not yet finished +1. Array slicing is partially supported -- possible, but not yet finished +1. The import system is partially supported -- possible, but not yet finished + ## Python syntax and tests coverage We have [handwritten tests](https://github.com/polystat/py2eo/tree/master/transpiler/src/test/resources/org/polystat/py2eo/transpiler) that are divided into groups by type: functional (also divided into groups by constructs in accordance with the language specification), integration tests (tests for the polystat analyzer), "negative" tests, etc. diff --git a/transpiler/src/test/scala/org/polystat/py2eo/transpiler/CPythonIT.scala b/transpiler/src/test/scala/org/polystat/py2eo/transpiler/CPythonIT.scala index 34e65ae5a..aa8e4ec48 100644 --- a/transpiler/src/test/scala/org/polystat/py2eo/transpiler/CPythonIT.scala +++ b/transpiler/src/test/scala/org/polystat/py2eo/transpiler/CPythonIT.scala @@ -1,7 +1,7 @@ package org.polystat.py2eo.transpiler import org.junit.jupiter.api.Assertions.fail -import org.junit.jupiter.api.{Order, Test, TestMethodOrder} +import org.junit.jupiter.api.{AfterEach, Order, Test, TestMethodOrder} import org.junit.jupiter.api.MethodOrderer.OrderAnnotation import scala.concurrent.ExecutionContext.Implicits.global @@ -15,28 +15,40 @@ import scala.util.Try @TestMethodOrder(classOf[OrderAnnotation]) final class CPythonIT extends Commons { - private val dirPath: Path = s"$testsPrefix/testParserPrinter" private val cpythonLink = "https://github.com/python/cpython" + private val directory = Directory.makeTemp(prefix = "org.polystat.py2eo.") private val blacklisted = Set( + // these are excluded because of some encoding problems in the lexer "test_unicode_identifiers.py", "test_source_encoding.py", "badsyntax_3131.py", "badsyntax_pep3120.py", - "module_koi8_r.py", "module_iso_8859_1.py" + "module_koi8_r.py", "module_iso_8859_1.py", + // most of these are excluded because they do tests by comparing stack traces as strings + // but code before parser-printer has different line numbers than code after => traces are + // always different => + // those tests cannot possibly pass + "test_traceback.py", "test_dis.py", "test_zipfile.py", + "test_multiprocessing_fork.py", "test_sys.py", + "test_import.yaml", "test_strtod.py", "test_trace.py", + "test_doctest.py", "test_concurrent_futures.py", "test_inspect.py", + "test_tracemalloc.py", "test_multiprocessing_spawn.py", + "test_sys_settrace.py", "test_multiprocessing_forkserver.py", + "test_compileall.py", "test_asyncio.py", "test_unittest.py", + "test_email.py", "test_tools.py", "test_atexit.py", "test_pdb.py", + "test_logging.py", "test_coroutines.py", "test_tasks.py", + + "test_grammar.py", "test_headerregistry.py" ) @Test @Order(1) def parserPrinterOnCPython(): Unit = { - val dir = Directory(dirPath) - dir.createDirectory(failIfExists = false) - - val eoFiles = Directory(dirPath / "afterParser") + val eoFiles = Directory(directory / "afterParser") eoFiles.createDirectory(failIfExists = false) - val cpython = Directory(dirPath / "cpython") - if (!cpython.exists) { - Process(s"git clone $cpythonLink ${cpython.name}", dirPath.jfile).!! - Process("git checkout v3.8.10", cpython.jfile).!! - } + val cpython = Directory(directory / "cpython") + + Process(s"git clone $cpythonLink ${cpython.name}", directory.jfile).!! + Process("git checkout v3.8.10", cpython.jfile).!! val testsDir = Directory(cpython / "Lib" / "test") val tests = testsDir.deepFiles.filter(_.extension == "py") @@ -58,9 +70,13 @@ final class CPythonIT extends Commons { for (f <- futures) Await.result(f, Duration.Inf) } + @AfterEach def cleanup(): Unit = { + directory.deleteRecursively + } + @Test @Order(2) def checkEOSyntax(): Unit = { - checkEOSyntaxInDirectory(Directory(dirPath / "afterParser").toString) + checkEOSyntaxInDirectory(Directory(directory / "afterParser").toString) } } \ No newline at end of file diff --git a/transpiler/src/test/scala/org/polystat/py2eo/transpiler/DjangoIT.scala b/transpiler/src/test/scala/org/polystat/py2eo/transpiler/DjangoIT.scala index f3e81772a..5d2adc85d 100644 --- a/transpiler/src/test/scala/org/polystat/py2eo/transpiler/DjangoIT.scala +++ b/transpiler/src/test/scala/org/polystat/py2eo/transpiler/DjangoIT.scala @@ -1,50 +1,48 @@ package org.polystat.py2eo.transpiler import org.junit.jupiter.api.MethodOrderer.OrderAnnotation -import org.junit.jupiter.api.{Order, Test, TestMethodOrder} +import org.junit.jupiter.api.{AfterEach, Order, Test, TestMethodOrder} import org.polystat.py2eo.parser.Statement -import org.polystat.py2eo.transpiler.Common.dfsFiles -import java.io.File -import java.nio.file.{Files, StandardCopyOption} -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, Future} import scala.reflect.io.Directory import scala.sys.process.Process @TestMethodOrder(classOf[OrderAnnotation]) class DjangoIT extends Commons { + private val djangoLink = "https://github.com/django/django" + private val directory = Directory.makeTemp(prefix = "org.polystat.py2eo.") + @Test @Order(1) - def genUnsupportedDjango() : Unit = { - val root = new File(testsPrefix) - val django = new File(testsPrefix + "/django") - if (!django.exists()) { - Process("git clone -b 4.0 https://github.com/django/django", root).!! + def genUnsupportedDjango(): Unit = { + val django = Directory(directory + "/django") + + Process(s"git clone -b 4.0 $djangoLink ${django.name}", directory.jfile).!! + + val tests = django.deepFiles.filter(_.extension == "py") + for (test <- tests) { + def db(s: Statement.T, str: String) = () // debugPrinter(test)(_, _) + + val name = test.name + val eoText = + Transpile.transpile(db)( + chopExtension(name), + Transpile.Parameters(wrapInAFunction = false, isModule = false), + readFile(test.jfile) + ) + writeFile(test.jfile, "genUnsupportedEO", ".eo", eoText) } - val test = dfsFiles(django).filter(f => f.getName.endsWith(".py")) - - test.map(test => - { - def db(s : Statement.T, str : String) = () // debugPrinter(test)(_, _) - val name = test.getName - val eoText = - Transpile.transpile(db)( - chopExtension(name), - Transpile.Parameters(wrapInAFunction = false, isModule = false), - readFile(test) - ) - writeFile(test, "genUnsupportedEO", ".eo", eoText) - } - ) + } + + @AfterEach def cleanup(): Unit = { + directory.deleteRecursively } @Test @Order(2) - def checkSyntaxForDjango() : Unit = { - checkEOSyntaxInDirectory(testsPrefix + "/django") + def checkSyntaxForDjango(): Unit = { + checkEOSyntaxInDirectory(Directory(directory + "/django").toString) } }