Merge branch 'master' into fixDjangoAndTodo

# Conflicts:
#	.github/workflows/github-ci.yml

.github/workflows/IntegrationTests.yaml

@@ -0,0 +1,22 @@
+name: Integration Tests
+
+on:
+  schedule:
+    - cron: '30 18 * * 1'
+
+jobs:
+  Verify:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 14
+        uses: actions/setup-java@v2
+        with:
+          java-version: '14'
+          distribution: 'adopt'
+      - name: Set up Python 3.8.10
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8.10'
+      - name: Run integration tests
+        run: mvn clean verify -B

.github/workflows/github-ci.yml

@@ -16,7 +16,7 @@ jobs:
           java-version: '14'
           distribution: 'adopt'
       - name: Build with Maven
-        run:  mvn clean compile -B -q
+        run:  mvn clean compile -B
 
   test:
     strategy:
@@ -35,7 +35,7 @@ jobs:
         with:
           python-version: '3.8.10'
       - name: Build with Maven
-        run:  mvn clean verify -B -DskipITs  
+        run:  mvn clean verify -B -DskipITs
       - name: archive artifacts
         uses: actions/upload-artifact@v2
         with:
@@ -109,7 +109,7 @@ jobs:
           name: artifacts
           path: ./
       - name: Build with Maven
-        run: cp -a transpiler/src/test/resources/org/polystat/py2eo/transpiler/runEO . && cp -a transpiler/target/results/* ./runEO && cp transpiler/src/main/eo/xmodules/*.eo ./runEO/xmodules/ && cp -a transpiler/src/main/eo/preface ./runEO && cd ./runEO && mvn clean test
+        run: cp -a transpiler/src/test/resources/org/polystat/py2eo/transpiler/runEO . && cp -a transpiler/target/results/* ./runEO && cp transpiler/src/main/eo/xmodules/*.eo ./runEO/xmodules/ && cp -a transpiler/src/main/eo/preface ./runEO && cd ./runEO && rm exceptions-finally-3.eo && mvn clean test && rm *.eo && cp transpiler/target/results/exceptions-finally-3.eo . && mvn clean test
       - name: upload artifacts
         uses: actions/upload-artifact@v2
         with:

README.md

@@ -13,13 +13,14 @@
 2. [Quick Start](#quick-start)
 3. [How to contribute](#how-to-contribute)
 4. [How to transpile Py to EO](#how-to-transpile-py-to-eo)
-5. [Python syntax and tests coverage](#python-syntax-and-tests-coverage)
-6. [Syntax support and passed tests info](#for-now-we-support-529-of-python-syntax-and-572-are-passed-successefully)
-7. [Big project transpilation results](#py2eo-is-capable-of-transpiling-more-than-hundreds-of-thousands-lines-of-python-code)
+5. [A list of not supported language features](#a-list-of-not-supported-language-features)
+6. [Python syntax and tests coverage](#python-syntax-and-tests-coverage)
+7. [Syntax support and passed tests info](#for-now-we-support-529-of-python-syntax-and-572-are-passed-successefully)
+8. [Big project transpilation results](#py2eo-is-capable-of-transpiling-more-than-hundreds-of-thousands-lines-of-python-code)
    - [Django](#django)
    - [CPython](#cpython)
-8. [Architecture and design](#architecture-and-design)
-9. [How we translate Python to EOLang](#how-do-we-project-python-to-eolang)
+9. [Architecture and design](#architecture-and-design)
+10. [How we translate Python to EOLang](#how-do-we-project-python-to-eolang)
    - [while](#while)
    - [while-try-break](#while-try-break)
    - [function definition](#function-definition)
@@ -143,6 +144,16 @@ $ docker run -v $(pwd):/eo yegor256/py2eo hello.py -o hello.eo
 
 This command will translate `hello.py` in the current directory, saving the output to the `hello.eo` file.
 
+## A list of not supported language features ##
+1. Any kind of yield, also coroutines and generators (incl generator expressions) -- no support in EO
+1. Threads, async, futures, await -- no support in EO
+1. Dynamic features of python (dynamic creation/change/lookup/deletion of variables, creation of classes with metaclasses etc., dynamic features of import) -- using completely dynamic features would make the output EO not statically analyzable
+1. Multiple inheritance -- not obvious how to do that for a general case, but with the EO delegation principle in mind
+1. The majority of standard library -- it is mostly written in C, so even if we support all of the python syntax, it is still a problem to support the library without rewriting it manually.
+1. Star expressions are mostly not supported -- possible, but not yet finished
+1. Array slicing is partially supported -- possible, but not yet finished
+1. The import system is partially supported -- possible, but not yet finished
+
 ## Python syntax and tests coverage
 
 We have [handwritten tests](https://github.com/polystat/py2eo/tree/master/transpiler/src/test/resources/org/polystat/py2eo/transpiler) that are divided into groups by type: functional (also divided into groups by constructs in accordance with the language specification), integration tests (tests for the polystat analyzer), "negative" tests, etc.

transpiler/src/test/scala/org/polystat/py2eo/transpiler/CPythonIT.scala

@@ -1,7 +1,7 @@
 package org.polystat.py2eo.transpiler
 
 import org.junit.jupiter.api.Assertions.fail
-import org.junit.jupiter.api.{Order, Test, TestMethodOrder}
+import org.junit.jupiter.api.{AfterEach, Order, Test, TestMethodOrder}
 import org.junit.jupiter.api.MethodOrderer.OrderAnnotation
 
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -15,28 +15,40 @@ import scala.util.Try
 @TestMethodOrder(classOf[OrderAnnotation])
 final class CPythonIT extends Commons {
 
-  private val dirPath: Path = s"$testsPrefix/testParserPrinter"
   private val cpythonLink = "https://github.com/python/cpython"
+  private val directory = Directory.makeTemp(prefix = "org.polystat.py2eo.")
   private val blacklisted = Set(
+    // these are excluded because of some encoding problems in the lexer
     "test_unicode_identifiers.py", "test_source_encoding.py",
     "badsyntax_3131.py", "badsyntax_pep3120.py",
-    "module_koi8_r.py", "module_iso_8859_1.py"
+    "module_koi8_r.py", "module_iso_8859_1.py",
+    // most of these are excluded because they do tests by comparing stack traces as strings
+    // but code before parser-printer has different line numbers than code after => traces are
+    // always different =>
+    // those tests cannot possibly pass
+    "test_traceback.py", "test_dis.py", "test_zipfile.py",
+    "test_multiprocessing_fork.py", "test_sys.py",
+    "test_import.yaml", "test_strtod.py", "test_trace.py",
+    "test_doctest.py", "test_concurrent_futures.py", "test_inspect.py",
+    "test_tracemalloc.py", "test_multiprocessing_spawn.py",
+    "test_sys_settrace.py", "test_multiprocessing_forkserver.py",
+    "test_compileall.py", "test_asyncio.py", "test_unittest.py",
+    "test_email.py", "test_tools.py", "test_atexit.py", "test_pdb.py",
+    "test_logging.py", "test_coroutines.py", "test_tasks.py",
+
+    "test_grammar.py", "test_headerregistry.py"
   )
 
   @Test
   @Order(1)
   def parserPrinterOnCPython(): Unit = {
-    val dir = Directory(dirPath)
-    dir.createDirectory(failIfExists = false)
-
-    val eoFiles = Directory(dirPath / "afterParser")
+    val eoFiles = Directory(directory / "afterParser")
     eoFiles.createDirectory(failIfExists = false)
 
-    val cpython = Directory(dirPath / "cpython")
-    if (!cpython.exists) {
-      Process(s"git clone $cpythonLink ${cpython.name}", dirPath.jfile).!!
-      Process("git checkout v3.8.10", cpython.jfile).!!
-    }
+    val cpython = Directory(directory / "cpython")
+
+    Process(s"git clone $cpythonLink ${cpython.name}", directory.jfile).!!
+    Process("git checkout v3.8.10", cpython.jfile).!!
 
     val testsDir = Directory(cpython / "Lib" / "test")
     val tests = testsDir.deepFiles.filter(_.extension == "py")
@@ -58,9 +70,13 @@ final class CPythonIT extends Commons {
     for (f <- futures) Await.result(f, Duration.Inf)
   }
 
+  @AfterEach def cleanup(): Unit = {
+    directory.deleteRecursively
+  }
+
   @Test
   @Order(2)
   def checkEOSyntax(): Unit = {
-    checkEOSyntaxInDirectory(Directory(dirPath / "afterParser").toString)
+    checkEOSyntaxInDirectory(Directory(directory / "afterParser").toString)
   }
 }

transpiler/src/test/scala/org/polystat/py2eo/transpiler/DjangoIT.scala

@@ -1,50 +1,48 @@
 package org.polystat.py2eo.transpiler
 
 import org.junit.jupiter.api.MethodOrderer.OrderAnnotation
-import org.junit.jupiter.api.{Order, Test, TestMethodOrder}
+import org.junit.jupiter.api.{AfterEach, Order, Test, TestMethodOrder}
 import org.polystat.py2eo.parser.Statement
-import org.polystat.py2eo.transpiler.Common.dfsFiles
 
-import java.io.File
-import java.nio.file.{Files, StandardCopyOption}
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.duration.Duration
-import scala.concurrent.{Await, Future}
 import scala.reflect.io.Directory
 import scala.sys.process.Process
 
 @TestMethodOrder(classOf[OrderAnnotation])
 class DjangoIT extends Commons {
 
+  private val djangoLink = "https://github.com/django/django"
+  private val directory = Directory.makeTemp(prefix = "org.polystat.py2eo.")
+
   @Test
   @Order(1)
-  def genUnsupportedDjango() : Unit = {
-    val root = new File(testsPrefix)
-    val django = new File(testsPrefix + "/django")
-    if (!django.exists()) {
-      Process("git clone -b 4.0 https://github.com/django/django", root).!!
+  def genUnsupportedDjango(): Unit = {
+    val django = Directory(directory + "/django")
+
+    Process(s"git clone -b 4.0 $djangoLink ${django.name}", directory.jfile).!!
+
+    val tests = django.deepFiles.filter(_.extension == "py")
+    for (test <- tests) {
+      def db(s: Statement.T, str: String) = () // debugPrinter(test)(_, _)
+
+      val name = test.name
+      val eoText =
+        Transpile.transpile(db)(
+          chopExtension(name),
+          Transpile.Parameters(wrapInAFunction = false, isModule = false),
+          readFile(test.jfile)
+        )
+      writeFile(test.jfile, "genUnsupportedEO", ".eo", eoText)
     }
-    val test = dfsFiles(django).filter(f => f.getName.endsWith(".py"))
-
-    test.map(test =>
-      {
-        def db(s : Statement.T, str : String) = () // debugPrinter(test)(_, _)
-        val name = test.getName
-        val eoText =
-          Transpile.transpile(db)(
-            chopExtension(name),
-            Transpile.Parameters(wrapInAFunction = false, isModule = false),
-            readFile(test)
-          )
-        writeFile(test, "genUnsupportedEO", ".eo", eoText)
-      }
-    )
+  }
+
+  @AfterEach def cleanup(): Unit = {
+    directory.deleteRecursively
   }
 
   @Test
   @Order(2)
-  def checkSyntaxForDjango() : Unit = {
-    checkEOSyntaxInDirectory(testsPrefix + "/django")
+  def checkSyntaxForDjango(): Unit = {
+    checkEOSyntaxInDirectory(Directory(directory + "/django").toString)
   }
 
 }