unix-ffi: Move more unix-only packages to unix-ffi.

With the dependencies captured in manifest.py, several packages in python-stdlib were still unix-only due to direct or transitive dependencies on unix-only or ffi modules. Or they just make no sense to run on microcontroller targets. In a few cases (e.g. base64) where possible, the unix dependency could be removed. Updates manifest.py to use the `unix_ffi=True` arg to `require()` for these libraries. Rename re-pcre to re now that unix-ffi is effectively its own namespace. Update unix-ffi/README.md, and strengthen the wording that the unix libraries are unmaintained. Signed-off-by: Jim Mussared <jim.mussared@gmail.com>
micropython · Sep 5, 2022 · f3cfc52 · f3cfc52
1 parent ce66e70
commit f3cfc52
Show file tree

Hide file tree

Showing 74 changed files with 51 additions and 119 deletions.
diff --git a/python-stdlib/base64/base64.py b/python-stdlib/base64/base64.py
@@ -96,7 +96,7 @@ def b64decode(s, altchars=None, validate=False):
         altchars = _bytes_from_decode_data(altchars)
         assert len(altchars) == 2, repr(altchars)
         s = s.translate(bytes.maketrans(altchars, b"+/"))
-    if validate and not re.match(b"^[A-Za-z0-9+/]*={0,2}$", s):
+    if validate and not re.match(b"^[A-Za-z0-9+/]*=*$", s):
         raise binascii.Error("Non-base64 digit found")
     return binascii.a2b_base64(s)
 

diff --git a/python-stdlib/base64/manifest.py b/python-stdlib/base64/manifest.py
@@ -1,7 +1,6 @@
 metadata(version="3.3.3-4")
 
 require("binascii")
-require("re-pcre")
 require("struct")
 
 module("base64.py")
diff --git a/python-stdlib/email.charset/manifest.py b/python-stdlib/email.charset/manifest.py
diff --git a/python-stdlib/email.feedparser/manifest.py b/python-stdlib/email.feedparser/manifest.py
diff --git a/python-stdlib/email.header/manifest.py b/python-stdlib/email.header/manifest.py
diff --git a/python-stdlib/email.internal/manifest.py b/python-stdlib/email.internal/manifest.py
diff --git a/python-stdlib/email.message/manifest.py b/python-stdlib/email.message/manifest.py
diff --git a/python-stdlib/email.parser/manifest.py b/python-stdlib/email.parser/manifest.py
diff --git a/python-stdlib/glob/manifest.py b/python-stdlib/glob/manifest.py
diff --git a/python-stdlib/html.parser/manifest.py b/python-stdlib/html.parser/manifest.py
diff --git a/python-stdlib/http.client/manifest.py b/python-stdlib/http.client/manifest.py
diff --git a/python-stdlib/uu/manifest.py b/python-stdlib/uu/manifest.py
@@ -1,6 +1,6 @@
 metadata(version="0.5.1")
 
 require("binascii")
-require("os")
+require("os.path")
 
 module("uu.py")
diff --git a/unix-ffi/README.md b/unix-ffi/README.md
@@ -1,17 +1,30 @@
 Unix-specific libraries
 =======================
 
-These are libraries that will only run on the Unix port of MicroPython. There is some limited support for the Windows port too.
+These are libraries that will only run on the Unix port of MicroPython, or are
+too big to be used on microcontrollers. There is some limited support for the
+Windows port too.
 
-**Note:** This directory is largely unmaintained, although large breaking changes are not expected.
+**Note:** This directory is unmaintained.
 
 Background
 ----------
 
 The libraries in this directory provide additional CPython compatibility using
 the host operating system's native libraries.
 
-This is implemented either by accessing the libraries directly via libffi, or by using built-in modules that are only available on the Unix port.
+This is implemented either by accessing the libraries directly via libffi, or
+by using built-in modules that are only available on the Unix port.
 
 In theory, this allows you to use MicroPython as a more complete drop-in
 replacement for CPython.
+
+Usage
+-----
+
+To use a unix-specific library, pass `unix_ffi=True` to `require()` in your
+manifest file.
+
+```py
+require("os", unix_ffi=True) # Use the unix-ffi version instead of python-stdlib.
+```
diff --git a/python-stdlib/_markupbase/_markupbase.py → unix-ffi/_markupbase/_markupbase.py b/python-stdlib/_markupbase/_markupbase.py → unix-ffi/_markupbase/_markupbase.py
diff --git a/python-stdlib/_markupbase/manifest.py → unix-ffi/_markupbase/manifest.py b/python-stdlib/_markupbase/manifest.py → unix-ffi/_markupbase/manifest.py
@@ -1,5 +1,5 @@
 metadata(version="3.3.3-1")
 
-require("re-pcre")
+require("re", unix_ffi=True)
 
 module("_markupbase.py")
diff --git a/python-stdlib/argparse/argparse.py → unix-ffi/argparse/argparse.py b/python-stdlib/argparse/argparse.py → unix-ffi/argparse/argparse.py
diff --git a/python-stdlib/argparse/manifest.py → unix-ffi/argparse/manifest.py b/python-stdlib/argparse/manifest.py → unix-ffi/argparse/manifest.py
diff --git a/python-stdlib/argparse/test_argparse.py → unix-ffi/argparse/test_argparse.py b/python-stdlib/argparse/test_argparse.py → unix-ffi/argparse/test_argparse.py
diff --git a/python-stdlib/cgi/cgi.py → unix-ffi/cgi/cgi.py b/python-stdlib/cgi/cgi.py → unix-ffi/cgi/cgi.py
diff --git a/python-stdlib/cgi/manifest.py → unix-ffi/cgi/manifest.py b/python-stdlib/cgi/manifest.py → unix-ffi/cgi/manifest.py
diff --git a/python-stdlib/email.charset/email/charset.py → unix-ffi/email.charset/email/charset.py b/python-stdlib/email.charset/email/charset.py → unix-ffi/email.charset/email/charset.py
diff --git a/...stdlib/email.encoders/email/base64mime.py → unix-ffi/email.encoders/email/base64mime.py b/...stdlib/email.encoders/email/base64mime.py → unix-ffi/email.encoders/email/base64mime.py
diff --git a/...n-stdlib/email.encoders/email/encoders.py → unix-ffi/email.encoders/email/encoders.py b/...n-stdlib/email.encoders/email/encoders.py → unix-ffi/email.encoders/email/encoders.py
diff --git a/...stdlib/email.encoders/email/quoprimime.py → unix-ffi/email.encoders/email/quoprimime.py b/...stdlib/email.encoders/email/quoprimime.py → unix-ffi/email.encoders/email/quoprimime.py
diff --git a/python-stdlib/email.encoders/manifest.py → unix-ffi/email.encoders/manifest.py b/python-stdlib/email.encoders/manifest.py → unix-ffi/email.encoders/manifest.py
@@ -3,7 +3,7 @@
 require("base64")
 require("binascii")
 require("quopri")
-require("re-pcre")
+require("re", unix_ffi=True)
 require("string")
 
 package("email")
diff --git a/python-stdlib/email.errors/email/errors.py → unix-ffi/email.errors/email/errors.py b/python-stdlib/email.errors/email/errors.py → unix-ffi/email.errors/email/errors.py
diff --git a/python-stdlib/email.errors/manifest.py → unix-ffi/email.errors/manifest.py b/python-stdlib/email.errors/manifest.py → unix-ffi/email.errors/manifest.py
diff --git a/...dlib/email.feedparser/email/feedparser.py → ...-ffi/email.feedparser/email/feedparser.py b/...dlib/email.feedparser/email/feedparser.py → ...-ffi/email.feedparser/email/feedparser.py
diff --git a/python-stdlib/email.header/email/header.py → unix-ffi/email.header/email/header.py b/python-stdlib/email.header/email/header.py → unix-ffi/email.header/email/header.py
diff --git a/...ib/email.internal/email/_encoded_words.py → ...fi/email.internal/email/_encoded_words.py b/...ib/email.internal/email/_encoded_words.py → ...fi/email.internal/email/_encoded_words.py
diff --git a/...stdlib/email.internal/email/_parseaddr.py → unix-ffi/email.internal/email/_parseaddr.py b/...stdlib/email.internal/email/_parseaddr.py → unix-ffi/email.internal/email/_parseaddr.py
diff --git a/...tdlib/email.internal/email/_policybase.py → unix-ffi/email.internal/email/_policybase.py b/...tdlib/email.internal/email/_policybase.py → unix-ffi/email.internal/email/_policybase.py
diff --git a/...n-stdlib/email.message/email/iterators.py → unix-ffi/email.message/email/iterators.py b/...n-stdlib/email.message/email/iterators.py → unix-ffi/email.message/email/iterators.py
diff --git a/python-stdlib/email.message/email/message.py → unix-ffi/email.message/email/message.py b/python-stdlib/email.message/email/message.py → unix-ffi/email.message/email/message.py
diff --git a/python-stdlib/email.parser/email/parser.py → unix-ffi/email.parser/email/parser.py b/python-stdlib/email.parser/email/parser.py → unix-ffi/email.parser/email/parser.py
diff --git a/python-stdlib/email.utils/email/utils.py → unix-ffi/email.utils/email/utils.py b/python-stdlib/email.utils/email/utils.py → unix-ffi/email.utils/email/utils.py
@@ -215,7 +215,10 @@ def make_msgid(idstring=None, domain=None):
     """
     timeval = time.time()
     utcdate = time.strftime("%Y%m%d%H%M%S", time.gmtime(timeval))
-    pid = os.getpid()
+    if hasattr(os, "getpid"):
+        pid = os.getpid()
+    else:
+        pid = 0
     randint = random.randrange(100000)
     if idstring is None:
         idstring = ""

diff --git a/python-stdlib/email.utils/manifest.py → unix-ffi/email.utils/manifest.py b/python-stdlib/email.utils/manifest.py → unix-ffi/email.utils/manifest.py
@@ -1,13 +1,13 @@
 metadata(version="3.3.3-2")
 
-require("os")
-require("re-pcre")
+require("os", unix_ffi=True)
+require("re", unix_ffi=True)
 require("base64")
 require("random")
 require("datetime")
-require("urllib.parse")
+require("urllib.parse", unix_ffi=True)
 require("warnings")
 require("quopri")
-require("email.charset")
+require("email.charset", unix_ffi=True)
 
 package("email")
diff --git a/unix-ffi/fcntl/manifest.py b/unix-ffi/fcntl/manifest.py
@@ -2,6 +2,6 @@
 
 # Originally written by Paul Sokolovsky.
 
-require("ffilib")
+require("ffilib", unix_ffi=True)
 
 module("fcntl.py")
diff --git a/python-stdlib/getopt/getopt.py → unix-ffi/getopt/getopt.py b/python-stdlib/getopt/getopt.py → unix-ffi/getopt/getopt.py
diff --git a/python-stdlib/getopt/manifest.py → unix-ffi/getopt/manifest.py b/python-stdlib/getopt/manifest.py → unix-ffi/getopt/manifest.py
@@ -1,5 +1,5 @@
 metadata(version="3.3.3-1")
 
-require("os")
+require("os", unix_ffi=True)
 
 module("getopt.py")
diff --git a/unix-ffi/gettext/manifest.py b/unix-ffi/gettext/manifest.py
@@ -2,6 +2,6 @@
 
 # Originally written by Riccardo Magliocchetti.
 
-require("ffilib")
+require("ffilib", unix_ffi=True)
 
 module("gettext.py")
diff --git a/python-stdlib/glob/glob.py → unix-ffi/glob/glob.py b/python-stdlib/glob/glob.py → unix-ffi/glob/glob.py
diff --git a/python-stdlib/glob/test_glob.py → unix-ffi/glob/test_glob.py b/python-stdlib/glob/test_glob.py → unix-ffi/glob/test_glob.py
diff --git a/python-stdlib/html.entities/html/entities.py → unix-ffi/html.entities/html/entities.py b/python-stdlib/html.entities/html/entities.py → unix-ffi/html.entities/html/entities.py
diff --git a/python-stdlib/html.entities/manifest.py → unix-ffi/html.entities/manifest.py b/python-stdlib/html.entities/manifest.py → unix-ffi/html.entities/manifest.py
diff --git a/python-stdlib/html.parser/html/parser.py → unix-ffi/html.parser/html/parser.py b/python-stdlib/html.parser/html/parser.py → unix-ffi/html.parser/html/parser.py
diff --git a/python-stdlib/http.client/example_client.py → unix-ffi/http.client/example_client.py b/python-stdlib/http.client/example_client.py → unix-ffi/http.client/example_client.py
diff --git a/python-stdlib/http.client/http/client.py → unix-ffi/http.client/http/client.py b/python-stdlib/http.client/http/client.py → unix-ffi/http.client/http/client.py
diff --git a/unix-ffi/machine/manifest.py b/unix-ffi/machine/manifest.py
@@ -2,8 +2,8 @@
 
 # Originally written by Paul Sokolovsky.
 
-require("ffilib")
-require("os")
-require("signal")
+require("ffilib", unix_ffi=True)
+require("os", unix_ffi=True)
+require("signal", unix_ffi=True)
 
 package("machine")
diff --git a/unix-ffi/multiprocessing/manifest.py b/unix-ffi/multiprocessing/manifest.py
@@ -2,8 +2,8 @@
 
 # Originally written by Paul Sokolovsky.
 
-require("os")
-require("select")
+require("os", unix_ffi=True)
+require("select", unix_ffi=True)
 require("pickle")
 
 module("multiprocessing.py")
diff --git a/unix-ffi/os/manifest.py b/unix-ffi/os/manifest.py
@@ -2,7 +2,7 @@
 
 # Originally written by Paul Sokolovsky.
 
-require("ffilib")
+require("ffilib", unix_ffi=True)
 require("errno")
 require("stat")
 

diff --git a/unix-ffi/pwd/manifest.py b/unix-ffi/pwd/manifest.py
@@ -2,6 +2,6 @@
 
 # Originally written by Riccardo Magliocchetti.
 
-require("ffilib")
+require("ffilib", unix_ffi=True)
 
 module("pwd.py")
diff --git a/unix-ffi/re-pcre/manifest.py → unix-ffi/re/manifest.py b/unix-ffi/re-pcre/manifest.py → unix-ffi/re/manifest.py
@@ -2,6 +2,6 @@
 
 # Originally written by Paul Sokolovsky.
 
-require("ffilib")
+require("ffilib", unix_ffi=True)
 
 module("re.py")
diff --git a/unix-ffi/re-pcre/re.py → unix-ffi/re/re.py b/unix-ffi/re-pcre/re.py → unix-ffi/re/re.py
diff --git a/unix-ffi/re-pcre/test_re.py → unix-ffi/re/test_re.py b/unix-ffi/re-pcre/test_re.py → unix-ffi/re/test_re.py
diff --git a/unix-ffi/select/manifest.py b/unix-ffi/select/manifest.py
@@ -2,7 +2,7 @@
 
 # Originally written by Paul Sokolovsky.
 
-require("os")
-require("ffilib")
+require("os", unix_ffi=True)
+require("ffilib", unix_ffi=True)
 
 module("select.py")
diff --git a/unix-ffi/signal/manifest.py b/unix-ffi/signal/manifest.py
@@ -2,6 +2,6 @@
 
 # Originally written by Paul Sokolovsky.
 
-require("ffilib")
+require("ffilib", unix_ffi=True)
 
 module("signal.py")
diff --git a/python-stdlib/socket/manifest.py → unix-ffi/socket/manifest.py b/python-stdlib/socket/manifest.py → unix-ffi/socket/manifest.py
diff --git a/python-stdlib/socket/socket.py → unix-ffi/socket/socket.py b/python-stdlib/socket/socket.py → unix-ffi/socket/socket.py
diff --git a/unix-ffi/sqlite3/manifest.py b/unix-ffi/sqlite3/manifest.py
@@ -2,6 +2,6 @@
 
 # Originally written by Paul Sokolovsky.
 
-require("ffilib")
+require("ffilib", unix_ffi=True)
 
 module("sqlite3.py")
diff --git a/micropython/test.support/manifest.py → unix-ffi/test.support/manifest.py b/micropython/test.support/manifest.py → unix-ffi/test.support/manifest.py
diff --git a/micropython/test.support/test/support.py → unix-ffi/test.support/test/support.py b/micropython/test.support/test/support.py → unix-ffi/test.support/test/support.py
diff --git a/unix-ffi/time/manifest.py b/unix-ffi/time/manifest.py
@@ -1,5 +1,5 @@
 metadata(version="0.5")
 
-require("ffilib")
+require("ffilib", unix_ffi=True)
 
 module("time.py")
diff --git a/python-stdlib/timeit/manifest.py → unix-ffi/timeit/manifest.py b/python-stdlib/timeit/manifest.py → unix-ffi/timeit/manifest.py
@@ -1,9 +1,9 @@
 metadata(version="3.3.3-3")
 
-require("getopt")
+require("getopt", unix_ffi=True)
 require("itertools")
 # require("linecache") TODO
-require("time")
+require("time", unix_ffi=True)
 require("traceback")
 
 module("timeit.py")
diff --git a/python-stdlib/timeit/timeit.py → unix-ffi/timeit/timeit.py b/python-stdlib/timeit/timeit.py → unix-ffi/timeit/timeit.py
diff --git a/unix-ffi/ucurses/manifest.py b/unix-ffi/ucurses/manifest.py
@@ -2,8 +2,8 @@
 
 # Originally written by Paul Sokolovsky.
 
-require("os")
-require("tty")
-require("select")
+require("os", unix_ffi=True)
+require("tty", unix_ffi=True)
+require("select", unix_ffi=True)
 
 package("ucurses")
diff --git a/python-stdlib/unittest/manifest.py → unix-ffi/unittest/manifest.py b/python-stdlib/unittest/manifest.py → unix-ffi/unittest/manifest.py
@@ -1,6 +1,6 @@
 metadata(version="0.9.0")
 
-require("argparse")
+require("argparse", unix_ffi=True)
 require("fnmatch")
 
 module("unittest.py")
diff --git a/python-stdlib/unittest/test_unittest.py → unix-ffi/unittest/test_unittest.py b/python-stdlib/unittest/test_unittest.py → unix-ffi/unittest/test_unittest.py
diff --git a/...stdlib/unittest/test_unittest_isolated.py → unix-ffi/unittest/test_unittest_isolated.py b/...stdlib/unittest/test_unittest_isolated.py → unix-ffi/unittest/test_unittest_isolated.py
diff --git a/python-stdlib/unittest/unittest.py → unix-ffi/unittest/unittest.py b/python-stdlib/unittest/unittest.py → unix-ffi/unittest/unittest.py
diff --git a/python-stdlib/unittest/unittest_discover.py → unix-ffi/unittest/unittest_discover.py b/python-stdlib/unittest/unittest_discover.py → unix-ffi/unittest/unittest_discover.py
diff --git a/python-stdlib/urllib.parse/manifest.py → unix-ffi/urllib.parse/manifest.py b/python-stdlib/urllib.parse/manifest.py → unix-ffi/urllib.parse/manifest.py
@@ -1,6 +1,6 @@
 metadata(version="0.5.2")
 
-require("re-pcre")
+require("re", unix_ffi=True)
 require("collections")
 require("collections.defaultdict")
 

diff --git a/python-stdlib/urllib.parse/test_urlparse.py → unix-ffi/urllib.parse/test_urlparse.py b/python-stdlib/urllib.parse/test_urlparse.py → unix-ffi/urllib.parse/test_urlparse.py
diff --git a/python-stdlib/urllib.parse/urllib/parse.py → unix-ffi/urllib.parse/urllib/parse.py b/python-stdlib/urllib.parse/urllib/parse.py → unix-ffi/urllib.parse/urllib/parse.py