/
sdist_upip.py
151 lines (125 loc) · 4.11 KB
/
sdist_upip.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
# This module is part of Pycopy https://github.com/pfalcon/pycopy
# and pycopy-lib https://github.com/pfalcon/pycopy-lib, projects to
# create a (very) lightweight full-stack Python distribution.
#
# Copyright (c) 2016-2019 Paul Sokolovsky
# Licence: MIT
#
# This module overrides distutils (also compatible with setuptools) "sdist"
# command to perform pre- and post-processing as required for Pycopy's
# upip package manager.
#
# Preprocessing steps:
# * Creation of Python resource module (R.py) from each top-level package's
# resources.
# Postprocessing steps:
# * Removing metadata files not used by upip (this includes setup.py)
# * Recompressing gzip archive with 4K dictionary size so it can be
# installed even on low-heap targets.
#
import sys
import os
import zlib
from subprocess import Popen, PIPE
import glob
import tarfile
import re
import io
from distutils.filelist import FileList
from setuptools.command.sdist import sdist as _sdist
def gzip_4k(inf, fname):
comp = zlib.compressobj(level=9, wbits=16 + 12)
with open(fname + ".out", "wb") as outf:
while 1:
data = inf.read(1024)
if not data:
break
outf.write(comp.compress(data))
outf.write(comp.flush())
os.rename(fname, fname + ".orig")
os.rename(fname + ".out", fname)
FILTERS = [
# include, exclude, repeat
(r".+\.egg-info/(PKG-INFO|requires\.txt)", r"setup.py$"),
(r".+\.py$", r"[^/]+$"),
(None, r".+\.egg-info/.+"),
]
outbuf = io.BytesIO()
def filter_tar(name):
fin = tarfile.open(name, "r:gz")
fout = tarfile.open(fileobj=outbuf, mode="w")
for info in fin:
# print(info)
if not "/" in info.name:
continue
fname = info.name.split("/", 1)[1]
include = None
for inc_re, exc_re in FILTERS:
if include is None and inc_re:
if re.match(inc_re, fname):
include = True
if include is None and exc_re:
if re.match(exc_re, fname):
include = False
if include is None:
include = True
if include:
print("including:", fname)
else:
print("excluding:", fname)
continue
farch = fin.extractfile(info)
fout.addfile(info, farch)
fout.close()
fin.close()
def make_resource_module(manifest_files):
resources = []
# Any non-python file included in manifest is resource
for fname in manifest_files:
ext = fname.rsplit(".", 1)
if len(ext) > 1:
ext = ext[1]
else:
ext = ""
if ext != "py":
resources.append(fname)
if resources:
print("creating resource module R.py")
resources.sort()
last_pkg = None
r_file = None
for fname in resources:
try:
pkg, res_name = fname.split("/", 1)
except ValueError:
print("not treating %s as a resource" % fname)
continue
if last_pkg != pkg:
last_pkg = pkg
if r_file:
r_file.write("}\n")
r_file.close()
r_file = open(pkg + "/R.py", "w")
r_file.write("R = {\n")
with open(fname, "rb") as f:
r_file.write("%r: %r,\n" % (res_name, f.read()))
if r_file:
r_file.write("}\n")
r_file.close()
class sdist(_sdist):
def run(self):
self.filelist = FileList()
self.get_file_list()
make_resource_module(self.filelist.files)
r = super().run()
assert len(self.archive_files) == 1
print("filtering files and recompressing with 4K dictionary")
filter_tar(self.archive_files[0])
outbuf.seek(0)
gzip_4k(outbuf, self.archive_files[0])
return r
# For testing only
if __name__ == "__main__":
filter_tar(sys.argv[1])
outbuf.seek(0)
gzip_4k(outbuf, sys.argv[1])