Skip to content

Commit 52ae2aa

Browse files
indygregindygreg
committed
windows: initial support for Windows
This is extremely hacky. We don't yet produce a tar file. The static/dynamic library configurations are all wrong. Essentially the reached milestone is we are able to invoke msbuild.exe to build CPython.
1 parent 53d81a9 commit 52ae2aa

File tree

4 files changed

+534
-0
lines changed

4 files changed

+534
-0
lines changed

‎build-windows.py‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#!/usr/bin/env python3
2+
# This Source Code Form is subject to the terms of the Mozilla Public
3+
# License, v. 2.0. If a copy of the MPL was not distributed with this
4+
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
5+
6+
import datetime
7+
import os
8+
import pathlib
9+
import subprocess
10+
import sys
11+
import venv
12+
13+
14+
ROOT = pathlib.Path(os.path.abspath(__file__)).parent
15+
BUILD = ROOT / 'build'
16+
DIST = ROOT / 'dist'
17+
VENV = BUILD / 'venv'
18+
PIP = VENV / 'Scripts' / 'pip.exe'
19+
PYTHON = VENV / 'Scripts' / 'python.exe'
20+
REQUIREMENTS = ROOT / 'requirements.win.txt'
21+
WINDOWS_DIR = ROOT / 'cpython-windows'
22+
23+
24+
def bootstrap():
25+
BUILD.mkdir(exist_ok=True)
26+
DIST.mkdir(exist_ok=True)
27+
28+
venv.create(VENV, with_pip=True)
29+
30+
subprocess.run([str(PIP), 'install', '-r', str(REQUIREMENTS)],
31+
check=True)
32+
33+
os.environ['PYBUILD_BOOTSTRAPPED'] = '1'
34+
os.environ['PATH'] = '%s;%s' % (str(VENV / 'bin'), os.environ['PATH'])
35+
os.environ['PYTHONPATH'] = str(ROOT)
36+
subprocess.run([str(PYTHON), __file__], check=True)
37+
38+
39+
def run():
40+
import zstandard
41+
from pythonbuild.downloads import DOWNLOADS
42+
from pythonbuild.utils import hash_path
43+
44+
now = datetime.datetime.utcnow()
45+
46+
env = dict(os.environ)
47+
env['PYTHONUNBUFFERED'] = '1'
48+
49+
subprocess.run([str(PYTHON), 'build.py'],
50+
cwd=str(WINDOWS_DIR), env=env, check=True,
51+
bufsize=0)
52+
53+
source_path = BUILD / 'cpython-windows.tar'
54+
dest_path = DIST / ('cpython-%s-windows-amd64-%s.tar.zst' % (
55+
DOWNLOADS['cpython-3.7']['version'], now.strftime('%Y%m%dT%H%M')))
56+
57+
#print('compressing Python archive to %s' % dest_path)
58+
#with source_path.open('rb') as ifh, dest_path.open('wb') as ofh:
59+
# cctx = zstandard.ZstdCompressor(level=15)
60+
# cctx.copy_stream(ifh, ofh, source_path.stat().st_size)
61+
#
62+
#sha256 = hash_path(dest_path)
63+
#print('%s has SHA256 %s' % (dest_path, sha256))
64+
65+
66+
if __name__ == '__main__':
67+
try:
68+
if 'PYBUILD_BOOTSTRAPPED' not in os.environ:
69+
bootstrap()
70+
else:
71+
run()
72+
except subprocess.CalledProcessError as e:
73+
sys.exit(e.returncode)

‎cpython-windows/build.py‎

Lines changed: 287 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,287 @@
1+
#!/usr/bin/env python3
2+
# This Source Code Form is subject to the terms of the Mozilla Public
3+
# License, v. 2.0. If a copy of the MPL was not distributed with this
4+
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
5+
6+
import os
7+
import pathlib
8+
import subprocess
9+
import sys
10+
import tempfile
11+
12+
from pythonbuild.downloads import (
13+
DOWNLOADS,
14+
)
15+
from pythonbuild.utils import (
16+
download_entry,
17+
extract_tar_to_directory,
18+
)
19+
20+
ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent
21+
BUILD = ROOT / 'build'
22+
SUPPORT = ROOT / 'cpython-windows'
23+
24+
LOG_PREFIX = [None]
25+
LOG_FH = [None]
26+
27+
28+
def log(msg):
29+
if isinstance(msg, bytes):
30+
msg_str = msg.decode('utf-8', 'replace')
31+
msg_bytes = msg
32+
else:
33+
msg_str = msg
34+
msg_bytes = msg.encode('utf-8', 'replace')
35+
36+
print('%s> %s' % (LOG_PREFIX[0], msg_str))
37+
38+
if LOG_FH[0]:
39+
LOG_FH[0].write(msg_bytes + b'\n')
40+
LOG_FH[0].flush()
41+
42+
43+
def exec_and_log(args, cwd, env):
44+
log('executing %s' % ' '.join(args))
45+
46+
p = subprocess.Popen(
47+
args,
48+
cwd=cwd,
49+
env=env,
50+
bufsize=1,
51+
stdout=subprocess.PIPE,
52+
stderr=subprocess.STDOUT)
53+
54+
for line in iter(p.stdout.readline, b''):
55+
log(line.rstrip())
56+
57+
p.wait()
58+
59+
if p.returncode:
60+
print('process exited %d' % p.returncode)
61+
import pdb; pdb.set_trace()
62+
sys.exit(p.returncode)
63+
64+
65+
def find_msbuild():
66+
vswhere = pathlib.Path(os.environ['ProgramFiles(x86)']) / 'Microsoft Visual Studio' / 'Installer' / 'vswhere.exe'
67+
68+
if not vswhere.exists():
69+
print('%s does not exist' % vswhere)
70+
sys.exit(1)
71+
72+
p = subprocess.check_output([str(vswhere), '-latest', '-property', 'installationPath'])
73+
74+
# Strictly speaking the output may not be UTF-8.
75+
p = pathlib.Path(p.strip().decode('utf-8'))
76+
77+
p = p / 'MSBuild' / '15.0' / 'Bin' / 'MSBuild.exe'
78+
79+
if not p.exists():
80+
print('%s does not exist' % p)
81+
sys.exit(1)
82+
83+
return p
84+
85+
86+
def static_replace_in_file(p: pathlib.Path, search, replace):
87+
"""Replace occurrences of a string in a file.
88+
89+
The updated file contents are written out in place.
90+
"""
91+
92+
with p.open('rb') as fh:
93+
data = fh.read()
94+
95+
# Build should be as deterministic as possible. Assert that wanted changes
96+
# actually occur.
97+
if search not in data:
98+
print('search string (%s) not in %s' % (search, p))
99+
sys.exit(1)
100+
101+
data = data.replace(search, replace)
102+
103+
with p.open('wb') as fh:
104+
fh.write(data)
105+
106+
def hack_props(td: pathlib.Path, pcbuild_path: pathlib.Path):
107+
# TODO can we pass props into msbuild.exe?
108+
109+
# Our dependencies are in different directories from what CPython's
110+
# build system expects. Modify the config file appropriately.
111+
112+
bzip2_version = DOWNLOADS['bzip2']['version']
113+
openssl_bin_version = DOWNLOADS['openssl-windows-bin']['version']
114+
sqlite_version = DOWNLOADS['sqlite']['version']
115+
xz_version = DOWNLOADS['xz']['version']
116+
zlib_version = DOWNLOADS['zlib']['version']
117+
tcltk_commit = DOWNLOADS['tk-windows-bin']['git_commit']
118+
119+
sqlite_path = td / ('sqlite-autoconf-%s' % sqlite_version)
120+
bzip2_path = td / ('bzip2-%s' % bzip2_version)
121+
openssl_bin_path = td / ('cpython-bin-deps-openssl-bin-%s' % openssl_bin_version)
122+
tcltk_path = td / ('cpython-bin-deps-%s' % tcltk_commit)
123+
xz_path = td / ('xz-%s' % xz_version)
124+
zlib_path = td / ('zlib-%s' % zlib_version)
125+
126+
python_props_path = pcbuild_path / 'python.props'
127+
lines = []
128+
129+
with python_props_path.open('rb') as fh:
130+
for line in fh:
131+
line = line.rstrip()
132+
133+
if b'<bz2Dir>' in line:
134+
line = b'<bz2Dir>%s\\</bz2Dir>' % bzip2_path
135+
136+
elif b'<lzmaDir>' in line:
137+
line = b'<lzmaDir>%s\\</lzmaDir>' % xz_path
138+
139+
# elif b'<opensslDir>' in line:
140+
# line = b'<opensslDir>%s</opensslDir>' % openssl_bin_path,
141+
142+
elif b'<opensslOutDir>' in line:
143+
line = b'<opensslOutDir>%s\\$(ArchName)\\</opensslOutDir>' % openssl_bin_path
144+
145+
elif b'<sqlite3Dir>' in line:
146+
line = b'<sqlite3Dir>%s\\</sqlite3Dir>' % sqlite_path
147+
148+
elif b'<zlibDir>' in line:
149+
line = b'<zlibDir>%s\\</zlibDir>' % zlib_path
150+
151+
lines.append(line)
152+
153+
with python_props_path.open('wb') as fh:
154+
fh.write(b'\n'.join(lines))
155+
156+
tcltkprops_path = pcbuild_path / 'tcltk.props'
157+
158+
static_replace_in_file(
159+
tcltkprops_path,
160+
br'<tcltkDir>$(ExternalsDir)tcltk-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\$(ArchName)\</tcltkDir>',
161+
br'<tcltkDir>%s\$(ArchName)\</tcltkDir>' % tcltk_path)
162+
163+
164+
def hack_project_files(td: pathlib.Path, pcbuild_path: pathlib.Path):
165+
"""Hacks Visual Studio project files to work with our build."""
166+
167+
hack_props(td, pcbuild_path)
168+
169+
# Our SQLite directory is named weirdly. This throws off version detection
170+
# in the project file. Replace the parsing logic with a static string.
171+
sqlite3_version = DOWNLOADS['sqlite']['actual_version'].encode('ascii')
172+
sqlite3_version_parts = sqlite3_version.split(b'.')
173+
sqlite3_path = pcbuild_path / 'sqlite3.vcxproj'
174+
static_replace_in_file(
175+
sqlite3_path,
176+
br'<_SqliteVersion>$([System.Text.RegularExpressions.Regex]::Match(`$(sqlite3Dir)`, `((\d+)\.(\d+)\.(\d+)\.(\d+))\\?$`).Groups)</_SqliteVersion>',
177+
br'<_SqliteVersion>%s</_SqliteVersion>' % sqlite3_version)
178+
static_replace_in_file(
179+
sqlite3_path,
180+
br'<SqliteVersion>$(_SqliteVersion.Split(`;`)[1])</SqliteVersion>',
181+
br'<SqliteVersion>%s</SqliteVersion>' % sqlite3_version)
182+
static_replace_in_file(
183+
sqlite3_path,
184+
br'<SqliteMajorVersion>$(_SqliteVersion.Split(`;`)[2])</SqliteMajorVersion>',
185+
br'<SqliteMajorVersion>%s</SqliteMajorVersion>' % sqlite3_version_parts[0])
186+
static_replace_in_file(
187+
sqlite3_path,
188+
br'<SqliteMinorVersion>$(_SqliteVersion.Split(`;`)[3])</SqliteMinorVersion>',
189+
br'<SqliteMinorVersion>%s</SqliteMinorVersion>' % sqlite3_version_parts[1])
190+
static_replace_in_file(
191+
sqlite3_path,
192+
br'<SqliteMicroVersion>$(_SqliteVersion.Split(`;`)[4])</SqliteMicroVersion>',
193+
br'<SqliteMicroVersion>%s</SqliteMicroVersion>' % sqlite3_version_parts[2])
194+
static_replace_in_file(
195+
sqlite3_path,
196+
br'<SqlitePatchVersion>$(_SqliteVersion.Split(`;`)[5])</SqlitePatchVersion>',
197+
br'<SqlitePatchVersion>%s</SqlitePatchVersion>' % sqlite3_version_parts[3])
198+
199+
# Our version of the xz sources is newer than what's in cpython-source-deps
200+
# and the xz sources changed the path to config.h. Hack the project file
201+
# accordingly.
202+
liblzma_path = pcbuild_path / 'liblzma.vcxproj'
203+
static_replace_in_file(
204+
liblzma_path,
205+
br'$(lzmaDir)windows;$(lzmaDir)src/liblzma/common;',
206+
br'$(lzmaDir)windows\vs2017;$(lzmaDir)src/liblzma/common;')
207+
static_replace_in_file(
208+
liblzma_path,
209+
br'<ClInclude Include="$(lzmaDir)windows\config.h" />',
210+
br'<ClInclude Include="$(lzmaDir)windows\vs2017\config.h" />')
211+
212+
213+
def run_msbuild(msbuild: pathlib.Path, pcbuild_path: pathlib.Path):
214+
python_version = DOWNLOADS['cpython-3.7']['version']
215+
216+
args = [
217+
str(msbuild),
218+
str(pcbuild_path / 'pcbuild.proj'),
219+
'/target:Build',
220+
'/property:Configuration=Release',
221+
'/property:Platform=x64',
222+
'/maxcpucount',
223+
'/nologo',
224+
'/verbosity:minimal',
225+
'/property:IncludeExternals=true',
226+
'/property:IncludeSSL=true',
227+
'/property:IncludeTkinter=true',
228+
'/property:OverrideVersion=%s' % python_version,
229+
]
230+
231+
exec_and_log(args, str(pcbuild_path), os.environ)
232+
233+
234+
def build_cpython():
235+
msbuild = find_msbuild()
236+
log('found MSBuild at %s' % msbuild)
237+
238+
# The python.props file keys off MSBUILD, so it needs to be set.
239+
os.environ['MSBUILD'] = str(msbuild)
240+
241+
bzip2_archive = download_entry('bzip2', BUILD)
242+
#openssl_archive = download_entry('openssl', BUILD)
243+
openssl_bin_archive = download_entry('openssl-windows-bin', BUILD)
244+
sqlite_archive = download_entry('sqlite', BUILD)
245+
tk_bin_archive = download_entry('tk-windows-bin', BUILD, local_name='tk-windows-bin.tar.gz')
246+
xz_archive = download_entry('xz', BUILD)
247+
zlib_archive = download_entry('zlib', BUILD)
248+
249+
python_archive = download_entry('cpython-3.7', BUILD)
250+
251+
python_version = DOWNLOADS['cpython-3.7']['version']
252+
253+
with tempfile.TemporaryDirectory() as td:
254+
td = pathlib.Path(td)
255+
256+
log('extracting CPython sources to %s' % td)
257+
extract_tar_to_directory(python_archive, td)
258+
259+
for a in (bzip2_archive, openssl_bin_archive, sqlite_archive,
260+
tk_bin_archive, xz_archive, zlib_archive):
261+
log('extracting %s to %s' % (a, td))
262+
extract_tar_to_directory(a, td)
263+
264+
cpython_source_path = td / ('Python-%s' % python_version)
265+
pcbuild_path = cpython_source_path / 'PCBuild'
266+
267+
hack_project_files(td, pcbuild_path)
268+
269+
run_msbuild(msbuild, pcbuild_path)
270+
271+
import pdb; pdb.set_trace()
272+
log('it worked!')
273+
274+
275+
def main():
276+
BUILD.mkdir(exist_ok=True)
277+
278+
log_path = BUILD / 'build.log'
279+
LOG_PREFIX[0] = 'cpython'
280+
281+
with log_path.open('wb') as log_fh:
282+
LOG_FH[0] = log_fh
283+
284+
build_cpython()
285+
286+
if __name__ == '__main__':
287+
sys.exit(main())

‎pythonbuild/downloads.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,12 @@
189189
'sha256': 'fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41',
190190
'version': '1.1.1a',
191191
},
192+
'openssl-windows-bin': {
193+
'url': 'https://github.com/python/cpython-bin-deps/archive/openssl-bin-1.1.0j.tar.gz',
194+
'size': 8274446,
195+
'sha256': 'c4684e220473fb2bdb0b95e43183c4701b6b103acac5ec23385e41a9a77fc9b1',
196+
'version': '1.1.0j',
197+
},
192198
'readline': {
193199
'url': 'ftp://ftp.gnu.org/gnu/readline/readline-6.3.tar.gz',
194200
'size': 2468560,
@@ -205,6 +211,7 @@
205211
'size': 2779667,
206212
'sha256': '5daa6a3fb7d1e8c767cd59c4ded8da6e4b00c61d3b466d0685e35c4dd6d7bf5d',
207213
'version': '3260000',
214+
'actual_version': '3.26.0.0',
208215
},
209216
'tcl': {
210217
'url': 'https://prdownloads.sourceforge.net/tcl/tcl8.6.9-src.tar.gz',
@@ -216,6 +223,13 @@
216223
'size': 4364603,
217224
'sha256': '8fcbcd958a8fd727e279f4cac00971eee2ce271dc741650b1fc33375fb74ebb4',
218225
},
226+
'tk-windows-bin': {
227+
'url': 'https://github.com/python/cpython-bin-deps/archive/86027ce3edda1284ae4bf6c2c580288366af4052.tar.gz',
228+
'size': 7162470,
229+
'sha256': '34400f7b76a13389a475fc1a4d6e33d5ca21dda6f6ff11b04759865814bdf3d2',
230+
'version': '6.6.9',
231+
'git_commit': '86027ce3edda1284ae4bf6c2c580288366af4052',
232+
},
219233
'uuid': {
220234
'url': 'https://sourceforge.net/projects/libuuid/files/libuuid-1.0.3.tar.gz',
221235
'size': 318256,

0 commit comments

Comments
 (0)