Skip to content
Permalink
Browse files

[2.7] bpo-30450: Pull Windows dependencies from GitHub rather than SVN (

GH-1783) (GH-3306)

The Windows build now depends on Python 3.6 to fetch externals, but it
will be downloaded via NuGet (which is downloaded via PowerShell) if it
is not available via `py -3.6`.  This means the only thing that must be
installed on a modern Windows box to do a full build of CPython with all
extensions is Visual Studio.

Cherry-picked from 51599e2, parts of 40a23e8, parts of 68d663c, d5cd21d, and possibly others that I've missed.

Also:

* Rename db -> bsddb for disambiguity

* Update sqlite3 to 3.14.2.0 since it's the version we use on 3.x, and it's simpler to just use it than to also upload the old version to cpython-source-deps

* Add PCbuild/*.ilk to .gitignore
  • Loading branch information...
zware committed Sep 4, 2017
1 parent 57d963b commit 986b7ffc650919b3022ccaa458a843bb8a95d2bd
@@ -10,6 +10,8 @@ build_script:
- cmd: PCbuild\python.exe -m test.pythoninfo
test_script:
- cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest -j2
environment:
HOST_PYTHON: C:\Python36\python.exe

# Only trigger AppVeyor if actual code or its configuration changes
only_commits:
@@ -44,6 +44,7 @@ PC/*/*.suo
PC/*/Win32-temp-*
PC/*/x64-temp-*
PC/*/amd64
PCbuild/*.ilk
PCbuild/*.user
PCbuild/*.suo
PCbuild/*.*sdf
@@ -5,8 +5,19 @@ pushd %~dp0

set this=%~n0

if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
if "%PYTHON%" EQU "" set PYTHON=py
call ..\PCBuild\find_python.bat %PYTHON%
if not defined SPHINXBUILD if defined PYTHON (
%PYTHON% -c "import sphinx" > nul 2> nul
if errorlevel 1 (
echo Installing sphinx with %PYTHON%
%PYTHON% -m pip install sphinx
if errorlevel 1 exit /B
)
set SPHINXBUILD=%PYTHON% -c "import sphinx, sys; sys.argv[0] = 'sphinx-build'; sphinx.main()"
)

if not defined PYTHON set PYTHON=py
if not defined SPHINXBUILD set SPHINXBUILD=sphinx-build

if DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles(x86)%
if NOT DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles%
@@ -73,7 +84,7 @@ goto end
if NOT "%PAPER%" == "" (
set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS%
)
cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%*
cmd /C "%SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%*"

if "%1" EQU "htmlhelp" (
if not exist "%HTMLHELP%" (
@@ -0,0 +1 @@
Pull build dependencies from GitHub rather than svn.python.org.
@@ -50,7 +50,7 @@
/>
<UserMacro
Name="bsddb47Dir"
Value="$(externalsDir)\db-4.7.25.0\build_windows"
Value="$(externalsDir)\bsddb-4.7.25.0\build_windows"
/>
<UserMacro
Name="bsddb47DepLibs"
@@ -74,7 +74,7 @@
/>
<UserMacro
Name="sqlite3Dir"
Value="$(externalsDir)\sqlite-3.8.11.0"
Value="$(externalsDir)\sqlite-3.14.2.0"
/>
<UserMacro
Name="bz2Dir"
@@ -5,12 +5,15 @@ echo.%~nx0 [flags and arguments] [quoted MSBuild options]
echo.
echo.Build CPython from the command line. Requires the appropriate
echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt).
echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is
echo.given.
echo.
echo.After the flags recognized by this script, up to 9 arguments to be passed
echo.directly to MSBuild may be passed. If the argument contains an '=', the
echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`)
echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`).
echo.Alternatively you can put extra flags for MSBuild in a file named
echo.`msbuild.rsp` in the `PCbuild` directory, one flag per line. This file
echo.will be picked automatically by MSBuild. Flags put in this file does not
echo.need to be quoted. You can still use environment variables inside the
echo.response file.
echo.
echo.Available flags:
echo. -h Display this help message
@@ -47,7 +50,6 @@ exit /b 127
:Run
setlocal
set platf=Win32
set vs_platf=x86
set conf=Release
set target=Build
set dir=%~dp0
@@ -56,10 +58,6 @@ set verbose=/nologo /v:m
set kill=
set do_pgo=
set pgo_job=-m test.regrtest --pgo
set on_64_bit=true

rem This may not be 100% accurate, but close enough.
if "%ProgramFiles(x86)%"=="" (set on_64_bit=false)

:CheckOpts
if "%~1"=="-h" goto Usage
@@ -89,18 +87,12 @@ if "%IncludeBsddb%"=="" set IncludeBsddb=true

if "%IncludeExternals%"=="true" call "%dir%get_externals.bat"

if "%platf%"=="x64" (
if "%on_64_bit%"=="true" (
rem This ought to always be correct these days...
set vs_platf=amd64
) else (
if "%do_pgo%"=="true" (
echo.ERROR: Cannot cross-compile with PGO
echo. 32bit operating system detected, if this is incorrect,
echo. make sure the ProgramFiles(x86^) environment variable is set
exit /b 1
)
set vs_platf=x86_amd64
if "%do_pgo%" EQU "true" if "%platf%" EQU "x64" (
if "%PROCESSOR_ARCHITEW6432%" NEQ "AMD64" if "%PROCESSOR_ARCHITECTURE%" NEQ "AMD64" (
echo.ERROR: Cannot cross-compile with PGO
echo. 32bit operating system detected. Ensure your PROCESSOR_ARCHITECTURE
echo. and PROCESSOR_ARCHITEW6432 environment variables are correct.
exit /b 1
)
)

@@ -109,26 +101,28 @@ if exist "%GIT%" set GITProperty=/p:GIT="%GIT%"
if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty=

rem Setup the environment
call "%dir%env.bat" %vs_platf% >nul
call "%dir%find_msbuild.bat" %MSBUILD%
if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)

if "%kill%"=="true" call :Kill

if "%do_pgo%"=="true" (
set conf=PGInstrument
call :Build
call :Build %1 %2 %3 %4 %5 %6 %7 %8 %9
del /s "%dir%\*.pgc"
del /s "%dir%\..\Lib\*.pyc"
echo on
call "%dir%\..\python.bat" %pgo_job%
@echo off
call :Kill
set conf=PGUpdate
set target=Build
)
goto Build

:Kill
echo on
msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
%MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
/p:Configuration=%conf% /p:Platform=%platf%^
/p:KillPython=true

@@ -140,7 +134,7 @@ rem Call on MSBuild to do the work, echo the command.
rem Passing %1-9 is not the preferred option, but argument parsing in
rem batch is, shall we say, "lackluster"
echo on
msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
%MSBUILD% "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
/p:Configuration=%conf% /p:Platform=%platf%^
/p:IncludeExternals=%IncludeExternals%^
/p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^
@@ -0,0 +1,52 @@
@rem
@rem Searches for MSBuild.exe. This is the only tool we need to initiate
@rem a build, so we no longer search for the full VC toolset.
@rem
@rem This file is supposed to modify the state of the caller (specifically
@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
@rem changing any other persistent state.
@rem

@rem No arguments provided means do full search
@if '%1' EQU '' goto :begin_search

@rem One argument may be the full path. Use a goto so we don't try to
@rem parse the next if statement - incorrect quoting in the multi-arg
@rem case can cause us to break immediately.
@if '%2' EQU '' goto :one_arg

@rem Entire command line may represent the full path if quoting failed.
@if exist "%*" (set MSBUILD="%*") & (set _Py_MSBuild_Source=environment) & goto :found
@goto :begin_search

:one_arg
@if exist "%~1" (set MSBUILD="%~1") & (set _Py_MSBuild_Source=environment) & goto :found

:begin_search
@set MSBUILD=

@rem If msbuild.exe is on the PATH, assume that the user wants that one.
@where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc"
@if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found

@rem VS 2015 and earlier register MSBuild separately, so we can find it.
@rem Prefer MSBuild 14.0 over MSBuild 15.0, since the latter may not be able to find a VC14 install.
@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul
@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @(
@if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe")
)
@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found

@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there.
@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul
@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @(
@if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe")
)
@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found


@exit /b 1

:found
@echo Using %MSBUILD% (found in the %_Py_MSBuild_Source%)
@set _Py_MSBuild_Source=
@@ -0,0 +1,75 @@
@rem
@rem Searches for python.exe and may download a private copy from nuget.
@rem
@rem This file is supposed to modify the state of the caller (specifically
@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
@rem changing any other persistent state.
@rem

@rem No arguments provided means do full search
@if '%1' EQU '' goto :begin_search

@rem One argument may be the full path. Use a goto so we don't try to
@rem parse the next if statement - incorrect quoting in the multi-arg
@rem case can cause us to break immediately.
@if '%2' EQU '' goto :one_arg

@rem Entire command line may represent the full path if quoting failed.
@if exist "%*" (set PYTHON="%*") & (set _Py_Python_Source=from environment) & goto :found
@goto :begin_search

:one_arg
@if exist "%~1" (set PYTHON="%~1") & (set _Py_Python_Source=from environment) & goto :found

:begin_search
@set PYTHON=

@set _Py_EXTERNALS_DIR=%EXTERNAL_DIR%
@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals)

@rem If we have Python in externals, use that one
@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found

@rem If HOST_PYTHON is recent enough, use that
@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -c "import sys; assert sys.version_info[:2] >= (3, 6)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found

@rem If py.exe finds a recent enough version, use that one
@py -3.6 -V >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found

@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
@set _Py_NUGET=%NUGET%
@set _Py_NUGET_URL=%NUGET_URL%
@set _Py_HOST_PYTHON=%HOST_PYTHON%
@if "%_Py_HOST_PYTHON%"=="" set _Py_HOST_PYTHON=py
@if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe)
@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl)
@if NOT exist "%_Py_NUGET%" (
@echo Downloading nuget...
@rem NB: Must use single quotes around NUGET here, NOT double!
@rem Otherwise, a space in the path would break things
@rem If it fails, retry with any available copy of Python
@powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%'
@if errorlevel 1 (
@%_Py_HOST_PYTHON% "%~dp0\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%"
)
)
@echo Installing Python via nuget...
@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%"
@rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
@if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found


@set _Py_Python_Source=
@set _Py_EXTERNALS_DIR=
@set _Py_NUGET=
@set _Py_NUGET_URL=
@set _Py_HOST_PYTHON=
@exit /b 1

:found
@echo Using %PYTHON% (%_Py_Python_Source%)
@set _Py_Python_Source=
@set _Py_EXTERNALS_DIR=
@set _Py_NUGET=
@set _Py_NUGET_URL=
@set _Py_HOST_PYTHON=
@@ -0,0 +1,60 @@
#!/usr/bin/env python3

import argparse
import os
import pathlib
import zipfile
from urllib.request import urlretrieve


def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose):
repo = f'cpython-{"bin" if binary else "source"}-deps'
url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip'
reporthook = None
if verbose:
reporthook = print
zip_dir.mkdir(parents=True, exist_ok=True)
filename, headers = urlretrieve(
url,
zip_dir / f'{commit_hash}.zip',
reporthook=reporthook,
)
return filename


def extract_zip(externals_dir, zip_path):
with zipfile.ZipFile(os.fspath(zip_path)) as zf:
zf.extractall(os.fspath(externals_dir))
return externals_dir / zf.namelist()[0].split('/')[0]


def parse_args():
p = argparse.ArgumentParser()
p.add_argument('-v', '--verbose', action='store_true')
p.add_argument('-b', '--binary', action='store_true',
help='Is the dependency in the binary repo?')
p.add_argument('-O', '--organization',
help='Organization owning the deps repos', default='python')
p.add_argument('-e', '--externals-dir', type=pathlib.Path,
help='Directory in which to store dependencies',
default=pathlib.Path(__file__).parent.parent / 'externals')
p.add_argument('tag',
help='tag of the dependency')
return p.parse_args()


def main():
args = parse_args()
zip_path = fetch_zip(
args.tag,
args.externals_dir / 'zips',
org=args.organization,
binary=args.binary,
verbose=args.verbose,
)
final_name = args.externals_dir / args.tag
extract_zip(args.externals_dir, zip_path).replace(final_name)


if __name__ == '__main__':
main()
Oops, something went wrong.

0 comments on commit 986b7ff

Please sign in to comment.
You can’t perform that action at this time.