From 738b1736d4238e5954db50b5f32bd1ec66edafea Mon Sep 17 00:00:00 2001
From: Steve Dower <steve.dower@python.org>
Date: Tue, 8 Jan 2019 13:56:28 +1100
Subject: [PATCH 1/5] bpo-35596: Use unchecked PYCs for the embeddable distro
 to avoid zipimport restrictions.

---
 .azure-pipelines/ci.yml                       |  1 +
 .azure-pipelines/windows-layout-steps.yml     | 35 +++++++++++++++++++
 .azure-pipelines/windows-steps.yml            |  6 ++++
 .../2019-01-08-13-56-01.bpo-35596.oFvhcm.rst  |  2 ++
 PC/layout/main.py                             | 20 +++++++----
 5 files changed, 57 insertions(+), 7 deletions(-)
 create mode 100644 .azure-pipelines/windows-layout-steps.yml
 create mode 100644 Misc/NEWS.d/next/Windows/2019-01-08-13-56-01.bpo-35596.oFvhcm.rst

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index 49a7bb6232aa26d..dda5197442b7b56 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -134,3 +134,4 @@ jobs:
 
   steps:
   - template: ./windows-steps.yml
+  - template: ./windows-layout-steps.yml
diff --git a/.azure-pipelines/windows-layout-steps.yml b/.azure-pipelines/windows-layout-steps.yml
new file mode 100644
index 000000000000000..5bc35ad85975cfd
--- /dev/null
+++ b/.azure-pipelines/windows-layout-steps.yml
@@ -0,0 +1,35 @@
+steps:
+- checkout: self
+  clean: true
+  fetchDepth: 5
+
+- task: DownloadPipelineArtifact@0
+  displayName: 'Download build artifacts'
+  inputs:
+    artifactName: windows-$(arch)
+    targetPath: $(Build.BinariesDirectory)\build
+
+- script: |
+    $py = ('python.exe', 'python_d.exe') | %{ Get-Item "${env:BUILD_BINARIESDIRECTORY}\$_" } | select -First 1
+    Write-Host '##vso[task.setVariable variable=PYTHON]"' + $py.FullName + '"'
+    Write-Host '##vso[task.setVariable variable=PYTHON_NAME]"' + $py.Name + '"'
+    Write-Host '##vso[task.setVariable variable=BUILD]"' + $py.Directory.FullName + '"'
+
+- script: $(PYTHON) PC\layout -vv -s "$(Build.SourcesDirectory)" -b $(BUILD) -t "$(Build.BinariesDirectory)\layout-tmp-embed-$(arch)" --copy "$(Build.BinariesDirectory)\layout-embed-$(arch)" --precompile --preset-embed --include-tests
+
+- script: $(PYTHON_NAME) -m test.pythoninfo
+  workingDirectory: $(Build.BinariesDirectory)\layout-embed-$(arch)
+  displayName: 'Embeddable layout info'
+
+- script: $(PYTHON) PC\layout -vv -s "$(Build.SourcesDirectory)" -b $(BUILD) -t "$(Build.BinariesDirectory)\layout-tmp-nuget-$(arch)" --copy "$(Build.BinariesDirectory)\layout-nuget-$(arch)" --preset-nuget --include-tests
+
+- script: $(PYTHON_NAME) -m test.pythoninfo
+  workingDirectory: $(Build.BinariesDirectory)\layout-nuget-$(arch)
+  displayName: 'Nuget layout info'
+
+- script: $(PYTHON) PC\layout -vv -s "$(Build.SourcesDirectory)" -b $(BUILD) -t "$(Build.BinariesDirectory)\layout-tmp-appx-$(arch)" --copy "$(Build.BinariesDirectory)\layout-appx-$(arch)" --precompile --preset-appx --include-tests
+  displayName: 'Create APPX layout'
+
+- script: $(PYTHON_NAME) -m test.pythoninfo
+  workingDirectory: $(Build.BinariesDirectory)\layout-appx-$(arch)
+  displayName: 'APPX layout info'
diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml
index cba00158ad131dd..b46619d2e3c8b60 100644
--- a/.azure-pipelines/windows-steps.yml
+++ b/.azure-pipelines/windows-steps.yml
@@ -32,3 +32,9 @@ steps:
     testRunTitle: $(testRunTitle)
     platform: $(testRunPlatform)
   condition: succeededOrFailed()
+
+- task: PublishPipelineArtifact@0
+  displayName: 'Publish build artifacts'
+  inputs:
+    artifactName: windows-$(arch)
+    targetPath: $(Py_OutDir)\$(arch)
diff --git a/Misc/NEWS.d/next/Windows/2019-01-08-13-56-01.bpo-35596.oFvhcm.rst b/Misc/NEWS.d/next/Windows/2019-01-08-13-56-01.bpo-35596.oFvhcm.rst
new file mode 100644
index 000000000000000..db4d8fa420d62a1
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-01-08-13-56-01.bpo-35596.oFvhcm.rst
@@ -0,0 +1,2 @@
+Use unchecked PYCs for the embeddable distro to avoid zipimport
+restrictions.
diff --git a/PC/layout/main.py b/PC/layout/main.py
index 7eaf201d532e368..d372fe50df3209a 100644
--- a/PC/layout/main.py
+++ b/PC/layout/main.py
@@ -240,12 +240,18 @@ def _c(d):
             yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat
 
 
-def _compile_one_py(src, dest, name, optimize):
+def _compile_one_py(src, dest, name, optimize, checked=True):
     import py_compile
 
     if dest is not None:
         dest = str(dest)
 
+    mode = (
+        py_compile.PycInvalidationMode.CHECKED_HASH
+        if checked
+        else py_compile.PycInvalidationMode.UNCHECKED_HASH
+    )
+
     try:
         return Path(
             py_compile.compile(
@@ -254,7 +260,7 @@ def _compile_one_py(src, dest, name, optimize):
                 str(name),
                 doraise=True,
                 optimize=optimize,
-                invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+                invalidation_mode=mode,
             )
         )
     except py_compile.PyCompileError:
@@ -262,16 +268,16 @@ def _compile_one_py(src, dest, name, optimize):
         return None
 
 
-def _py_temp_compile(src, ns, dest_dir=None):
+def _py_temp_compile(src, ns, dest_dir=None, checked=True):
     if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS:
         return None
 
     dest = (dest_dir or ns.temp) / (src.stem + ".py")
-    return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2)
+    return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2, checked=checked)
 
 
-def _write_to_zip(zf, dest, src, ns):
-    pyc = _py_temp_compile(src, ns)
+def _write_to_zip(zf, dest, src, ns, checked=True):
+    pyc = _py_temp_compile(src, ns, checked=checked)
     if pyc:
         try:
             zf.write(str(pyc), dest.with_suffix(".pyc"))
@@ -321,7 +327,7 @@ def generate_source_files(ns):
         ns.temp.mkdir(parents=True, exist_ok=True)
         with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
             for dest, src in get_lib_layout(ns):
-                _write_to_zip(zf, dest, src, ns)
+                _write_to_zip(zf, dest, src, ns, checked=False)
 
     if ns.include_underpth:
         log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp)

From eab1b318422957f03e26b5d042eb81ce82440ef3 Mon Sep 17 00:00:00 2001
From: Steve Dower <steve.dower@python.org>
Date: Tue, 8 Jan 2019 14:15:29 +1100
Subject: [PATCH 2/5] Fix CI steps

---
 .azure-pipelines/ci.yml | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index dda5197442b7b56..2b5f40affddeef5 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -134,4 +134,28 @@ jobs:
 
   steps:
   - template: ./windows-steps.yml
+
+- job: Windows_Layout_Tests
+  displayName: Windows Layout Tests
+  dependsOn: Windows_CI_Tests
+  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+  pool:
+    vmImage: vs2017-win2016
+
+  strategy:
+    matrix:
+      win32:
+        arch: win32
+        buildOpt:
+        testRunTitle: '$(Build.SourceBranchName)-win32'
+        testRunPlatform: win32
+      win64:
+        arch: amd64
+        buildOpt: '-p x64'
+        testRunTitle: '$(Build.SourceBranchName)-win64'
+        testRunPlatform: win64
+    maxParallel: 2
+
+  steps:
   - template: ./windows-layout-steps.yml

From 321409eb704f20b0d5af28a6a5586e36766871e0 Mon Sep 17 00:00:00 2001
From: Steve Dower <steve.dower@python.org>
Date: Tue, 8 Jan 2019 14:34:28 +1100
Subject: [PATCH 3/5] Remove unnecessary condition

---
 .azure-pipelines/ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index 2b5f40affddeef5..5aab1e1472b0608 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -138,7 +138,6 @@ jobs:
 - job: Windows_Layout_Tests
   displayName: Windows Layout Tests
   dependsOn: Windows_CI_Tests
-  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
     vmImage: vs2017-win2016

From bf2656c1ddbd158e28e4629fc5c5a6ca8b82a035 Mon Sep 17 00:00:00 2001
From: Steve Dower <steve.dower@python.org>
Date: Tue, 8 Jan 2019 15:07:01 +1100
Subject: [PATCH 4/5] Simplify test steps

---
 .azure-pipelines/ci.yml                   | 30 +++++-----------
 .azure-pipelines/windows-layout-steps.yml | 42 +++++------------------
 .azure-pipelines/windows-steps.yml        |  6 ----
 3 files changed, 17 insertions(+), 61 deletions(-)

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index 5aab1e1472b0608..78075bcfc1477c7 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -135,26 +135,12 @@ jobs:
   steps:
   - template: ./windows-steps.yml
 
-- job: Windows_Layout_Tests
-  displayName: Windows Layout Tests
-  dependsOn: Windows_CI_Tests
-
-  pool:
-    vmImage: vs2017-win2016
-
-  strategy:
-    matrix:
-      win32:
-        arch: win32
-        buildOpt:
-        testRunTitle: '$(Build.SourceBranchName)-win32'
-        testRunPlatform: win32
-      win64:
-        arch: amd64
-        buildOpt: '-p x64'
-        testRunTitle: '$(Build.SourceBranchName)-win64'
-        testRunPlatform: win64
-    maxParallel: 2
-
-  steps:
   - template: ./windows-layout-steps.yml
+    parameters:
+      kind: nuget
+  - template: ./windows-layout-steps.yml
+    parameters:
+      kind: embed
+  - template: ./windows-layout-steps.yml
+    parameters:
+      kind: appx
diff --git a/.azure-pipelines/windows-layout-steps.yml b/.azure-pipelines/windows-layout-steps.yml
index 5bc35ad85975cfd..f509b322186522c 100644
--- a/.azure-pipelines/windows-layout-steps.yml
+++ b/.azure-pipelines/windows-layout-steps.yml
@@ -1,35 +1,11 @@
-steps:
-- checkout: self
-  clean: true
-  fetchDepth: 5
-
-- task: DownloadPipelineArtifact@0
-  displayName: 'Download build artifacts'
-  inputs:
-    artifactName: windows-$(arch)
-    targetPath: $(Build.BinariesDirectory)\build
-
-- script: |
-    $py = ('python.exe', 'python_d.exe') | %{ Get-Item "${env:BUILD_BINARIESDIRECTORY}\$_" } | select -First 1
-    Write-Host '##vso[task.setVariable variable=PYTHON]"' + $py.FullName + '"'
-    Write-Host '##vso[task.setVariable variable=PYTHON_NAME]"' + $py.Name + '"'
-    Write-Host '##vso[task.setVariable variable=BUILD]"' + $py.Directory.FullName + '"'
-
-- script: $(PYTHON) PC\layout -vv -s "$(Build.SourcesDirectory)" -b $(BUILD) -t "$(Build.BinariesDirectory)\layout-tmp-embed-$(arch)" --copy "$(Build.BinariesDirectory)\layout-embed-$(arch)" --precompile --preset-embed --include-tests
+parameters:
+  kind: nuget
+  extraOpts: --precompile
 
-- script: $(PYTHON_NAME) -m test.pythoninfo
-  workingDirectory: $(Build.BinariesDirectory)\layout-embed-$(arch)
-  displayName: 'Embeddable layout info'
-
-- script: $(PYTHON) PC\layout -vv -s "$(Build.SourcesDirectory)" -b $(BUILD) -t "$(Build.BinariesDirectory)\layout-tmp-nuget-$(arch)" --copy "$(Build.BinariesDirectory)\layout-nuget-$(arch)" --preset-nuget --include-tests
-
-- script: $(PYTHON_NAME) -m test.pythoninfo
-  workingDirectory: $(Build.BinariesDirectory)\layout-nuget-$(arch)
-  displayName: 'Nuget layout info'
-
-- script: $(PYTHON) PC\layout -vv -s "$(Build.SourcesDirectory)" -b $(BUILD) -t "$(Build.BinariesDirectory)\layout-tmp-appx-$(arch)" --copy "$(Build.BinariesDirectory)\layout-appx-$(arch)" --precompile --preset-appx --include-tests
-  displayName: 'Create APPX layout'
+steps:
+- script: .\python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-${{ parameters['kind'] }}-$(arch)" --copy "$(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)" ${{ parameters['extraOpts'] }} --preset-${{ parameters['kind'] }} --include-tests
+  displayName: 'Create ${{ parameters['kind'] }} layout'
 
-- script: $(PYTHON_NAME) -m test.pythoninfo
-  workingDirectory: $(Build.BinariesDirectory)\layout-appx-$(arch)
-  displayName: 'APPX layout info'
+- script: .\python.exe -m test.pythoninfo
+  workingDirectory: $(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)
+  displayName: 'Show layout info (${{ parameters['kind'] }})'
diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml
index b46619d2e3c8b60..cba00158ad131dd 100644
--- a/.azure-pipelines/windows-steps.yml
+++ b/.azure-pipelines/windows-steps.yml
@@ -32,9 +32,3 @@ steps:
     testRunTitle: $(testRunTitle)
     platform: $(testRunPlatform)
   condition: succeededOrFailed()
-
-- task: PublishPipelineArtifact@0
-  displayName: 'Publish build artifacts'
-  inputs:
-    artifactName: windows-$(arch)
-    targetPath: $(Py_OutDir)\$(arch)

From a8fa1f7ea5d80d85eb4359b405d471a3190a26a2 Mon Sep 17 00:00:00 2001
From: Steve Dower <steve.dower@python.org>
Date: Tue, 8 Jan 2019 15:09:19 +1100
Subject: [PATCH 5/5] Fix YAML

---
 .azure-pipelines/windows-layout-steps.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.azure-pipelines/windows-layout-steps.yml b/.azure-pipelines/windows-layout-steps.yml
index f509b322186522c..62e5259375f5007 100644
--- a/.azure-pipelines/windows-layout-steps.yml
+++ b/.azure-pipelines/windows-layout-steps.yml
@@ -4,8 +4,8 @@ parameters:
 
 steps:
 - script: .\python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-${{ parameters['kind'] }}-$(arch)" --copy "$(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)" ${{ parameters['extraOpts'] }} --preset-${{ parameters['kind'] }} --include-tests
-  displayName: 'Create ${{ parameters['kind'] }} layout'
+  displayName: Create ${{ parameters['kind'] }} layout
 
 - script: .\python.exe -m test.pythoninfo
   workingDirectory: $(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)
-  displayName: 'Show layout info (${{ parameters['kind'] }})'
+  displayName: Show layout info (${{ parameters['kind'] }})