Merge pull request #250 from glennsarti/fix-pipe-reader

(MODULES-8748) Improve pipe reading in the PowerShell Manager

Author: Erick Banks

.gitignore

@@ -22,3 +22,7 @@
 /convert_report.txt
 /update_report.txt
 .DS_Store
+.project
+.vscode/
+.envrc
+/inventory.yaml

.pdkignore

@@ -22,6 +22,10 @@
 /convert_report.txt
 /update_report.txt
 .DS_Store
+.project
+.vscode/
+.envrc
+/inventory.yaml
 /appveyor.yml
 /.fixtures.yml
 /Gemfile
@@ -30,6 +34,7 @@
 /.gitlab-ci.yml
 /.pdkignore
 /Rakefile
+/rakelib/
 /.rspec
 /.rubocop.yml
 /.travis.yml

.project

@@ -27,5 +27,9 @@ spec/default_facts.yml:
 .travis.yml:
   unmanaged: true
 
+# Due to https://github.com/puppetlabs/pdk-templates/issues/229 we can't manage Appveyor yet.
+appveyor.yml:
+  unmanaged: true
+
 .gitlab-ci.yml:
   delete: true

.sync.yml

@@ -1,13 +1,19 @@
 ---
+os:
+  - linux
+  # OSX Only tests on the latest Puppet Gem, not the full matrix as there's no need to double up
+  # testing effort here. We are only concerned about whether the Mac OSX edition of PowerShell Core
+  # will work with our PowerShell manager code.
+  - osx
+
 dist: trusty
 language: ruby
 cache: bundler
 before_install:
   # Additional instructions
-  - curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
-  - curl https://packages.microsoft.com/config/ubuntu/14.04/prod.list | sudo tee /etc/apt/sources.list.d/microsoft.list
-  - sudo apt-get update
-  - sudo apt-get install -y powershell
+  - bash <(curl -s https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/install-powershell.sh) -skip-sudo-check
+  # Output the PowerShell Core version information
+  - pwsh -NoProfile -NoLogo -NonInteractive -Command \$PSVersionTable
   - if [ $BUNDLER_VERSION ]; then
       gem install -v $BUNDLER_VERSION bundler --no-rdoc --no-ri;
     fi
@@ -24,18 +30,17 @@ rvm:
 env:
   global:
     - BEAKER_PUPPET_COLLECTION=puppet6 PUPPET_GEM_VERSION="~> 6.0"
+    - CHECK=parallel_spec
 matrix:
   fast_finish: true
   include:
     -
       env: CHECK="syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop"
     -
-      env: CHECK=parallel_spec
-    -
-      env: PUPPET_GEM_VERSION="~> 5.0" CHECK=parallel_spec
+      env: PUPPET_GEM_VERSION="~> 5.0"
       rvm: 2.4.4
     -
-      env: PUPPET_GEM_VERSION="~> 4.0" CHECK=parallel_spec RUBYGEMS_VERSION=2.7.8 BUNDLER_VERSION=1.17.3
+      env: PUPPET_GEM_VERSION="~> 4.0" RUBYGEMS_VERSION=2.7.8 BUNDLER_VERSION=1.17.3
       rvm: 2.1.9
 branches:
   only:

.travis.yml

@@ -22,7 +22,7 @@ end
 
 def changelog_future_release
   return unless Rake.application.top_level_tasks.include? "changelog"
-  returnVal = JSON.load(File.read('metadata.json'))['version']
+  returnVal = "v%s" % JSON.load(File.read('metadata.json'))['version']
   raise "unable to find the future_release (version) in metadata.json" if returnVal.nil?
   puts "GitHubChangelogGenerator future_release:#{returnVal}"
   returnVal

Rakefile

@@ -39,6 +39,11 @@ install:
   - set PATH=C:\Ruby%RUBY_VERSION%\bin;%PATH%
   - bundle install --jobs 4 --retry 2 --without system_tests
   - type Gemfile.lock
+  - ps: "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; Invoke-WebRequest -Uri https://github.com/PowerShell/PowerShell/raw/master/tools/install-powershell.ps1 -UseBasicParsing -OutFile install-pwsh.ps1"
+  - ps: "& ./install-pwsh.ps1"
+  - set PATH=%LOCALAPPDATA%\Microsoft\powershell;%PATH%
+  - pwsh -NoProfile -NoLogo -NonInteractive -Command $PSVersionTable
+  - powershell -NoProfile -NoLogo -NonInteractive -Command $PSVersionTable
 build: off
 test_script:
   - bundle exec puppet -V

appveyor.yml

@@ -274,7 +274,7 @@ def self.instance_key(cmd, args, options)
       def self.is_readable?(stream, timeout = 0.5)
         raise Errno::EPIPE if !is_stream_valid?(stream)
         read_ready = IO.select([stream], [], [], timeout)
-        read_ready && stream == read_ready[0][0]
+        read_ready && stream == read_ready[0][0] && !stream.eof?
       end
 
       # when a stream has been closed by handle, but Ruby still has a file
@@ -395,17 +395,16 @@ def read_streams
           # read a Little Endian 32-bit integer for length of response
           expected_response_length = pipe.sysread(4).unpack('V').first
 
-          if expected_response_length == 0
-            nil
-          else
-            # reads the expected bytes as a binary string or fails
-            buffer = ""
-            # Reads in the pipe data 8K, or less, at a time
-            while (buffer.length < expected_response_length)
-              buffer << pipe.sysread([expected_response_length - buffer.length, 8192].min)
-            end
-            buffer
+          next nil if expected_response_length == 0
+          # reads the expected bytes as a binary string or fails
+          buffer = ""
+          # sysread may not return all of the requested bytes due to buffering or the
+          # underlying IO system. Keep reading from the pipe until all the bytes are read
+          loop do
+            buffer.concat(pipe.sysread(expected_response_length - buffer.length))
+            break if buffer.length >= expected_response_length
           end
+          buffer
         end
 
         Puppet.debug "Waited #{Time.now - start_time} total seconds."

lib/puppet_x/puppetlabs/powershell/powershell_manager.rb

@@ -723,10 +723,23 @@ function ConvertTo-PipeCommand
 
   # Read blocks until all bytes are read or EOF / broken pipe hit - tested with 5MB and worked fine
   $parsed.RawData = New-Object Byte[] $parsed.Length
-  $read = $Stream.Read($parsed.RawData, 0, $parsed.Length)
-  if ($read -lt $parsed.Length)
+  $readBytes = 0
+  do {
+    $attempt = $attempt + 1
+    # This will block if there's not enough data in the pipe
+    $read = $Stream.Read($parsed.RawData, $readBytes, $parsed.Length - $readBytes)
+    if ($read -eq 0)
+    {
+      throw "Catastrophic failure: Expected $($parsed.Length - $readBytesh) raw bytes, but the pipe reached an end of stream"
+    }
+
+    $readBytes = $readBytes + $read
+    Write-SystemDebugMessage -Message "Read $($read) bytes from the pipe"
+  } while ($readBytes -lt $parsed.Length)
+
+  if ($readBytes -lt $parsed.Length)
   {
-    throw "Catastrophic failure: Expected $($parsed.Length) raw bytes, only received $read"
+    throw "Catastrophic failure: Expected $($parsed.Length) raw bytes, only received $readBytes"
   }
 
   # turn the raw bytes into the expected encoded string!

lib/puppet_x/templates/powershell/init_ps.ps1

@@ -76,5 +76,5 @@
   ],
   "pdk-version": "1.8.0",
   "template-url": "https://github.com/puppetlabs/pdk-templates",
-  "template-ref": "heads/master-0-g7281db5"
-}
+  "template-ref": "heads/master-0-ga6554ab"
+}