Replaced run-csharp with unified build system

builder/csharp_builder.rb

@@ -0,0 +1,93 @@
+require 'fileutils'
+require 'set'
+
+require_relative 'partial_builder'
+require_relative 'shellconfig'
+
+class CSharpBuilder < PartialBuilder
+  def initialize
+    super
+
+    @mode = :make_posix
+    @mode = :msbuild_windows if ENV['APPVEYOR']
+
+    @spec_dir = 'spec/csharp/kaitai_struct_csharp_tests'
+    @compiled_dir = 'compiled/csharp'
+    @project_file = "#{@spec_dir}/kaitai_struct_csharp_tests.csproj"
+    @project_template = "#{@spec_dir}/kaitai_struct_csharp_tests.csproj.in"
+
+    @test_out_dir = "#{@config['TEST_OUT_DIR']}/csharp"
+
+    detect_tools
+  end
+
+  def detect_tools
+    # msbuild
+    if system("msbuild /version")
+      @msbuild = 'msbuild'
+    elsif system("xbuild /version")
+      @msbuild = 'xbuild'
+    else
+      raise 'Unable to find msbuild/xbuild, bailing out'
+    end
+
+    # If we're running in AppVeyor, add extra logger args
+    @msbuild_args = []
+    if ENV['APPVEYOR']
+      @msbuild_args << '/logger:C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll'
+    end
+  end
+
+  def list_mandatory_files
+    convert_slashes([
+      'Properties/AssemblyInfo.cs',
+      'CommonSpec.cs',
+    ])
+  end
+
+  def list_disposable_files
+    orig_pwd = Dir.pwd
+
+    rel_files = Dir.glob("#{@spec_dir}/tests/**/*.cs") + Dir.glob("#{@compiled_dir}/**/*.cs")
+    abs_files = rel_files.map { |fn| "#{orig_pwd}/#{fn}" }
+    convert_slashes(abs_files)
+  end
+
+  def create_project(mand_files, disp_files)
+    tmpl = File.read(@project_template)
+    files_xml = (mand_files + disp_files).map { |x| "    <Compile Include=\"#{x}\" />" }.join("\n")
+    project = tmpl.gsub(/%%%FILES%%%/, files_xml)
+    File.write(@project_file, project)
+  end
+
+  def build_project(log_file)
+    cli = [@msbuild] + @msbuild_args + ["#{@spec_dir}/kaitai_struct_csharp_tests.csproj"]
+    puts "run-csharp: running msbuild: #{cli.inspect}"
+    run_and_tee({}, cli, log_file).exitstatus
+  end
+
+  def parse_failed_build(log_file)
+    list = Set.new
+
+    File.open(log_file, 'r') { |f|
+      f.each_line { |l|
+        l.chomp!
+        if l =~ /^\s*(.*?)\((\d+),(\d+)\): error (.*?): (.*)$/
+          filename = $1
+          #row = $2
+          #col = $3
+          #code = $4
+          #msg = $5
+          list << filename
+        end
+      }
+    }
+
+    convert_slashes(list)
+  end
+
+  private
+  def convert_slashes(list)
+    list.sort.map { |f| f.gsub(/\//, '\\') }
+  end
+end

builder/partial_builder.rb

@@ -0,0 +1,127 @@
+require 'set'
+require 'open3'
+
+require_relative 'shellconfig'
+
+##
+# Common base for all compiled languages, which follow the same
+# routine to achieve "partial compilation":
+#
+# 1. Get a full list of files to compile:
+#   * list of "disposable" files, removal of which from compilation
+#     might get a partial successful build (compiled/* and specs/*)
+#   * list of "mandatory" files (typically, runtime and test unit
+#     framework), which cannot be removed at all
+# 2. Construct a project file with full list of files
+# 3. Build that project file
+# 4. If successful, we're done, woo-hoo
+# 5. If failed:
+#   * Parse project build to get list of files to remove from project
+#     in attempt to get it to compile
+#   * Build updated project file with these files removed
+#   * Retry from step 3
+class PartialBuilder
+  def initialize
+    @config = ShellConfig.new
+    log 'initialized'
+  end
+
+  def command_line(arg)
+    if arg == []
+      run
+    elsif arg == ['--project']
+      create_project(list_mandatory_files, list_disposable_files)
+    else
+      puts "Usage: [--project]"
+      exit 1
+    end
+  end
+
+  def run
+    raise "test_out_dir is undefined" unless @test_out_dir
+
+    mand_files = Set.new(list_mandatory_files)
+    disp_files = Set.new(list_disposable_files)
+
+    attempt = 1
+    loop {
+      log "create project with #{disp_files.size} files"
+      create_project(mand_files, disp_files)
+
+      build_log = "#{@test_out_dir}/build-#{attempt}.log"
+      log "build attempt #{attempt} (log: #{build_log})"
+      if build_project(build_log) == 0
+        log "success"
+        return true
+      else
+        log "build failed"
+        bad_files = Set.new(parse_failed_build(build_log))
+        if bad_files.empty?
+          log "build fails, but unable to detect any bad files"
+          log "unable to recover, bailing out :("
+          return false
+        end
+        mand_int = mand_files.intersection(bad_files)
+        if not mand_int.empty?
+          log "errors detected in mandatory files:"
+          log mand_int.sort.to_a.join("\n")
+          log "unable to recover, bailing out :("
+          return false
+        end
+
+        # Test if all "bad files" are actually in disposable files
+        leftover = bad_files - disp_files
+        if not leftover.empty?
+          log "errors detected in bogus files:"
+          log leftover.sort.to_a.join("\n")
+          log "unable to recover, bailing out :("
+          return false
+        end
+
+        disp_files -= bad_files
+        attempt += 1
+      end
+    }
+  end
+
+  def list_mandatory_files
+    raise NotImplementedError
+  end
+
+  def list_disposable_files
+    raise NotImplementedError
+  end
+
+  def create_project(disp_files, mand_files)
+    raise NotImplementedError
+  end
+
+  def build_project(log_file)
+    raise NotImplementedError
+  end
+
+  def parse_failed_build(log_file)
+    raise NotImplementedError
+  end
+
+  # ====================================================================
+
+  def log(msg)
+    puts "#### #{self.class}: #{msg}"
+  end
+
+  def run_and_tee(environment, cmd, stdout_file)
+    process_status = nil
+    File.open(stdout_file, 'w') { |f|
+      Open3.popen2e(environment, *cmd) { |stdin, out, wait_thr|
+        while line = out.gets
+          puts line
+          f.puts line
+        end
+        process_status = wait_thr.value
+      }
+    }
+    log "process_status: #{process_status.inspect}"
+    return process_status
+  end
+end

builder/shellconfig.rb

@@ -0,0 +1,19 @@
+class ShellConfig
+  def initialize(filename = 'config')
+    @entries = {}
+    File.open(filename, 'r') { |f|
+      f.each_line { |l|
+        l.chomp!
+        l.strip!
+        l.gsub!(/#.*$/, '')
+        if l =~ /^([A-Za-z0-9_]+)=(.*?)$/
+          @entries[$1] = $2
+        end
+      }
+    }
+  end
+
+  def [](key)
+    @entries[key]
+  end
+end

builder/specs/csharp/1/config

@@ -0,0 +1,16 @@
+COMPILER_DIR=../compiler
+FORMATS_KSY_DIR=formats
+FORMATS_COMPILED_DIR=compiled
+FORMATS_REPO_DIR=../formats
+
+CSHARP_RUNTIME_DIR=../runtime/csharp
+JAVA_RUNTIME_DIR=../runtime/java
+JAVA_TESTNG_JAR=$HOME/.m2/repository/org/testng/testng/6.9.10/testng-6.9.10.jar:$HOME/.m2/repository/com/beust/jcommander/1.48/jcommander-1.48.jar
+JAVASCRIPT_RUNTIME_DIR=../runtime/javascript
+JAVASCRIPT_MODULES_DIR=node_modules
+LUA_RUNTIME_DIR=../runtime/lua
+PERL_RUNTIME_DIR=../runtime/perl/lib
+PYTHON_RUNTIME_DIR=../runtime/python
+RUBY_RUNTIME_DIR=../runtime/ruby/lib
+
+TEST_OUT_DIR=test_out

builder/specs/csharp/csharp_builder_spec.rb

@@ -0,0 +1,22 @@
+require_relative '../../csharp_builder'
+
+$spec_dir = File.dirname(__FILE__)
+
+RSpec.describe CSharpBuilder do
+  context '1' do
+    Dir.chdir("#{$spec_dir}/1")
+    r = CSharpBuilder.new
+
+    describe '#list_mandatory_files' do
+      it 'shows no mandatory files' do
+        expect(r.list_mandatory_files.to_a.sort).to eq ["CommonSpec.cs", "Properties\\AssemblyInfo.cs"]
+      end
+    end
+
+    describe '#parse_failed_build' do
+      it 'parses failed build information' do
+        expect(r.parse_failed_build('test_out/csharp/build-1.log').to_a.sort).to eq ['/home/greycat/git/kaitai_struct/tests/compiled/cpp_stl_11/io_local_var.cpp']
+      end
+    end
+  end
+end