ssh notation support

lib/terraspace_bundler.rb

@@ -1,10 +1,12 @@
 $:.unshift(File.expand_path("../", __FILE__))
+require "active_support"
 require "active_support/core_ext/class"
 require "active_support/core_ext/hash"
 require "active_support/core_ext/string"
 require "active_support/ordered_options"
 require "dsl_evaluator"
 require "fileutils"
+require "json"
 require "memoist"
 require "rainbow/ext/string"
 require "singleton"

lib/terraspace_bundler/dsl/syntax.rb

@@ -1,7 +1,7 @@
 class TerraspaceBundler::Dsl
   module Syntax
-    def org(url)
-      config.org = url
+    def org(value)
+      config.org = value
     end
     alias_method :user, :org
 

lib/terraspace_bundler/mod.rb

@@ -1,7 +1,9 @@
 module TerraspaceBundler
   class Mod
+    extend Memoist
     extend Props::Extension
     props :export_to, :name, :sha, :source, :subfolder, :type, :url, :clone_with
+    delegate :repo, :org, :repo_folder, :org_folder, to: :org_repo
 
     include Concerns::StackConcern
     include Concerns::LocalConcern
@@ -18,22 +20,6 @@ def checkout_version
       @version || @ref || @tag || @branch
     end
 
-    # use url instead of source because for registry modules, the repo name is different
-    def repo
-      url_words[-1].sub(/\.git$/,'')
-    end
-
-    # https://github.com/tongueroo/pet - 2nd to last word
-    # git@github.com:tongueroo/pet - 2nd to last word without chars before :
-    def org
-      s = url_words[-2] # second to last word
-      s.split(':').last # in case of git@github.com:tongueroo/pet form
-    end
-
-    def full_repo
-      "#{org}/#{repo}"
-    end
-
     def latest_sha
       fetcher = Fetcher.new(self).instance
       fetcher.run
@@ -44,15 +30,18 @@ def vcs_provider
       if url.include?('http')
         # "https://github.com/org/repo"  => github.com
         url.match(%r{http[s]?://(.*?)/})[1]
-      else # git@
+      elsif url.include?('http') # git@
         # "git@github.com:org/repo"      => github.com
         url.match(%r{git@(.*?):})[1]
+      else # ssh://user@domain.com/path/to/repo
+        'none'
       end
     end
 
   private
-    def url_words
-      url.split('/')
+    def org_repo
+      OrgRepo.new(url)
     end
+    memoize :org_repo
   end
 end

lib/terraspace_bundler/mod/concerns/notation_concern.rb

@@ -11,7 +11,8 @@ def remove_ref_notation(source)
     end
 
     def remove_subfolder_notation(source)
-      parts = clean_notation(source).split('//')
+      clean = clean_notation(source)
+      parts = clean.split('//')
       if parts.size == 2 # has subfolder
         source.split('//')[0..-2].join('//') # remove only subfolder, keep rest of original source
       else
@@ -20,7 +21,8 @@ def remove_subfolder_notation(source)
     end
 
     def subfolder(source)
-      parts = clean_notation(source).split('//')
+      clean = clean_notation(source)
+      parts = clean.split('//')
       if parts.size == 2 # has subfolder
         remove_ref_notation(parts.last)
       end
@@ -35,8 +37,23 @@ def ref(source)
       end
     end
 
+
     def clean_notation(source)
-      source.sub(/.*::/,'').sub(%r{http[s?]://},'').sub(%r{git@(.*?):},'') # also remove git@ notation
+      clean = source
+        .sub(/.*::/,'')            # remove git::
+        .sub(%r{http[s?]://},'')   # remove https://
+        .sub(%r{git@(.*?):},'')    # remove git@word
+      remove_host(clean)
+    end
+
+    def remove_host(clean)
+      return clean unless clean.include?('ssh://')
+      if clean.count(':') == 1
+        uri = URI(clean)
+        uri.path
+      else
+        clean.split(':').last
+      end
     end
   end
 end

lib/terraspace_bundler/mod/concerns/path_concern.rb

@@ -18,11 +18,11 @@ def stage_root
     end
 
     def cache_path(name)
-      [cache_root, parent_stage_folder, name].compact.join('/')
+      [cache_root, @mod.type, name].compact.join('/')
     end
 
     def stage_path(name)
-      [stage_root, parent_stage_folder, name].compact.join('/')
+      [stage_root, @mod.type, name].compact.join('/')
     end
 
     # Fetcher: Downloader/Local copies to a slightly different folder.
@@ -41,21 +41,8 @@ def rel_dest_dir
       when 'http'
         path = type_path # https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH/TO/module.zip
         remove_ext(path) # terraform-example-modules/modules/example-module
-      when -> (_) { @mod.source.include?('git::') }
-        @mod.name      # example-module
-      else # inferred git, registry
-        @mod.full_repo #  tongueroo/example-module
-      end
-    end
-
-    def parent_stage_folder
-      case @mod.type
-      when 'local'
-        'local'
-      when 'http'
-        'http'
-      else # gcs, s3, git, registry
-        @mod.vcs_provider
+      else # inferred git, registry, git::, ssh://, git::ssh://
+        @mod.repo_folder #  tongueroo/example-module
       end
     end
 

lib/terraspace_bundler/mod/fetcher/git.rb

@@ -4,11 +4,11 @@ class Git < Base
 
     def run
       setup_tmp
-      org_path = cache_path(@mod.org)
-      FileUtils.mkdir_p(org_path)
+      org_folder = cache_path(@mod.org_folder)
+      FileUtils.mkdir_p(org_folder)
 
-      Dir.chdir(org_path) do
-        logger.debug "Current root dir: #{org_path}"
+      Dir.chdir(org_folder) do
+        logger.debug "Current root dir: #{org_folder}"
         clone unless File.exist?(@mod.repo)
 
         Dir.chdir(@mod.repo) do
@@ -45,12 +45,13 @@ def default_branch
       if found
         found.split(':').last.strip
       else
-        'master'
+        ENV['TS_GIT_DEFAULT_BRANCH'] || 'master'
       end
     end
 
     def switch_version(version)
       stage_path = stage_path(rel_dest_dir)
+      logger.debug "stage_path #{stage_path}"
       Dir.chdir(stage_path) do
         git "checkout #{version}"
         @sha = git("rev-parse HEAD").strip

lib/terraspace_bundler/mod/org_repo.rb

@@ -0,0 +1,65 @@
+class TerraspaceBundler::Mod
+  class OrgRepo
+    def initialize(url)
+      @url = url.sub('ssh://','') # important to copy so dont change the string reference
+    end
+
+    def repo
+      org_repo_words[-1]
+    end
+
+    def org
+      s = org_folder.split('/').last
+      s ? s.split('/').last : 'none'
+    end
+
+    def org_folder
+      org_repo_words[-2] # second to last word
+    end
+
+    def repo_folder
+      org_repo_words.join('/')
+    end
+
+    def org_repo_words
+      if @url.include?(':') && !@url.match(%r{http[s?]://}) # user@host:repo git@github.com:org/repo
+        folder, repo = handle_string_with_colon
+      else # IE: https://github.com/org/repo, org/repo, etc
+        parts = @url.split('/')
+        folder = parts[0..-2].join('/')
+        repo = parts.last
+      end
+
+      org_path = clean_folder(folder)
+      repo = strip_dot_git(repo)
+      [org_path, repo]
+    end
+
+    def clean_folder(folder)
+      folder.sub(%r{.*@},'')           # remove user@
+            .sub(%r{http[s?]://},'')   # remove https://
+    end
+
+    # user@host:repo git@github.com:org/repo
+    def handle_string_with_colon
+      host, path = @url.split(':')
+      if path.size == 2
+        folder, repo = path.split(':')
+      else
+        folder = join(host, File.dirname(path))
+        repo = File.basename(path)
+      end
+      [folder, repo]
+    end
+
+    def join(*path)
+      path.compact!
+      path[1] = path[1].sub('/','') if path[1].starts_with?('/')
+      path.reject(&:blank?).join('/')
+    end
+
+    def strip_dot_git(s)
+      s.sub(/\.git$/,'')
+    end
+  end
+end

lib/terraspace_bundler/mod/props.rb

@@ -66,6 +66,20 @@ def http_source_url
     end
 
     # git_source_url is normalized
+    #
+    # See: https://stackoverflow.com/questions/6167905/git-clone-through-ssh
+    #
+    #    ssh://username@host.xz/absolute/path/to/repo.git/ - just a forward slash for absolute path on server
+    #    username@host.xz:relative/path/to/repo.git/ - just a colon (it mustn't have the ssh:// for relative path on server (relative to home dir of username on server machine)
+    #
+    # When colon is separator for relative path do not want ssh prepended
+    #
+    #   git clone ec2-user@localhost:repo
+    #
+    # When slash is separator for absolute path want ssh prepended
+    #
+    #   git clone ssh://ec2-user@localhost/path/to/repo => valid URI
+    #
     def git_source_url
       if @source.include?('http') || @source.include?(':')
         # Examples:
@@ -76,7 +90,15 @@ def git_source_url
         #   mod "example3", source: "git::https://example.com/example-module.git"
         #
         # sub to allow for generic git repo notiation
-        @source.sub('git::','')
+        source = @source.sub('git::','')
+        if source.include?('ssh://')
+          if source.count(':') == 1
+            return source
+          else
+            return source.sub('ssh://', '')
+          end
+        end
+        source
       else
         # Examples:
         #   mod "pet", source: "tongueroo/pet"

lib/terraspace_bundler/mod/props/typer.rb

@@ -12,7 +12,9 @@ def initialize(props)
 
     # IE: git or registry
     def type
-      if source.include?('::')
+      if source.include?('ssh://')
+        "git"
+      elsif source.include?('::')
         source.split('::').first # IE: git:: s3:: gcs::
       elsif local?
         "local"

readme/qa/Terrafile

@@ -0,0 +1,39 @@
+# Here's an example Terrafile that can be used to qa the different types of sources.
+# Based on: https://terraspace.cloud/docs/terrafile/sources/
+# Note: Actually need to make sure that have access to the sources, IE: s3, gcs, github, etc
+# This is still useful as an example to help with QA.
+
+# Use modules from the Terraform registry
+mod "sqs", source: "terraform-aws-modules/sqs/aws"
+mod "s3", source: "git@github.com:boltops-tools/terraform-aws-s3"
+mod "s3test", source: "boltops-tools/terraform-aws-s3"
+
+mod "test1", source: "git::https://github.com/boltops-tools/terraform-aws-s3"
+mod "test2", source: "git::ssh://ec2-user@localhost:/home/ec2-user/environment/repo"
+mod "test3", source: "git::ssh://ec2-user@localhost:environment/repo"
+mod "test4", source: "git::ssh://localhost:environment/repo"
+mod "test5", source: "ssh://ec2-user@localhost/home/ec2-user/environment/repo"
+mod "test6", source: "ssh://ec2-user@localhost:/home/ec2-user/environment/repo"
+mod "test7", source: "ssh://ec2-user@localhost:/home/ec2-user/environment/repo//subfolder"
+mod "test8", source: "ssh://ec2-user@localhost:environment/repo"
+mod "test9", source: "ssh://ec2-user@localhost:repo"
+
+mod "pet1", source: "tongueroo/pet"
+mod "pet2", source: "https://github.com/tongueroo/pet"
+mod "example1", source: "git@bitbucket.org:tongueroo/example-module.git"
+mod "example2", source: "git@gitlab.com:tongueroo/example-module"
+
+org "tongueroo"
+mod "pet3", source: "pet" # inferred org
+
+mod "sg", source: "terraform-aws-modules/security-group/aws" # terraform registry public example
+mod "pet4", source: "app.terraform.io/boltops/pet/random" # , clone_with: "https"
+
+mod "demo1", source: "s3::https://s3-us-west-2.amazonaws.com/demo-terraform-test/modules/example-module.tgz"
+mod "demo2", source: "s3::https://s3-us-west-2.amazonaws.com/demo-terraform-test/modules/example-module.zip"
+mod "demo3", source: "gcs::https://www.googleapis.com/storage/v1/terraform-example-modules/modules/example-module.zip//subfolder"
+mod "demo4", source: "gcs::https://www.googleapis.com/storage/v1/terraform-example-modules/modules/example-module.tgz//subfolder"
+
+mod "local1", source: "/home/ec2-user/environment/downloads/example-module"
+mod "local2", source: "./example-module"
+mod "local3", source: "~/environment/infra-terrafile/example-module"

spec/terraform_bundler/mod/concerns/notation_concern_spec.rb

@@ -0,0 +1,35 @@
+class NotationTest
+  include TB::Mod::Concerns::NotationConcern
+end
+
+describe NotationTest do
+  let(:notation) { described_class.new }
+
+  context "clean_notation" do
+    it "cleans" do
+      source = "git::ssh://ec2-user@localhost:/home/ec2-user/environment/repo"
+      cleaned = notation.clean_notation(source)
+      expect(cleaned).to eq "/home/ec2-user/environment/repo"
+
+      source = "ssh://ec2-user@localhost:/home/ec2-user/environment/repo"
+      cleaned = notation.clean_notation(source)
+      expect(cleaned).to eq "/home/ec2-user/environment/repo"
+
+      source = "ssh://ec2-user@localhost:/home/ec2-user/environment/repo//subfolder"
+      cleaned = notation.clean_notation(source)
+      expect(cleaned).to eq "/home/ec2-user/environment/repo//subfolder"
+
+      source = "git::ssh://ec2-user@localhost:environment/repo"
+      cleaned = notation.clean_notation(source)
+      expect(cleaned).to eq "environment/repo"
+
+      source = "ssh://ec2-user@localhost:~/environment/repo"
+      cleaned = notation.clean_notation(source)
+      expect(cleaned).to eq "~/environment/repo"
+
+      source = "git::ssh://localhost:environment/repo"
+      cleaned = notation.clean_notation(source)
+      expect(cleaned).to eq "environment/repo"
+    end
+  end
+end