refactor!(loader): download git repo archive instead of clone & pull

Dockerfile.test

@@ -7,7 +7,9 @@ ARG PLACE_COMMIT="DEV"
 
 # - Add trusted CAs for communicating with external services
 # - Add watchexec for running tests on change
-RUN apk add --no-cache \
+RUN apk upgrade --no-cache apk \
+    && \
+    apk add --no-cache \
         bash \
         ca-certificates \
     && \

README.md

@@ -53,3 +53,4 @@ See [`CONTRIBUTING.md`](./CONTRIBUTING.md).
 ## Contributors
 
 - [Caspian Baska](https://github.com/caspiano) - creator and maintainer
+- [Tassja Kriek](https://github.com/tassja) - contributor and maintainer

shard.lock

@@ -29,6 +29,10 @@ shards:
     git: https://github.com/maiha/crc16.cr.git
     version: 0.1.0
 
+  crest:
+    git: https://github.com/mamantoha/crest.git
+    version: 1.2.1
+
   cron_parser:
     git: https://github.com/kostya/cron_parser.git
     version: 0.4.0
@@ -37,6 +41,10 @@ shards:
     git: https://github.com/spider-gazelle/crunits.git
     version: 1.1.0
 
+  crystar:
+    git: https://github.com/naqvis/crystar.git
+    version: 0.2.0
+
   db:
     git: https://github.com/crystal-lang/crystal-db.git
     version: 0.11.0
@@ -65,18 +73,38 @@ shards:
     git: https://github.com/crystal-community/future.cr.git
     version: 1.0.0
 
+  gitlab:
+    git: https://github.com/icyleaf/gitlab.cr.git
+    version: 0.7.0
+
   habitat:
     git: https://github.com/luckyframework/habitat.git
     version: 0.4.7
 
+  halite:
+    git: https://github.com/icyleaf/halite.git
+    version: 0.12.0
+
+  hash_file:
+    git: https://github.com/tassja/hash_file.git
+    version: 0.4
+
   hound-dog:
     git: https://github.com/place-labs/hound-dog.git
     version: 2.9.0
 
+  http-client-digest_auth:
+    git: https://github.com/mamantoha/http-client-digest_auth.git
+    version: 0.6.0
+
   http-params-serializable:
     git: https://github.com/place-labs/http-params-serializable.git
     version: 0.5.0
 
+  http_proxy:
+    git: https://github.com/mamantoha/http_proxy.git
+    version: 0.9.0
+
   ipaddress:
     git: https://github.com/sija/ipaddress.cr.git
     version: 0.2.1
@@ -105,6 +133,10 @@ shards:
     git: https://github.com/spider-gazelle/neuroplastic.git
     version: 1.11.0
 
+  octokit:
+    git: https://github.com/place-labs/octokit.cr.git
+    version: 0.1.6
+
   open_api:
     git: https://github.com/elbywan/open_api.cr.git
     version: 1.3.0

shard.yml

@@ -4,6 +4,7 @@ crystal: ">= 1.1.1"
 license: MIT
 authors:
   - Caspian Baska <caspian@place.tech>
+  - Tassja Kriek <tassja@place.tech>
 
 targets:
   frontends:
@@ -14,6 +15,22 @@ dependencies:
     github: spider-gazelle/action-controller
     version: ~> 4.4
 
+  crest:
+    github: mamantoha/crest
+
+  crystar:
+    github: naqvis/crystar
+
+  gitlab:
+    github: icyleaf/gitlab.cr
+
+  hash_file:
+    github: tassja/hash_file
+
+  octokit:
+    github: place-labs/octokit.cr
+    version: ~> 0.1.2
+
   placeos-compiler:
     github: placeos/compiler
     version: ~> 4.8
@@ -35,7 +52,6 @@ dependencies:
 
   tasker:
     github: spider-gazelle/tasker
-
 development_dependencies:
   ameba:
     github: crystal-ameba/ameba

spec/api/remotes_spec.cr

@@ -0,0 +1,36 @@
+require "../helper"
+
+module PlaceOS::FrontendLoader::Api
+  describe Remotes do
+    with_server do
+      remotes_base = "/api/frontend-loader/v1/remotes"
+      it "lists releases for a given repository" do
+        encoded_url = URI.encode_www_form("https://www.github.com/PlaceOS/frontend-loader")
+        route = "#{remotes_base}/#{encoded_url}/releases"
+        result = curl("GET", route)
+        Array(String).from_json(result.body).includes?("v0.11.2").should be_true
+      end
+
+      it "lists commits for a given repository" do
+        encoded_url = URI.encode_www_form("https://www.github.com/PlaceOS/frontend-loader")
+        route = "#{remotes_base}/#{encoded_url}/commits"
+        result = curl("GET", route)
+        Array(PlaceOS::FrontendLoader::Remote::Commit).from_json(result.body).should_not be_empty
+      end
+
+      it "lists branches for a given repository" do
+        encoded_url = URI.encode_www_form("https://www.github.com/PlaceOS/frontend-loader")
+        route = "#{remotes_base}/#{encoded_url}/branches"
+        result = curl("GET", route)
+        Array(String).from_json(result.body).includes?("master").should be_true
+      end
+
+      it "lists tags for a given repository" do
+        encoded_url = URI.encode_www_form("https://www.github.com/PlaceOS/frontend-loader")
+        route = "#{remotes_base}/#{encoded_url}/tags"
+        result = curl("GET", route)
+        Array(String).from_json(result.body).includes?("v1.3.0").should be_true
+      end
+    end
+  end
+end

spec/api/repositories_spec.cr

@@ -10,6 +10,16 @@ module PlaceOS::FrontendLoader::Api
       commits.should_not be_empty
     end
 
+    it "gets the default branch if not master" do
+      repository = example_repository(TEST_FOLDER, uri: "https://www.github.com/placeos/backoffice")
+
+      loader = Loader.new
+      loader.process_resource(:created, repository).success?.should be_true
+      branch = Api::Repositories.default_branch(repository.folder_name, loader: loader)
+      commits = Api::Repositories.commits(repository.folder_name, branch, loader: loader).not_nil!
+      commits.should_not be_empty
+    end
+
     it "lists branches for a loaded repository" do
       repository = example_repository(TEST_FOLDER)
       loader = Loader.new
@@ -30,6 +40,14 @@ module PlaceOS::FrontendLoader::Api
       loaded[repository.folder_name].should_not eq("HEAD")
     end
 
+    it "lists releases for a loaded repository" do
+      repository = example_repository(TEST_FOLDER)
+      loader = Loader.new
+      loader.process_resource(:created, repository).success?.should be_true
+      releases = Api::Repositories.releases(repository.folder_name, loader: loader).not_nil!
+      releases.should_not be_empty
+    end
+
     describe "query" do
       it "does not mutate the managed repositories" do
         branch = "test-fixture"
@@ -43,12 +61,11 @@ module PlaceOS::FrontendLoader::Api
         expected_path = File.join(loader.content_directory, folder)
 
         Dir.exists?(expected_path).should be_true
-        Compiler::Git.current_repository_commit(folder, loader.content_directory).should eq checked_out_commit
+        Api::Repositories.current_commit(expected_path).should eq checked_out_commit
         Api::Repositories.branches(folder, loader).not_nil!.should_not be_empty
         Api::Repositories.commits(folder, branch, loader: loader).not_nil!.should_not be_empty
         Api::Repositories.commits(folder, "master", loader: loader).not_nil!.should_not be_empty
-
-        Compiler::Git.current_repository_commit(folder, loader.content_directory).should eq checked_out_commit
+        Api::Repositories.current_commit(expected_path).should eq checked_out_commit
       end
     end
   end

spec/gitlab_spec.cr

@@ -0,0 +1,19 @@
+require "./helper"
+
+module PlaceOS::FrontendLoader
+  LAB_TEST_FOLDER = "test-gitlab"
+  describe GitLab do
+    before_each do
+      reset
+    end
+    repository = example_repository(LAB_TEST_FOLDER, uri: "https://gitlab.com/bdowney/ansible-demo/")
+    expected_path = File.join(TEST_DIR, repository.folder_name)
+
+    it "downloads a GitLab archive" do
+      ref = PlaceOS::FrontendLoader::Remote::Reference.new(repository.uri, branch: "master")
+      actioner = PlaceOS::FrontendLoader::GitLab.new
+      actioner.download(ref: ref, path: expected_path)
+      Dir.exists?(File.join(expected_path, "decks")).should be_true
+    end
+  end
+end

spec/helper.cr

@@ -26,7 +26,7 @@ TEST_FOLDER = "test-repo"
 
 def example_repository(
   folder_name : String = UUID.random.to_s[0..8],
-  uri : String = "https://github.com/placeos/compiler",
+  uri : String = "https://www.github.com/placeos/compiler",
   commit : String = "HEAD",
   branch : String = "master"
 )
@@ -35,6 +35,7 @@ def example_repository(
     existing.uri = uri unless existing.uri == uri
     existing.branch = branch unless existing.branch == branch
     existing.commit_hash = commit unless existing.commit_hash == commit
+    existing.save!
     existing
   else
     PlaceOS::Model::Generator.repository(type: :interface).tap do |repository|
@@ -43,6 +44,6 @@ def example_repository(
       repository.folder_name = folder_name
       repository.commit_hash = commit
       repository.branch = branch
-    end
+    end.save!
   end
 end

spec/loader_spec.cr

@@ -85,7 +85,7 @@ module PlaceOS::FrontendLoader
     end
 
     it "supports changing a uri" do
-      expected_uri = "https://github.com/placeOS/private-drivers"
+      expected_uri = "https://www.github.com/placeOS/private-drivers"
       repository.username = "robot@place.tech"
 
       loader = Loader.new
@@ -94,12 +94,12 @@ module PlaceOS::FrontendLoader
 
       repository.clear_changes_information
       repository.uri = expected_uri
+      repository.save!
       loader.process_resource(:updated, repository).success?.should be_true
 
       Dir.exists?(expected_path).should be_true
-
-      url = Compiler::Git.run_git(expected_path, {"remote", "get-url", "origin"}).output.to_s
-      url.strip.should end_with("private-drivers")
+      Api::Repositories.current_repo(expected_path).should end_with("private-drivers")
+      File.exists?("/app/test-www/test-repo/README.md").should be_true
     end
 
     describe "branches" do
@@ -123,14 +123,29 @@ module PlaceOS::FrontendLoader
 
         loader.process_resource(:created, repository).success?.should be_true
         Dir.exists?(expected_path).should be_true
-        Compiler::Git.current_branch(expected_path).should eq branch
-
+        Api::Repositories.current_branch(expected_path).should eq branch
         repository.clear_changes_information
         repository.branch = updated_branch
-
         loader.process_resource(:updated, repository).success?.should be_true
         Dir.exists?(expected_path).should be_true
-        Compiler::Git.current_branch(expected_path).should eq updated_branch
+        Api::Repositories.current_branch(expected_path).should eq updated_branch
+      end
+
+      it "downloads a release asset" do
+        actioner = PlaceOS::FrontendLoader::Github.new
+        actioner.download_latest_asset("tassja/octokit.cr", Dir.current.to_s)
+        File.exists?("new_file.txt").should be_true
+      end
+
+      it "downloads the release asset on repo flag" do
+        repository = example_repository(LAB_TEST_FOLDER, uri: "https://www.github.com/tassja/octokit.cr")
+        repository.release = true
+        repository.save!
+        ref = PlaceOS::FrontendLoader::Remote::Reference.from_repository(repository)
+        expected_path = File.join(TEST_DIR, repository.folder_name)
+        actioner = PlaceOS::FrontendLoader::Github.new
+        actioner.download(ref: ref, path: expected_path)
+        File.exists?(File.join(expected_path, "new_file.txt")).should be_true
       end
     end
   end

src/constants.cr

@@ -17,8 +17,12 @@ module PlaceOS::FrontendLoader
   # settings for `./placeos-frontend-loader/loader.cr`
   WWW      = ENV["PLACE_LOADER_WWW"]? || "www"
   CRON     = ENV["PLACE_LOADER_CRON"]? || "0 * * * *"
-  GIT_USER = ENV["PLACE_LOADER_GIT_USER"]?
-  GIT_PASS = ENV["PLACE_LOADER_GIT_PASS"]?
+  GIT_USER = ENV["PLACE_LOADER_GIT_USER"]? || ""
+  GIT_PASS = ENV["PLACE_LOADER_GIT_PASS"]? || ""
+
+  GITLAB_TOKEN = ENV["GITLAB_TOKEN"]? || ""
+
+  BASE_REF = "https://www.github.com/PlaceOS/www-core"
 
   # NOTE:: following used in `./placeos-frontend-loader/client.cr`
   # URI.parse(ENV["PLACE_LOADER_URI"]? || "http://127.0.0.1:3000")