refactor: migrate to postgres

.ameba.yml

@@ -0,0 +1,2 @@
+Lint/NotNil:
+  Enabled: false

docker-compose.yml

@@ -7,10 +7,12 @@ x-deployment-env: &deployment-env
   SG_ENV: ${SG_ENV:-development}
   TZ: $TZ
 
-x-rethinkdb-client-env: &rethinkdb-client-env
-  RETHINKDB_HOST: ${RETHINKDB_HOST:-rethink}
-  RETHINKDB_PORT: ${RETHINKDB_PORT:-28015}
-  RETHINKDB_DB: ${RETHINKDB_DB:-place_development}
+x-postgresdb-client-env: &postgresdb-client-env
+  PG_HOST: ${PG_HOST:-postgres}
+  PG_PORT: ${PG_PORT:-5432}
+  PG_DB: ${PG_DB:-place_development}
+  PG_USER: ${PG_USER:-postgres}
+  PG_PASSWORD: ${PG_PASSWORD:-password}
 
 services:
   test: # Frontends
@@ -25,17 +27,28 @@ services:
       - ${PWD}/shard.override.yml:/app/shard.override.yml
       - ${PWD}/shard.yml:/app/shard.yml.input
     depends_on:
-      - rethink
-    security_opt:
-      - seccomp:unconfined
+      - postgres
     environment:
       # Environment
       GITHUB_ACTION: ${GITHUB_ACTION-}
       <<: *deployment-env
       # Service Hosts
-      <<: *rethinkdb-client-env
+      <<: *postgresdb-client-env
 
-  rethink:
-    image: rethinkdb:${RETHINKDB_VERSION:-2.4}
-    restart: always
-    hostname: rethink
+  postgres:
+    hostname: postgres
+    image: postgres
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 30s
+      timeout: 30s
+      retries: 3
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: place_development
+    healthcheck:
+      test: /usr/bin/pg_isready
+      interval: 5s
+    ports:
+      - 5432:5432

shard.lock

@@ -57,6 +57,10 @@ shards:
     git: https://github.com/wyhaines/defined.cr.git
     version: 0.3.6
 
+  eventbus:
+    git: https://github.com/spider-gazelle/eventbus.git
+    version: 0.9.9+git.commit.086b2ba92475b88e8481b0387eb56c735cbfd7bd
+
   exception_page:
     git: https://github.com/crystal-loot/exception_page.git
     version: 0.3.0
@@ -145,17 +149,29 @@ shards:
     git: https://github.com/wyhaines/parsedate.cr.git
     version: 0.1.2
 
+  pg:
+    git: https://github.com/will/crystal-pg.git
+    version: 0.26.0
+
+  pg-orm:
+    git: https://github.com/spider-gazelle/pg-orm.git
+    version: 1.0.0+git.commit.1ba70d1ae9d6e40a9ab1973590ccb1a1b0c3a661
+
   placeos-log-backend:
     git: https://github.com/place-labs/log-backend.git
     version: 0.11.0
 
-  placeos-models:
+  placeos-models: # Overridden
     git: https://github.com/placeos/models.git
-    version: 8.13.1
+    version: 8.12.2+git.commit.795e54cc2864d759f61dac0da422318a285d5ff4
 
-  placeos-resource:
+  placeos-resource: # Overridden
     git: https://github.com/place-labs/resource.git
-    version: 2.5.4
+    version: 2.5.4+git.commit.165b87a8318fbde63a27b505b147e5d57ba9ece7
+
+  pool:
+    git: https://github.com/ysbaddaden/pool.git
+    version: 0.2.4
 
   promise:
     git: https://github.com/spider-gazelle/promise.git
@@ -169,13 +185,9 @@ shards:
     git: https://github.com/sija/raven.cr.git
     version: 1.9.2+git.commit.de91bb38858124270ea06be61641153b8947e18f
 
-  rethinkdb:
-    git: https://github.com/kingsleyh/crystal-rethinkdb.git
-    version: 0.3.1
-
-  rethinkdb-orm:
-    git: https://github.com/spider-gazelle/rethinkdb-orm.git
-    version: 6.0.4
+  redis:
+    git: https://github.com/stefanwille/crystal-redis.git
+    version: 2.8.3
 
   retriable: # Overridden
     git: https://github.com/sija/retriable.cr.git

shard.override.yml

@@ -3,3 +3,10 @@ dependencies:
     github: Sija/retriable.cr
   pars:
     github: spider-gazelle/pars
+
+  placeos-models:
+    github: placeos/models
+    branch: refactor/pg-migration
+  placeos-resource:
+    github: place-labs/resource
+    branch: refactor/pg-migration

spec/helper.cr

@@ -10,8 +10,34 @@ require "spec"
 
 TEST_DIR = "/app/test-www"
 
+PgORM::Database.configure { |_| }
+PgORM::Database.connection do |db|
+  db.exec <<-SQL
+    DROP TABLE IF EXISTS "repo"
+  SQL
+
+  db.exec <<-SQL
+  CREATE TABLE IF NOT EXISTS "repo"(
+    created_at TIMESTAMPTZ NOT NULL,
+    updated_at TIMESTAMPTZ NOT NULL,
+    name TEXT NOT NULL,
+    description TEXT NOT NULL,
+    folder_name TEXT NOT NULL,
+    uri TEXT NOT NULL,
+    commit_hash TEXT NOT NULL,
+    branch TEXT NOT NULL,
+    deployed_commit_hash TEXT,
+    release BOOLEAN NOT NULL,
+    username TEXT,
+    password TEXT,
+    repo_type INTEGER NOT NULL,
+    id TEXT NOT NULL PRIMARY KEY
+  );
+  SQL
+end
+
 Spec.before_suite do
-  Log.builder.bind "*", :trace, PlaceOS::LogBackend.log_backend
+  Log.builder.bind "*", :info, PlaceOS::LogBackend.log_backend
   reset
 end
 

spec/loader_spec.cr

@@ -42,13 +42,16 @@ module PlaceOS::FrontendLoader
         repository.password = old_token
         repository.save!
 
-        changes = [] of RethinkORM::Changefeed::Change(PlaceOS::Model::Repository)
+        changes = [] of PlaceOS::Model::Repository::ChangeFeed::Change(PlaceOS::Model::Repository)
+        changefeed = Model::Repository.changes
         spawn do
-          Model::Repository.changes.each do |change|
+          changefeed.each do |change|
             changes << change
           end
         end
 
+        sleep 1
+
         loader = Loader.new
 
         loader.process_resource(:created, repository).success?.should be_true
@@ -60,7 +63,7 @@ module PlaceOS::FrontendLoader
         repository = repository.class.find!(repository.id.not_nil!)
         loader.process_resource(:updated, repository).success?.should be_true
 
-        sleep 10.seconds
+        changefeed.stop
         changes.size.should eq 2
       end
     end
@@ -104,7 +107,7 @@ module PlaceOS::FrontendLoader
       loader.process_resource(:updated, repository).success?.should be_true
 
       Dir.exists?(expected_path).should be_true
-      File.exists?("/app/test-www/test-repo/README.md").should be_true
+      File.exists?("#{TEST_DIR}/test-repo/README.md").should be_true
     end
 
     describe "branches" do

src/app.cr

@@ -71,6 +71,14 @@ end
 
 require "./config"
 
+# Configure the database connection. First check if PG_DATABASE_URL environment variable
+# is set. If not, assume database configuration are set via individual environment variables
+if pg_url = ENV["PG_DATABASE_URL"]?
+  PgORM::Database.parse(pg_url)
+else
+  PgORM::Database.configure { |_| }
+end
+
 # Configure the loader
 
 PlaceOS::FrontendLoader::Loader.configure do |settings|

src/placeos-frontend-loader/api/repositories.cr

@@ -36,9 +36,8 @@ module PlaceOS::FrontendLoader::Api
       return if @repo_cache
 
       # attempt to find the record in the database
-      repo_details = ::PlaceOS::Model::Repository.collection_query do |table|
-        table.get_all(folder_name, index: :folder_name)
-      end.to_a.select!(&.repo_type.interface?).first?
+      repo_details = ::PlaceOS::Model::Repository.where(folder_name: folder_name)
+        .to_a.select!(&.repo_type.interface?).first?
 
       raise Error::NotFound.new("unable to find repository at #{folder_name}") unless repo_details
       @repo_cache = GitRepository.new(repo_details.uri, repo_details.username, repo_details.decrypt_password)

src/placeos-frontend-loader/loader.cr

@@ -1,6 +1,6 @@
 require "file_utils"
 require "habitat"
-require "placeos-models/repository"
+require "placeos-models"
 require "placeos-resource"
 require "tasker"