diff --git a/.ameba.yml b/.ameba.yml new file mode 100644 index 0000000..0be859e --- /dev/null +++ b/.ameba.yml @@ -0,0 +1,2 @@ +Lint/NotNil: + Enabled: false \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index c016776..c5d26be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,10 +7,12 @@ x-deployment-env: &deployment-env SG_ENV: ${SG_ENV:-development} TZ: $TZ -x-rethinkdb-client-env: &rethinkdb-client-env - RETHINKDB_HOST: ${RETHINKDB_HOST:-rethink} - RETHINKDB_PORT: ${RETHINKDB_PORT:-28015} - RETHINKDB_DB: ${RETHINKDB_DB:-place_development} +x-postgresdb-client-env: &postgresdb-client-env + PG_HOST: ${PG_HOST:-postgres} + PG_PORT: ${PG_PORT:-5432} + PG_DB: ${PG_DB:-place_development} + PG_USER: ${PG_USER:-postgres} + PG_PASSWORD: ${PG_PASSWORD:-password} services: test: # Frontends @@ -25,17 +27,28 @@ services: - ${PWD}/shard.override.yml:/app/shard.override.yml - ${PWD}/shard.yml:/app/shard.yml.input depends_on: - - rethink - security_opt: - - seccomp:unconfined + - postgres environment: # Environment GITHUB_ACTION: ${GITHUB_ACTION-} <<: *deployment-env # Service Hosts - <<: *rethinkdb-client-env + <<: *postgresdb-client-env - rethink: - image: rethinkdb:${RETHINKDB_VERSION:-2.4} - restart: always - hostname: rethink + postgres: + hostname: postgres + image: postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 30s + timeout: 30s + retries: 3 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: place_development + healthcheck: + test: /usr/bin/pg_isready + interval: 5s + ports: + - 5432:5432 diff --git a/shard.lock b/shard.lock index b8f9f42..1f63294 100644 --- a/shard.lock +++ b/shard.lock @@ -57,6 +57,10 @@ shards: git: https://github.com/wyhaines/defined.cr.git version: 0.3.6 + eventbus: + git: https://github.com/spider-gazelle/eventbus.git + version: 0.9.9+git.commit.086b2ba92475b88e8481b0387eb56c735cbfd7bd + exception_page: git: https://github.com/crystal-loot/exception_page.git version: 0.3.0 @@ -145,17 +149,29 @@ shards: git: https://github.com/wyhaines/parsedate.cr.git version: 0.1.2 + pg: + git: https://github.com/will/crystal-pg.git + version: 0.26.0 + + pg-orm: + git: https://github.com/spider-gazelle/pg-orm.git + version: 1.0.0+git.commit.1ba70d1ae9d6e40a9ab1973590ccb1a1b0c3a661 + placeos-log-backend: git: https://github.com/place-labs/log-backend.git version: 0.11.0 - placeos-models: + placeos-models: # Overridden git: https://github.com/placeos/models.git - version: 8.13.1 + version: 8.12.2+git.commit.795e54cc2864d759f61dac0da422318a285d5ff4 - placeos-resource: + placeos-resource: # Overridden git: https://github.com/place-labs/resource.git - version: 2.5.4 + version: 2.5.4+git.commit.165b87a8318fbde63a27b505b147e5d57ba9ece7 + + pool: + git: https://github.com/ysbaddaden/pool.git + version: 0.2.4 promise: git: https://github.com/spider-gazelle/promise.git @@ -169,13 +185,9 @@ shards: git: https://github.com/sija/raven.cr.git version: 1.9.2+git.commit.de91bb38858124270ea06be61641153b8947e18f - rethinkdb: - git: https://github.com/kingsleyh/crystal-rethinkdb.git - version: 0.3.1 - - rethinkdb-orm: - git: https://github.com/spider-gazelle/rethinkdb-orm.git - version: 6.0.4 + redis: + git: https://github.com/stefanwille/crystal-redis.git + version: 2.8.3 retriable: # Overridden git: https://github.com/sija/retriable.cr.git diff --git a/shard.override.yml b/shard.override.yml index 74859e0..cbe45db 100644 --- a/shard.override.yml +++ b/shard.override.yml @@ -3,3 +3,10 @@ dependencies: github: Sija/retriable.cr pars: github: spider-gazelle/pars + + placeos-models: + github: placeos/models + branch: refactor/pg-migration + placeos-resource: + github: place-labs/resource + branch: refactor/pg-migration \ No newline at end of file diff --git a/spec/helper.cr b/spec/helper.cr index 7805930..2e7c8e1 100644 --- a/spec/helper.cr +++ b/spec/helper.cr @@ -10,8 +10,34 @@ require "spec" TEST_DIR = "/app/test-www" +PgORM::Database.configure { |_| } +PgORM::Database.connection do |db| + db.exec <<-SQL + DROP TABLE IF EXISTS "repo" + SQL + + db.exec <<-SQL + CREATE TABLE IF NOT EXISTS "repo"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + folder_name TEXT NOT NULL, + uri TEXT NOT NULL, + commit_hash TEXT NOT NULL, + branch TEXT NOT NULL, + deployed_commit_hash TEXT, + release BOOLEAN NOT NULL, + username TEXT, + password TEXT, + repo_type INTEGER NOT NULL, + id TEXT NOT NULL PRIMARY KEY + ); + SQL +end + Spec.before_suite do - Log.builder.bind "*", :trace, PlaceOS::LogBackend.log_backend + Log.builder.bind "*", :info, PlaceOS::LogBackend.log_backend reset end diff --git a/spec/loader_spec.cr b/spec/loader_spec.cr index a80592c..fdb8737 100644 --- a/spec/loader_spec.cr +++ b/spec/loader_spec.cr @@ -42,13 +42,16 @@ module PlaceOS::FrontendLoader repository.password = old_token repository.save! - changes = [] of RethinkORM::Changefeed::Change(PlaceOS::Model::Repository) + changes = [] of PlaceOS::Model::Repository::ChangeFeed::Change(PlaceOS::Model::Repository) + changefeed = Model::Repository.changes spawn do - Model::Repository.changes.each do |change| + changefeed.each do |change| changes << change end end + sleep 1 + loader = Loader.new loader.process_resource(:created, repository).success?.should be_true @@ -60,7 +63,7 @@ module PlaceOS::FrontendLoader repository = repository.class.find!(repository.id.not_nil!) loader.process_resource(:updated, repository).success?.should be_true - sleep 10.seconds + changefeed.stop changes.size.should eq 2 end end @@ -104,7 +107,7 @@ module PlaceOS::FrontendLoader loader.process_resource(:updated, repository).success?.should be_true Dir.exists?(expected_path).should be_true - File.exists?("/app/test-www/test-repo/README.md").should be_true + File.exists?("#{TEST_DIR}/test-repo/README.md").should be_true end describe "branches" do diff --git a/src/app.cr b/src/app.cr index 21c1cdf..fb6d223 100644 --- a/src/app.cr +++ b/src/app.cr @@ -71,6 +71,14 @@ end require "./config" +# Configure the database connection. First check if PG_DATABASE_URL environment variable +# is set. If not, assume database configuration are set via individual environment variables +if pg_url = ENV["PG_DATABASE_URL"]? + PgORM::Database.parse(pg_url) +else + PgORM::Database.configure { |_| } +end + # Configure the loader PlaceOS::FrontendLoader::Loader.configure do |settings| diff --git a/src/placeos-frontend-loader/api/repositories.cr b/src/placeos-frontend-loader/api/repositories.cr index ce085b7..678f663 100644 --- a/src/placeos-frontend-loader/api/repositories.cr +++ b/src/placeos-frontend-loader/api/repositories.cr @@ -36,9 +36,8 @@ module PlaceOS::FrontendLoader::Api return if @repo_cache # attempt to find the record in the database - repo_details = ::PlaceOS::Model::Repository.collection_query do |table| - table.get_all(folder_name, index: :folder_name) - end.to_a.select!(&.repo_type.interface?).first? + repo_details = ::PlaceOS::Model::Repository.where(folder_name: folder_name) + .to_a.select!(&.repo_type.interface?).first? raise Error::NotFound.new("unable to find repository at #{folder_name}") unless repo_details @repo_cache = GitRepository.new(repo_details.uri, repo_details.username, repo_details.decrypt_password) diff --git a/src/placeos-frontend-loader/loader.cr b/src/placeos-frontend-loader/loader.cr index 68368fd..3a785d1 100644 --- a/src/placeos-frontend-loader/loader.cr +++ b/src/placeos-frontend-loader/loader.cr @@ -1,6 +1,6 @@ require "file_utils" require "habitat" -require "placeos-models/repository" +require "placeos-models" require "placeos-resource" require "tasker"