diff --git a/docker-compose.yml b/docker-compose.yml index c5d26be..f57614d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,6 @@ version: "3.7" x-deployment-env: &deployment-env ENV: ${ENV:-development} SG_ENV: ${SG_ENV:-development} - TZ: $TZ x-postgresdb-client-env: &postgresdb-client-env PG_HOST: ${PG_HOST:-postgres} @@ -28,16 +27,15 @@ services: - ${PWD}/shard.yml:/app/shard.yml.input depends_on: - postgres + - migrator environment: # Environment GITHUB_ACTION: ${GITHUB_ACTION-} - <<: *deployment-env - # Service Hosts - <<: *postgresdb-client-env + <<: [*deployment-env, *postgresdb-client-env] postgres: hostname: postgres - image: postgres + image: postgres:15-alpine healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 30s @@ -47,8 +45,16 @@ services: POSTGRES_USER: postgres POSTGRES_PASSWORD: password POSTGRES_DB: place_development - healthcheck: - test: /usr/bin/pg_isready - interval: 5s ports: - 5432:5432 + + migrator: + build: + context: ${PWD}/spec/migration + container_name: migrator + depends_on: + postgres: + condition: service_healthy + environment: + GITHUB_ACTION: ${GITHUB_ACTION:-} + PG_DATABASE_URL: ${PG_DATABASE_URL:-postgresql://postgres:password@postgres:5432/place_development} \ No newline at end of file diff --git a/shard.lock b/shard.lock index 8bacfdc..6f57816 100644 --- a/shard.lock +++ b/shard.lock @@ -31,7 +31,7 @@ shards: connect-proxy: git: https://github.com/spider-gazelle/connect-proxy.git - version: 2.0.0 + version: 2.0.1 cron_parser: git: https://github.com/kostya/cron_parser.git @@ -39,11 +39,11 @@ shards: crystar: git: https://github.com/naqvis/crystar.git - version: 0.3.1 + version: 0.4.0 csuuid: git: https://github.com/wyhaines/csuuid.cr.git - version: 1.0.0+git.commit.a4cf9615c6518cf27c68a1755a8c2ac4ae4fe987 + version: 1.0.1+git.commit.b71cf5c899dd5cde6aff8e922bdabd5e2dfab585 db: git: https://github.com/crystal-lang/crystal-db.git @@ -59,7 +59,7 @@ shards: email: git: https://github.com/arcage/crystal-email.git - version: 0.7.0 + version: 0.7.1 eventbus: git: https://github.com/spider-gazelle/eventbus.git @@ -71,7 +71,7 @@ shards: faker: git: https://github.com/askn/faker.git - version: 0.8.0 + version: 0.9.0 future: git: https://github.com/crystal-community/future.cr.git @@ -79,7 +79,7 @@ shards: git-repository: git: https://github.com/place-labs/git-repository.git - version: 1.3.1 + version: 1.4.0 google: git: https://github.com/placeos/google.git @@ -127,7 +127,7 @@ shards: office365: git: https://github.com/placeos/office365.git - version: 1.24.0 + version: 1.25.3 open_api: git: https://github.com/elbywan/open_api.cr.git @@ -175,7 +175,7 @@ shards: place_calendar: git: https://github.com/placeos/calendar.git - version: 4.20.1 + version: 4.22.1 placeos-log-backend: git: https://github.com/place-labs/log-backend.git @@ -183,7 +183,7 @@ shards: placeos-models: git: https://github.com/placeos/models.git - version: 9.45.0 + version: 9.48.0 placeos-resource: git: https://github.com/place-labs/resource.git diff --git a/spec/helper.cr b/spec/helper.cr index 2e7c8e1..4cef250 100644 --- a/spec/helper.cr +++ b/spec/helper.cr @@ -10,31 +10,8 @@ require "spec" TEST_DIR = "/app/test-www" +# Configure DB PgORM::Database.configure { |_| } -PgORM::Database.connection do |db| - db.exec <<-SQL - DROP TABLE IF EXISTS "repo" - SQL - - db.exec <<-SQL - CREATE TABLE IF NOT EXISTS "repo"( - created_at TIMESTAMPTZ NOT NULL, - updated_at TIMESTAMPTZ NOT NULL, - name TEXT NOT NULL, - description TEXT NOT NULL, - folder_name TEXT NOT NULL, - uri TEXT NOT NULL, - commit_hash TEXT NOT NULL, - branch TEXT NOT NULL, - deployed_commit_hash TEXT, - release BOOLEAN NOT NULL, - username TEXT, - password TEXT, - repo_type INTEGER NOT NULL, - id TEXT NOT NULL PRIMARY KEY - ); - SQL -end Spec.before_suite do Log.builder.bind "*", :info, PlaceOS::LogBackend.log_backend diff --git a/spec/loader_spec.cr b/spec/loader_spec.cr index 8f4fb20..203f210 100644 --- a/spec/loader_spec.cr +++ b/spec/loader_spec.cr @@ -38,10 +38,6 @@ module PlaceOS::FrontendLoader old_token = "fake_password" new_token = "fake_password_electric_boogaloo" - repository = example_repository(commit: "f7c6d8fb810c2be78722249e06bbfbda3d30d355") - repository.password = old_token - repository.save! - changes = [] of PlaceOS::Model::Repository::ChangeFeed::Change(PlaceOS::Model::Repository) changefeed = Model::Repository.changes spawn do @@ -50,12 +46,17 @@ module PlaceOS::FrontendLoader end end - sleep 1 + Fiber.yield + + repository = example_repository(commit: "f7c6d8fb810c2be78722249e06bbfbda3d30d355") + repository.password = old_token + repository.save! loader = Loader.new loader.process_resource(:created, repository).success?.should be_true repository.reload! + repository.password = new_token repository.password_will_change! repository.save! @@ -64,7 +65,7 @@ module PlaceOS::FrontendLoader loader.process_resource(:updated, repository).success?.should be_true changefeed.stop - changes.size.should eq 2 + changes.size.should eq 3 repository.reload! encrypted = repository.password.not_nil! @@ -146,5 +147,42 @@ module PlaceOS::FrontendLoader Dir.exists?(expected_path).should be_true end end + + describe "error handling" do + it "should not break on non-existent repo/branch" do + loader = Loader.new + + branch = "doesnt-exist" + repository.branch = branch + + loader.process_resource(:created, repository).success?.should be_false + Dir.exists?(expected_path).should be_false + repository = Model::Repository.find!(repository.id.as(String)) + repository.has_runtime_error.should be_true + repository.error_message.should_not be_nil + end + + it "should clear error flag when branch is correct" do + loader = Loader.new + updated_branch = "master" + branch = "doesnt-exist" + repository.branch = branch + + loader.process_resource(:created, repository).success?.should be_false + Dir.exists?(expected_path).should be_false + repository = Model::Repository.find!(repository.id.as(String)) + repository.has_runtime_error.should be_true + repository.error_message.should_not be_nil + + repository.branch = updated_branch + repository.save! + loader.process_resource(:updated, repository).success?.should be_true + Dir.exists?(expected_path).should be_true + + repository = Model::Repository.find!(repository.id.as(String)) + repository.has_runtime_error.should be_false + repository.error_message.should be_nil + end + end end end diff --git a/spec/migration/.dockerignore b/spec/migration/.dockerignore new file mode 100644 index 0000000..00e03f5 --- /dev/null +++ b/spec/migration/.dockerignore @@ -0,0 +1,2 @@ +bin +lib \ No newline at end of file diff --git a/spec/migration/.gitignore b/spec/migration/.gitignore new file mode 100644 index 0000000..0bb75ea --- /dev/null +++ b/spec/migration/.gitignore @@ -0,0 +1,5 @@ +/docs/ +/lib/ +/bin/ +/.shards/ +*.dwarf diff --git a/spec/migration/Dockerfile b/spec/migration/Dockerfile new file mode 100644 index 0000000..57ff5e2 --- /dev/null +++ b/spec/migration/Dockerfile @@ -0,0 +1,14 @@ +FROM placeos/crystal:latest + +WORKDIR /app + +COPY . /app + +RUN git clone https://github.com/PlaceOS/models +RUN mv ./models/migration/db ./db + +ENV PATH /app/bin:$PATH + +RUN shards build + +ENTRYPOINT [ "/app/run.sh" ] diff --git a/spec/migration/README.md b/spec/migration/README.md new file mode 100644 index 0000000..c43d139 --- /dev/null +++ b/spec/migration/README.md @@ -0,0 +1,3 @@ +#PlaceOS Models Migration Script + +This folder contains source code for scaffolding PlaceOS models schema and running migration. diff --git a/spec/migration/run.sh b/spec/migration/run.sh new file mode 100755 index 0000000..5b76a70 --- /dev/null +++ b/spec/migration/run.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -eu + +bin/migration up \ No newline at end of file diff --git a/spec/migration/shard.lock b/spec/migration/shard.lock new file mode 100644 index 0000000..33adf43 --- /dev/null +++ b/spec/migration/shard.lock @@ -0,0 +1,14 @@ +version: 2.0 +shards: + db: + git: https://github.com/crystal-lang/crystal-db.git + version: 0.10.1 + + micrate: + git: https://github.com/amberframework/micrate.git + version: 0.12.0 + + pg: + git: https://github.com/will/crystal-pg.git + version: 0.25.0 + diff --git a/spec/migration/shard.yml b/spec/migration/shard.yml new file mode 100644 index 0000000..ae3873b --- /dev/null +++ b/spec/migration/shard.yml @@ -0,0 +1,16 @@ +name: placeos-models-migration +version: 0.9.9 +crystal: ~> 1.4 +license: MIT + + +targets: + migration: + main: src/migration.cr + +dependencies: + micrate: + github: amberframework/micrate + version: "0.12.0" + pg: + github: will/crystal-pg diff --git a/spec/migration/src/migration.cr b/spec/migration/src/migration.cr new file mode 100644 index 0000000..d59fe70 --- /dev/null +++ b/spec/migration/src/migration.cr @@ -0,0 +1,5 @@ +require "micrate" +require "pg" + +Micrate::DB.connection_url = ENV["PG_DATABASE_URL"] +Micrate::Cli.run diff --git a/src/placeos-frontend-loader/loader.cr b/src/placeos-frontend-loader/loader.cr index 030f8ec..f24c77a 100644 --- a/src/placeos-frontend-loader/loader.cr +++ b/src/placeos-frontend-loader/loader.cr @@ -142,6 +142,11 @@ module PlaceOS::FrontendLoader return Resource::Result::Skipped end + # Skip load for error indicator update actions + if (changes = repository.changed_attributes) && (changes[:has_runtime_error]? || changes[:error_message]?) + return Resource::Result::Skipped + end + # Load the repository Loader.load( repository: repository, @@ -184,6 +189,7 @@ module PlaceOS::FrontendLoader end end + # ameba:disable Metrics/CyclomaticComplexity def self.load( repository : Model::Repository, content_directory : String @@ -192,16 +198,26 @@ module PlaceOS::FrontendLoader # check for any relevant changes rebuild_cache, old_folder_name = check_for_changes(repository) - + has_error = false + error_message = nil # rebuild caches cache = if rebuild_cache - GitRepository.new(repository.uri, repository.username, repository.decrypt_password, repository.branch) + begin + GitRepository.new(repository.uri, repository.username, repository.decrypt_password, repository.branch) + rescue ex : Exception + has_error = true + error_message = ex.message + Log.error(exception: ex) { {message: "Error loading repository", repository: repository.name, branch: repository.branch} } + nil + end else repo_cache = id_lookup[repository.id] old_commit_hash = repo_cache.commit repo_cache.cache end + Model::Repository.update(repository.id, {has_runtime_error: has_error, error_message: error_message}) if has_error || repository.has_runtime_error + return Resource::Result::Error unless cache # grab the required commit content_directory = File.expand_path(content_directory) repository_directory = File.join(content_directory, repository.folder_name)