From f9b817f05b4c3e34c13c196a6273a298ae1361b5 Mon Sep 17 00:00:00 2001 From: loiswells97 Date: Fri, 10 Feb 2023 11:44:20 +0000 Subject: [PATCH 01/17] Create draft PR for #126 From c812ba5182388861fa80aa09fd7a1f172b13b115 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Fri, 10 Feb 2023 16:05:00 +0000 Subject: [PATCH 02/17] Dummy importer code --- lib/project_importer_example.rb | 97 +++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 lib/project_importer_example.rb diff --git a/lib/project_importer_example.rb b/lib/project_importer_example.rb new file mode 100644 index 000000000..80ecaa7ee --- /dev/null +++ b/lib/project_importer_example.rb @@ -0,0 +1,97 @@ + + +class ProjectImporter + attr_reader :name, :identifier, :images, :components, :type + + components = [ + { + filename: "index" + extension: "html" + contents: "" + default: true + }, + { + filename: "styles" + extension: "css" + contents: "html { color: pink } " + default: false + }] + + images = { + [ + filename: "foo.png" + contents: # binary + ] + } + + def initialize(name:, identifier:, type:, components:, images: []) + @name = name + @identifier = identifier + @components = components + @images = images + @type = type + end + + def import! + Project.transaction do + delete_components + + components.each do |component| + project_component = Component.new(*component) + project.components << project_component + end + + delete_removed_images + project_images.each do |image_name| + attach_image_if_needed(project, image_name, dir) + end + + project.save! + end + end + + private + + def project + @project ||= Project.find_or_initialze_by(identifier:) + end + + def delete_components + project.components.each(&:destroy) + end + + def delete_removed_images + existing_images = project.images.map { |x| x.blob.filename.to_s } + diff = existing_images - images.map{|x| x[:filename]} + return if diff.empty? + + diff.each do |filename| + img = project.images.find { |i| i.blob.filename == filename } + img.purge! + end + end + + def attach_images_if_needed + existing_image = project.images.find { |i| i.blob.filename == image_name } + + if existing_image + return if existing_image.blob.checksum == image_checksum(image_name) + + existing_image.purge! + end + + project.images.attach!(io: image.io, + filename: image.filename) + end + + + def image_checksum(io) + OpenSSL::Digest.new('MD5').tap do |checksum| + while (chunk = io.read(5.megabytes)) + checksum << chunk + end + + io.rewind + end.base64digest + end + end From 76da45a477f05c26321f5d42ee4af1c20bee7224 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Fri, 10 Feb 2023 17:03:36 +0000 Subject: [PATCH 03/17] reduce rake task yml use --- app/models/component.rb | 1 - app/models/project.rb | 2 +- ...210142728_remove_component_index_column.rb | 5 ++ db/schema.rb | 63 ++++++++++++++++++- lib/tasks/projects.rake | 55 +++++++++------- 5 files changed, 99 insertions(+), 27 deletions(-) create mode 100644 db/migrate/20230210142728_remove_component_index_column.rb diff --git a/app/models/component.rb b/app/models/component.rb index eb8eb2dc8..4bbce5ec9 100644 --- a/app/models/component.rb +++ b/app/models/component.rb @@ -4,7 +4,6 @@ class Component < ApplicationRecord belongs_to :project validates :name, presence: true validates :extension, presence: true - validates :index, presence: true, uniqueness: { scope: :project_id } validate :default_component_protected_properties, on: :update private diff --git a/app/models/project.rb b/app/models/project.rb index 5297329cc..d7ab81ebd 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -6,7 +6,7 @@ class Project < ApplicationRecord before_validation :check_unique_not_null, on: :create validates :identifier, presence: true, uniqueness: true belongs_to :parent, class_name: 'Project', foreign_key: 'remixed_from_id', optional: true, inverse_of: :remixes - has_many :components, -> { order(:index) }, dependent: :destroy, inverse_of: :project + has_many :components, -> { order(default: :desc, name: :asc) }, dependent: :destroy, inverse_of: :project has_many :remixes, class_name: 'Project', foreign_key: 'remixed_from_id', dependent: :nullify, inverse_of: :parent has_many_attached :images accepts_nested_attributes_for :components diff --git a/db/migrate/20230210142728_remove_component_index_column.rb b/db/migrate/20230210142728_remove_component_index_column.rb new file mode 100644 index 000000000..25a2540de --- /dev/null +++ b/db/migrate/20230210142728_remove_component_index_column.rb @@ -0,0 +1,5 @@ +class RemoveComponentIndexColumn < ActiveRecord::Migration[7.0] + def change + remove_column :components, :index + end +end diff --git a/db/schema.rb b/db/schema.rb index 468f8e243..19abe38d3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2022_03_11_121518) do +ActiveRecord::Schema[7.0].define(version: 2023_02_10_142728) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -50,12 +50,69 @@ t.string "content" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "index" t.boolean "default", default: false, null: false - t.index ["index", "project_id"], name: "index_components_on_index_and_project_id", unique: true t.index ["project_id"], name: "index_components_on_project_id" end + create_table "good_job_batches", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.text "description" + t.jsonb "serialized_properties" + t.text "on_finish" + t.text "on_success" + t.text "on_discard" + t.text "callback_queue_name" + t.integer "callback_priority" + t.datetime "enqueued_at" + t.datetime "discarded_at" + t.datetime "finished_at" + end + + create_table "good_job_processes", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.jsonb "state" + end + + create_table "good_job_settings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.text "key" + t.jsonb "value" + t.index ["key"], name: "index_good_job_settings_on_key", unique: true + end + + create_table "good_jobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.text "queue_name" + t.integer "priority" + t.jsonb "serialized_params" + t.datetime "scheduled_at" + t.datetime "performed_at" + t.datetime "finished_at" + t.text "error" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.uuid "active_job_id" + t.text "concurrency_key" + t.text "cron_key" + t.uuid "retried_good_job_id" + t.datetime "cron_at" + t.uuid "batch_id" + t.uuid "batch_callback_id" + t.index ["active_job_id", "created_at"], name: "index_good_jobs_on_active_job_id_and_created_at" + t.index ["active_job_id"], name: "index_good_jobs_on_active_job_id" + t.index ["batch_callback_id"], name: "index_good_jobs_on_batch_callback_id", where: "(batch_callback_id IS NOT NULL)" + t.index ["batch_id"], name: "index_good_jobs_on_batch_id", where: "(batch_id IS NOT NULL)" + t.index ["concurrency_key"], name: "index_good_jobs_on_concurrency_key_when_unfinished", where: "(finished_at IS NULL)" + t.index ["cron_key", "created_at"], name: "index_good_jobs_on_cron_key_and_created_at" + t.index ["cron_key", "cron_at"], name: "index_good_jobs_on_cron_key_and_cron_at", unique: true + t.index ["finished_at"], name: "index_good_jobs_jobs_on_finished_at", where: "((retried_good_job_id IS NULL) AND (finished_at IS NOT NULL))" + t.index ["priority", "created_at"], name: "index_good_jobs_jobs_on_priority_created_at_when_unfinished", order: { priority: "DESC NULLS LAST" }, where: "(finished_at IS NULL)" + t.index ["queue_name", "scheduled_at"], name: "index_good_jobs_on_queue_name_and_scheduled_at", where: "(finished_at IS NULL)" + t.index ["scheduled_at"], name: "index_good_jobs_on_scheduled_at", where: "(finished_at IS NULL)" + end + create_table "projects", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "user_id" t.string "name" diff --git a/lib/tasks/projects.rake b/lib/tasks/projects.rake index 23307857b..910ff02e6 100644 --- a/lib/tasks/projects.rake +++ b/lib/tasks/projects.rake @@ -5,26 +5,36 @@ require 'yaml' namespace :projects do desc 'Import starter projects' task create_starter: :environment do + code_formats = [".py", '.csv', '.txt'] + image_formats = ['.png', '.jpg', '.jpeg'] + Dir.each_child("#{File.dirname(__FILE__)}/project_components") do |dir| proj_config = YAML.safe_load(File.read("#{File.dirname(__FILE__)}/project_components/#{dir}/project_config.yml")) project = find_project(proj_config) - components = proj_config['COMPONENTS'] - components.each do |component| - name = component['name'] - extension = component['extension'] - code = File.read(File.dirname(__FILE__) + "/project_components/#{dir}/#{component['location']}") - index = component['index'] - default = component['default'] - project_component = Component.new(name:, extension:, content: code, index:, default:) - project.components << project_component - end + files = Dir.children("#{File.dirname(__FILE__)}/project_components/#{dir}") + code_files = files.filter{ |file| code_formats.include? File.extname(file) } + image_files = files.filter{ |file| image_formats.include? File.extname(file) } - project_images = proj_config['IMAGES'] || [] - delete_removed_images(project, project_images) - project_images.each do |image_name| - attach_image_if_needed(project, image_name, dir) + code_files.each do |file| + name = File.basename(file, '.*') + extension = File.extname(file).delete('.') + code = File.read(File.dirname(__FILE__) + "/project_components/#{dir}/#{File.basename(file)}") + default = (File.basename(file)=='main.py') + project_component = Component.new(name:, extension:, content: code, default:) + project.components << project_component + end + delete_removed_images(project, image_files) + image_files.each do |image| + attach_image_if_needed(project, image) end + # project_images = proj_config['IMAGES'] || [] + # delete_removed_images(project, project_images) + # project_images.each do |image_name| + # attach_image_if_needed(project, image_name, dir) + # end + # puts project.identifier + # puts project.components.length project.save end end @@ -44,8 +54,9 @@ def find_project(proj_config) end def delete_removed_images(project, images_to_attach) - existing_images = project.images.map { |x| x.blob.filename.to_s } - diff = existing_images - images_to_attach + new_image_names = images_to_attach.map { |image| File.basename(image) } + existing_image_names = project.images.map { |x| x.blob.filename.to_s } + diff = existing_image_names - new_image_names return if diff.empty? diff.each do |filename| @@ -54,20 +65,20 @@ def delete_removed_images(project, images_to_attach) end end -def attach_image_if_needed(project, image_name, dir) +def attach_image_if_needed(project, image) + image_name = File.basename(image) existing_image = project.images.find { |i| i.blob.filename == image_name } if existing_image - return if existing_image.blob.checksum == image_checksum(image_name, dir) + return if existing_image.blob.checksum == image_checksum(image) existing_image.purge end - project.images.attach(io: File.open(File.dirname(__FILE__) + "/project_components/#{dir}/#{image_name}"), - filename: image_name) + project.images.attach(image) end -def image_checksum(image_name, dir) - io = File.open(File.dirname(__FILE__) + "/project_components/#{dir}/#{image_name}") +def image_checksum(image) + io = File.open(image) OpenSSL::Digest.new('MD5').tap do |checksum| while (chunk = io.read(5.megabytes)) checksum << chunk From cf72cd7f86dd98179073edea0e7f11689ab88834 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Fri, 10 Feb 2023 17:29:29 +0000 Subject: [PATCH 04/17] fixing lots of broken tests --- app/views/api/projects/show.json.jbuilder | 2 +- lib/concepts/project/operations/create_remix.rb | 2 +- lib/tasks/projects.rake | 13 +++++++------ spec/concepts/project/create_remix_spec.rb | 12 +++++------- spec/concepts/project/create_spec.rb | 1 - .../project/update_default_component_spec.rb | 3 +-- .../project/update_delete_components_spec.rb | 3 +-- spec/concepts/project/update_invalid_spec.rb | 11 ++++------- spec/concepts/project/update_spec.rb | 14 +++++--------- spec/factories/component.rb | 2 -- spec/models/component_spec.rb | 2 -- spec/request/projects/update_spec.rb | 3 +-- 12 files changed, 26 insertions(+), 42 deletions(-) diff --git a/app/views/api/projects/show.json.jbuilder b/app/views/api/projects/show.json.jbuilder index 75dbff2c3..61a0c9ed8 100644 --- a/app/views/api/projects/show.json.jbuilder +++ b/app/views/api/projects/show.json.jbuilder @@ -4,7 +4,7 @@ json.call(@project, :identifier, :project_type, :name, :user_id) json.parent(@project.parent, :name, :identifier) if @project.parent -json.components @project.components, :id, :name, :extension, :content, :index +json.components @project.components, :id, :name, :extension, :content json.image_list @project.images do |image| json.filename image.filename diff --git a/lib/concepts/project/operations/create_remix.rb b/lib/concepts/project/operations/create_remix.rb index d8373ab90..38a9cdf2c 100644 --- a/lib/concepts/project/operations/create_remix.rb +++ b/lib/concepts/project/operations/create_remix.rb @@ -42,7 +42,7 @@ def create_remix(original_project, params, user_id) end params[:components].each do |x| - remix.components.build(x.slice(:name, :extension, :content, :index)) + remix.components.build(x.slice(:name, :extension, :content)) end remix diff --git a/lib/tasks/projects.rake b/lib/tasks/projects.rake index 910ff02e6..13db72a6e 100644 --- a/lib/tasks/projects.rake +++ b/lib/tasks/projects.rake @@ -25,7 +25,7 @@ namespace :projects do end delete_removed_images(project, image_files) image_files.each do |image| - attach_image_if_needed(project, image) + attach_image_if_needed(project, image, dir) end # project_images = proj_config['IMAGES'] || [] @@ -65,20 +65,21 @@ def delete_removed_images(project, images_to_attach) end end -def attach_image_if_needed(project, image) +def attach_image_if_needed(project, image, dir) image_name = File.basename(image) existing_image = project.images.find { |i| i.blob.filename == image_name } if existing_image - return if existing_image.blob.checksum == image_checksum(image) + return if existing_image.blob.checksum == image_checksum(image_name, dir) existing_image.purge end - project.images.attach(image) + project.images.attach(io: File.open(File.dirname(__FILE__) + "/project_components/#{dir}/#{image_name}"), + filename: image_name) end -def image_checksum(image) - io = File.open(image) +def image_checksum(image_name, dir) + io = File.open(File.dirname(__FILE__) + "/project_components/#{dir}/#{image_name}") OpenSSL::Digest.new('MD5').tap do |checksum| while (chunk = io.read(5.megabytes)) checksum << chunk diff --git a/spec/concepts/project/create_remix_spec.rb b/spec/concepts/project/create_remix_spec.rb index 74099800d..91df29564 100644 --- a/spec/concepts/project/create_remix_spec.rb +++ b/spec/concepts/project/create_remix_spec.rb @@ -17,8 +17,7 @@ id: component.id, name: component.name, extension: component.extension, - content: 'some updated component content', - index: component.index + content: 'some updated component content' } ] } @@ -82,7 +81,7 @@ end context 'when a new component has been added before remixing' do - let(:new_component_params) { { name: 'added_component', extension: 'py', content: 'some added component content', index: 9999 } } + let(:new_component_params) { { name: 'added_component', extension: 'py', content: 'some added component content'} } before do remix_params[:components] << new_component_params @@ -94,7 +93,7 @@ it 'persists the new component' do remixed_project = create_remix[:project] - expect(remixed_project.components.last.attributes.symbolize_keys).to include(new_component_params) + expect(remixed_project.components.first.attributes.symbolize_keys).to include(new_component_params) end end @@ -136,7 +135,7 @@ end context 'when project components are invalid' do - let(:invalid_component_params) { { name: 'added_component', extension: 'py', content: '' } } + let(:invalid_component_params) { { name: 'added_component', content: '' } } before do remix_params[:components] << invalid_component_params @@ -156,8 +155,7 @@ def component_props(component) { name: component.name, content: component.content, - extension: component.extension, - index: component.index + extension: component.extension } end end diff --git a/spec/concepts/project/create_spec.rb b/spec/concepts/project/create_spec.rb index d1f4d92d1..aa7d35019 100644 --- a/spec/concepts/project/create_spec.rb +++ b/spec/concepts/project/create_spec.rb @@ -25,7 +25,6 @@ name: 'main', extension: 'py', content: 'print("hello world")', - index: 0, default: true }], image_list: [], diff --git a/spec/concepts/project/update_default_component_spec.rb b/spec/concepts/project/update_default_component_spec.rb index 415202355..5960fcc61 100644 --- a/spec/concepts/project/update_default_component_spec.rb +++ b/spec/concepts/project/update_default_component_spec.rb @@ -40,8 +40,7 @@ :id, :name, :content, - :extension, - :index + :extension ) end diff --git a/spec/concepts/project/update_delete_components_spec.rb b/spec/concepts/project/update_delete_components_spec.rb index 4b1a9b998..18a1aadb0 100644 --- a/spec/concepts/project/update_delete_components_spec.rb +++ b/spec/concepts/project/update_delete_components_spec.rb @@ -14,8 +14,7 @@ :id, :name, :content, - :extension, - :index + :extension ) end diff --git a/spec/concepts/project/update_invalid_spec.rb b/spec/concepts/project/update_invalid_spec.rb index 8b60be486..7c2560f56 100644 --- a/spec/concepts/project/update_invalid_spec.rb +++ b/spec/concepts/project/update_invalid_spec.rb @@ -20,8 +20,7 @@ id: editable_component.id, name: nil, content: 'updated content', - extension: 'py', - index: 5 + extension: 'py' } end @@ -46,7 +45,7 @@ end def component_properties_hash(component) - component.attributes.symbolize_keys.slice(:name, :content, :extension, :index) + component.attributes.symbolize_keys.slice(:name, :content, :extension) end def default_component_hash @@ -54,8 +53,7 @@ def default_component_hash :id, :name, :content, - :extension, - :index + :extension ) end @@ -63,8 +61,7 @@ def new_component_hash { name: 'new component', content: 'new component content', - extension: 'py', - index: 99 + extension: 'py' } end end diff --git a/spec/concepts/project/update_spec.rb b/spec/concepts/project/update_spec.rb index f28347f75..4598e578e 100644 --- a/spec/concepts/project/update_spec.rb +++ b/spec/concepts/project/update_spec.rb @@ -21,8 +21,7 @@ id: editable_component.id, name: 'updated component name', content: 'updated content', - extension: 'py', - index: 5 + extension: 'py' } end @@ -40,7 +39,7 @@ it 'updates component properties' do expect { update } .to change { component_properties_hash(editable_component.reload) } - .to(edited_component_hash.slice(:name, :content, :extension, :index)) + .to(edited_component_hash.slice(:name, :content, :extension)) end end @@ -83,8 +82,7 @@ def component_properties_hash(component) component.attributes.symbolize_keys.slice( :name, :content, - :extension, - :index + :extension ) end @@ -93,8 +91,7 @@ def default_component_hash :id, :name, :content, - :extension, - :index + :extension ) end @@ -102,8 +99,7 @@ def new_component_hash { name: 'new component', content: 'new component content', - extension: 'py', - index: 99 + extension: 'py' } end end diff --git a/spec/factories/component.rb b/spec/factories/component.rb index debe18f42..f0d2c06ad 100644 --- a/spec/factories/component.rb +++ b/spec/factories/component.rb @@ -4,14 +4,12 @@ factory :component do name { Faker::Lorem.word } extension { 'py' } - sequence(:index) { |n| n } default { false } content { Faker::Lorem.paragraph } project factory :default_python_component do name { 'main' } - index { 0 } default { true } end end diff --git a/spec/models/component_spec.rb b/spec/models/component_spec.rb index 308465152..506ab8f73 100644 --- a/spec/models/component_spec.rb +++ b/spec/models/component_spec.rb @@ -8,8 +8,6 @@ it { is_expected.to belong_to(:project) } it { is_expected.to validate_presence_of(:name) } it { is_expected.to validate_presence_of(:extension) } - it { is_expected.to validate_presence_of(:index) } - it { is_expected.to validate_uniqueness_of(:index).scoped_to(:project_id) } context 'when default component' do let(:component) { create(:default_python_component) } diff --git a/spec/request/projects/update_spec.rb b/spec/request/projects/update_spec.rb index 06e8e9359..5b527f9b8 100644 --- a/spec/request/projects/update_spec.rb +++ b/spec/request/projects/update_spec.rb @@ -14,8 +14,7 @@ :id, :name, :content, - :extension, - :index + :extension ) end From 7789a580cf312c8bae32162a8468e89e418aa324 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Fri, 10 Feb 2023 17:37:30 +0000 Subject: [PATCH 05/17] some rubocop fixes --- app/views/api/projects/show.json.jbuilder | 2 +- lib/project_importer_example.rb | 132 ++++++++++----------- lib/tasks/projects.rake | 20 ++-- spec/concepts/project/create_remix_spec.rb | 2 +- 4 files changed, 78 insertions(+), 78 deletions(-) diff --git a/app/views/api/projects/show.json.jbuilder b/app/views/api/projects/show.json.jbuilder index 61a0c9ed8..de8b6fa8c 100644 --- a/app/views/api/projects/show.json.jbuilder +++ b/app/views/api/projects/show.json.jbuilder @@ -4,7 +4,7 @@ json.call(@project, :identifier, :project_type, :name, :user_id) json.parent(@project.parent, :name, :identifier) if @project.parent -json.components @project.components, :id, :name, :extension, :content +json.components @project.components, :id, :name, :extension, :content json.image_list @project.images do |image| json.filename image.filename diff --git a/lib/project_importer_example.rb b/lib/project_importer_example.rb index 80ecaa7ee..338b742f4 100644 --- a/lib/project_importer_example.rb +++ b/lib/project_importer_example.rb @@ -1,39 +1,40 @@ - +# frozen_string_literal: true class ProjectImporter - attr_reader :name, :identifier, :images, :components, :type - - components = [ - { - filename: "index" - extension: "html" - contents: "" - default: true - }, - { - filename: "styles" - extension: "css" - contents: "html { color: pink } " - default: false - }] - - images = { - [ - filename: "foo.png" - contents: # binary - ] - } - - def initialize(name:, identifier:, type:, components:, images: []) - @name = name - @identifier = identifier - @components = components - @images = images - @type = type - end + attr_reader :name, :identifier, :images, :components, :type + + # Example of components and images input data structure + # components = [ + # { + # filename: "index" + # extension: "html" + # contents: "" + # default: true + # }, + # { + # filename: "styles" + # extension: "css" + # contents: "html { color: pink } " + # default: false + # }] + + # images = { + # [ + # filename: "foo.png" + # contents: # binary + # ] + # } + + def initialize(name:, identifier:, type:, components:, images: []) + @name = name + @identifier = identifier + @components = components + @images = images + @type = type + end - def import! - Project.transaction do + def import! + Project.transaction do delete_components components.each do |component| @@ -47,51 +48,50 @@ def import! end project.save! - end end + end - private + private - def project - @project ||= Project.find_or_initialze_by(identifier:) - end + def project + @project ||= Project.find_or_initialze_by(identifier:) + end - def delete_components - project.components.each(&:destroy) - end + def delete_components + project.components.each(&:destroy) + end - def delete_removed_images - existing_images = project.images.map { |x| x.blob.filename.to_s } - diff = existing_images - images.map{|x| x[:filename]} - return if diff.empty? + def delete_removed_images + existing_images = project.images.map { |x| x.blob.filename.to_s } + diff = existing_images - images.pluck(:filename) + return if diff.empty? - diff.each do |filename| - img = project.images.find { |i| i.blob.filename == filename } - img.purge! - end + diff.each do |filename| + img = project.images.find { |i| i.blob.filename == filename } + img.purge! end + end - def attach_images_if_needed - existing_image = project.images.find { |i| i.blob.filename == image_name } - - if existing_image - return if existing_image.blob.checksum == image_checksum(image_name) + def attach_images_if_needed + existing_image = project.images.find { |i| i.blob.filename == image_name } - existing_image.purge! - end + if existing_image + return if existing_image.blob.checksum == image_checksum(image_name) - project.images.attach!(io: image.io, - filename: image.filename) + existing_image.purge! end + project.images.attach!(io: image.io, + filename: image.filename) + end - def image_checksum(io) - OpenSSL::Digest.new('MD5').tap do |checksum| - while (chunk = io.read(5.megabytes)) - checksum << chunk - end + def image_checksum(io) + OpenSSL::Digest.new('MD5').tap do |checksum| + while (chunk = io.read(5.megabytes)) + checksum << chunk + end - io.rewind - end.base64digest - end + io.rewind + end.base64digest end +end diff --git a/lib/tasks/projects.rake b/lib/tasks/projects.rake index 13db72a6e..024152ff8 100644 --- a/lib/tasks/projects.rake +++ b/lib/tasks/projects.rake @@ -5,26 +5,26 @@ require 'yaml' namespace :projects do desc 'Import starter projects' task create_starter: :environment do - code_formats = [".py", '.csv', '.txt'] + code_formats = ['.py', '.csv', '.txt'] image_formats = ['.png', '.jpg', '.jpeg'] Dir.each_child("#{File.dirname(__FILE__)}/project_components") do |dir| proj_config = YAML.safe_load(File.read("#{File.dirname(__FILE__)}/project_components/#{dir}/project_config.yml")) project = find_project(proj_config) files = Dir.children("#{File.dirname(__FILE__)}/project_components/#{dir}") - code_files = files.filter{ |file| code_formats.include? File.extname(file) } - image_files = files.filter{ |file| image_formats.include? File.extname(file) } + code_files = files.filter { |file| code_formats.include? File.extname(file) } + image_files = files.filter { |file| image_formats.include? File.extname(file) } code_files.each do |file| - name = File.basename(file, '.*') - extension = File.extname(file).delete('.') - code = File.read(File.dirname(__FILE__) + "/project_components/#{dir}/#{File.basename(file)}") - default = (File.basename(file)=='main.py') - project_component = Component.new(name:, extension:, content: code, default:) - project.components << project_component + name = File.basename(file, '.*') + extension = File.extname(file).delete('.') + code = File.read(File.dirname(__FILE__) + "/project_components/#{dir}/#{File.basename(file)}") + default = (File.basename(file) == 'main.py') + project_component = Component.new(name:, extension:, content: code, default:) + project.components << project_component end delete_removed_images(project, image_files) - image_files.each do |image| + image_files.each do |image| attach_image_if_needed(project, image, dir) end diff --git a/spec/concepts/project/create_remix_spec.rb b/spec/concepts/project/create_remix_spec.rb index 91df29564..3c1307e16 100644 --- a/spec/concepts/project/create_remix_spec.rb +++ b/spec/concepts/project/create_remix_spec.rb @@ -81,7 +81,7 @@ end context 'when a new component has been added before remixing' do - let(:new_component_params) { { name: 'added_component', extension: 'py', content: 'some added component content'} } + let(:new_component_params) { { name: 'added_component', extension: 'py', content: 'some added component content' } } before do remix_params[:components] << new_component_params From bfee6b02861c9c209d346774adba8a7af311d0f3 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Mon, 13 Feb 2023 15:58:23 +0000 Subject: [PATCH 06/17] getting project importer refactor working --- ...mporter_example.rb => project_importer.rb} | 45 +++------ lib/tasks/projects.rake | 93 ++++++------------- 2 files changed, 37 insertions(+), 101 deletions(-) rename lib/{project_importer_example.rb => project_importer.rb} (56%) diff --git a/lib/project_importer_example.rb b/lib/project_importer.rb similarity index 56% rename from lib/project_importer_example.rb rename to lib/project_importer.rb index 338b742f4..4e7d7a0d4 100644 --- a/lib/project_importer_example.rb +++ b/lib/project_importer.rb @@ -3,28 +3,6 @@ class ProjectImporter attr_reader :name, :identifier, :images, :components, :type - # Example of components and images input data structure - # components = [ - # { - # filename: "index" - # extension: "html" - # contents: "" - # default: true - # }, - # { - # filename: "styles" - # extension: "css" - # contents: "html { color: pink } " - # default: false - # }] - - # images = { - # [ - # filename: "foo.png" - # contents: # binary - # ] - # } - def initialize(name:, identifier:, type:, components:, images: []) @name = name @identifier = identifier @@ -38,14 +16,12 @@ def import! delete_components components.each do |component| - project_component = Component.new(*component) + project_component = Component.new(**component) project.components << project_component end delete_removed_images - project_images.each do |image_name| - attach_image_if_needed(project, image_name, dir) - end + attach_images_if_needed project.save! end @@ -54,7 +30,7 @@ def import! private def project - @project ||= Project.find_or_initialze_by(identifier:) + @project ||= Project.find_or_initialize_by(identifier:) end def delete_components @@ -73,16 +49,17 @@ def delete_removed_images end def attach_images_if_needed - existing_image = project.images.find { |i| i.blob.filename == image_name } + images.each do |image| + existing_image = project.images.find { |i| i.blob.filename == image[:filename] } - if existing_image - return if existing_image.blob.checksum == image_checksum(image_name) + if existing_image + return if existing_image.blob.checksum == image_checksum(image[:io]) - existing_image.purge! - end + existing_image.purge! + end - project.images.attach!(io: image.io, - filename: image.filename) + project.images.attach!(*image) + end end def image_checksum(io) diff --git a/lib/tasks/projects.rake b/lib/tasks/projects.rake index 024152ff8..2ab4da2ef 100644 --- a/lib/tasks/projects.rake +++ b/lib/tasks/projects.rake @@ -1,90 +1,49 @@ # frozen_string_literal: true require 'yaml' +require 'project_importer' + +CODE_FORMATS = ['.py', '.csv', '.txt'].freeze +IMAGE_FORMATS = ['.png', '.jpg', '.jpeg'].freeze namespace :projects do desc 'Import starter projects' task create_starter: :environment do - code_formats = ['.py', '.csv', '.txt'] - image_formats = ['.png', '.jpg', '.jpeg'] - Dir.each_child("#{File.dirname(__FILE__)}/project_components") do |dir| proj_config = YAML.safe_load(File.read("#{File.dirname(__FILE__)}/project_components/#{dir}/project_config.yml")) - project = find_project(proj_config) files = Dir.children("#{File.dirname(__FILE__)}/project_components/#{dir}") - code_files = files.filter { |file| code_formats.include? File.extname(file) } - image_files = files.filter { |file| image_formats.include? File.extname(file) } + code_files = files.filter { |file| CODE_FORMATS.include? File.extname(file) } + image_files = files.filter { |file| IMAGE_FORMATS.include? File.extname(file) } + components = [] code_files.each do |file| - name = File.basename(file, '.*') - extension = File.extname(file).delete('.') - code = File.read(File.dirname(__FILE__) + "/project_components/#{dir}/#{File.basename(file)}") - default = (File.basename(file) == 'main.py') - project_component = Component.new(name:, extension:, content: code, default:) - project.components << project_component + components << component(file, dir) end - delete_removed_images(project, image_files) - image_files.each do |image| - attach_image_if_needed(project, image, dir) + + images = [] + image_files.each do |file| + images << image(file, dir) end - # project_images = proj_config['IMAGES'] || [] - # delete_removed_images(project, project_images) - # project_images.each do |image_name| - # attach_image_if_needed(project, image_name, dir) - # end - # puts project.identifier - # puts project.components.length - project.save + project_importer = ProjectImporter.new(name: proj_config['NAME'], identifier: proj_config['IDENTIFIER'], + type: proj_config['TYPE'] ||= 'python', components:, images:) + project_importer.import! end end end -def find_project(proj_config) - if Project.find_by(identifier: proj_config['IDENTIFIER']).nil? - project = Project.new(identifier: proj_config['IDENTIFIER'], name: proj_config['NAME'], - project_type: proj_config['TYPE'] ||= 'python') - else - project = Project.find_by(identifier: proj_config['IDENTIFIER']) - project.name = proj_config['NAME'] - project.components.each(&:destroy) - end - - project -end - -def delete_removed_images(project, images_to_attach) - new_image_names = images_to_attach.map { |image| File.basename(image) } - existing_image_names = project.images.map { |x| x.blob.filename.to_s } - diff = existing_image_names - new_image_names - return if diff.empty? +private - diff.each do |filename| - img = project.images.find { |i| i.blob.filename == filename } - img.purge + def component(file, dir) + name = File.basename(file, '.*') + extension = File.extname(file).delete('.') + code = File.read(File.dirname(__FILE__) + "/project_components/#{dir}/#{File.basename(file)}") + default = (File.basename(file) == 'main.py') + component = { name:, extension:, content: code, default: } end -end -def attach_image_if_needed(project, image, dir) - image_name = File.basename(image) - existing_image = project.images.find { |i| i.blob.filename == image_name } - - if existing_image - return if existing_image.blob.checksum == image_checksum(image_name, dir) - - existing_image.purge + def image(file, dir) + filename = File.basename(file) + io = File.open(File.dirname(__FILE__) + "/project_components/#{dir}/#{filename}") + image = { filename:, io: } end - project.images.attach(io: File.open(File.dirname(__FILE__) + "/project_components/#{dir}/#{image_name}"), - filename: image_name) -end - -def image_checksum(image_name, dir) - io = File.open(File.dirname(__FILE__) + "/project_components/#{dir}/#{image_name}") - OpenSSL::Digest.new('MD5').tap do |checksum| - while (chunk = io.read(5.megabytes)) - checksum << chunk - end - - io.rewind - end.base64digest -end From 062dbe369057ff8cde569e7a392465f2613206a3 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Mon, 13 Feb 2023 17:02:37 +0000 Subject: [PATCH 07/17] fixing rubocop --- lib/project_importer.rb | 21 +++++++++++++-------- lib/tasks/projects.rake | 24 ++++++++++++------------ 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index 4e7d7a0d4..24f41e343 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -38,30 +38,35 @@ def delete_components end def delete_removed_images - existing_images = project.images.map { |x| x.blob.filename.to_s } - diff = existing_images - images.pluck(:filename) - return if diff.empty? + return if removed_image_names.empty? - diff.each do |filename| + removed_image_names.each do |filename| img = project.images.find { |i| i.blob.filename == filename } img.purge! end end + def removed_image_names + existing_images = project.images.map { |x| x.blob.filename.to_s } + existing_images - images.pluck(:filename) + end + def attach_images_if_needed images.each do |image| - existing_image = project.images.find { |i| i.blob.filename == image[:filename] } - + existing_image = find_existing_image(filename) if existing_image - return if existing_image.blob.checksum == image_checksum(image[:io]) + next if existing_image.blob.checksum == image_checksum(image[:io]) existing_image.purge! end - project.images.attach!(*image) end end + def find_existing_image(filename) + project.images.find { |i| i.blob.filename == filename } + end + def image_checksum(io) OpenSSL::Digest.new('MD5').tap do |checksum| while (chunk = io.read(5.megabytes)) diff --git a/lib/tasks/projects.rake b/lib/tasks/projects.rake index 2ab4da2ef..ffc039bba 100644 --- a/lib/tasks/projects.rake +++ b/lib/tasks/projects.rake @@ -34,16 +34,16 @@ end private - def component(file, dir) - name = File.basename(file, '.*') - extension = File.extname(file).delete('.') - code = File.read(File.dirname(__FILE__) + "/project_components/#{dir}/#{File.basename(file)}") - default = (File.basename(file) == 'main.py') - component = { name:, extension:, content: code, default: } - end +def component(file, dir) + name = File.basename(file, '.*') + extension = File.extname(file).delete('.') + code = File.read(File.dirname(__FILE__) + "/project_components/#{dir}/#{File.basename(file)}") + default = (File.basename(file) == 'main.py') + component = { name:, extension:, content: code, default: } +end - def image(file, dir) - filename = File.basename(file) - io = File.open(File.dirname(__FILE__) + "/project_components/#{dir}/#{filename}") - image = { filename:, io: } - end +def image(file, dir) + filename = File.basename(file) + io = File.open(File.dirname(__FILE__) + "/project_components/#{dir}/#{filename}") + image = { filename:, io: } +end From 140d4cdcf7edfcbecccbd5e9ff352e67b7b785a1 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Mon, 13 Feb 2023 17:14:20 +0000 Subject: [PATCH 08/17] fixing --- lib/project_importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index 24f41e343..aa8aadd1d 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -53,7 +53,7 @@ def removed_image_names def attach_images_if_needed images.each do |image| - existing_image = find_existing_image(filename) + existing_image = find_existing_image(image[:filename]) if existing_image next if existing_image.blob.checksum == image_checksum(image[:io]) From 7d0b008134fafc4d0d5ab7fe05296371c0830f46 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Mon, 13 Feb 2023 17:27:49 +0000 Subject: [PATCH 09/17] removed schema rb, will regenerate --- db/schema.rb | 136 --------------------------------------------------- 1 file changed, 136 deletions(-) delete mode 100644 db/schema.rb diff --git a/db/schema.rb b/db/schema.rb deleted file mode 100644 index 19abe38d3..000000000 --- a/db/schema.rb +++ /dev/null @@ -1,136 +0,0 @@ -# This file is auto-generated from the current state of the database. Instead -# of editing this file, please use the migrations feature of Active Record to -# incrementally modify your database, and then regenerate this schema definition. -# -# This file is the source Rails uses to define your schema when running `bin/rails -# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to -# be faster and is potentially less error prone than running all of your -# migrations from scratch. Old migrations may fail to apply correctly if those -# migrations use external dependencies or application code. -# -# It's strongly recommended that you check this file into your version control system. - -ActiveRecord::Schema[7.0].define(version: 2023_02_10_142728) do - # These are extensions that must be enabled in order to support this database - enable_extension "pgcrypto" - enable_extension "plpgsql" - - create_table "active_storage_attachments", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.string "name", null: false - t.string "record_type", null: false - t.uuid "record_id", null: false - t.uuid "blob_id", null: false - t.datetime "created_at", null: false - t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" - t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true - end - - create_table "active_storage_blobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.string "key", null: false - t.string "filename", null: false - t.string "content_type" - t.text "metadata" - t.string "service_name", null: false - t.bigint "byte_size", null: false - t.string "checksum" - t.datetime "created_at", null: false - t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true - end - - create_table "active_storage_variant_records", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.uuid "blob_id", null: false - t.string "variation_digest", null: false - t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true - end - - create_table "components", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.uuid "project_id" - t.string "name", null: false - t.string "extension", null: false - t.string "content" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.boolean "default", default: false, null: false - t.index ["project_id"], name: "index_components_on_project_id" - end - - create_table "good_job_batches", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.text "description" - t.jsonb "serialized_properties" - t.text "on_finish" - t.text "on_success" - t.text "on_discard" - t.text "callback_queue_name" - t.integer "callback_priority" - t.datetime "enqueued_at" - t.datetime "discarded_at" - t.datetime "finished_at" - end - - create_table "good_job_processes", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.jsonb "state" - end - - create_table "good_job_settings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.text "key" - t.jsonb "value" - t.index ["key"], name: "index_good_job_settings_on_key", unique: true - end - - create_table "good_jobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.text "queue_name" - t.integer "priority" - t.jsonb "serialized_params" - t.datetime "scheduled_at" - t.datetime "performed_at" - t.datetime "finished_at" - t.text "error" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.uuid "active_job_id" - t.text "concurrency_key" - t.text "cron_key" - t.uuid "retried_good_job_id" - t.datetime "cron_at" - t.uuid "batch_id" - t.uuid "batch_callback_id" - t.index ["active_job_id", "created_at"], name: "index_good_jobs_on_active_job_id_and_created_at" - t.index ["active_job_id"], name: "index_good_jobs_on_active_job_id" - t.index ["batch_callback_id"], name: "index_good_jobs_on_batch_callback_id", where: "(batch_callback_id IS NOT NULL)" - t.index ["batch_id"], name: "index_good_jobs_on_batch_id", where: "(batch_id IS NOT NULL)" - t.index ["concurrency_key"], name: "index_good_jobs_on_concurrency_key_when_unfinished", where: "(finished_at IS NULL)" - t.index ["cron_key", "created_at"], name: "index_good_jobs_on_cron_key_and_created_at" - t.index ["cron_key", "cron_at"], name: "index_good_jobs_on_cron_key_and_cron_at", unique: true - t.index ["finished_at"], name: "index_good_jobs_jobs_on_finished_at", where: "((retried_good_job_id IS NULL) AND (finished_at IS NOT NULL))" - t.index ["priority", "created_at"], name: "index_good_jobs_jobs_on_priority_created_at_when_unfinished", order: { priority: "DESC NULLS LAST" }, where: "(finished_at IS NULL)" - t.index ["queue_name", "scheduled_at"], name: "index_good_jobs_on_queue_name_and_scheduled_at", where: "(finished_at IS NULL)" - t.index ["scheduled_at"], name: "index_good_jobs_on_scheduled_at", where: "(finished_at IS NULL)" - end - - create_table "projects", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.uuid "user_id" - t.string "name" - t.string "identifier", null: false - t.string "project_type", default: "python", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.uuid "remixed_from_id" - t.index ["identifier"], name: "index_projects_on_identifier", unique: true - t.index ["remixed_from_id"], name: "index_projects_on_remixed_from_id" - end - - create_table "words", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.string "word" - t.index ["word"], name: "index_words_on_word" - end - - add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" - add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" - add_foreign_key "components", "projects" -end From e4bc378e6fa8ff742c41f962e07cb60aaf64df07 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Mon, 13 Feb 2023 17:31:15 +0000 Subject: [PATCH 10/17] regenerating schema rb --- db/schema.rb | 136 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 db/schema.rb diff --git a/db/schema.rb b/db/schema.rb new file mode 100644 index 000000000..19abe38d3 --- /dev/null +++ b/db/schema.rb @@ -0,0 +1,136 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema[7.0].define(version: 2023_02_10_142728) do + # These are extensions that must be enabled in order to support this database + enable_extension "pgcrypto" + enable_extension "plpgsql" + + create_table "active_storage_attachments", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.string "name", null: false + t.string "record_type", null: false + t.uuid "record_id", null: false + t.uuid "blob_id", null: false + t.datetime "created_at", null: false + t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" + t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true + end + + create_table "active_storage_blobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.string "key", null: false + t.string "filename", null: false + t.string "content_type" + t.text "metadata" + t.string "service_name", null: false + t.bigint "byte_size", null: false + t.string "checksum" + t.datetime "created_at", null: false + t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true + end + + create_table "active_storage_variant_records", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "blob_id", null: false + t.string "variation_digest", null: false + t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true + end + + create_table "components", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "project_id" + t.string "name", null: false + t.string "extension", null: false + t.string "content" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.boolean "default", default: false, null: false + t.index ["project_id"], name: "index_components_on_project_id" + end + + create_table "good_job_batches", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.text "description" + t.jsonb "serialized_properties" + t.text "on_finish" + t.text "on_success" + t.text "on_discard" + t.text "callback_queue_name" + t.integer "callback_priority" + t.datetime "enqueued_at" + t.datetime "discarded_at" + t.datetime "finished_at" + end + + create_table "good_job_processes", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.jsonb "state" + end + + create_table "good_job_settings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.text "key" + t.jsonb "value" + t.index ["key"], name: "index_good_job_settings_on_key", unique: true + end + + create_table "good_jobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.text "queue_name" + t.integer "priority" + t.jsonb "serialized_params" + t.datetime "scheduled_at" + t.datetime "performed_at" + t.datetime "finished_at" + t.text "error" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.uuid "active_job_id" + t.text "concurrency_key" + t.text "cron_key" + t.uuid "retried_good_job_id" + t.datetime "cron_at" + t.uuid "batch_id" + t.uuid "batch_callback_id" + t.index ["active_job_id", "created_at"], name: "index_good_jobs_on_active_job_id_and_created_at" + t.index ["active_job_id"], name: "index_good_jobs_on_active_job_id" + t.index ["batch_callback_id"], name: "index_good_jobs_on_batch_callback_id", where: "(batch_callback_id IS NOT NULL)" + t.index ["batch_id"], name: "index_good_jobs_on_batch_id", where: "(batch_id IS NOT NULL)" + t.index ["concurrency_key"], name: "index_good_jobs_on_concurrency_key_when_unfinished", where: "(finished_at IS NULL)" + t.index ["cron_key", "created_at"], name: "index_good_jobs_on_cron_key_and_created_at" + t.index ["cron_key", "cron_at"], name: "index_good_jobs_on_cron_key_and_cron_at", unique: true + t.index ["finished_at"], name: "index_good_jobs_jobs_on_finished_at", where: "((retried_good_job_id IS NULL) AND (finished_at IS NOT NULL))" + t.index ["priority", "created_at"], name: "index_good_jobs_jobs_on_priority_created_at_when_unfinished", order: { priority: "DESC NULLS LAST" }, where: "(finished_at IS NULL)" + t.index ["queue_name", "scheduled_at"], name: "index_good_jobs_on_queue_name_and_scheduled_at", where: "(finished_at IS NULL)" + t.index ["scheduled_at"], name: "index_good_jobs_on_scheduled_at", where: "(finished_at IS NULL)" + end + + create_table "projects", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "user_id" + t.string "name" + t.string "identifier", null: false + t.string "project_type", default: "python", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.uuid "remixed_from_id" + t.index ["identifier"], name: "index_projects_on_identifier", unique: true + t.index ["remixed_from_id"], name: "index_projects_on_remixed_from_id" + end + + create_table "words", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.string "word" + t.index ["word"], name: "index_words_on_word" + end + + add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" + add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" + add_foreign_key "components", "projects" +end From 8ce4726bdfabf5d08cd889b231694a5020767943 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Tue, 14 Feb 2023 17:10:11 +0000 Subject: [PATCH 11/17] Adding tests for the project importer --- lib/project_importer.rb | 8 +++-- spec/project_importer_spec.rb | 62 +++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 spec/project_importer_spec.rb diff --git a/lib/project_importer.rb b/lib/project_importer.rb index aa8aadd1d..baef57617 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -13,6 +13,8 @@ def initialize(name:, identifier:, type:, components:, images: []) def import! Project.transaction do + project.name = name + project.project_type = type delete_components components.each do |component| @@ -42,7 +44,7 @@ def delete_removed_images removed_image_names.each do |filename| img = project.images.find { |i| i.blob.filename == filename } - img.purge! + img.purge end end @@ -57,9 +59,9 @@ def attach_images_if_needed if existing_image next if existing_image.blob.checksum == image_checksum(image[:io]) - existing_image.purge! + existing_image.purge end - project.images.attach!(*image) + project.images.attach(**image) end end diff --git a/spec/project_importer_spec.rb b/spec/project_importer_spec.rb new file mode 100644 index 000000000..ce3aedc19 --- /dev/null +++ b/spec/project_importer_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'project_importer' + +RSpec.describe ProjectImporter do + let(:importer) do + described_class.new( + name: 'My amazing project', + identifier: 'my-amazing-project', + type: 'python', + components: [ + { name: 'main', extension: 'py', content: 'print(\'hello\')', default: true }, + { name: 'amazing', extension: 'py', content: 'print(\'this is amazing\')' } + ], + images: [ + { filename: 'my-amazing-image.png', io: File.open('spec/fixtures/files/test_image_1.png') } + ] + ) + end + + context 'when the project does not already exist in the database' do + it 'saves the project to the database' do + expect { importer.import! }.to change(Project, :count).by(1) + end + end + + context 'when the project already exists in the database' do + let!(:project) { create( + :project, + :with_default_component, + :with_components, + :with_attached_image, + component_count: 2, + identifier: 'my-amazing-project' + ) } + + it 'does not change number of saved projects' do + expect { importer.import! }.not_to change(Project, :count) + end + + it 'renames project' do + expect { importer.import! }.to change {project.reload.name}.to(importer.name) + end + + it 'deletes removed components' do + expect { importer.import! }.to change {project.components.count}.from(3).to(2) + end + + it 'updates existing components' do + expect { importer.import! }.to change {project.reload.components[0].content }.to('print(\'hello\')') + end + + it 'creates new components' do + expect {importer.import! }.to change {project.reload.components[1].name}.to('amazing') + end + + it 'updates images' do + expect { importer.import! }.to change {project.reload.images[0].filename.to_s}.to('my-amazing-image.png') + end + end +end From 99ef10affb57b71f2d249ff4aa48b51a93bbd97b Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Tue, 14 Feb 2023 17:21:35 +0000 Subject: [PATCH 12/17] adding more project importer tests --- spec/project_importer_spec.rb | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/spec/project_importer_spec.rb b/spec/project_importer_spec.rb index ce3aedc19..c3fa4a305 100644 --- a/spec/project_importer_spec.rb +++ b/spec/project_importer_spec.rb @@ -20,9 +20,32 @@ end context 'when the project does not already exist in the database' do + let(:project) { Project.find_by(identifier: importer.identifier) } + it 'saves the project to the database' do expect { importer.import! }.to change(Project, :count).by(1) end + + it 'names the project correctly' do + importer.import! + expect(project.name).to eq(importer.name) + end + + it 'gives the project the correct type' do + importer.import! + expect(project.project_type).to eq(importer.type) + end + + it 'creates the project components' do + importer.import! + expect(project.components.count).to eq(2) + end + + it 'creates the project images' do + importer.import! + expect(project.images.count).to eq(1) + end + end context 'when the project already exists in the database' do From da07c2fa449c3d82150f4a125f08b891bbd1382f Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Thu, 16 Feb 2023 12:38:18 +0000 Subject: [PATCH 13/17] Add very basic rake task test --- .rubocop.yml | 4 ++++ lib/project_importer.rb | 22 +++++++++++++-------- spec/project_importer_spec.rb | 33 ++++++++++++++++--------------- spec/tasks/create_starter_spec.rb | 13 ++++++++++++ 4 files changed, 48 insertions(+), 24 deletions(-) create mode 100644 spec/tasks/create_starter_spec.rb diff --git a/.rubocop.yml b/.rubocop.yml index dd7673317..2585eb3dd 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -14,3 +14,7 @@ inherit_from: inherit_mode: merge: - Exclude + +RSpec/DescribeClass: + Exclude: + - spec/tasks/* diff --git a/lib/project_importer.rb b/lib/project_importer.rb index baef57617..9912e5593 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -13,15 +13,9 @@ def initialize(name:, identifier:, type:, components:, images: []) def import! Project.transaction do - project.name = name - project.project_type = type + setup_project delete_components - - components.each do |component| - project_component = Component.new(**component) - project.components << project_component - end - + create_components delete_removed_images attach_images_if_needed @@ -35,10 +29,22 @@ def project @project ||= Project.find_or_initialize_by(identifier:) end + def setup_project + project.name = name + project.project_type = type + end + def delete_components project.components.each(&:destroy) end + def create_components + components.each do |component| + project_component = Component.new(**component) + project.components << project_component + end + end + def delete_removed_images return if removed_image_names.empty? diff --git a/spec/project_importer_spec.rb b/spec/project_importer_spec.rb index c3fa4a305..b6b8572e0 100644 --- a/spec/project_importer_spec.rb +++ b/spec/project_importer_spec.rb @@ -13,7 +13,7 @@ { name: 'main', extension: 'py', content: 'print(\'hello\')', default: true }, { name: 'amazing', extension: 'py', content: 'print(\'this is amazing\')' } ], - images: [ + images: [ { filename: 'my-amazing-image.png', io: File.open('spec/fixtures/files/test_image_1.png') } ] ) @@ -45,41 +45,42 @@ importer.import! expect(project.images.count).to eq(1) end - end context 'when the project already exists in the database' do - let!(:project) { create( - :project, - :with_default_component, - :with_components, - :with_attached_image, - component_count: 2, - identifier: 'my-amazing-project' - ) } + let!(:project) do + create( + :project, + :with_default_component, + :with_components, + :with_attached_image, + component_count: 2, + identifier: 'my-amazing-project' + ) + end it 'does not change number of saved projects' do expect { importer.import! }.not_to change(Project, :count) end it 'renames project' do - expect { importer.import! }.to change {project.reload.name}.to(importer.name) + expect { importer.import! }.to change { project.reload.name }.to(importer.name) end it 'deletes removed components' do - expect { importer.import! }.to change {project.components.count}.from(3).to(2) + expect { importer.import! }.to change { project.components.count }.from(3).to(2) end - it 'updates existing components' do - expect { importer.import! }.to change {project.reload.components[0].content }.to('print(\'hello\')') + it 'updates existing components' do + expect { importer.import! }.to change { project.reload.components[0].content }.to('print(\'hello\')') end it 'creates new components' do - expect {importer.import! }.to change {project.reload.components[1].name}.to('amazing') + expect { importer.import! }.to change { project.reload.components[1].name }.to('amazing') end it 'updates images' do - expect { importer.import! }.to change {project.reload.images[0].filename.to_s}.to('my-amazing-image.png') + expect { importer.import! }.to change { project.reload.images[0].filename.to_s }.to('my-amazing-image.png') end end end diff --git a/spec/tasks/create_starter_spec.rb b/spec/tasks/create_starter_spec.rb new file mode 100644 index 000000000..314100032 --- /dev/null +++ b/spec/tasks/create_starter_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'rails_helper' +Rails.application.load_tasks + +describe 'projects:create_starter' do + it 'runs' do + project_config = { 'NAME' => 'My amazing project', 'IDENTIFIER' => 'my-amazing-project', 'TYPE' => 'python' } + allow(YAML).to receive(:safe_load).and_return(project_config) + allow(File).to receive(:read).and_return('print("hello")') + expect { Rake::Task['projects:create_starter'].invoke }.not_to raise_error + end +end From b118c5a189e82ede21b4bcd8c4a321745aa02fe2 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Thu, 16 Feb 2023 16:43:31 +0000 Subject: [PATCH 14/17] removing rubocop exclusion --- .rubocop.yml | 4 ---- spec/tasks/create_starter_spec.rb | 4 +++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 2585eb3dd..dd7673317 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -14,7 +14,3 @@ inherit_from: inherit_mode: merge: - Exclude - -RSpec/DescribeClass: - Exclude: - - spec/tasks/* diff --git a/spec/tasks/create_starter_spec.rb b/spec/tasks/create_starter_spec.rb index 314100032..43b793a5e 100644 --- a/spec/tasks/create_starter_spec.rb +++ b/spec/tasks/create_starter_spec.rb @@ -3,7 +3,9 @@ require 'rails_helper' Rails.application.load_tasks -describe 'projects:create_starter' do +describe 'projects:create_starter', type: :task do + subject { task.execute } + it 'runs' do project_config = { 'NAME' => 'My amazing project', 'IDENTIFIER' => 'my-amazing-project', 'TYPE' => 'python' } allow(YAML).to receive(:safe_load).and_return(project_config) From 56df676834f91cc187d208127e53eba6dcb53c6d Mon Sep 17 00:00:00 2001 From: Izzy Smillie Date: Thu, 16 Feb 2023 17:19:54 +0000 Subject: [PATCH 15/17] Calling projectImporter --- spec/tasks/create_starter_spec.rb | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/spec/tasks/create_starter_spec.rb b/spec/tasks/create_starter_spec.rb index 43b793a5e..114231a82 100644 --- a/spec/tasks/create_starter_spec.rb +++ b/spec/tasks/create_starter_spec.rb @@ -1,15 +1,28 @@ # frozen_string_literal: true require 'rails_helper' +require 'project_importer' + Rails.application.load_tasks describe 'projects:create_starter', type: :task do subject { task.execute } + let(:project_config) { { 'NAME' => 'My amazing project', 'IDENTIFIER' => 'my-amazing-project', 'TYPE' => 'python' } } + let(:components) { [] } + let(:images) { [] } + it 'runs' do - project_config = { 'NAME' => 'My amazing project', 'IDENTIFIER' => 'my-amazing-project', 'TYPE' => 'python' } allow(YAML).to receive(:safe_load).and_return(project_config) allow(File).to receive(:read).and_return('print("hello")') expect { Rake::Task['projects:create_starter'].invoke }.not_to raise_error end + + it 'calls the ProjectImporter' do + expected_config = { components: [], identifier: 'my-amazing-project', images: [], name: 'My amazing project', type: 'python' } + allow(ProjectImporter).to receive(:new) + ProjectImporter.new(name: project_config['NAME'], identifier: project_config['IDENTIFIER'], + type: project_config['TYPE'] ||= 'python', components:, images:) + expect(ProjectImporter).to have_received(:new).with(expected_config) + end end From 92f7011b5def562fbf769d3e3331ec44bc00d980 Mon Sep 17 00:00:00 2001 From: Izzy Smillie Date: Thu, 16 Feb 2023 17:22:28 +0000 Subject: [PATCH 16/17] fix errors --- spec/tasks/create_starter_spec.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spec/tasks/create_starter_spec.rb b/spec/tasks/create_starter_spec.rb index 114231a82..3b62aa29b 100644 --- a/spec/tasks/create_starter_spec.rb +++ b/spec/tasks/create_starter_spec.rb @@ -9,8 +9,6 @@ subject { task.execute } let(:project_config) { { 'NAME' => 'My amazing project', 'IDENTIFIER' => 'my-amazing-project', 'TYPE' => 'python' } } - let(:components) { [] } - let(:images) { [] } it 'runs' do allow(YAML).to receive(:safe_load).and_return(project_config) @@ -20,9 +18,10 @@ it 'calls the ProjectImporter' do expected_config = { components: [], identifier: 'my-amazing-project', images: [], name: 'My amazing project', type: 'python' } + allow(ProjectImporter).to receive(:new) ProjectImporter.new(name: project_config['NAME'], identifier: project_config['IDENTIFIER'], - type: project_config['TYPE'] ||= 'python', components:, images:) + type: project_config['TYPE'] ||= 'python', components: [], images: []) expect(ProjectImporter).to have_received(:new).with(expected_config) end end From 3bd061403fda8264fa4bc5f015e9842988b3c674 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Fri, 17 Feb 2023 10:46:37 +0000 Subject: [PATCH 17/17] removing test --- spec/tasks/create_starter_spec.rb | 9 --------- 1 file changed, 9 deletions(-) diff --git a/spec/tasks/create_starter_spec.rb b/spec/tasks/create_starter_spec.rb index 3b62aa29b..02614d5bc 100644 --- a/spec/tasks/create_starter_spec.rb +++ b/spec/tasks/create_starter_spec.rb @@ -15,13 +15,4 @@ allow(File).to receive(:read).and_return('print("hello")') expect { Rake::Task['projects:create_starter'].invoke }.not_to raise_error end - - it 'calls the ProjectImporter' do - expected_config = { components: [], identifier: 'my-amazing-project', images: [], name: 'My amazing project', type: 'python' } - - allow(ProjectImporter).to receive(:new) - ProjectImporter.new(name: project_config['NAME'], identifier: project_config['IDENTIFIER'], - type: project_config['TYPE'] ||= 'python', components: [], images: []) - expect(ProjectImporter).to have_received(:new).with(expected_config) - end end