diff --git a/.gitignore b/.gitignore index 19d4297..0007158 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ log/*.pid log/*.out log/*.gz scripts/.ipynb_checkpoints/* +app/db/test.sqlite3* diff --git a/app/Dockerfile.app b/app/Dockerfile.app index b26af60..ed113ba 100644 --- a/app/Dockerfile.app +++ b/app/Dockerfile.app @@ -1,4 +1,7 @@ -FROM ruby:3.2.2-slim-bullseye AS base +ARG TARGET_PLATFORM=linux/amd64 +ARG IMAGE_TAG=3.3.7-slim-bullseye + +FROM --platform=${TARGET_PLATFORM} ruby:${IMAGE_TAG} AS base RUN apt-get update -qq \ && apt-get install -y \ @@ -23,7 +26,7 @@ COPY Gemfile Gemfile.lock ./ RUN bundle config set --local without 'test' && \ bundle install -FROM base AS final +FROM --platform=$TARGET_PLATFORM base AS final COPY --from=dependencies /usr/local/bundle /usr/local/bundle @@ -33,4 +36,4 @@ COPY . . EXPOSE 80 -CMD ["bundle", "exec", "puma", "-p", "80"] \ No newline at end of file +CMD ["bundle", "exec", "puma", "-p", "80"] diff --git a/app/Dockerfile.test b/app/Dockerfile.test index 06cdd2d..3469a5c 100644 --- a/app/Dockerfile.test +++ b/app/Dockerfile.test @@ -1,4 +1,7 @@ -FROM ruby:3.2.2-slim-bullseye +ARG TARGET_PLATFORM=linux/amd64 +ARG IMAGE_TAG=3.3.7-slim-bullseye + +FROM --platform=${TARGET_PLATFORM} ruby:${IMAGE_TAG} # Download dependencies for testing RUN apt-get update -qq \ @@ -59,4 +62,4 @@ RUN bundle config set --local with 'test' && \ COPY . . -RUN ["rake", "test"] \ No newline at end of file +RUN ["rake", "test"] diff --git a/app/Gemfile b/app/Gemfile index 28ad3b6..e8bd9e3 100755 --- a/app/Gemfile +++ b/app/Gemfile @@ -1,11 +1,11 @@ source 'https://rubygems.org' -ruby '3.2.2' +ruby '3.3.7' gem 'sinatra' gem 'puma' -gem 'activerecord' +gem 'activerecord', '7.2.2.1' gem 'mysql2' gem 'will_paginate' diff --git a/app/Gemfile.lock b/app/Gemfile.lock index 636fe11..c207834 100644 --- a/app/Gemfile.lock +++ b/app/Gemfile.lock @@ -1,43 +1,47 @@ GEM remote: https://rubygems.org/ specs: - activemodel (7.1.3.2) - activesupport (= 7.1.3.2) - activerecord (7.1.3.2) - activemodel (= 7.1.3.2) - activesupport (= 7.1.3.2) + activemodel (7.2.2.1) + activesupport (= 7.2.2.1) + activerecord (7.2.2.1) + activemodel (= 7.2.2.1) + activesupport (= 7.2.2.1) timeout (>= 0.4.0) - activesupport (7.1.3.2) + activesupport (7.2.2.1) base64 + benchmark (>= 0.3) bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) + concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + logger (>= 1.4.2) minitest (>= 5.1) - mutex_m - tzinfo (~> 2.0) - addressable (2.8.6) - public_suffix (>= 2.0.2, < 6.0) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) base64 (0.2.0) - bigdecimal (3.1.7) - capybara (3.39.2) + benchmark (0.4.0) + bigdecimal (3.1.9) + capybara (3.40.0) addressable matrix mini_mime (>= 0.1.3) - nokogiri (~> 1.8) + nokogiri (~> 1.11) rack (>= 1.6.0) rack-test (>= 0.6.3) regexp_parser (>= 1.5, < 3.0) xpath (~> 3.2) - concurrent-ruby (1.2.3) - connection_pool (2.4.1) - date (3.3.4) + concurrent-ruby (1.3.5) + connection_pool (2.5.0) + date (3.4.1) drb (2.2.1) fakeredis (0.9.2) redis (~> 4.8) - i18n (1.14.4) + i18n (1.14.7) concurrent-ruby (~> 1.0) + logger (1.6.6) mail (2.8.1) mini_mime (>= 0.1.1) net-imap @@ -45,61 +49,98 @@ GEM net-smtp matrix (0.4.2) mini_mime (1.1.5) - minitest (5.22.3) - mustermann (3.0.0) + minitest (5.25.4) + mustermann (3.0.3) ruby2_keywords (~> 0.0.1) - mutex_m (0.2.0) - mysql2 (0.5.5) - net-imap (0.4.8) + mysql2 (0.5.6) + net-imap (0.5.6) date net-protocol net-pop (0.1.2) net-protocol net-protocol (0.2.2) timeout - net-smtp (0.4.0) + net-smtp (0.5.1) net-protocol - nio4r (2.7.0) - nokogiri (1.16.3-x86_64-linux) + nio4r (2.7.4) + nokogiri (1.18.2-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.18.2-aarch64-linux-musl) + racc (~> 1.4) + nokogiri (1.18.2-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.18.2-arm-linux-musl) + racc (~> 1.4) + nokogiri (1.18.2-arm64-darwin) + racc (~> 1.4) + nokogiri (1.18.2-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.18.2-x86_64-linux-gnu) + racc (~> 1.4) + nokogiri (1.18.2-x86_64-linux-musl) racc (~> 1.4) - public_suffix (5.0.4) - puma (6.4.0) + public_suffix (6.0.1) + puma (6.6.0) nio4r (~> 2.0) - racc (1.7.3) - rack (2.2.9) - rack-protection (3.1.0) - rack (~> 2.2, >= 2.2.4) - rack-test (2.1.0) + racc (1.8.1) + rack (3.1.10) + rack-protection (4.1.1) + base64 (>= 0.1.0) + logger (>= 1.6.0) + rack (>= 3.0.0, < 4) + rack-session (2.1.0) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rack-test (2.2.0) rack (>= 1.3) redis (4.8.1) - regexp_parser (2.8.3) - rexml (3.2.6) + regexp_parser (2.10.0) + rexml (3.4.0) ruby2_keywords (0.0.5) - rubyzip (2.3.2) - selenium-webdriver (4.16.0) + rubyzip (2.4.1) + securerandom (0.4.1) + selenium-webdriver (4.28.0) + base64 (~> 0.2) + logger (~> 1.4) rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) - sinatra (3.1.0) + sinatra (4.1.1) + logger (>= 1.6.0) mustermann (~> 3.0) - rack (~> 2.2, >= 2.2.4) - rack-protection (= 3.1.0) + rack (>= 3.0.0, < 4) + rack-protection (= 4.1.1) + rack-session (>= 2.0.0, < 3) tilt (~> 2.0) - sqlite3 (1.6.9-x86_64-linux) - tilt (2.3.0) - timeout (0.4.1) + sqlite3 (2.5.0-aarch64-linux-gnu) + sqlite3 (2.5.0-aarch64-linux-musl) + sqlite3 (2.5.0-arm-linux-gnu) + sqlite3 (2.5.0-arm-linux-musl) + sqlite3 (2.5.0-arm64-darwin) + sqlite3 (2.5.0-x86_64-darwin) + sqlite3 (2.5.0-x86_64-linux-gnu) + sqlite3 (2.5.0-x86_64-linux-musl) + tilt (2.6.0) + timeout (0.4.3) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - websocket (1.2.10) - will_paginate (4.0.0) + websocket (1.2.11) + will_paginate (4.0.1) xpath (3.2.0) nokogiri (~> 1.8) PLATFORMS - x86_64-linux + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl + arm64-darwin + x86_64-darwin + x86_64-linux-gnu + x86_64-linux-musl DEPENDENCIES - activerecord + activerecord (= 7.2.2.1) capybara fakeredis mail @@ -115,7 +156,7 @@ DEPENDENCIES will_paginate RUBY VERSION - ruby 3.2.2p53 + ruby 3.3.7p123 BUNDLED WITH - 2.4.10 \ No newline at end of file + 2.6.3 diff --git a/app/README.md b/app/README.md index 1b4fc79..148af20 100644 --- a/app/README.md +++ b/app/README.md @@ -1,4 +1,6 @@ -## Running the Application +# APP README + +## Configuring - Make sure you have the repository cloned (`git clone https://github.com/cdlib/pid.git`). - Navigate to the root directory of the project (where `docker-compose.yml` exists). @@ -6,6 +8,9 @@ > Generate a 64-character hex for the session secret. The SMTP stuff is optional. - Replace `/app/config/*.yml.example` files with `*.yml` versions. > The `.example` files are primarily for reference, though they will be sufficient to get the app running; you can modify the values to implement your own configuration. Naturally, if you wish to connect to an external database, you may need to be on VPN. + +## Docker Installation +### Running the Appplication - Using `docker-compose`: - Run `docker-compose up --build` to build and start the application. > This will build the Redis container as well as the application container, but only after making sure the tests pass. You can modify the `docker-compose.yml` file to skip the tests. @@ -22,7 +27,7 @@ - There's another script to checks for duplicate URLs. This is not as crucial to the functioning of the application as Redis, but you can run it by following the steps above, except replace the command in the final step with `ruby ruby_scripts/detect_duplicate_urls.rb`. -## Running Tests +### Running Tests Assuming you've cloned the repository (`git clone https://github.com/cdlib/pid.git`), tests are automatically run with `docker-compose up --build`. To run tests manually: - If you want to run all the tests, navigate to the root directory and run `docker-compose build test`. @@ -30,6 +35,106 @@ Assuming you've cloned the repository (`git clone https://github.com/cdlib/pid.g - `RUN ["rake", "test_client", "TEST=test/client/test_user_views.rb"]` - `RUN ["ruby", "-I", "test", "test/integration/test_pid_controller.rb", "-n", "test_post_pid"]` +## Mac OS X Installation + +These instructions assume you are working on a Mac with Homebrew installed. + +1. **Clone the Repository & Prepare Environment Files** + + `git clone https://github.com/cdlib/pid.git cd pid # + + * Replace .env.example with your .env file and update values. + + * Copy example and setup environment variables. + > `cp .env.example .env` + + > **Note:** + > - You will need access to a pid database locally or on a host. + > - You will need access to a redis in not using docker + > - For the session secret, generate a 64-character hexadecimal string (which represents 32 bytes): `openssl rand -hex 32` + +2. **Set Up Ruby Using rbenv** + Install rbenv via Homebrew if you haven't already: + + `brew install rbenv` + + Initialize rbenv (follow the instructions that appear to add it to your shell profile, e.g., in `~/.zshrc`): + + + `rbenv init` + + Install Ruby 3.2.2 (or version in the Gemfile): + + `rbenv install 3.2.2` + + Then, in your project directory, set the local Ruby version: + + `cd path/to/pid/app rbenv local 3.2.2` + +3. **Install Bundler and Dependencies** + Install Bundler: + + `gem install bundler` + + `bundle install` + +4. **Install MySQL Client** + The application requires the MySQL client for the native `mysql2` gem. There may be an error during install. If so, install it with Homebrew: + + `brew install mysql-client` + + Because `mysql-client` is keg-only, add it to your PATH. Append this line to your shell profile (`~/.zshrc`): + + `echo 'export PATH="/opt/homebrew/opt/mysql-client/bin:$PATH"' >> ~/.zshrc` + + Then, reload your shell: + + `source ~/.zshrc` + + `export LDFLAGS="-L/opt/homebrew/opt/mysql-client/lib" export CPPFLAGS="-I/opt/homebrew/opt/mysql-client/include"` + +5. **Ensure zstd Is Installed** + (If it isn’t already installed, you can do so with:) + + `brew install zstd` + + Verify that its libraries are in `/opt/homebrew/opt/zstd/lib`. + +6. **Manually Install the mysql2 Gem** + The gemfile specifies a particular version (e.g., `0.5.6`). Install it manually using the proper flags to ensure it finds both MySQL and zstd libraries. Replace `0.5.6` with the version specified in your Gemfile.lock if different. + `export LIBRARY_PATH="/opt/homebrew/opt/zstd/lib:$LIBRARY_PATH"` + + `gem install mysql2 --version 0.5.6 -- \ --with-mysql-config=/opt/homebrew/opt/mysql-client/bin/mysql_config \ --with-ldflags="-L/opt/homebrew/opt/zstd/lib" \ --with-cppflags="-I/opt/homebrew/opt/zstd/include"` + +7. **Run the Application or Tests** + + `rake test` + + > Note: The unit tests should work without Redis or Mysql connection. + + or start the app: + + `ruby app.rb` + + > Note: Running the application requires Redis and Mysql connections. + +8. **Issues with Selenium ChromeUI tests** + + If you have issues with Selenium Chrome web tests you will need to install it. + + Install Google Chrome if not already on system. + + Install Chromedriver: + + `brew install --cask chromedriver` + + Verify the installation path: + + `which chromedriver` + + The expected path is typically `/opt/homebrew/bin/chromedriver`. + + ## Database Structure TODO: Generate SQL statements to create tables and indexes. @@ -46,9 +151,4 @@ MySQL Table Overview - **Invalid_url_reports** - A table that stores a list of invalid URLs (i.e. Pinging the URL returns an HTTP >= 400 status code). This table is populated by a job that is kicked off by Cron each weekend. - **Skip_checks** - A table that stores the domains of URLs were a contractually not allowed to run the invalid URLs check against -Please refer to the schema file `/app/db/schema.rb` to see the structure of the database. The schema file was generated by Active Record while connected to the actual database for the service. It is currently used to initialize the database for testing with SQLite, so it should be a good reference, though unfortunately it doesn't include the indexes and constraints. Please contact one of the developers for more information. - -## Developer Contacts - -- Lam Pham (lam.pham@ucop.edu) -- Charlie Collett (charlie.collett@ucop.edu) \ No newline at end of file +Please refer to the schema file `/app/db/schema.rb` to see the structure of the database. The schema file was generated by Active Record while connected to the actual database for the service. It is currently used to initialize the database for testing with SQLite, so it should be a good reference, though unfortunately it doesn't include the indexes and constraints. Please contact one of the developers for more information. \ No newline at end of file diff --git a/app/test/test_helper.rb b/app/test/test_helper.rb index 1dfcf72..6354617 100755 --- a/app/test/test_helper.rb +++ b/app/test/test_helper.rb @@ -16,10 +16,26 @@ Capybara.app = PidApp -chrome_binary_path = '/opt/google/chrome/chrome-linux64/chrome' -chrome_driver_path = '/usr/local/bin/chromedriver-linux64/chromedriver' +# In test_helper.rb + +if RUBY_PLATFORM.include?('linux') || ENV['DOCKER'] == 'true' || File.exist?('/.dockerenv') + # When running in a Docker container (or on Linux) + chrome_binary_path = '/opt/google/chrome/chrome-linux64/chrome' + chrome_driver_path = '/usr/local/bin/chromedriver-linux64/chromedriver' +elsif RUBY_PLATFORM.include?('darwin') + # When running on a Mac + chrome_binary_path = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' + chrome_driver_path = '/opt/homebrew/bin/chromedriver' +else + raise "Unsupported platform for Chrome driver configuration." +end + Selenium::WebDriver::Chrome::Service.driver_path = chrome_driver_path +# Optionally, if you need to set the Chrome binary: +Selenium::WebDriver::Chrome.path = chrome_binary_path + + Capybara.register_driver :chrome_headless do |app| options = Selenium::WebDriver::Chrome::Options.new options.add_argument('--headless') diff --git a/docker-compose.yml b/docker-compose.yml index a7979c4..25b9782 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,5 @@ services: test: - platform: linux/amd64 build: context: ./app # Build context is the app directory dockerfile: Dockerfile.test # Use Dockerfile.test @@ -10,7 +9,6 @@ services: - .env # Load environment variables from the .env file app: - platform: linux/amd64 build: context: ./app # Build context is the app directory dockerfile: Dockerfile.app # Use Dockerfile.app diff --git a/redis/Dockerfile b/redis/Dockerfile index d4d190a..1356ed1 100644 --- a/redis/Dockerfile +++ b/redis/Dockerfile @@ -1,4 +1,7 @@ -FROM redis:latest +ARG TARGET_PLATFORM=linux/amd64 +ARG IMAGE_TAG=latest + +FROM --platform=${TARGET_PLATFORM} redis:${IMAGE_TAG} COPY redis.conf /usr/local/etc/redis/redis.conf