Merge branch 'master' into pl/add-mainlang-defaults-refactored

.dockerignore

@@ -0,0 +1,11 @@
+# Git
+.git
+.gitignore
+# Logs
+log/*
+# Temp files
+tmp/*
+# Editor temp files
+*.swp
+*.swo
+test/solr

.github/workflows/ruby-unit-tests.yml

@@ -10,11 +10,14 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Set up solr configsets
-      run: test/solr/generate_ncbo_configsets.sh
+      working-directory: ./test
+      run: solr/generate_ncbo_configsets.sh
     - name: create config.rb file
       run: cp config/config.rb.sample config/config.rb
     - name: Build docker-compose
+      working-directory: ./test
       run: docker-compose build
     - name: Run unit tests
-      run: docker-compose up --exit-code-from unit-test
+      working-directory: ./test
+      run: docker-compose run ld-unit-test wait-for-it ld-solr-ut:8983 -- bundle exec rake test TESTOPTS='-v'
 

Dockerfile

@@ -1,12 +1,12 @@
 FROM ruby:2.6
 
-RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends vim openjdk-11-jre-headless raptor2-utils
+RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends openjdk-11-jre-headless raptor2-utils wait-for-it
 # The Gemfile Caching Trick
 RUN mkdir -p /srv/ontoportal/ontologies_linked_data
-COPY Gemfile* /srv/ontoportal/ontologies_linked_data/
+COPY Gemfile* *.gemspec /srv/ontoportal/ontologies_linked_data/
 WORKDIR /srv/ontoportal/ontologies_linked_data
 RUN gem install bundler -v "$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1)"
-RUN bundle install --binstubs
+RUN bundle install
 
 COPY . /srv/ontoportal/ontologies_linked_data
 

config/config.rb.sample

@@ -6,7 +6,7 @@ LinkedData.config do |config|
   config.repository_folder = "./test/data/ontology_files/repo"
   config.rest_url_prefix   = "http://data.bioontology.org/"
   config.enable_security   = false
-
+  config.java_max_heap_size         = '10240M'
   #PURL server config parameters
   config.enable_purl            = false
   config.purl_host              = "purl.bioontology.org"

lib/ontologies_linked_data/config/config.rb

@@ -15,67 +15,69 @@ def config(&block)
     overide_connect_goo = false
 
     # Set defaults
-    @settings.goo_backend_name              ||= "4store"
+    @settings.goo_backend_name              ||= '4store'
     @settings.goo_port                      ||= 9000
-    @settings.goo_host                      ||= "localhost"
-    @settings.goo_path_query                ||= "/sparql/"
-    @settings.goo_path_data                 ||= "/data/"
-    @settings.goo_path_update               ||= "/update/"
-    @settings.search_server_url             ||= "http://localhost:8983/solr/term_search_core1"
-    @settings.property_search_server_url    ||= "http://localhost:8983/solr/prop_search_core1"
-    @settings.repository_folder             ||= "./test/data/ontology_files/repo"
-    @settings.rest_url_prefix               ||= "http://data.bioontology.org/"
+    @settings.goo_host                      ||= 'localhost'
+    @settings.goo_path_query                ||= '/sparql/'
+    @settings.goo_path_data                 ||= '/data/'
+    @settings.goo_path_update               ||= '/update/'
+    @settings.search_server_url             ||= 'http://localhost:8983/solr/term_search_core1'
+    @settings.property_search_server_url    ||= 'http://localhost:8983/solr/prop_search_core1'
+    @settings.repository_folder             ||= './test/data/ontology_files/repo'
+    @settings.rest_url_prefix                ||= 'http://data.bioontology.org/'
     @settings.enable_security               ||= false
     @settings.enable_slices                 ||= false
 
+    # Java/JVM options
+    @settings.java_max_heap_size            ||= '10240M'
     ### these params should be not ussed any more
     # removed so that dependencies shout
     #
     # @settings.redis_host                    ||= "localhost"
     # @settings.redis_port                    ||= 6379
     # ###
 
-    @settings.ui_host                       ||= "bioportal.bioontology.org"
+    @settings.ui_host                       ||= 'bioportal.bioontology.org'
     @settings.replace_url_prefix            ||= false
-    @settings.id_url_prefix                 ||= "http://data.bioontology.org/"
+    @settings.id_url_prefix                 ||= 'http://data.bioontology.org/'
     @settings.queries_debug                 ||= false
     @settings.enable_monitoring             ||= false
-    @settings.cube_host                     ||= "localhost"
+    @settings.cube_host                     ||= 'localhost'
     @settings.cube_port                     ||= 1180
 
     # Caching http
     @settings.enable_http_cache             ||= false
-    @settings.http_redis_host               ||= "localhost"
+    @settings.http_redis_host               ||= 'localhost'
     @settings.http_redis_port               ||= 6379
 
     #Caching goo
-    @settings.goo_redis_host                ||= "localhost"
+    @settings.goo_redis_host                ||= 'localhost'
     @settings.goo_redis_port                ||= 6379
 
     #Ontology Analytics Redis
-    @settings.ontology_analytics_redis_host ||= "localhost"
+    @settings.ontology_analytics_redis_host ||= 'localhost'
     @settings.ontology_analytics_redis_port ||= 6379
 
     # PURL server config parameters
     @settings.enable_purl                   ||= false
-    @settings.purl_host                     ||= "purl.bioontology.org"
+    @settings.purl_host                     ||= 'purl.bioontology.org'
     @settings.purl_port                     ||= 80
-    @settings.purl_username                 ||= ""
-    @settings.purl_password                 ||= ""
-    @settings.purl_maintainers              ||= ""
-    @settings.purl_target_url_prefix        ||= "http://bioportal.bioontology.org"
+    @settings.purl_username                 ||= ''
+    @settings.purl_password                 ||= ''
+    @settings.purl_maintainers              ||= ''
+    @settings.purl_target_url_prefix        ||= 'http://bioportal.bioontology.org'
 
     # Email settings
     @settings.enable_notifications          ||= false
-    @settings.email_sender                  ||= "admin@example.org" # Default sender for emails
-    @settings.email_override                ||= "test.email@example.org" # By default, all email gets sent here. Disable with email_override_disable.
+    @settings.email_sender                  ||= 'admin@example.org' # Default sender for emails
+    @settings.email_override                ||= 'test.email@example.org' # By default, all email gets sent here. Disable with email_override_disable.
     @settings.email_disable_override        ||= false
-    @settings.smtp_host                     ||= "localhost"
+    @settings.smtp_host                     ||= 'localhost'
     @settings.smtp_port                     ||= 25
-    @settings.smtp_user                     ||= "user"
-    @settings.smtp_password                 ||= "password"
+    @settings.smtp_user                     ||= 'user'
+    @settings.smtp_password                 ||= 'password'
     @settings.smtp_auth_type                ||= :none # :none, :plain, :login, :cram_md5
-    @settings.smtp_domain                   ||= "localhost.localhost"
+    @settings.smtp_domain                   ||= 'localhost.localhost'
     @settings.enable_starttls_auto          ||= false # set to true for use with gmail
 
     # number of times to retry a query when empty records are returned
@@ -89,19 +91,19 @@ def config(&block)
 
     unless @settings.redis_host.nil?
       puts "Error: 'redis_host' is not a valid conf parameter."
-      puts "        Redis databases were split into multiple hosts (09/22/13)."
-      raise Exception, "redis_host is not a valid conf parameter."
+      puts '        Redis databases were split into multiple hosts (09/22/13).'
+      raise Exception, 'redis_host is not a valid conf parameter.'
     end
 
     # Check to make sure url prefix has trailing slash
-    @settings.rest_url_prefix = @settings.rest_url_prefix + "/" unless @settings.rest_url_prefix[-1].eql?("/")
+    @settings.rest_url_prefix = @settings.rest_url_prefix + '/' unless @settings.rest_url_prefix[-1].eql?('/')
 
     puts "(LD) >> Using rdf store #{@settings.goo_host}:#{@settings.goo_port}"
     puts "(LD) >> Using term search server at #{@settings.search_server_url}"
     puts "(LD) >> Using property search server at #{@settings.property_search_server_url}"
-    puts "(LD) >> Using HTTP Redis instance at "+
+    puts '(LD) >> Using HTTP Redis instance at '+
             "#{@settings.http_redis_host}:#{@settings.http_redis_port}"
-    puts "(LD) >> Using Goo Redis instance at "+
+    puts '(LD) >> Using Goo Redis instance at '+
             "#{@settings.goo_redis_host}:#{@settings.goo_redis_port}"
 
     connect_goo unless overide_connect_goo

lib/ontologies_linked_data/diff/bubastis_diff.rb

@@ -43,6 +43,7 @@ def initialize(input_fileOld, input_fileNew)
         @input_fileNew = input_fileNew
         @output_repo = File.expand_path(@input_fileNew).gsub(File.basename(@input_fileNew),'')
         @file_diff_path = nil
+        @java_heap_size = LinkedData.settings.java_max_heap_size
       end
 
       def setup_environment
@@ -83,7 +84,7 @@ def call_bubastis_java_cmd
         end
         errors_log = File.join([@output_repo, "bubastis_diff_errors.log"])
         File.delete errors_log if File.exist? errors_log
-        java_cmd = "java -DentityExpansionLimit=1500000 -Xmx5120M -jar #{@bubastis_jar_path} #{options.join(' ')}"
+        java_cmd = "java -DentityExpansionLimit=1500000 -Xmx#{@java_heap_size} -jar #{@bubastis_jar_path} #{options.join(' ')}"
         Diff.logger.info("Java call [#{java_cmd}]")
         stdout,stderr,status = Open3.capture3(java_cmd)
         if not status.success?

lib/ontologies_linked_data/parser/owlapi.rb

@@ -13,14 +13,15 @@ class RDFFileNotGeneratedException < Parser::ParserException
 
     class OWLAPICommand
       def initialize(input_file, output_repo, opts = {})
-        @owlapi_wrapper_jar_path = LinkedData.bindir + "/owlapi-wrapper-1.3.7.jar"
+        @owlapi_wrapper_jar_path = LinkedData.bindir + "/owlapi-wrapper-1.3.8.jar"
         @input_file = input_file
         @output_repo = output_repo
         @master_file = opts[:master_file]
         @logger = opts[:logger] || Parser.logger
         @file_triples_path = nil
         @missing_imports = nil
         @reasoning = true
+        @java_heap_size = LinkedData.settings.java_max_heap_size
       end
 
       def setup_environment
@@ -71,7 +72,7 @@ def call_owlapi_java_command
         if File.exist? errors_log
           File.delete errors_log
         end
-        command_call = "java -DentityExpansionLimit=2500000 -Xmx10240M -jar #{@owlapi_wrapper_jar_path} #{options}"
+        command_call = "java -DentityExpansionLimit=2500000 -Xmx#{@java_heap_size} -jar #{@owlapi_wrapper_jar_path} #{options}"
         @logger.info("Java call [#{command_call}]")
         Open3.popen3(command_call) do |i,o,e,w|
           i.close

docker-compose.yml → test/docker-compose.yml

@@ -1,50 +1,42 @@
 version: '3'
 
 services:
-  unit-test:
-    # volumes:
-    #  - .:/srv/ontoportal/ontologies_linked_data
-    build: .
-    #command: "bundle exec rake test TESTOPTS='-v' TEST='./test/parser/test_owl_api_command.rb'"
+  ld-unit-test:
+    build: ../.
     environment:
+      - OVERRIDE_CONNECT_GOO=true
       - GOO_BACKEND_NAME=4store
       - GOO_PORT=9000
-      - GOO_HOST=4store
+      - GOO_HOST=4store-ut
       - GOO_PATH_QUERY=/sparql/
       - GOO_PATH_DATA=/data/
       - GOO_PATH_UPDATE=/update/
-      - OVERRIDE_CONNECT_GOO=true
-      - REDIS_HOST=redis
+      - REDIS_HOST=redis-ut
       - REDIS_PORT=6379
-      - SOLR_HOST=solr
-    command: "bundle exec rake test TESTOPTS='-v'"
+      - SOLR_HOST=ld-solr-ut
+    command: "rake test TESTOPTS='-v'"
     depends_on:
-      - solr
-      - redis
-      - 4store
+      - ld-solr-ut
+      - redis-ut
+      - 4store-ut
 
-  redis:
+  redis-ut:
     image: redis
 
-  4store:
+  4store-ut:
     image: bde2020/4store
-    ports:
-        - "9000:9000"
     command: >
       bash -c "4s-backend-setup --segments 4 ontoportal_kb
       && 4s-backend ontoportal_kb
       && 4s-httpd -D -s-1 -p 9000 ontoportal_kb"
 
-  solr:
-    image: solr:8.8
+  ld-solr-ut:
+    image: solr:8
     volumes:
-      - ./test/solr/configsets:/configsets:ro
-    ports:
-      - "8983:8983"
+      - ./solr/configsets:/configsets:ro
+    #ports:
+    #  - "8983:8983"
     command: >
       bash -c "precreate-core term_search_core1 /configsets/term_search
       && precreate-core prop_search_core1 /configsets/property_search
       && solr-foreground"
-
-#  mgrep:
-#    image: ontoportal/mgrep-ncbo:0.1

test/run-unit-tests.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# sample script to run unit tests with docker
+
+# add config for unit testing
+[ -f ../config/config.rb ] || cp ../config/config.rb.sample ../config/config.rb
+
+#generate solr configsets for solr container
+solr/generate_ncbo_configsets.sh
+
+# build docker containers
+docker-compose build
+
+#docker-compose up --exit-code-from unit-test
+docker-compose run --rm ld-unit-test wait-for-it ld-solr-ut:8983 -- bundle exec rake test TESTOPTS='-v'
+

test/solr/generate_ncbo_configsets.sh

@@ -2,18 +2,22 @@
 # generates solr configsets by merging _default configset with config files in config/solr
 # _default is copied from sorl distribuion solr-8.10.1/server/solr/configsets/_default/
 
+#cd solr/configsets
+ld_config='../config/solr'
+[ -d solr/configsets/property_search ] && rm -Rf solr/configsets/property_search
+[ -d solr/configsets/term_search ] && rm -Rf solr/configsets/term_search
+if [[ ! -d ${ld_config}/property_search ]]; then
+  echo 'cant find ld solr config sets'
+  exit 1
+fi
+if [[ ! -d solr/configsets/_default/conf ]]; then
+  echo 'cant find default solr configset' 
+  exit 1
+fi
+mkdir -p solr/configsets/property_search/conf
+mkdir -p solr/configsets/term_search/conf
+cp -a solr/configsets/_default/conf/* solr/configsets/property_search/conf/
+cp -a solr/configsets/_default/conf/* solr/configsets/term_search/conf/
+cp -a $ld_config/property_search/* solr/configsets/property_search/conf
+cp -a $ld_config/term_search/* solr/configsets/term_search/conf
 
-pushd test/solr/configsets
-ld_config='../../../config/solr/'
-ls -l $ld_config
-pwd
-[ -d property_search ] && rm -Rf property_search
-[ -d term_search ] && rm -Rf property_search
-[ -d $ld_config/property_search ] || echo "cant find ontologies_linked_data project" 
-mkdir -p property_search/conf
-mkdir -p term_search/conf
-cp -a _default/conf/* property_search/conf/
-cp -a _default/conf/* term_search/conf/
-cp -a $ld_config/property_search/* property_search/conf
-cp -a $ld_config/term_search/* term_search/conf
-popd